diff --git a/.github/workflows/startos-iso.yaml b/.github/workflows/startos-iso.yaml index 010c79594..bc0fb626e 100644 --- a/.github/workflows/startos-iso.yaml +++ b/.github/workflows/startos-iso.yaml @@ -45,7 +45,7 @@ on: - next/* env: - NODEJS_VERSION: "18.15.0" + NODEJS_VERSION: "20.16.0" ENVIRONMENT: '${{ fromJson(format(''["{0}", ""]'', github.event.inputs.environment || ''dev''))[github.event.inputs.environment == ''NONE''] }}' jobs: @@ -75,6 +75,11 @@ jobs: with: submodules: recursive + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - uses: actions/setup-node@v4 with: node-version: ${{ env.NODEJS_VERSION }} @@ -148,6 +153,11 @@ jobs: with: submodules: recursive + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install dependencies run: | sudo apt-get update diff --git a/Makefile b/Makefile index e63419898..113b18f98 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ format: test: | test-core test-sdk test-container-runtime test-core: $(CORE_SRC) $(ENVIRONMENT_FILE) - cd core && cargo build --features=test && cargo test --features=test + ./core/run-tests.sh test-sdk: $(shell git ls-files sdk) sdk/lib/osBindings cd sdk && make test @@ -231,7 +231,7 @@ sdk/lib/osBindings: core/startos/bindings core/startos/bindings: $(shell git ls-files core) $(ENVIRONMENT_FILE) rm -rf core/startos/bindings - (cd core/ && cargo test --features=test 'export_bindings_') + ./core/build-ts.sh touch core/startos/bindings sdk/dist: $(shell git ls-files sdk) sdk/lib/osBindings diff --git a/build-cargo-dep.sh b/build-cargo-dep.sh index c32e4f8ae..922dfbdf9 100755 --- a/build-cargo-dep.sh +++ b/build-cargo-dep.sh @@ -18,7 +18,7 @@ if [ -z "$ARCH" ]; then fi DOCKER_PLATFORM="linux/${ARCH}" -if [ "$ARCH" = aarch64 ]; then +if [ "$ARCH" = aarch64 ] || [ "$ARCH" = arm64 ]; then DOCKER_PLATFORM="linux/arm64" elif [ "$ARCH" = x86_64 ]; then DOCKER_PLATFORM="linux/amd64" diff --git a/build/dpkg-deps/depends b/build/dpkg-deps/depends index 3ccaee4d6..cd29714b2 100644 --- a/build/dpkg-deps/depends +++ b/build/dpkg-deps/depends @@ -47,6 +47,7 @@ smartmontools socat sqlite3 squashfs-tools +squashfs-tools-ng sudo systemd systemd-resolved diff --git a/container-runtime/package-lock.json b/container-runtime/package-lock.json index e2af8830d..73d95bdae 100644 --- a/container-runtime/package-lock.json +++ b/container-runtime/package-lock.json @@ -20,6 +20,7 @@ "node-fetch": "^3.1.0", "ts-matches": "^5.5.1", "tslib": "^2.5.3", + "tslog": "^4.9.3", "typescript": "^5.1.3", "yaml": "^2.3.1" }, @@ -5527,6 +5528,17 @@ "version": "2.6.3", "license": "0BSD" }, + "node_modules/tslog": { + "version": "4.9.3", + "resolved": "https://registry.npmjs.org/tslog/-/tslog-4.9.3.tgz", + "integrity": "sha512-oDWuGVONxhVEBtschLf2cs/Jy8i7h1T+CpdkTNWQgdAF7DhRo2G8vMCgILKe7ojdEkLhICWgI1LYSSKaJsRgcw==", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/fullstack-build/tslog?sponsor=1" + } + }, "node_modules/type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", diff --git a/container-runtime/src/Adapters/EffectCreator.ts b/container-runtime/src/Adapters/EffectCreator.ts index 0ef299151..e0390b1e1 100644 --- a/container-runtime/src/Adapters/EffectCreator.ts +++ b/container-runtime/src/Adapters/EffectCreator.ts @@ -1,4 +1,4 @@ -import { types as T } from "@start9labs/start-sdk" +import { types as T, utils } from "@start9labs/start-sdk" import * as net from "net" import { object, string, number, literals, some, unknown } from "ts-matches" import { Effects } from "../Models/Effects" @@ -40,7 +40,7 @@ export type EffectContext = { const rpcRoundFor = (procedureId: string | null) => - ( + ( method: K, params: Record, ) => { @@ -65,7 +65,10 @@ const rpcRoundFor = ) if (testRpcError(res)) { let message = res.error.message - console.error("Error in host RPC:", { method, params }) + console.error( + "Error in host RPC:", + utils.asError({ method, params }), + ) if (string.test(res.error.data)) { message += ": " + res.error.data console.error(`Details: ${res.error.data}`) @@ -107,65 +110,65 @@ function makeEffects(context: EffectContext): Effects { }) as ReturnType }, clearBindings(...[]: Parameters) { - return rpcRound("clearBindings", {}) as ReturnType< + return rpcRound("clear-bindings", {}) as ReturnType< T.Effects["clearBindings"] > }, clearServiceInterfaces( ...[]: Parameters ) { - return rpcRound("clearServiceInterfaces", {}) as ReturnType< + return rpcRound("clear-service-interfaces", {}) as ReturnType< T.Effects["clearServiceInterfaces"] > }, getInstalledPackages(...[]: Parameters) { - return rpcRound("getInstalledPackages", {}) as ReturnType< + return rpcRound("get-installed-packages", {}) as ReturnType< T.Effects["getInstalledPackages"] > }, - createOverlayedImage(options: { - imageId: string - }): Promise<[string, string]> { - return rpcRound("createOverlayedImage", options) as ReturnType< - T.Effects["createOverlayedImage"] - > - }, - destroyOverlayedImage(options: { guid: string }): Promise { - return rpcRound("destroyOverlayedImage", options) as ReturnType< - T.Effects["destroyOverlayedImage"] - > + subcontainer: { + createFs(options: { imageId: string }) { + return rpcRound("subcontainer.create-fs", options) as ReturnType< + T.Effects["subcontainer"]["createFs"] + > + }, + destroyFs(options: { guid: string }): Promise { + return rpcRound("subcontainer.destroy-fs", options) as ReturnType< + T.Effects["subcontainer"]["destroyFs"] + > + }, }, executeAction(...[options]: Parameters) { - return rpcRound("executeAction", options) as ReturnType< + return rpcRound("execute-action", options) as ReturnType< T.Effects["executeAction"] > }, exportAction(...[options]: Parameters) { - return rpcRound("exportAction", options) as ReturnType< + return rpcRound("export-action", options) as ReturnType< T.Effects["exportAction"] > }, exportServiceInterface: (( ...[options]: Parameters ) => { - return rpcRound("exportServiceInterface", options) as ReturnType< + return rpcRound("export-service-interface", options) as ReturnType< T.Effects["exportServiceInterface"] > }) as Effects["exportServiceInterface"], exposeForDependents( ...[options]: Parameters ) { - return rpcRound("exposeForDependents", options) as ReturnType< + return rpcRound("expose-for-dependents", options) as ReturnType< T.Effects["exposeForDependents"] > }, getConfigured(...[]: Parameters) { - return rpcRound("getConfigured", {}) as ReturnType< + return rpcRound("get-configured", {}) as ReturnType< T.Effects["getConfigured"] > }, getContainerIp(...[]: Parameters) { - return rpcRound("getContainerIp", {}) as ReturnType< + return rpcRound("get-container-ip", {}) as ReturnType< T.Effects["getContainerIp"] > }, @@ -174,21 +177,21 @@ function makeEffects(context: EffectContext): Effects { ...allOptions, callback: context.callbacks?.addCallback(allOptions.callback) || null, } - return rpcRound("getHostInfo", options) as ReturnType< + return rpcRound("get-host-info", options) as ReturnType< T.Effects["getHostInfo"] > as any }) as Effects["getHostInfo"], getServiceInterface( ...[options]: Parameters ) { - return rpcRound("getServiceInterface", { + return rpcRound("get-service-interface", { ...options, callback: context.callbacks?.addCallback(options.callback) || null, }) as ReturnType }, getPrimaryUrl(...[options]: Parameters) { - return rpcRound("getPrimaryUrl", { + return rpcRound("get-primary-url", { ...options, callback: context.callbacks?.addCallback(options.callback) || null, }) as ReturnType @@ -196,22 +199,22 @@ function makeEffects(context: EffectContext): Effects { getServicePortForward( ...[options]: Parameters ) { - return rpcRound("getServicePortForward", options) as ReturnType< + return rpcRound("get-service-port-forward", options) as ReturnType< T.Effects["getServicePortForward"] > }, getSslCertificate(options: Parameters[0]) { - return rpcRound("getSslCertificate", options) as ReturnType< + return rpcRound("get-ssl-certificate", options) as ReturnType< T.Effects["getSslCertificate"] > }, getSslKey(options: Parameters[0]) { - return rpcRound("getSslKey", options) as ReturnType< + return rpcRound("get-ssl-key", options) as ReturnType< T.Effects["getSslKey"] > }, getSystemSmtp(...[options]: Parameters) { - return rpcRound("getSystemSmtp", { + return rpcRound("get-system-smtp", { ...options, callback: context.callbacks?.addCallback(options.callback) || null, }) as ReturnType @@ -219,7 +222,7 @@ function makeEffects(context: EffectContext): Effects { listServiceInterfaces( ...[options]: Parameters ) { - return rpcRound("listServiceInterfaces", { + return rpcRound("list-service-interfaces", { ...options, callback: context.callbacks?.addCallback(options.callback) || null, }) as ReturnType @@ -228,7 +231,7 @@ function makeEffects(context: EffectContext): Effects { return rpcRound("mount", options) as ReturnType }, clearActions(...[]: Parameters) { - return rpcRound("clearActions", {}) as ReturnType< + return rpcRound("clear-actions", {}) as ReturnType< T.Effects["clearActions"] > }, @@ -236,37 +239,39 @@ function makeEffects(context: EffectContext): Effects { return rpcRound("restart", {}) as ReturnType }, setConfigured(...[configured]: Parameters) { - return rpcRound("setConfigured", { configured }) as ReturnType< + return rpcRound("set-configured", { configured }) as ReturnType< T.Effects["setConfigured"] > }, setDependencies( dependencies: Parameters[0], ): ReturnType { - return rpcRound("setDependencies", dependencies) as ReturnType< + return rpcRound("set-dependencies", dependencies) as ReturnType< T.Effects["setDependencies"] > }, checkDependencies( options: Parameters[0], ): ReturnType { - return rpcRound("checkDependencies", options) as ReturnType< + return rpcRound("check-dependencies", options) as ReturnType< T.Effects["checkDependencies"] > }, getDependencies(): ReturnType { - return rpcRound("getDependencies", {}) as ReturnType< + return rpcRound("get-dependencies", {}) as ReturnType< T.Effects["getDependencies"] > }, setHealth(...[options]: Parameters) { - return rpcRound("setHealth", options) as ReturnType< + return rpcRound("set-health", options) as ReturnType< T.Effects["setHealth"] > }, setMainStatus(o: { status: "running" | "stopped" }): Promise { - return rpcRound("setMainStatus", o) as ReturnType + return rpcRound("set-main-status", o) as ReturnType< + T.Effects["setHealth"] + > }, shutdown(...[]: Parameters) { @@ -274,13 +279,23 @@ function makeEffects(context: EffectContext): Effects { }, store: { get: async (options: any) => - rpcRound("getStore", { + rpcRound("store.get", { ...options, callback: context.callbacks?.addCallback(options.callback) || null, }) as any, set: async (options: any) => - rpcRound("setStore", options) as ReturnType, + rpcRound("store.set", options) as ReturnType, } as T.Effects["store"], + getDataVersion() { + return rpcRound("get-data-version", {}) as ReturnType< + T.Effects["getDataVersion"] + > + }, + setDataVersion(...[options]: Parameters) { + return rpcRound("set-data-version", options) as ReturnType< + T.Effects["setDataVersion"] + > + }, } return self } diff --git a/container-runtime/src/Adapters/RpcListener.ts b/container-runtime/src/Adapters/RpcListener.ts index 28f578149..860f1c066 100644 --- a/container-runtime/src/Adapters/RpcListener.ts +++ b/container-runtime/src/Adapters/RpcListener.ts @@ -19,7 +19,7 @@ import * as fs from "fs" import { CallbackHolder } from "../Models/CallbackHolder" import { AllGetDependencies } from "../Interfaces/AllGetDependencies" -import { jsonPath } from "../Models/JsonPath" +import { jsonPath, unNestPath } from "../Models/JsonPath" import { RunningMain, System } from "../Interfaces/System" import { MakeMainEffects, @@ -52,6 +52,8 @@ const SOCKET_PARENT = "/media/startos/rpc" const SOCKET_PATH = "/media/startos/rpc/service.sock" const jsonrpc = "2.0" as const +const isResult = object({ result: any }).test + const idType = some(string, number, literal(null)) type IdType = null | string | number const runType = object({ @@ -64,7 +66,7 @@ const runType = object({ input: any, timeout: number, }, - ["timeout", "input"], + ["timeout"], ), }) const sandboxRunType = object({ @@ -77,7 +79,7 @@ const sandboxRunType = object({ input: any, timeout: number, }, - ["timeout", "input"], + ["timeout"], ), }) const callbackType = object({ @@ -226,27 +228,25 @@ export class RpcListener { const system = this.system const procedure = jsonPath.unsafeCast(params.procedure) const effects = this.getDependencies.makeProcedureEffects()(params.id) - return handleRpc( - id, - system.execute(effects, { - procedure, - input: params.input, - timeout: params.timeout, - }), - ) + const input = params.input + const timeout = params.timeout + const result = getResult(procedure, system, effects, timeout, input) + + return handleRpc(id, result) }) .when(sandboxRunType, async ({ id, params }) => { const system = this.system const procedure = jsonPath.unsafeCast(params.procedure) const effects = this.makeProcedureEffects(params.id) - return handleRpc( - id, - system.sandbox(effects, { - procedure, - input: params.input, - timeout: params.timeout, - }), + const result = getResult( + procedure, + system, + effects, + params.input, + params.input, ) + + return handleRpc(id, result) }) .when(callbackType, async ({ params: { callback, args } }) => { this.system.callCallback(callback, args) @@ -280,7 +280,7 @@ export class RpcListener { (async () => { if (!this._system) { const system = await this.getDependencies.system() - await system.init() + await system.containerInit() this._system = system } })().then((result) => ({ result })), @@ -342,3 +342,97 @@ export class RpcListener { }) } } +function getResult( + procedure: typeof jsonPath._TYPE, + system: System, + effects: T.Effects, + timeout: number | undefined, + input: any, +) { + const ensureResultTypeShape = ( + result: + | void + | T.ConfigRes + | T.PropertiesReturn + | T.ActionMetadata[] + | T.ActionResult, + ): { result: any } => { + if (isResult(result)) return result + return { result } + } + return (async () => { + switch (procedure) { + case "/backup/create": + return system.createBackup(effects, timeout || null) + case "/backup/restore": + return system.restoreBackup(effects, timeout || null) + case "/config/get": + return system.getConfig(effects, timeout || null) + case "/config/set": + return system.setConfig(effects, input, timeout || null) + case "/properties": + return system.properties(effects, timeout || null) + case "/actions/metadata": + return system.actionsMetadata(effects) + case "/init": + return system.packageInit( + effects, + string.optional().unsafeCast(input), + timeout || null, + ) + case "/uninit": + return system.packageUninit( + effects, + string.optional().unsafeCast(input), + timeout || null, + ) + default: + const procedures = unNestPath(procedure) + switch (true) { + case procedures[1] === "actions" && procedures[3] === "get": + return system.action(effects, procedures[2], input, timeout || null) + case procedures[1] === "actions" && procedures[3] === "run": + return system.action(effects, procedures[2], input, timeout || null) + case procedures[1] === "dependencies" && procedures[3] === "query": + return system.dependenciesAutoconfig( + effects, + procedures[2], + input, + timeout || null, + ) + + case procedures[1] === "dependencies" && procedures[3] === "update": + return system.dependenciesAutoconfig( + effects, + procedures[2], + input, + timeout || null, + ) + } + } + })().then(ensureResultTypeShape, (error) => + matches(error) + .when( + object( + { + error: string, + code: number, + }, + ["code"], + { code: 0 }, + ), + (error) => ({ + error: { + code: error.code, + message: error.error, + }, + }), + ) + .defaultToLazy(() => ({ + error: { + code: 0, + message: String(error), + }, + })), + ) +} diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts index 012a70eee..805f9b531 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts @@ -1,40 +1,60 @@ import * as fs from "fs/promises" import * as cp from "child_process" -import { Overlay, types as T } from "@start9labs/start-sdk" +import { SubContainer, types as T } from "@start9labs/start-sdk" import { promisify } from "util" import { DockerProcedure, VolumeId } from "../../../Models/DockerProcedure" import { Volume } from "./matchVolume" +import { + CommandOptions, + ExecOptions, + ExecSpawnable, +} from "@start9labs/start-sdk/cjs/lib/util/SubContainer" export const exec = promisify(cp.exec) export const execFile = promisify(cp.execFile) export class DockerProcedureContainer { - private constructor(readonly overlay: Overlay) {} - // static async readonlyOf(data: DockerProcedure) { - // return DockerProcedureContainer.of(data, ["-o", "ro"]) - // } + private constructor(private readonly subcontainer: ExecSpawnable) {} + static async of( effects: T.Effects, packageId: string, data: DockerProcedure, volumes: { [id: VolumeId]: Volume }, + options: { subcontainer?: ExecSpawnable } = {}, ) { - const overlay = await Overlay.of(effects, { id: data.image }) + const subcontainer = + options?.subcontainer ?? + (await DockerProcedureContainer.createSubContainer( + effects, + packageId, + data, + volumes, + )) + return new DockerProcedureContainer(subcontainer) + } + static async createSubContainer( + effects: T.Effects, + packageId: string, + data: DockerProcedure, + volumes: { [id: VolumeId]: Volume }, + ) { + const subcontainer = await SubContainer.of(effects, { id: data.image }) if (data.mounts) { const mounts = data.mounts for (const mount in mounts) { const path = mounts[mount].startsWith("/") - ? `${overlay.rootfs}${mounts[mount]}` - : `${overlay.rootfs}/${mounts[mount]}` + ? `${subcontainer.rootfs}${mounts[mount]}` + : `${subcontainer.rootfs}/${mounts[mount]}` await fs.mkdir(path, { recursive: true }) const volumeMount = volumes[mount] if (volumeMount.type === "data") { - await overlay.mount( + await subcontainer.mount( { type: "volume", id: mount, subpath: null, readonly: false }, mounts[mount], ) } else if (volumeMount.type === "assets") { - await overlay.mount( + await subcontainer.mount( { type: "assets", id: mount, subpath: null }, mounts[mount], ) @@ -80,25 +100,35 @@ export class DockerProcedureContainer { }) .catch(console.warn) } else if (volumeMount.type === "backup") { - await overlay.mount({ type: "backup", subpath: null }, mounts[mount]) + await subcontainer.mount( + { type: "backup", subpath: null }, + mounts[mount], + ) } } } - - return new DockerProcedureContainer(overlay) + return subcontainer } - async exec(commands: string[]) { + async exec( + commands: string[], + options?: CommandOptions & ExecOptions, + timeoutMs?: number | null, + ) { try { - return await this.overlay.exec(commands) + return await this.subcontainer.exec(commands, options, timeoutMs) } finally { - await this.overlay.destroy() + await this.subcontainer.destroy?.() } } - async execFail(commands: string[], timeoutMs: number | null) { + async execFail( + commands: string[], + timeoutMs: number | null, + options?: CommandOptions & ExecOptions, + ) { try { - const res = await this.overlay.exec(commands, {}, timeoutMs) + const res = await this.subcontainer.exec(commands, options, timeoutMs) if (res.exitCode !== 0) { const codeOrSignal = res.exitCode !== null @@ -110,11 +140,11 @@ export class DockerProcedureContainer { } return res } finally { - await this.overlay.destroy() + await this.subcontainer.destroy?.() } } async spawn(commands: string[]): Promise { - return await this.overlay.spawn(commands) + return await this.subcontainer.spawn(commands) } } diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts index f8f0a2d6e..e5aaacfdb 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts @@ -5,6 +5,8 @@ import { T, utils } from "@start9labs/start-sdk" import { Daemon } from "@start9labs/start-sdk/cjs/lib/mainFn/Daemon" import { Effects } from "../../../Models/Effects" import { off } from "node:process" +import { CommandController } from "@start9labs/start-sdk/cjs/lib/mainFn/CommandController" +import { asError } from "@start9labs/start-sdk/cjs/lib/util" const EMBASSY_HEALTH_INTERVAL = 15 * 1000 const EMBASSY_PROPERTIES_LOOP = 30 * 1000 @@ -14,6 +16,9 @@ const EMBASSY_PROPERTIES_LOOP = 30 * 1000 * Also, this has an ability to clean itself up too if need be. */ export class MainLoop { + get mainSubContainerHandle() { + return this.mainEvent?.daemon?.subContainerHandle + } private healthLoops?: { name: string interval: NodeJS.Timeout @@ -48,26 +53,32 @@ export class MainLoop { await this.setupInterfaces(effects) await effects.setMainStatus({ status: "running" }) const jsMain = (this.system.moduleCode as any)?.jsMain - const dockerProcedureContainer = await DockerProcedureContainer.of( - effects, - this.system.manifest.id, - this.system.manifest.main, - this.system.manifest.volumes, - ) if (jsMain) { throw new Error("Unreachable") } - const daemon = await Daemon.of()( - this.effects, - { id: this.system.manifest.main.image }, - currentCommand, - { - overlay: dockerProcedureContainer.overlay, - sigtermTimeout: utils.inMs( - this.system.manifest.main["sigterm-timeout"], - ), - }, - ) + const daemon = new Daemon(async () => { + const subcontainer = await DockerProcedureContainer.createSubContainer( + effects, + this.system.manifest.id, + this.system.manifest.main, + this.system.manifest.volumes, + ) + return CommandController.of()( + this.effects, + subcontainer, + currentCommand, + { + runAsInit: true, + env: { + TINI_SUBREAPER: "true", + }, + sigtermTimeout: utils.inMs( + this.system.manifest.main["sigterm-timeout"], + ), + }, + ) + }) + daemon.start() return { daemon, @@ -123,7 +134,9 @@ export class MainLoop { const main = await mainEvent delete this.mainEvent delete this.healthLoops - await main?.daemon.stop().catch((e) => console.error(e)) + await main?.daemon + .stop() + .catch((e) => console.error(`Main loop error`, utils.asError(e))) this.effects.setMainStatus({ status: "stopped" }) if (healthLoops) healthLoops.forEach((x) => clearInterval(x.interval)) } @@ -134,27 +147,42 @@ export class MainLoop { const start = Date.now() return Object.entries(manifest["health-checks"]).map( ([healthId, value]) => { + effects + .setHealth({ + id: healthId, + name: value.name, + result: "starting", + message: null, + }) + .catch((e) => console.error(asError(e))) const interval = setInterval(async () => { const actionProcedure = value const timeChanged = Date.now() - start if (actionProcedure.type === "docker") { - const container = await DockerProcedureContainer.of( - effects, - manifest.id, - actionProcedure, - manifest.volumes, + const subcontainer = actionProcedure.inject + ? this.mainSubContainerHandle + : undefined + // prettier-ignore + const container = + await DockerProcedureContainer.of( + effects, + manifest.id, + actionProcedure, + manifest.volumes, + { + subcontainer, + } + ) + const executed = await container.exec( + [actionProcedure.entrypoint, ...actionProcedure.args], + { input: JSON.stringify(timeChanged) }, ) - const executed = await container.exec([ - actionProcedure.entrypoint, - ...actionProcedure.args, - JSON.stringify(timeChanged), - ]) if (executed.exitCode === 0) { await effects.setHealth({ id: healthId, name: value.name, result: "success", - message: actionProcedure["success-message"], + message: actionProcedure["success-message"] ?? null, }) return } diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/__fixtures__/giteaManifest.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/__fixtures__/giteaManifest.ts new file mode 100644 index 000000000..1b3a8ba94 --- /dev/null +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/__fixtures__/giteaManifest.ts @@ -0,0 +1,123 @@ +export default { + "eos-version": "0.3.5.1", + id: "gitea", + "git-hash": "91fada3edf30357a2e75c281d32f8888c87fcc2d\n", + title: "Gitea", + version: "1.22.0", + description: { + short: "A painless self-hosted Git service.", + long: "Gitea is a community managed lightweight code hosting solution written in Go. It is published under the MIT license.\n", + }, + assets: { + license: "LICENSE", + instructions: "instructions.md", + icon: "icon.png", + "docker-images": null, + assets: null, + scripts: null, + }, + build: ["make"], + "release-notes": + "* Upstream code update\n* Fix deprecated config options\n* Full list of upstream changes available [here](https://github.com/go-gitea/gitea/compare/v1.21.8...v1.22.0)\n", + license: "MIT", + "wrapper-repo": "https://github.com/Start9Labs/gitea-startos", + "upstream-repo": "https://github.com/go-gitea/gitea", + "support-site": "https://docs.gitea.io/en-us/", + "marketing-site": "https://gitea.io/en-us/", + "donation-url": null, + alerts: { + install: null, + uninstall: null, + restore: null, + start: null, + stop: null, + }, + main: { + type: "docker", + image: "main", + system: false, + entrypoint: "/usr/local/bin/docker_entrypoint.sh", + args: [], + inject: false, + mounts: { main: "/data" }, + "io-format": null, + "sigterm-timeout": null, + "shm-size-mb": null, + "gpu-acceleration": false, + }, + "health-checks": { + "user-signups-off": { + name: "User Signups Off", + "success-message": null, + type: "script", + args: [], + timeout: null, + }, + web: { + name: "Web & Git HTTP Tor Interfaces", + "success-message": + "Gitea is ready to be visited in a web browser and git can be used with SSH over TOR.", + type: "script", + args: [], + timeout: null, + }, + }, + config: { + get: { type: "script", args: [] }, + set: { type: "script", args: [] }, + }, + properties: { type: "script", args: [] }, + volumes: { main: { type: "data" } }, + interfaces: { + main: { + name: "Web UI / Git HTTPS/SSH", + description: + "Port 80: Browser Interface and HTTP Git Interface / Port 22: Git SSH Interface", + "tor-config": { "port-mapping": { "22": "22", "80": "3000" } }, + "lan-config": { "443": { ssl: true, internal: 3000 } }, + ui: true, + protocols: ["tcp", "http", "ssh", "git"], + }, + }, + backup: { + create: { + type: "docker", + image: "compat", + system: true, + entrypoint: "compat", + args: ["duplicity", "create", "/mnt/backup", "/root/data"], + inject: false, + mounts: { BACKUP: "/mnt/backup", main: "/root/data" }, + "io-format": "yaml", + "sigterm-timeout": null, + "shm-size-mb": null, + "gpu-acceleration": false, + }, + restore: { + type: "docker", + image: "compat", + system: true, + entrypoint: "compat", + args: ["duplicity", "restore", "/mnt/backup", "/root/data"], + inject: false, + mounts: { BACKUP: "/mnt/backup", main: "/root/data" }, + "io-format": "yaml", + "sigterm-timeout": null, + "shm-size-mb": null, + "gpu-acceleration": false, + }, + }, + migrations: { + from: { "*": { type: "script", args: ["from"] } }, + to: { "*": { type: "script", args: ["to"] } }, + }, + actions: {}, + dependencies: {}, + containers: null, + replaces: [], + "hardware-requirements": { + device: {}, + ram: null, + arch: ["x86_64", "aarch64"], + }, +} diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/__fixtures__/nostrConfig2.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/__fixtures__/nostrConfig2.ts new file mode 100644 index 000000000..0cea482c7 --- /dev/null +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/__fixtures__/nostrConfig2.ts @@ -0,0 +1,187 @@ +export default { + "tor-address": { + name: "Tor Address", + description: "The Tor address for the websocket server.", + type: "pointer", + subtype: "package", + "package-id": "nostr", + target: "tor-address", + interface: "websocket", + }, + "lan-address": { + name: "Tor Address", + description: "The LAN address for the websocket server.", + type: "pointer", + subtype: "package", + "package-id": "nostr", + target: "lan-address", + interface: "websocket", + }, + "relay-type": { + type: "union", + name: "Relay Type", + warning: + "Running a public relay carries risk. Your relay can be spammed, resulting in large amounts of disk usage.", + tag: { + id: "type", + name: "Relay Type", + description: + "Private or public. A private relay (highly recommended) restricts write access to specific pubkeys. Anyone can write to a public relay.", + "variant-names": { private: "Private", public: "Public" }, + }, + default: "private", + variants: { + private: { + pubkey_whitelist: { + name: "Pubkey Whitelist (hex)", + description: + "A list of pubkeys that are permitted to publish through your relay. A minimum, you need to enter your own Nostr hex (not npub) pubkey. Go to https://damus.io/key/ to convert from npub to hex.", + type: "list", + range: "[1,*)", + subtype: "string", + spec: { + placeholder: "hex (not npub) pubkey", + pattern: "[0-9a-fA-F]{64}", + "pattern-description": + "Must be a valid 64-digit hexadecimal value (ie a Nostr hex pubkey, not an npub). Go to https://damus.io/key/ to convert npub to hex.", + }, + default: [], + }, + }, + public: { + info: { + name: "Relay Info", + description: "General public info about your relay", + type: "object", + spec: { + name: { + name: "Relay Name", + description: "Your relay's human-readable identifier", + type: "string", + nullable: true, + placeholder: "Bob's Public Relay", + pattern: ".{3,32}", + "pattern-description": + "Must be at least 3 character and no more than 32 characters", + masked: false, + }, + description: { + name: "Relay Description", + description: "A more detailed description for your relay", + type: "string", + nullable: true, + placeholder: "The best relay in town", + pattern: ".{6,256}", + "pattern-description": + "Must be at least 6 character and no more than 256 characters", + masked: false, + }, + pubkey: { + name: "Admin contact pubkey (hex)", + description: + "The Nostr hex (not npub) pubkey of the relay administrator", + type: "string", + nullable: true, + placeholder: "hex (not npub) pubkey", + pattern: "[0-9a-fA-F]{64}", + "pattern-description": + "Must be a valid 64-digit hexadecimal value (ie a Nostr hex pubkey, not an npub). Go to https://damus.io/key/ to convert npub to hex.", + masked: false, + }, + contact: { + name: "Admin contact email", + description: "The email address of the relay administrator", + type: "string", + nullable: true, + pattern: "[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+.[a-zA-Z0-9-.]+", + "pattern-description": "Must be a valid email address.", + masked: false, + }, + }, + }, + limits: { + name: "Limits", + description: + "Data limits to protect your relay from using too many resources", + type: "object", + spec: { + messages_per_sec: { + name: "Messages Per Second Limit", + description: + "Limit events created per second, averaged over one minute. Note: this is for the server as a whole, not per connection.", + type: "number", + nullable: false, + range: "[1,*)", + integral: true, + default: 2, + units: "messages/sec", + }, + subscriptions_per_min: { + name: "Subscriptions Per Minute Limit", + description: + "Limit client subscriptions created per second, averaged over one minute. Strongly recommended to set this to a low value such as 10 to ensure fair service.", + type: "number", + nullable: false, + range: "[1,*)", + integral: true, + default: 10, + units: "subscriptions", + }, + max_blocking_threads: { + name: "Max Blocking Threads", + description: + "Maximum number of blocking threads used for database connections.", + type: "number", + nullable: false, + range: "[0,*)", + integral: true, + units: "threads", + default: 16, + }, + max_event_bytes: { + name: "Max Event Size", + description: + "Limit the maximum size of an EVENT message. Set to 0 for unlimited", + type: "number", + nullable: false, + range: "[0,*)", + integral: true, + units: "bytes", + default: 131072, + }, + max_ws_message_bytes: { + name: "Max Websocket Message Size", + description: "Maximum WebSocket message in bytes.", + type: "number", + nullable: false, + range: "[0,*)", + integral: true, + units: "bytes", + default: 131072, + }, + max_ws_frame_bytes: { + name: "Max Websocket Frame Size", + description: "Maximum WebSocket frame size in bytes.", + type: "number", + nullable: false, + range: "[0,*)", + integral: true, + units: "bytes", + default: 131072, + }, + event_kind_blacklist: { + name: "Event Kind Blacklist", + description: + "Events with these kinds will be discarded. For a list of event kinds, see here: https://github.com/nostr-protocol/nips#event-kinds", + type: "list", + range: "[0,*)", + subtype: "number", + spec: { integral: true, placeholder: 30023, range: "(0,100000]" }, + default: [], + }, + }, + }, + }, + }, + }, +} diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/__fixtures__/synapseManifest.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/__fixtures__/synapseManifest.ts new file mode 100644 index 000000000..18b520097 --- /dev/null +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/__fixtures__/synapseManifest.ts @@ -0,0 +1,191 @@ +export default { + id: "synapse", + title: "Synapse", + version: "1.98.0", + "release-notes": + "* Upstream code update\n* Synapse Admin updated to the latest version - ([full changelog](https://github.com/Awesome-Technologies/synapse-admin/compare/0.8.7...0.9.1))\n* Instructions update\n* Updated package and upstream repositories links\n* Full list of upstream changes available [here](https://github.com/element-hq/synapse/compare/v1.95.1...v1.98.0)\n", + license: "Apache-2.0", + "wrapper-repo": "https://github.com/Start9Labs/synapse-startos", + "upstream-repo": "https://github.com/element-hq/synapse", + "support-site": "https://github.com/element-hq/synapse/issues", + "marketing-site": "https://matrix.org/", + build: ["make"], + description: { + short: + "Synapse is a battle-tested implementation of the Matrix protocol, the killer of all messaging apps.", + long: "Synapse is the battle-tested, reference implementation of the Matrix protocol. Matrix is a next-generation, federated, full-featured, encrypted, independent messaging system. There are no trusted third parties involved. (see matrix.org for details).", + }, + assets: { + license: "LICENSE", + icon: "icon.png", + instructions: "instructions.md", + }, + main: { + type: "docker", + image: "main", + entrypoint: "docker_entrypoint.sh", + args: [], + mounts: { + main: "/data", + cert: "/mnt/cert", + "admin-cert": "/mnt/admin-cert", + }, + }, + "health-checks": { + federation: { + name: "Federation", + type: "docker", + image: "main", + system: false, + entrypoint: "check-federation.sh", + args: [], + mounts: {}, + "io-format": "json", + inject: true, + }, + "synapse-admin": { + name: "Admin interface", + "success-message": + "Synapse Admin is ready to be visited in a web browser.", + type: "docker", + image: "main", + system: false, + entrypoint: "check-ui.sh", + args: [], + mounts: {}, + "io-format": "yaml", + inject: true, + }, + "user-signups-off": { + name: "User Signups Off", + type: "docker", + image: "main", + system: false, + entrypoint: "user-signups-off.sh", + args: [], + mounts: {}, + "io-format": "yaml", + inject: true, + }, + }, + config: { + get: { + type: "script", + }, + set: { + type: "script", + }, + }, + properties: { + type: "script", + }, + volumes: { + main: { + type: "data", + }, + cert: { + type: "certificate", + "interface-id": "main", + }, + "admin-cert": { + type: "certificate", + "interface-id": "admin", + }, + }, + alerts: { + start: + "After your first run, Synapse needs a little time to establish a stable TOR connection over federation. We kindly ask for your patience during this process. Remember, great things take time! 🕒", + }, + interfaces: { + main: { + name: "Homeserver Address", + description: + "Used by clients and other servers to connect with your homeserver", + "tor-config": { + "port-mapping": { + "80": "80", + "443": "443", + "8448": "8448", + }, + }, + ui: false, + protocols: ["tcp", "http", "matrix"], + }, + admin: { + name: "Admin Portal", + description: "A web application for administering your Synapse server", + "tor-config": { + "port-mapping": { + "80": "8080", + "443": "4433", + }, + }, + "lan-config": { + "443": { + ssl: true, + internal: 8080, + }, + }, + ui: true, + protocols: ["tcp", "http"], + }, + }, + dependencies: {}, + backup: { + create: { + type: "docker", + image: "compat", + system: true, + entrypoint: "compat", + args: ["duplicity", "create", "/mnt/backup", "/data"], + mounts: { + BACKUP: "/mnt/backup", + main: "/data", + }, + }, + restore: { + type: "docker", + image: "compat", + system: true, + entrypoint: "compat", + args: ["duplicity", "restore", "/mnt/backup", "/data"], + mounts: { + BACKUP: "/mnt/backup", + main: "/data", + }, + }, + }, + actions: { + "reset-first-user": { + name: "Reset First User", + description: + "This action will reset the password of the first user in your database to a random value.", + "allowed-statuses": ["stopped"], + implementation: { + type: "docker", + image: "main", + system: false, + entrypoint: "docker_entrypoint.sh", + args: ["reset-first-user"], + mounts: { + main: "/data", + }, + "io-format": "json", + }, + }, + }, + migrations: { + from: { + "*": { + type: "script", + args: ["from"], + }, + }, + to: { + "*": { + type: "script", + args: ["to"], + }, + }, + }, +} diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/__snapshots__/transformConfigSpec.test.ts.snap b/container-runtime/src/Adapters/Systems/SystemForEmbassy/__snapshots__/transformConfigSpec.test.ts.snap index 9eb6e97cf..01e2d0763 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/__snapshots__/transformConfigSpec.test.ts.snap +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/__snapshots__/transformConfigSpec.test.ts.snap @@ -750,6 +750,283 @@ exports[`transformConfigSpec transformConfigSpec(nostr) 1`] = ` } `; +exports[`transformConfigSpec transformConfigSpec(nostr2) 1`] = ` +{ + "relay-type": { + "default": "private", + "description": "Private or public. A private relay (highly recommended) restricts write access to specific pubkeys. Anyone can write to a public relay.", + "disabled": false, + "immutable": false, + "name": "Relay Type", + "required": true, + "type": "union", + "variants": { + "private": { + "name": "Private", + "spec": { + "pubkey_whitelist": { + "default": [], + "description": "A list of pubkeys that are permitted to publish through your relay. A minimum, you need to enter your own Nostr hex (not npub) pubkey. Go to https://damus.io/key/ to convert from npub to hex.", + "disabled": false, + "maxLength": null, + "minLength": 1, + "name": "Pubkey Whitelist (hex)", + "spec": { + "generate": null, + "inputmode": "text", + "masked": false, + "maxLength": null, + "minLength": null, + "patterns": [ + { + "description": "Must be a valid 64-digit hexadecimal value (ie a Nostr hex pubkey, not an npub). Go to https://damus.io/key/ to convert npub to hex.", + "regex": "[0-9a-fA-F]{64}", + }, + ], + "placeholder": "hex (not npub) pubkey", + "type": "text", + }, + "type": "list", + "warning": null, + }, + }, + }, + "public": { + "name": "Public", + "spec": { + "info": { + "description": "General public info about your relay", + "name": "Relay Info", + "spec": { + "contact": { + "default": null, + "description": "The email address of the relay administrator", + "disabled": false, + "generate": null, + "immutable": false, + "inputmode": "text", + "masked": false, + "maxLength": null, + "minLength": null, + "name": "Admin contact email", + "patterns": [ + { + "description": "Must be a valid email address.", + "regex": "[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+.[a-zA-Z0-9-.]+", + }, + ], + "placeholder": null, + "required": false, + "type": "text", + "warning": null, + }, + "description": { + "default": null, + "description": "A more detailed description for your relay", + "disabled": false, + "generate": null, + "immutable": false, + "inputmode": "text", + "masked": false, + "maxLength": null, + "minLength": null, + "name": "Relay Description", + "patterns": [ + { + "description": "Must be at least 6 character and no more than 256 characters", + "regex": ".{6,256}", + }, + ], + "placeholder": "The best relay in town", + "required": false, + "type": "text", + "warning": null, + }, + "name": { + "default": null, + "description": "Your relay's human-readable identifier", + "disabled": false, + "generate": null, + "immutable": false, + "inputmode": "text", + "masked": false, + "maxLength": null, + "minLength": null, + "name": "Relay Name", + "patterns": [ + { + "description": "Must be at least 3 character and no more than 32 characters", + "regex": ".{3,32}", + }, + ], + "placeholder": "Bob's Public Relay", + "required": false, + "type": "text", + "warning": null, + }, + "pubkey": { + "default": null, + "description": "The Nostr hex (not npub) pubkey of the relay administrator", + "disabled": false, + "generate": null, + "immutable": false, + "inputmode": "text", + "masked": false, + "maxLength": null, + "minLength": null, + "name": "Admin contact pubkey (hex)", + "patterns": [ + { + "description": "Must be a valid 64-digit hexadecimal value (ie a Nostr hex pubkey, not an npub). Go to https://damus.io/key/ to convert npub to hex.", + "regex": "[0-9a-fA-F]{64}", + }, + ], + "placeholder": "hex (not npub) pubkey", + "required": false, + "type": "text", + "warning": null, + }, + }, + "type": "object", + "warning": null, + }, + "limits": { + "description": "Data limits to protect your relay from using too many resources", + "name": "Limits", + "spec": { + "event_kind_blacklist": { + "default": [], + "description": "Events with these kinds will be discarded. For a list of event kinds, see here: https://github.com/nostr-protocol/nips#event-kinds", + "disabled": false, + "maxLength": null, + "minLength": null, + "name": "Event Kind Blacklist", + "spec": { + "generate": null, + "inputmode": "text", + "masked": false, + "maxLength": null, + "minLength": null, + "patterns": [ + { + "description": "Integral number type", + "regex": "[0-9]+", + }, + ], + "placeholder": "30023", + "type": "text", + }, + "type": "list", + "warning": null, + }, + "max_blocking_threads": { + "default": 16, + "description": "Maximum number of blocking threads used for database connections.", + "disabled": false, + "immutable": false, + "integer": true, + "max": null, + "min": null, + "name": "Max Blocking Threads", + "placeholder": null, + "required": true, + "step": null, + "type": "number", + "units": "threads", + "warning": null, + }, + "max_event_bytes": { + "default": 131072, + "description": "Limit the maximum size of an EVENT message. Set to 0 for unlimited", + "disabled": false, + "immutable": false, + "integer": true, + "max": null, + "min": null, + "name": "Max Event Size", + "placeholder": null, + "required": true, + "step": null, + "type": "number", + "units": "bytes", + "warning": null, + }, + "max_ws_frame_bytes": { + "default": 131072, + "description": "Maximum WebSocket frame size in bytes.", + "disabled": false, + "immutable": false, + "integer": true, + "max": null, + "min": null, + "name": "Max Websocket Frame Size", + "placeholder": null, + "required": true, + "step": null, + "type": "number", + "units": "bytes", + "warning": null, + }, + "max_ws_message_bytes": { + "default": 131072, + "description": "Maximum WebSocket message in bytes.", + "disabled": false, + "immutable": false, + "integer": true, + "max": null, + "min": null, + "name": "Max Websocket Message Size", + "placeholder": null, + "required": true, + "step": null, + "type": "number", + "units": "bytes", + "warning": null, + }, + "messages_per_sec": { + "default": 2, + "description": "Limit events created per second, averaged over one minute. Note: this is for the server as a whole, not per connection.", + "disabled": false, + "immutable": false, + "integer": true, + "max": null, + "min": 1, + "name": "Messages Per Second Limit", + "placeholder": null, + "required": true, + "step": null, + "type": "number", + "units": "messages/sec", + "warning": null, + }, + "subscriptions_per_min": { + "default": 10, + "description": "Limit client subscriptions created per second, averaged over one minute. Strongly recommended to set this to a low value such as 10 to ensure fair service.", + "disabled": false, + "immutable": false, + "integer": true, + "max": null, + "min": 1, + "name": "Subscriptions Per Minute Limit", + "placeholder": null, + "required": true, + "step": null, + "type": "number", + "units": "subscriptions", + "warning": null, + }, + }, + "type": "object", + "warning": null, + }, + }, + }, + }, + "warning": null, + }, +} +`; + exports[`transformConfigSpec transformConfigSpec(searNXG) 1`] = ` { "enable-metrics": { diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index ffdb02988..1e0c34189 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -61,6 +61,42 @@ const MANIFEST_LOCATION = "/usr/lib/startos/package/embassyManifest.json" export const EMBASSY_JS_LOCATION = "/usr/lib/startos/package/embassy.js" const EMBASSY_POINTER_PATH_PREFIX = "/embassyConfig" as StorePath +const matchResult = object({ + result: any, +}) +const matchError = object({ + error: string, +}) +const matchErrorCode = object<{ + "error-code": [number, string] | readonly [number, string] +}>({ + "error-code": tuple(number, string), +}) + +const assertNever = ( + x: never, + message = "Not expecting to get here: ", +): never => { + throw new Error(message + JSON.stringify(x)) +} +/** + Should be changing the type for specific properties, and this is mostly a transformation for the old return types to the newer one. +*/ +const fromReturnType = (a: U.ResultType): A => { + if (matchResult.test(a)) { + return a.result + } + if (matchError.test(a)) { + console.info({ passedErrorStack: new Error().stack, error: a.error }) + throw { error: a.error } + } + if (matchErrorCode.test(a)) { + const [code, message] = a["error-code"] + throw { error: message, code } + } + return assertNever(a) +} + const matchSetResult = object( { "depends-on": dictionary([string, array(string)]), @@ -194,7 +230,7 @@ export class SystemForEmbassy implements System { const moduleCode = await import(EMBASSY_JS_LOCATION) .catch((_) => require(EMBASSY_JS_LOCATION)) .catch(async (_) => { - console.error("Could not load the js") + console.error(utils.asError("Could not load the js")) console.error({ exists: await fs.stat(EMBASSY_JS_LOCATION), }) @@ -206,12 +242,49 @@ export class SystemForEmbassy implements System { moduleCode, ) } + constructor( readonly manifest: Manifest, readonly moduleCode: Partial, ) {} - async init(): Promise {} + async actionsMetadata(effects: T.Effects): Promise { + const actions = Object.entries(this.manifest.actions ?? {}) + return Promise.all( + actions.map(async ([actionId, action]): Promise => { + const name = action.name ?? actionId + const description = action.description + const warning = action.warning ?? null + const disabled = false + const input = (await convertToNewConfig(action["input-spec"] as any)) + .spec + const hasRunning = !!action["allowed-statuses"].find( + (x) => x === "running", + ) + const hasStopped = !!action["allowed-statuses"].find( + (x) => x === "stopped", + ) + // prettier-ignore + const allowedStatuses = + hasRunning && hasStopped ? "any": + hasRunning ? "onlyRunning" : + "onlyStopped" + + const group = null + return { + name, + description, + warning, + disabled, + allowedStatuses, + group, + input, + } + }), + ) + } + + async containerInit(): Promise {} async exit(): Promise { if (this.currentRunning) await this.currentRunning.clean() @@ -235,141 +308,7 @@ export class SystemForEmbassy implements System { } } - async execute( - effects: Effects, - options: { - procedure: JsonPath - input?: unknown - timeout?: number | undefined - }, - ): Promise { - return this._execute(effects, options) - .then((x) => - matches(x) - .when( - object({ - result: any, - }), - (x) => x, - ) - .when( - object({ - error: string, - }), - (x) => ({ - error: { - code: 0, - message: x.error, - }, - }), - ) - .when( - object({ - "error-code": tuple(number, string), - }), - ({ "error-code": [code, message] }) => ({ - error: { - code, - message, - }, - }), - ) - .defaultTo({ result: x }), - ) - .catch((error: unknown) => { - if (error instanceof Error) - return { - error: { - code: 0, - message: error.name, - data: { - details: error.message, - debug: `${error?.cause ?? "[noCause]"}:${error?.stack ?? "[noStack]"}`, - }, - }, - } - if (matchRpcResult.test(error)) return error - return { - error: { - code: 0, - message: String(error), - }, - } - }) - } - async _execute( - effects: Effects, - options: { - procedure: JsonPath - input?: unknown - timeout?: number | undefined - }, - ): Promise { - const input = options.input - switch (options.procedure) { - case "/backup/create": - return this.createBackup(effects, options.timeout || null) - case "/backup/restore": - return this.restoreBackup(effects, options.timeout || null) - case "/config/get": - return this.getConfig(effects, options.timeout || null) - case "/config/set": - return this.setConfig(effects, input, options.timeout || null) - case "/properties": - return this.properties(effects, options.timeout || null) - case "/actions/metadata": - return todo() - case "/init": - return this.initProcedure( - effects, - string.optional().unsafeCast(input), - options.timeout || null, - ) - case "/uninit": - return this.uninit( - effects, - string.optional().unsafeCast(input), - options.timeout || null, - ) - default: - const procedures = unNestPath(options.procedure) - switch (true) { - case procedures[1] === "actions" && procedures[3] === "get": - return this.action( - effects, - procedures[2], - input, - options.timeout || null, - ) - case procedures[1] === "actions" && procedures[3] === "run": - return this.action( - effects, - procedures[2], - input, - options.timeout || null, - ) - case procedures[1] === "dependencies" && procedures[3] === "query": - return null - - case procedures[1] === "dependencies" && procedures[3] === "update": - return this.dependenciesAutoconfig( - effects, - procedures[2], - input, - options.timeout || null, - ) - } - } - throw new Error(`Could not find the path for ${options.procedure}`) - } - async sandbox( - effects: Effects, - options: { procedure: Procedure; input: unknown; timeout?: number }, - ): Promise { - return this.execute(effects, options) - } - - private async initProcedure( + async packageInit( effects: Effects, previousVersion: Optional, timeoutMs: number | null, @@ -445,7 +384,6 @@ export class SystemForEmbassy implements System { id: `${id}-${internal}`, description: interfaceValue.description, hasPrimary: false, - disabled: false, type: interfaceValue.ui && (origin.scheme === "http" || origin.sslScheme === "https") @@ -490,7 +428,7 @@ export class SystemForEmbassy implements System { }) } } - private async uninit( + async packageUninit( effects: Effects, nextVersion: Optional, timeoutMs: number | null, @@ -499,7 +437,7 @@ export class SystemForEmbassy implements System { await effects.setMainStatus({ status: "stopped" }) } - private async createBackup( + async createBackup( effects: Effects, timeoutMs: number | null, ): Promise { @@ -520,7 +458,7 @@ export class SystemForEmbassy implements System { await moduleCode.createBackup?.(polyfillEffects(effects, this.manifest)) } } - private async restoreBackup( + async restoreBackup( effects: Effects, timeoutMs: number | null, ): Promise { @@ -544,7 +482,7 @@ export class SystemForEmbassy implements System { await moduleCode.restoreBackup?.(polyfillEffects(effects, this.manifest)) } } - private async getConfig( + async getConfig( effects: Effects, timeoutMs: number | null, ): Promise { @@ -585,7 +523,7 @@ export class SystemForEmbassy implements System { )) as any } } - private async setConfig( + async setConfig( effects: Effects, newConfigWithoutPointers: unknown, timeoutMs: number | null, @@ -677,7 +615,7 @@ export class SystemForEmbassy implements System { }) } - private async migration( + async migration( effects: Effects, fromVersion: string, timeoutMs: number | null, @@ -749,10 +687,10 @@ export class SystemForEmbassy implements System { } return { configured: true } } - private async properties( + async properties( effects: Effects, timeoutMs: number | null, - ): Promise> { + ): Promise { // TODO BLU-J set the properties ever so often const setConfigValue = this.manifest.properties if (!setConfigValue) throw new Error("There is no properties") @@ -780,23 +718,80 @@ export class SystemForEmbassy implements System { if (!method) throw new Error("Expecting that the method properties exists") const properties = matchProperties.unsafeCast( - await method(polyfillEffects(effects, this.manifest)).then((x) => { - if ("result" in x) return x.result - if ("error" in x) throw new Error("Error getting config: " + x.error) - throw new Error("Error getting config: " + x["error-code"][1]) - }), + await method(polyfillEffects(effects, this.manifest)).then( + fromReturnType, + ), ) return asProperty(properties.data) } throw new Error(`Unknown type in the fetch properties: ${setConfigValue}`) } - private async action( + async action( effects: Effects, actionId: string, formData: unknown, timeoutMs: number | null, ): Promise { const actionProcedure = this.manifest.actions?.[actionId]?.implementation + const toActionResult = ({ + message, + value = "", + copyable, + qr, + }: U.ActionResult): T.ActionResult => ({ + version: "0", + message, + value, + copyable, + qr, + }) + if (!actionProcedure) throw Error("Action not found") + if (actionProcedure.type === "docker") { + const subcontainer = actionProcedure.inject + ? this.currentRunning?.mainSubContainerHandle + : undefined + const container = await DockerProcedureContainer.of( + effects, + this.manifest.id, + actionProcedure, + this.manifest.volumes, + { + subcontainer, + }, + ) + return toActionResult( + JSON.parse( + ( + await container.execFail( + [ + actionProcedure.entrypoint, + ...actionProcedure.args, + JSON.stringify(formData), + ], + timeoutMs, + ) + ).stdout.toString(), + ), + ) + } else { + const moduleCode = await this.moduleCode + const method = moduleCode.action?.[actionId] + if (!method) throw new Error("Expecting that the method action exists") + return await method( + polyfillEffects(effects, this.manifest), + formData as any, + ) + .then(fromReturnType) + .then(toActionResult) + } + } + async dependenciesCheck( + effects: Effects, + id: string, + oldConfig: unknown, + timeoutMs: number | null, + ): Promise { + const actionProcedure = this.manifest.dependencies?.[id]?.config?.check if (!actionProcedure) return { message: "Action not found", value: null } if (actionProcedure.type === "docker") { const container = await DockerProcedureContainer.of( @@ -811,27 +806,32 @@ export class SystemForEmbassy implements System { [ actionProcedure.entrypoint, ...actionProcedure.args, - JSON.stringify(formData), + JSON.stringify(oldConfig), ], timeoutMs, ) ).stdout.toString(), ) - } else { + } else if (actionProcedure.type === "script") { const moduleCode = await this.moduleCode - const method = moduleCode.action?.[actionId] - if (!method) throw new Error("Expecting that the method action exists") + const method = moduleCode.dependencies?.[id]?.check + if (!method) + throw new Error( + `Expecting that the method dependency check ${id} exists`, + ) return (await method( polyfillEffects(effects, this.manifest), - formData as any, + oldConfig as any, ).then((x) => { if ("result" in x) return x.result if ("error" in x) throw new Error("Error getting config: " + x.error) throw new Error("Error getting config: " + x["error-code"][1]) })) as any + } else { + return {} } } - private async dependenciesAutoconfig( + async dependenciesAutoconfig( effects: Effects, id: string, input: unknown, @@ -982,7 +982,10 @@ async function updateConfig( }) .once() .catch((x) => { - console.error("Could not get the service interface", x) + console.error( + "Could not get the service interface", + utils.asError(x), + ) return null }) const catchFn = (fn: () => X) => { diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.test.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.test.ts new file mode 100644 index 000000000..3730dd3b6 --- /dev/null +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.test.ts @@ -0,0 +1,12 @@ +import { matchManifest } from "./matchManifest" +import giteaManifest from "./__fixtures__/giteaManifest" +import synapseManifest from "./__fixtures__/synapseManifest" + +describe("matchManifest", () => { + test("gittea", () => { + matchManifest.unsafeCast(giteaManifest) + }) + test("synapse", () => { + matchManifest.unsafeCast(synapseManifest) + }) +}) diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts index 8ce6cabbc..bd8856b42 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts @@ -55,10 +55,13 @@ export const matchManifest = object( string, every( matchProcedure, - object({ - name: string, - ["success-message"]: string, - }), + object( + { + name: string, + ["success-message"]: string, + }, + ["success-message"], + ), ), ]), config: object({ diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts index 6481a7a56..c212722e6 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts @@ -3,7 +3,7 @@ import * as oet from "./oldEmbassyTypes" import { Volume } from "../../../Models/Volume" import * as child_process from "child_process" import { promisify } from "util" -import { daemons, startSdk, T } from "@start9labs/start-sdk" +import { daemons, startSdk, T, utils } from "@start9labs/start-sdk" import "isomorphic-fetch" import { Manifest } from "./matchManifest" import { DockerProcedureContainer } from "./DockerProcedureContainer" @@ -124,20 +124,18 @@ export const polyfillEffects = ( wait(): Promise> term(): Promise } { - const dockerProcedureContainer = DockerProcedureContainer.of( + const promiseSubcontainer = DockerProcedureContainer.createSubContainer( effects, manifest.id, manifest.main, manifest.volumes, ) - const daemon = dockerProcedureContainer.then((dockerProcedureContainer) => + const daemon = promiseSubcontainer.then((subcontainer) => daemons.runCommand()( effects, - { id: manifest.main.image }, + subcontainer, [input.command, ...(input.args || [])], - { - overlay: dockerProcedureContainer.overlay, - }, + {}, ), ) return { @@ -224,16 +222,16 @@ export const polyfillEffects = ( return new Promise((resolve) => setTimeout(resolve, timeMs)) }, trace(whatToPrint: string): void { - console.trace(whatToPrint) + console.trace(utils.asError(whatToPrint)) }, warn(whatToPrint: string): void { - console.warn(whatToPrint) + console.warn(utils.asError(whatToPrint)) }, error(whatToPrint: string): void { - console.error(whatToPrint) + console.error(utils.asError(whatToPrint)) }, debug(whatToPrint: string): void { - console.debug(whatToPrint) + console.debug(utils.asError(whatToPrint)) }, info(whatToPrint: string): void { console.log(false) @@ -357,7 +355,7 @@ export const polyfillEffects = ( }) spawned.stderr.on("data", (data: unknown) => { - console.error(String(data)) + console.error(`polyfill.runAsync`, utils.asError(data)) }) const id = async () => { diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/transformConfigSpec.test.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/transformConfigSpec.test.ts index 79caef377..93b43910b 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/transformConfigSpec.test.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/transformConfigSpec.test.ts @@ -3,6 +3,7 @@ import fixtureEmbasyPagesConfig from "./__fixtures__/embasyPagesConfig" import searNXG from "./__fixtures__/searNXG" import bitcoind from "./__fixtures__/bitcoind" import nostr from "./__fixtures__/nostr" +import nostrConfig2 from "./__fixtures__/nostrConfig2" describe("transformConfigSpec", () => { test("matchOldConfigSpec(embassyPages.homepage.variants[web-page])", () => { @@ -30,4 +31,8 @@ describe("transformConfigSpec", () => { const spec = matchOldConfigSpec.unsafeCast(nostr) expect(transformConfigSpec(spec)).toMatchSnapshot() }) + test("transformConfigSpec(nostr2)", () => { + const spec = matchOldConfigSpec.unsafeCast(nostrConfig2) + expect(transformConfigSpec(spec)).toMatchSnapshot() + }) }) diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/transformConfigSpec.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/transformConfigSpec.ts index 706e0b941..5ce601c57 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/transformConfigSpec.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/transformConfigSpec.ts @@ -74,7 +74,7 @@ export function transformConfigSpec(oldSpec: OldConfigSpec): CT.InputSpec { integer: oldVal.integral, step: null, units: oldVal.units || null, - placeholder: oldVal.placeholder || null, + placeholder: oldVal.placeholder ? String(oldVal.placeholder) : null, } } else if (oldVal.type === "object") { newVal = { @@ -267,6 +267,31 @@ function getListSpec( {}, ), } + } else if (isNumberList(oldVal)) { + return { + ...partial, + type: "list", + default: oldVal.default.map(String) as string[], + spec: { + type: "text", + patterns: oldVal.spec.integral + ? [{ regex: "[0-9]+", description: "Integral number type" }] + : [ + { + regex: "[-+]?[0-9]*\\.?[0-9]+", + description: "Number type", + }, + ], + minLength: null, + maxLength: null, + masked: false, + generate: null, + inputmode: "text", + placeholder: oldVal.spec.placeholder + ? String(oldVal.spec.placeholder) + : null, + }, + } } else if (isStringList(oldVal)) { return { ...partial, @@ -337,11 +362,16 @@ function isStringList( ): val is OldValueSpecList & { subtype: "string" } { return val.subtype === "string" } +function isNumberList( + val: OldValueSpecList, +): val is OldValueSpecList & { subtype: "number" } { + return val.subtype === "number" +} function isObjectList( val: OldValueSpecList, ): val is OldValueSpecList & { subtype: "object" } { - if (["number", "union"].includes(val.subtype)) { + if (["union"].includes(val.subtype)) { throw new Error("Invalid list subtype. enum, string, and object permitted.") } return val.subtype === "object" @@ -398,7 +428,7 @@ export const matchOldValueSpecNumber = object( description: string, warning: string, units: string, - placeholder: string, + placeholder: anyOf(number, string), }, ["default", "description", "warning", "units", "placeholder"], ) @@ -499,6 +529,15 @@ const matchOldListValueSpecEnum = object({ values: array(string), "value-names": dictionary([string, string]), }) +const matchOldListValueSpecNumber = object( + { + range: string, + integral: boolean, + units: string, + placeholder: anyOf(number, string), + }, + ["units", "placeholder"], +) // represents a spec for a list const matchOldValueSpecList = every( @@ -531,6 +570,10 @@ const matchOldValueSpecList = every( subtype: literals("object"), spec: matchOldListValueSpecObject, }), + object({ + subtype: literals("number"), + spec: matchOldListValueSpecNumber, + }), ), ) type OldValueSpecList = typeof matchOldValueSpecList._TYPE diff --git a/container-runtime/src/Adapters/Systems/SystemForStartOs.ts b/container-runtime/src/Adapters/Systems/SystemForStartOs.ts index 029b504c0..51d91abb5 100644 --- a/container-runtime/src/Adapters/Systems/SystemForStartOs.ts +++ b/container-runtime/src/Adapters/Systems/SystemForStartOs.ts @@ -4,10 +4,11 @@ import matches, { any, number, object, string, tuple } from "ts-matches" import { Effects } from "../../Models/Effects" import { RpcResult, matchRpcResult } from "../RpcListener" import { duration } from "../../Models/Duration" -import { T } from "@start9labs/start-sdk" +import { T, utils } from "@start9labs/start-sdk" import { Volume } from "../../Models/Volume" import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk" import { CallbackHolder } from "../../Models/CallbackHolder" +import { Optional } from "ts-matches/lib/parsers/interfaces" export const STARTOS_JS_LOCATION = "/usr/lib/startos/package/index.js" @@ -25,6 +26,107 @@ export class SystemForStartOs implements System { } constructor(readonly abi: T.ABI) {} + containerInit(): Promise { + throw new Error("Method not implemented.") + } + async packageInit( + effects: Effects, + previousVersion: Optional = null, + timeoutMs: number | null = null, + ): Promise { + return void (await this.abi.init({ effects })) + } + async packageUninit( + effects: Effects, + nextVersion: Optional = null, + timeoutMs: number | null = null, + ): Promise { + return void (await this.abi.uninit({ effects, nextVersion })) + } + async createBackup( + effects: T.Effects, + timeoutMs: number | null, + ): Promise { + return void (await this.abi.createBackup({ + effects, + pathMaker: ((options) => + new Volume(options.volume, options.path).path) as T.PathMaker, + })) + } + async restoreBackup( + effects: T.Effects, + timeoutMs: number | null, + ): Promise { + return void (await this.abi.restoreBackup({ + effects, + pathMaker: ((options) => + new Volume(options.volume, options.path).path) as T.PathMaker, + })) + } + getConfig( + effects: T.Effects, + timeoutMs: number | null, + ): Promise { + return this.abi.getConfig({ effects }) + } + async setConfig( + effects: Effects, + input: { effects: Effects; input: Record }, + timeoutMs: number | null, + ): Promise { + const _: unknown = await this.abi.setConfig({ effects, input }) + return + } + migration( + effects: Effects, + fromVersion: string, + timeoutMs: number | null, + ): Promise { + throw new Error("Method not implemented.") + } + properties( + effects: Effects, + timeoutMs: number | null, + ): Promise { + throw new Error("Method not implemented.") + } + async action( + effects: Effects, + id: string, + formData: unknown, + timeoutMs: number | null, + ): Promise { + const action = (await this.abi.actions({ effects }))[id] + if (!action) throw new Error(`Action ${id} not found`) + return action.run({ effects }) + } + dependenciesCheck( + effects: Effects, + id: string, + oldConfig: unknown, + timeoutMs: number | null, + ): Promise { + const dependencyConfig = this.abi.dependencyConfig[id] + if (!dependencyConfig) throw new Error(`dependencyConfig ${id} not found`) + return dependencyConfig.query({ effects }) + } + async dependenciesAutoconfig( + effects: Effects, + id: string, + remoteConfig: unknown, + timeoutMs: number | null, + ): Promise { + const dependencyConfig = this.abi.dependencyConfig[id] + if (!dependencyConfig) throw new Error(`dependencyConfig ${id} not found`) + const queryResults = await this.getConfig(effects, timeoutMs) + return void (await dependencyConfig.update({ + queryResults, + remoteConfig, + })) // TODO + } + async actionsMetadata(effects: T.Effects): Promise { + return this.abi.actionsMetadata({ effects }) + } async init(): Promise {} @@ -57,7 +159,9 @@ export class SystemForStartOs implements System { if (this.runningMain) { this.runningMain.callbacks .callCallback(callback, args) - .catch((error) => console.error(`callback ${callback} failed`, error)) + .catch((error) => + console.error(`callback ${callback} failed`, utils.asError(error)), + ) } else { console.warn(`callback ${callback} ignored because system is not running`) } @@ -70,157 +174,4 @@ export class SystemForStartOs implements System { this.runningMain = undefined } } - - async execute( - effects: Effects, - options: { - procedure: Procedure - input?: unknown - timeout?: number | undefined - }, - ): Promise { - return this._execute(effects, options) - .then((x) => - matches(x) - .when( - object({ - result: any, - }), - (x) => x, - ) - .when( - object({ - error: string, - }), - (x) => ({ - error: { - code: 0, - message: x.error, - }, - }), - ) - .when( - object({ - "error-code": tuple(number, string), - }), - ({ "error-code": [code, message] }) => ({ - error: { - code, - message, - }, - }), - ) - .defaultTo({ result: x }), - ) - .catch((error: unknown) => { - if (error instanceof Error) - return { - error: { - code: 0, - message: error.name, - data: { - details: error.message, - debug: `${error?.cause ?? "[noCause]"}:${error?.stack ?? "[noStack]"}`, - }, - }, - } - if (matchRpcResult.test(error)) return error - return { - error: { - code: 0, - message: String(error), - }, - } - }) - } - async _execute( - effects: Effects | MainEffects, - options: { - procedure: Procedure - input?: unknown - timeout?: number | undefined - }, - ): Promise { - switch (options.procedure) { - case "/init": { - const previousVersion = - string.optional().unsafeCast(options.input) || null - return this.abi.init({ effects, previousVersion }) - } - case "/uninit": { - const nextVersion = string.optional().unsafeCast(options.input) || null - return this.abi.uninit({ effects, nextVersion }) - } - // case "/main/start": { - // - // } - // case "/main/stop": { - // if (this.onTerm) await this.onTerm() - // await effects.setMainStatus({ status: "stopped" }) - // delete this.onTerm - // return duration(30, "s") - // } - case "/config/set": { - const input = options.input as any // TODO - return this.abi.setConfig({ effects, input }) - } - case "/config/get": { - return this.abi.getConfig({ effects }) - } - case "/backup/create": - return this.abi.createBackup({ - effects, - pathMaker: ((options) => - new Volume(options.volume, options.path).path) as T.PathMaker, - }) - case "/backup/restore": - return this.abi.restoreBackup({ - effects, - pathMaker: ((options) => - new Volume(options.volume, options.path).path) as T.PathMaker, - }) - case "/actions/metadata": { - return this.abi.actionsMetadata({ effects }) - } - case "/properties": { - throw new Error("TODO") - } - default: - const procedures = unNestPath(options.procedure) - const id = procedures[2] - switch (true) { - case procedures[1] === "actions" && procedures[3] === "get": { - const action = (await this.abi.actions({ effects }))[id] - if (!action) throw new Error(`Action ${id} not found`) - return action.getConfig({ effects }) - } - case procedures[1] === "actions" && procedures[3] === "run": { - const action = (await this.abi.actions({ effects }))[id] - if (!action) throw new Error(`Action ${id} not found`) - return action.run({ effects, input: options.input as any }) // TODO - } - case procedures[1] === "dependencies" && procedures[3] === "query": { - const dependencyConfig = this.abi.dependencyConfig[id] - if (!dependencyConfig) - throw new Error(`dependencyConfig ${id} not found`) - const localConfig = options.input - return dependencyConfig.query({ effects }) - } - case procedures[1] === "dependencies" && procedures[3] === "update": { - const dependencyConfig = this.abi.dependencyConfig[id] - if (!dependencyConfig) - throw new Error(`dependencyConfig ${id} not found`) - return dependencyConfig.update(options.input as any) // TODO - } - } - return - } - } - - async sandbox( - effects: Effects, - options: { procedure: Procedure; input?: unknown; timeout?: number }, - ): Promise { - return this.execute(effects, options) - } } diff --git a/container-runtime/src/Interfaces/System.ts b/container-runtime/src/Interfaces/System.ts index 01fd3c5ff..1348b79e9 100644 --- a/container-runtime/src/Interfaces/System.ts +++ b/container-runtime/src/Interfaces/System.ts @@ -3,6 +3,7 @@ import { RpcResult } from "../Adapters/RpcListener" import { Effects } from "../Models/Effects" import { CallbackHolder } from "../Models/CallbackHolder" import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk" +import { Optional } from "ts-matches/lib/parsers/interfaces" export type Procedure = | "/init" @@ -22,28 +23,60 @@ export type ExecuteResult = | { ok: unknown } | { err: { code: number; message: string } } export type System = { - init(): Promise + containerInit(): Promise start(effects: MainEffects): Promise callCallback(callback: number, args: any[]): void stop(): Promise - execute( + packageInit( effects: Effects, - options: { - procedure: Procedure - input: unknown - timeout?: number - }, - ): Promise - sandbox( + previousVersion: Optional, + timeoutMs: number | null, + ): Promise + packageUninit( effects: Effects, - options: { - procedure: Procedure - input: unknown - timeout?: number - }, - ): Promise + nextVersion: Optional, + timeoutMs: number | null, + ): Promise + + createBackup(effects: T.Effects, timeoutMs: number | null): Promise + restoreBackup(effects: T.Effects, timeoutMs: number | null): Promise + getConfig(effects: T.Effects, timeoutMs: number | null): Promise + setConfig( + effects: Effects, + input: { effects: Effects; input: Record }, + timeoutMs: number | null, + ): Promise + migration( + effects: Effects, + fromVersion: string, + timeoutMs: number | null, + ): Promise + properties( + effects: Effects, + timeoutMs: number | null, + ): Promise + action( + effects: Effects, + actionId: string, + formData: unknown, + timeoutMs: number | null, + ): Promise + + dependenciesCheck( + effects: Effects, + id: string, + oldConfig: unknown, + timeoutMs: number | null, + ): Promise + dependenciesAutoconfig( + effects: Effects, + id: string, + oldConfig: unknown, + timeoutMs: number | null, + ): Promise + actionsMetadata(effects: T.Effects): Promise exit(): Promise } diff --git a/core/Cargo.lock b/core/Cargo.lock index 2108ac851..1fccce36d 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -231,7 +231,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -242,7 +242,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e89b6941c2d1a7045538884d6e760ccfffdf8e1ffc2613d8efa74305e1f3752" +checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" dependencies = [ "bindgen", "cc", @@ -532,8 +532,8 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.72", - "which", + "syn 2.0.74", + "which 4.4.2", ] [[package]] @@ -675,9 +675,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.1" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cache-padded" @@ -687,12 +687,13 @@ checksum = "981520c98f422fcc584dc1a95c334e6953900b9106bc47a9839b81790009eb21" [[package]] name = "cc" -version = "1.1.6" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" +checksum = "68064e60dbf1f17005c2fde4d07c16d8baa506fd7ffed8ccab702d93617975c7" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -799,9 +800,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.11" +version = "4.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" +checksum = "11d8838454fda655dafd3accb2b6e2bea645b9e4078abe84a22ceb947235c5cc" dependencies = [ "clap_builder", "clap_derive", @@ -809,9 +810,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.11" +version = "4.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" +checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" dependencies = [ "anstream", "anstyle", @@ -821,14 +822,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -839,9 +840,9 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "cmake" -version = "0.1.50" +version = "0.1.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" dependencies = [ "cc", ] @@ -1035,15 +1036,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" dependencies = [ "libc", ] @@ -1117,16 +1118,16 @@ checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crossterm" -version = "0.27.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ "bitflags 2.6.0", "crossterm_winapi", "futures-core", - "libc", - "mio 0.8.11", + "mio", "parking_lot", + "rustix", "signal-hook", "signal-hook-mio", "winapi", @@ -1236,7 +1237,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1260,7 +1261,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1271,7 +1272,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1302,7 +1303,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1325,7 +1326,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1348,7 +1349,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1416,9 +1417,9 @@ dependencies = [ [[package]] name = "dunce" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "dyn-clone" @@ -1570,7 +1571,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1579,6 +1580,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + [[package]] name = "errno" version = "0.3.9" @@ -1589,6 +1601,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -1667,14 +1689,14 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "filetime" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", - "windows-sys 0.52.0", + "libredox", + "windows-sys 0.59.0", ] [[package]] @@ -1685,9 +1707,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920" dependencies = [ "crc32fast", "miniz_oxide", @@ -1828,7 +1850,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1941,7 +1963,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.2.6", + "indexmap 2.4.0", "slab", "tokio", "tokio-util", @@ -1960,7 +1982,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.2.6", + "indexmap 2.4.0", "slab", "tokio", "tokio-util", @@ -2068,6 +2090,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -2076,9 +2104,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hifijson" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18ae468bcb4dfecf0e4949ee28abbc99076b6a0077f51ddbc94dbfff8e6a870c" +checksum = "9958ab3ce3170c061a27679916bd9b969eceeb5e8b120438e6751d0987655c42" [[package]] name = "hkdf" @@ -2273,9 +2301,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" dependencies = [ "bytes", "futures-channel", @@ -2432,9 +2460,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -2475,9 +2503,9 @@ dependencies = [ [[package]] name = "integer-encoding" -version = "4.0.0" +version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "924df4f0e24e2e7f9cdd90babb0b96f93b20f3ecfa949ea9e6613756b8c8e1bf" +checksum = "0d762194228a2f1c11063e46e32e5acb96e66e906382b9eb5441f2e0504bbd5a" dependencies = [ "async-trait", "tokio", @@ -2504,11 +2532,11 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ - "hermit-abi", + "hermit-abi 0.4.0", "libc", "windows-sys 0.52.0", ] @@ -2647,9 +2675,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -2799,6 +2827,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", + "redox_syscall 0.5.3", ] [[package]] @@ -2945,25 +2974,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.11" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ + "hermit-abi 0.3.9", "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.48.0", -] - -[[package]] -name = "mio" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" -dependencies = [ - "hermit-abi", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -3029,14 +3047,26 @@ checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" [[package]] name = "new_mime_guess" -version = "4.0.1" +version = "4.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d684d1b59e0dc07b37e2203ef576987473288f530082512aff850585c61b1f" +checksum = "02a2dfb3559d53e90b709376af1c379462f7fb3085a0177deb73e6ea0d99eff4" dependencies = [ "mime", "unicase", ] +[[package]] +name = "nix" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa9b4819da1bc61c0ea48b63b7bc8604064dd43013e7cc325df098d49cd7c18a" +dependencies = [ + "bitflags 1.3.2", + "cc", + "cfg-if", + "libc", +] + [[package]] name = "nix" version = "0.24.3" @@ -3219,29 +3249,29 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] [[package]] name = "num_enum" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" dependencies = [ "num_enum_derive", ] [[package]] name = "num_enum_derive" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -3307,7 +3337,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -3514,7 +3544,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -3535,7 +3565,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.2.6", + "indexmap 2.4.0", ] [[package]] @@ -3570,7 +3600,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -3626,9 +3656,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "precomputed-hash" @@ -3643,7 +3676,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -3687,6 +3720,32 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" +dependencies = [ + "bitflags 2.6.0", + "chrono", + "flate2", + "hex", + "lazy_static", + "procfs-core", + "rustix", +] + +[[package]] +name = "procfs-core" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" +dependencies = [ + "bitflags 2.6.0", + "chrono", + "hex", +] + [[package]] name = "proptest" version = "1.5.0" @@ -3715,7 +3774,7 @@ checksum = "6ff7ff745a347b87471d859a377a9a404361e7efc2a971d73424a6d183c0fc77" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -3738,7 +3797,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -3938,9 +3997,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", @@ -4009,7 +4068,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile 2.1.2", + "rustls-pemfile 2.1.3", "serde", "serde_json", "serde_urlencoded", @@ -4079,7 +4138,7 @@ dependencies = [ [[package]] name = "rpc-toolkit" version = "0.2.3" -source = "git+https://github.com/Start9Labs/rpc-toolkit.git?branch=refactor/no-dyn-ctx#f608480034942f1f521ab95949ab33fbc51d99a9" +source = "git+https://github.com/Start9Labs/rpc-toolkit.git?branch=refactor/no-dyn-ctx#60a974a29c5e6380f7bbfbc1b4716f6d2b20b189" dependencies = [ "async-stream", "async-trait", @@ -4174,7 +4233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.6.0", - "errno", + "errno 0.3.9", "libc", "linux-raw-sys", "windows-sys 0.52.0", @@ -4217,9 +4276,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ "base64 0.22.1", "rustls-pki-types", @@ -4227,9 +4286,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" @@ -4273,9 +4332,9 @@ dependencies = [ [[package]] name = "rustyline-async" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6eb06391513b2184f0a5405c11a4a0a5302e8be442f4c5c35267187c2b37d5" +checksum = "bc9396d834c31f9fddd716e7c279e7cb70207092a1e59767918610f5c560c6eb" dependencies = [ "crossterm", "futures-channel", @@ -4375,9 +4434,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.204" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" dependencies = [ "serde_derive", ] @@ -4401,23 +4460,24 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "itoa", + "memchr", "ryu", "serde", ] @@ -4463,7 +4523,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.2.6", + "indexmap 2.4.0", "serde", "serde_derive", "serde_json", @@ -4480,7 +4540,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -4489,7 +4549,7 @@ version = "0.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ce6afeda22f0b55dde2c34897bce76a629587348480384231205c14b59a01f" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "itoa", "libyml", "log", @@ -4578,12 +4638,12 @@ dependencies = [ [[package]] name = "signal-hook-mio" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" dependencies = [ "libc", - "mio 0.8.11", + "mio", "signal-hook", ] @@ -4724,7 +4784,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap 2.2.6", + "indexmap 2.4.0", "log", "memchr", "once_cell", @@ -4913,7 +4973,7 @@ dependencies = [ "quote", "regex-syntax 0.6.29", "strsim 0.10.0", - "syn 2.0.72", + "syn 2.0.74", "unicode-width", ] @@ -4961,7 +5021,7 @@ dependencies = [ [[package]] name = "start-os" -version = "0.3.6-alpha.3" +version = "0.3.6-alpha.5" dependencies = [ "aes", "async-compression", @@ -5006,7 +5066,7 @@ dependencies = [ "imbl", "imbl-value", "include_dir", - "indexmap 2.2.6", + "indexmap 2.4.0", "indicatif", "integer-encoding", "ipnet", @@ -5039,6 +5099,7 @@ dependencies = [ "pin-project", "pkcs8", "prettytable-rs", + "procfs", "proptest", "proptest-derive", "rand 0.8.5", @@ -5057,6 +5118,7 @@ dependencies = [ "serde_yml", "sha2 0.10.8", "shell-words", + "signal-hook", "simple-logging", "socket2", "sqlx", @@ -5072,7 +5134,7 @@ dependencies = [ "tokio-tar", "tokio-tungstenite 0.23.1", "tokio-util", - "toml 0.8.16", + "toml 0.8.19", "torut", "tower-service", "tracing", @@ -5083,9 +5145,12 @@ dependencies = [ "trust-dns-server", "ts-rs", "typed-builder", + "unix-named-pipe", + "unshare", "url", "urlencoding", "uuid", + "which 6.0.3", "zeroize", ] @@ -5150,9 +5215,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.72" +version = "2.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" dependencies = [ "proc-macro2", "quote", @@ -5211,14 +5276,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.10.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5279,7 +5345,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -5360,14 +5426,14 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.1" +version = "1.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" dependencies = [ "backtrace", "bytes", "libc", - "mio 1.0.1", + "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", @@ -5395,7 +5461,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -5421,9 +5487,9 @@ dependencies = [ [[package]] name = "tokio-socks" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51165dfa029d2a65969413a6cc96f354b86b464498702f174a4efa13608fd8c0" +checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" dependencies = [ "either", "futures-util", @@ -5510,21 +5576,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.17", + "toml_edit 0.22.20", ] [[package]] name = "toml_datetime" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] @@ -5535,7 +5601,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "serde", "serde_spanned", "toml_datetime", @@ -5548,22 +5614,22 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "toml_datetime", "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.22.17" +version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16" +checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.4.0", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.16", + "winnow 0.6.18", ] [[package]] @@ -5634,15 +5700,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -5664,7 +5730,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -5816,7 +5882,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", "termcolor", ] @@ -5876,7 +5942,7 @@ checksum = "1f718dfaf347dcb5b983bfc87608144b0bad87970aebcbea5ce44d2a30c08e63" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -5963,6 +6029,26 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "unix-named-pipe" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ad653da8f36ac5825ba06642b5a3cce14a4e52c6a5fab4a8928d53f4426dae2" +dependencies = [ + "errno 0.2.8", + "libc", +] + +[[package]] +name = "unshare" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ceda295552a1eda89f8a748237654ad76b9c87e383fc07af5c4e423eb8e7b9b" +dependencies = [ + "libc", + "nix 0.20.0", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -6074,34 +6160,35 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ "cfg-if", "js-sys", @@ -6111,9 +6198,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6121,22 +6208,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-streams" @@ -6153,9 +6240,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", @@ -6179,6 +6266,18 @@ dependencies = [ "rustix", ] +[[package]] +name = "which" +version = "6.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ee928febd44d98f2f459a4a79bd4d928591333a494a10a868418ac1b39cf1f" +dependencies = [ + "either", + "home", + "rustix", + "winsafe", +] + [[package]] name = "whoami" version = "1.5.1" @@ -6207,11 +6306,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6247,6 +6346,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -6379,9 +6487,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.16" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b480ae9340fc261e6be3e95a1ba86d54ae3f9171132a73ce8d4bbaf68339507c" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" dependencies = [ "memchr", ] @@ -6396,6 +6504,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winsafe" +version = "0.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" + [[package]] name = "wyz" version = "0.2.0" @@ -6470,6 +6584,7 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] @@ -6481,7 +6596,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -6501,7 +6616,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -6515,18 +6630,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.12+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/core/build-containerbox.sh b/core/build-containerbox.sh index e4a8f6e7a..e81efcc97 100755 --- a/core/build-containerbox.sh +++ b/core/build-containerbox.sh @@ -2,13 +2,17 @@ cd "$(dirname "${BASH_SOURCE[0]}")" -set -e +set -ea shopt -s expand_aliases if [ -z "$ARCH" ]; then ARCH=$(uname -m) fi +if [ "$ARCH" = "arm64" ]; then + ARCH="aarch64" +fi + USE_TTY= if tty -s; then USE_TTY="-it" @@ -24,16 +28,9 @@ fi alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$HOME/.cargo/git":/root/.cargo/git -v "$(pwd)":/home/rust/src -w /home/rust/src -P messense/rust-musl-cross:$ARCH-musl' -set +e -fail= echo "FEATURES=\"$FEATURES\"" echo "RUSTFLAGS=\"$RUSTFLAGS\"" -if ! rust-musl-builder sh -c "cd core && cargo build --release --no-default-features --features container-runtime,$FEATURES --locked --bin containerbox --target=$ARCH-unknown-linux-musl && chown -R $UID:$UID target && chown -R $UID:$UID /root/.cargo"; then - fail=true -fi -set -e -cd core - -if [ -n "$fail" ]; then - exit 1 -fi +rust-musl-builder sh -c "cd core && cargo build --release --no-default-features --features container-runtime,$FEATURES --locked --bin containerbox --target=$ARCH-unknown-linux-musl" +if [ "$(ls -nd core/target/$ARCH-unknown-linux-musl/release/containerbox | awk '{ print $3 }')" != "$UID" ]; then + rust-musl-builder sh -c "cd core && chown -R $UID:$UID target && chown -R $UID:$UID /root/.cargo" +fi \ No newline at end of file diff --git a/core/build-registrybox.sh b/core/build-registrybox.sh index 9db57dd80..3659b372a 100755 --- a/core/build-registrybox.sh +++ b/core/build-registrybox.sh @@ -2,13 +2,17 @@ cd "$(dirname "${BASH_SOURCE[0]}")" -set -e +set -ea shopt -s expand_aliases if [ -z "$ARCH" ]; then ARCH=$(uname -m) fi +if [ "$ARCH" = "arm64" ]; then + ARCH="aarch64" +fi + USE_TTY= if tty -s; then USE_TTY="-it" @@ -24,16 +28,9 @@ fi alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$HOME/.cargo/git":/root/.cargo/git -v "$(pwd)":/home/rust/src -w /home/rust/src -P messense/rust-musl-cross:$ARCH-musl' -set +e -fail= echo "FEATURES=\"$FEATURES\"" echo "RUSTFLAGS=\"$RUSTFLAGS\"" -if ! rust-musl-builder sh -c "cd core && cargo build --release --no-default-features --features cli,registry,$FEATURES --locked --bin registrybox --target=$ARCH-unknown-linux-musl && chown -R $UID:$UID target && chown -R $UID:$UID /root/.cargo"; then - fail=true -fi -set -e -cd core - -if [ -n "$fail" ]; then - exit 1 +rust-musl-builder sh -c "cd core && cargo build --release --no-default-features --features cli,registry,$FEATURES --locked --bin registrybox --target=$ARCH-unknown-linux-musl" +if [ "$(ls -nd core/target/$ARCH-unknown-linux-musl/release/registrybox | awk '{ print $3 }')" != "$UID" ]; then + rust-musl-builder sh -c "cd core && chown -R $UID:$UID target && chown -R $UID:$UID /root/.cargo" fi diff --git a/core/build-startbox.sh b/core/build-startbox.sh index 55a455f09..9fad6fa3d 100755 --- a/core/build-startbox.sh +++ b/core/build-startbox.sh @@ -2,13 +2,17 @@ cd "$(dirname "${BASH_SOURCE[0]}")" -set -e +set -ea shopt -s expand_aliases if [ -z "$ARCH" ]; then ARCH=$(uname -m) fi +if [ "$ARCH" = "arm64" ]; then + ARCH="aarch64" +fi + USE_TTY= if tty -s; then USE_TTY="-it" @@ -24,16 +28,9 @@ fi alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$HOME/.cargo/git":/root/.cargo/git -v "$(pwd)":/home/rust/src -w /home/rust/src -P messense/rust-musl-cross:$ARCH-musl' -set +e -fail= echo "FEATURES=\"$FEATURES\"" echo "RUSTFLAGS=\"$RUSTFLAGS\"" -if ! rust-musl-builder sh -c "cd core && cargo build --release --no-default-features --features cli,daemon,$FEATURES --locked --bin startbox --target=$ARCH-unknown-linux-musl && chown -R $UID:$UID target && chown -R $UID:$UID /root/.cargo"; then - fail=true -fi -set -e -cd core - -if [ -n "$fail" ]; then - exit 1 -fi +rust-musl-builder sh -c "cd core && cargo build --release --no-default-features --features cli,daemon,$FEATURES --locked --bin startbox --target=$ARCH-unknown-linux-musl" +if [ "$(ls -nd core/target/$ARCH-unknown-linux-musl/release/startbox | awk '{ print $3 }')" != "$UID" ]; then + rust-musl-builder sh -c "cd core && chown -R $UID:$UID target && chown -R $UID:$UID /root/.cargo" +fi \ No newline at end of file diff --git a/core/build-ts.sh b/core/build-ts.sh new file mode 100755 index 000000000..c9890bfe7 --- /dev/null +++ b/core/build-ts.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +cd "$(dirname "${BASH_SOURCE[0]}")" + +set -ea +shopt -s expand_aliases + +if [ -z "$ARCH" ]; then + ARCH=$(uname -m) +fi + +if [ "$ARCH" = "arm64" ]; then + ARCH="aarch64" +fi + +USE_TTY= +if tty -s; then + USE_TTY="-it" +fi + +cd .. +FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')" +RUSTFLAGS="" + +if [[ "${ENVIRONMENT}" =~ (^|-)unstable($|-) ]]; then + RUSTFLAGS="--cfg tokio_unstable" +fi + +alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$HOME/.cargo/git":/root/.cargo/git -v "$(pwd)":/home/rust/src -w /home/rust/src -P messense/rust-musl-cross:$ARCH-musl' + +echo "FEATURES=\"$FEATURES\"" +echo "RUSTFLAGS=\"$RUSTFLAGS\"" +rust-musl-builder sh -c "cd core && cargo test --release --features=test,$FEATURES 'export_bindings_' && chown \$UID:\$UID startos/bindings" +if [ "$(ls -nd core/startos/bindings | awk '{ print $3 }')" != "$UID" ]; then + rust-musl-builder sh -c "cd core && chown -R $UID:$UID startos/bindings && chown -R $UID:$UID target && chown -R $UID:$UID /root/.cargo" +fi \ No newline at end of file diff --git a/core/install-cli.sh b/core/install-cli.sh index 620600d92..b278947a3 100755 --- a/core/install-cli.sh +++ b/core/install-cli.sh @@ -2,14 +2,18 @@ cd "$(dirname "${BASH_SOURCE[0]}")" -set -e +set -ea shopt -s expand_aliases web="../web/dist/static" [ -d "$web" ] || mkdir -p "$web" if [ -z "$PLATFORM" ]; then - export PLATFORM=$(uname -m) + PLATFORM=$(uname -m) +fi + +if [ "$PLATFORM" = "arm64" ]; then + PLATFORM="aarch64" fi cargo install --path=./startos --no-default-features --features=cli,docker,registry --bin start-cli --locked diff --git a/core/run-tests.sh b/core/run-tests.sh new file mode 100755 index 000000000..02ec34d55 --- /dev/null +++ b/core/run-tests.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +cd "$(dirname "${BASH_SOURCE[0]}")" + +set -ea +shopt -s expand_aliases + +if [ -z "$ARCH" ]; then + ARCH=$(uname -m) +fi + +if [ "$ARCH" = "arm64" ]; then + ARCH="aarch64" +fi + +USE_TTY= +if tty -s; then + USE_TTY="-it" +fi + +cd .. +FEATURES="$(echo $ENVIRONMENT | sed 's/-/,/g')" +RUSTFLAGS="" + +if [[ "${ENVIRONMENT}" =~ (^|-)unstable($|-) ]]; then + RUSTFLAGS="--cfg tokio_unstable" +fi + +alias 'rust-musl-builder'='docker run $USE_TTY --rm -e "RUSTFLAGS=$RUSTFLAGS" -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$HOME/.cargo/git":/root/.cargo/git -v "$(pwd)":/home/rust/src -w /home/rust/src -P messense/rust-musl-cross:$ARCH-musl' + +echo "FEATURES=\"$FEATURES\"" +echo "RUSTFLAGS=\"$RUSTFLAGS\"" +rust-musl-builder sh -c "apt-get update && apt-get install -y rsync && cd core && cargo test --release --features=test,$FEATURES --workspace --locked --target=$ARCH-unknown-linux-musl -- --skip export_bindings_ && chown \$UID:\$UID target" +if [ "$(ls -nd core/target | awk '{ print $3 }')" != "$UID" ]; then + rust-musl-builder sh -c "cd core && chown -R $UID:$UID target && chown -R $UID:$UID /root/.cargo" +fi \ No newline at end of file diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index 1e1cd4737..d0228ba31 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -14,7 +14,7 @@ keywords = [ name = "start-os" readme = "README.md" repository = "https://github.com/Start9Labs/start-os" -version = "0.3.6-alpha.3" +version = "0.3.6-alpha.5" license = "MIT" [lib] @@ -39,7 +39,7 @@ path = "src/main.rs" [features] cli = [] -container-runtime = [] +container-runtime = ["procfs", "unshare"] daemon = [] registry = [] default = ["cli", "daemon"] @@ -130,7 +130,14 @@ log = "0.4.20" mbrman = "0.5.2" models = { version = "*", path = "../models" } new_mime_guess = "4" -nix = { version = "0.29.0", features = ["user", "process", "signal", "fs"] } +nix = { version = "0.29.0", features = [ + "fs", + "mount", + "process", + "sched", + "signal", + "user", +] } nom = "7.1.3" num = "0.4.1" num_enum = "0.7.0" @@ -146,6 +153,7 @@ pbkdf2 = "0.12.2" pin-project = "1.1.3" pkcs8 = { version = "0.10.2", features = ["std"] } prettytable-rs = "0.10.0" +procfs = { version = "0.16.0", optional = true } proptest = "1.3.1" proptest-derive = "0.5.0" rand = { version = "0.8.5", features = ["std"] } @@ -166,6 +174,7 @@ serde_with = { version = "3.4.0", features = ["macros", "json"] } serde_yaml = { package = "serde_yml", version = "0.0.10" } sha2 = "0.10.2" shell-words = "1" +signal-hook = "0.3.17" simple-logging = "2.0.2" socket2 = "0.5.7" sqlx = { version = "0.7.2", features = [ @@ -197,6 +206,9 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } trust-dns-server = "0.23.1" ts-rs = { git = "https://github.com/dr-bonez/ts-rs.git", branch = "feature/top-level-as" } # "8.1.0" typed-builder = "0.18.0" +which = "6.0.3" +unix-named-pipe = "0.2.0" +unshare = { version = "0.7.0", optional = true } url = { version = "2.4.1", features = ["serde"] } urlencoding = "2.1.3" uuid = { version = "1.4.1", features = ["v4"] } diff --git a/core/startos/src/context/rpc.rs b/core/startos/src/context/rpc.rs index 0db681d3b..5330c58bc 100644 --- a/core/startos/src/context/rpc.rs +++ b/core/startos/src/context/rpc.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::future::Future; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::ops::Deref; use std::path::PathBuf; @@ -6,6 +7,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; +use chrono::{TimeDelta, Utc}; +use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; use josekit::jwk::Jwk; use reqwest::{Client, Proxy}; @@ -29,7 +32,7 @@ use crate::net::utils::{find_eth_iface, find_wifi_iface}; use crate::net::wifi::WpaCli; use crate::prelude::*; use crate::progress::{FullProgressTracker, PhaseProgressTrackerHandle}; -use crate::rpc_continuations::{OpenAuthedContinuations, RpcContinuations}; +use crate::rpc_continuations::{Guid, OpenAuthedContinuations, RpcContinuations}; use crate::service::effects::callbacks::ServiceCallbacks; use crate::service::ServiceMap; use crate::shutdown::Shutdown; @@ -63,6 +66,7 @@ pub struct RpcContextSeed { pub client: Client, pub hardware: Hardware, pub start_time: Instant, + pub crons: SyncMutex>>, #[cfg(feature = "dev")] pub dev: Dev, } @@ -94,12 +98,14 @@ impl InitRpcContextPhases { } pub struct CleanupInitPhases { + cleanup_sessions: PhaseProgressTrackerHandle, init_services: PhaseProgressTrackerHandle, check_dependencies: PhaseProgressTrackerHandle, } impl CleanupInitPhases { pub fn new(handle: &FullProgressTracker) -> Self { Self { + cleanup_sessions: handle.add_phase("Cleaning up sessions".into(), Some(1)), init_services: handle.add_phase("Initializing services".into(), Some(10)), check_dependencies: handle.add_phase("Checking dependencies".into(), Some(1)), } @@ -174,6 +180,8 @@ impl RpcContext { let ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024; read_device_info.complete(); + let crons = SyncMutex::new(BTreeMap::new()); + if !db .peek() .await @@ -183,18 +191,24 @@ impl RpcContext { .de()? { let db = db.clone(); - tokio::spawn(async move { - while !check_time_is_synchronized().await.unwrap() { - tokio::time::sleep(Duration::from_secs(30)).await; - } - db.mutate(|v| { - v.as_public_mut() - .as_server_info_mut() - .as_ntp_synced_mut() - .ser(&true) - }) - .await - .unwrap() + crons.mutate(|c| { + c.insert( + Guid::new(), + tokio::spawn(async move { + while !check_time_is_synchronized().await.unwrap() { + tokio::time::sleep(Duration::from_secs(30)).await; + } + db.mutate(|v| { + v.as_public_mut() + .as_server_info_mut() + .as_ntp_synced_mut() + .ser(&true) + }) + .await + .unwrap() + }) + .into(), + ) }); } @@ -259,6 +273,7 @@ impl RpcContext { .with_kind(crate::ErrorKind::ParseUrl)?, hardware: Hardware { devices, ram }, start_time: Instant::now(), + crons, #[cfg(feature = "dev")] dev: Dev { lxc: Mutex::new(BTreeMap::new()), @@ -273,6 +288,7 @@ impl RpcContext { #[instrument(skip_all)] pub async fn shutdown(self) -> Result<(), Error> { + self.crons.mutate(|c| std::mem::take(c)); self.services.shutdown_all().await?; self.is_closed.store(true, Ordering::SeqCst); tracing::info!("RPC Context is shutdown"); @@ -280,14 +296,75 @@ impl RpcContext { Ok(()) } + pub fn add_cron + Send + 'static>(&self, fut: F) -> Guid { + let guid = Guid::new(); + self.crons + .mutate(|c| c.insert(guid.clone(), tokio::spawn(fut).into())); + guid + } + #[instrument(skip_all)] pub async fn cleanup_and_initialize( &self, CleanupInitPhases { + mut cleanup_sessions, init_services, mut check_dependencies, }: CleanupInitPhases, ) -> Result<(), Error> { + cleanup_sessions.start(); + self.db + .mutate(|db| { + if db.as_public().as_server_info().as_ntp_synced().de()? { + for id in db.as_private().as_sessions().keys()? { + if Utc::now() + - db.as_private() + .as_sessions() + .as_idx(&id) + .unwrap() + .de()? + .last_active + > TimeDelta::days(30) + { + db.as_private_mut().as_sessions_mut().remove(&id)?; + } + } + } + Ok(()) + }) + .await?; + let db = self.db.clone(); + self.add_cron(async move { + loop { + tokio::time::sleep(Duration::from_secs(86400)).await; + if let Err(e) = db + .mutate(|db| { + if db.as_public().as_server_info().as_ntp_synced().de()? { + for id in db.as_private().as_sessions().keys()? { + if Utc::now() + - db.as_private() + .as_sessions() + .as_idx(&id) + .unwrap() + .de()? + .last_active + > TimeDelta::days(30) + { + db.as_private_mut().as_sessions_mut().remove(&id)?; + } + } + } + Ok(()) + }) + .await + { + tracing::error!("Error in session cleanup cron: {e}"); + tracing::debug!("{e:?}"); + } + } + }); + cleanup_sessions.complete(); + self.services.init(&self, init_services).await?; tracing::info!("Initialized Package Managers"); diff --git a/core/startos/src/db/model/package.rs b/core/startos/src/db/model/package.rs index cb537a2b5..957e42c54 100644 --- a/core/startos/src/db/model/package.rs +++ b/core/startos/src/db/model/package.rs @@ -3,7 +3,9 @@ use std::collections::{BTreeMap, BTreeSet}; use chrono::{DateTime, Utc}; use exver::VersionRange; use imbl_value::InternedString; -use models::{ActionId, DataUrl, HealthCheckId, HostId, PackageId, ServiceInterfaceId}; +use models::{ + ActionId, DataUrl, HealthCheckId, HostId, PackageId, ServiceInterfaceId, VersionString, +}; use patch_db::json_ptr::JsonPointer; use patch_db::HasModel; use reqwest::Url; @@ -335,6 +337,7 @@ pub struct ActionMetadata { #[ts(export)] pub struct PackageDataEntry { pub state_info: PackageState, + pub data_version: Option, pub status: Status, #[ts(type = "string | null")] pub registry: Option, diff --git a/core/startos/src/lxc/mod.rs b/core/startos/src/lxc/mod.rs index b8ce9c703..480f7a24c 100644 --- a/core/startos/src/lxc/mod.rs +++ b/core/startos/src/lxc/mod.rs @@ -268,9 +268,10 @@ impl LxcContainer { .invoke(ErrorKind::Docker) .await?, )?; - let out_str = output.trim(); - if !out_str.is_empty() { - return Ok(out_str.parse()?); + for line in output.lines() { + if let Ok(ip) = line.trim().parse() { + return Ok(ip); + } } if start.elapsed() > CONTAINER_DHCP_TIMEOUT { return Err(Error::new( diff --git a/core/startos/src/net/service_interface.rs b/core/startos/src/net/service_interface.rs index dbe228ef2..b1824140b 100644 --- a/core/startos/src/net/service_interface.rs +++ b/core/startos/src/net/service_interface.rs @@ -67,7 +67,6 @@ pub struct ServiceInterface { pub name: String, pub description: String, pub has_primary: bool, - pub disabled: bool, pub masked: bool, pub address_info: AddressInfo, #[serde(rename = "type")] diff --git a/core/startos/src/s9pk/v1/manifest.rs b/core/startos/src/s9pk/v1/manifest.rs index 4a9956f9f..9b3eb9895 100644 --- a/core/startos/src/s9pk/v1/manifest.rs +++ b/core/startos/src/s9pk/v1/manifest.rs @@ -21,7 +21,7 @@ pub struct Manifest { #[serde(default)] pub git_hash: Option, pub title: String, - pub version: exver::emver::Version, + pub version: String, pub description: Description, #[serde(default)] pub assets: Assets, diff --git a/core/startos/src/s9pk/v2/compat.rs b/core/startos/src/s9pk/v2/compat.rs index 22250419a..8e62c69d0 100644 --- a/core/startos/src/s9pk/v2/compat.rs +++ b/core/startos/src/s9pk/v2/compat.rs @@ -1,8 +1,9 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; +use std::str::FromStr; use std::sync::Arc; -use exver::ExtendedVersion; +use exver::{ExtendedVersion, VersionRange}; use models::ImageId; use tokio::io::{AsyncRead, AsyncSeek, AsyncWriteExt}; use tokio::process::Command; @@ -44,9 +45,9 @@ impl S9pk> { // manifest.json let manifest_raw = reader.manifest().await?; let manifest = from_value::(manifest_raw.clone())?; - let mut new_manifest = Manifest::from(manifest.clone()); + let mut new_manifest = Manifest::try_from(manifest.clone())?; - let images: BTreeMap = manifest + let images: BTreeSet<(ImageId, bool)> = manifest .package_procedures() .filter_map(|p| { if let PackageProcedure::Docker(p) = p { @@ -89,8 +90,6 @@ impl S9pk> { // images for arch in reader.docker_arches().await? { - let images_dir = tmp_dir.join("images").join(&arch); - tokio::fs::create_dir_all(&images_dir).await?; Command::new(CONTAINER_TOOL) .arg("load") .input(Some(&mut reader.docker_images(&arch).await?)) @@ -194,15 +193,22 @@ impl S9pk> { } } -impl From for Manifest { - fn from(value: ManifestV1) -> Self { +impl TryFrom for Manifest { + type Error = Error; + fn try_from(value: ManifestV1) -> Result { let default_url = value.upstream_repo.clone(); - Self { + Ok(Self { id: value.id, title: value.title.into(), - version: ExtendedVersion::from(value.version).into(), + version: ExtendedVersion::from( + exver::emver::Version::from_str(&value.version) + .with_kind(ErrorKind::Deserialization)?, + ) + .into(), satisfies: BTreeSet::new(), release_notes: value.release_notes, + can_migrate_from: VersionRange::any(), + can_migrate_to: VersionRange::none(), license: value.license.into(), wrapper_repo: value.wrapper_repo, upstream_repo: value.upstream_repo, @@ -244,6 +250,6 @@ impl From for Manifest { git_hash: value.git_hash, os_version: value.eos_version, has_config: value.config.is_some(), - } + }) } } diff --git a/core/startos/src/s9pk/v2/manifest.rs b/core/startos/src/s9pk/v2/manifest.rs index a10a65ddb..1f24a0b73 100644 --- a/core/startos/src/s9pk/v2/manifest.rs +++ b/core/startos/src/s9pk/v2/manifest.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use color_eyre::eyre::eyre; -use exver::Version; +use exver::{Version, VersionRange}; use helpers::const_true; use imbl_value::InternedString; pub use models::PackageId; @@ -37,6 +37,10 @@ pub struct Manifest { pub satisfies: BTreeSet, pub release_notes: String, #[ts(type = "string")] + pub can_migrate_to: VersionRange, + #[ts(type = "string")] + pub can_migrate_from: VersionRange, + #[ts(type = "string")] pub license: InternedString, // type of license #[ts(type = "string")] pub wrapper_repo: Url, @@ -159,8 +163,8 @@ impl Manifest { #[ts(export)] pub struct HardwareRequirements { #[serde(default)] - #[ts(type = "{ device?: string, processor?: string }")] - pub device: BTreeMap, + #[ts(type = "{ display?: string, processor?: string }")] + pub device: BTreeMap, // TODO: array #[ts(type = "number | null")] pub ram: Option, #[ts(type = "string[] | null")] diff --git a/core/startos/src/s9pk/v2/pack.rs b/core/startos/src/s9pk/v2/pack.rs index 06a47b9d0..aa0fd39f2 100644 --- a/core/startos/src/s9pk/v2/pack.rs +++ b/core/startos/src/s9pk/v2/pack.rs @@ -60,14 +60,20 @@ impl SqfsDir { .get_or_try_init(|| async move { let guid = Guid::new(); let path = self.tmpdir.join(guid.as_ref()).with_extension("squashfs"); - let mut cmd = Command::new("mksquashfs"); if self.path.extension().and_then(|s| s.to_str()) == Some("tar") { - cmd.arg("-tar"); + Command::new("tar2sqfs") + .arg(&path) + .input(Some(&mut open_file(&self.path).await?)) + .invoke(ErrorKind::Filesystem) + .await?; + } else { + Command::new("mksquashfs") + .arg(&self.path) + .arg(&path) + .invoke(ErrorKind::Filesystem) + .await?; } - cmd.arg(&self.path) - .arg(&path) - .invoke(ErrorKind::Filesystem) - .await?; + Ok(MultiCursorFile::from( open_file(&path) .await @@ -507,7 +513,7 @@ impl ImageSource { Command::new(CONTAINER_TOOL) .arg("export") .arg(container.trim()) - .pipe(Command::new("mksquashfs").arg("-").arg(&dest).arg("-tar")) + .pipe(Command::new("tar2sqfs").arg(&dest)) .capture(false) .invoke(ErrorKind::Docker) .await?; diff --git a/core/startos/src/service/effects/dependency.rs b/core/startos/src/service/effects/dependency.rs index ad5ec2e9b..26582d061 100644 --- a/core/startos/src/service/effects/dependency.rs +++ b/core/startos/src/service/effects/dependency.rs @@ -4,8 +4,10 @@ use std::str::FromStr; use clap::builder::ValueParserFactory; use exver::VersionRange; +use imbl::OrdMap; +use imbl_value::InternedString; use itertools::Itertools; -use models::{HealthCheckId, PackageId, VolumeId}; +use models::{HealthCheckId, PackageId, VersionString, VolumeId}; use patch_db::json_ptr::JsonPointer; use tokio::process::Command; @@ -17,7 +19,7 @@ use crate::disk::mount::filesystem::idmapped::IdMapped; use crate::disk::mount::filesystem::{FileSystem, MountType}; use crate::rpc_continuations::Guid; use crate::service::effects::prelude::*; -use crate::status::health_check::HealthCheckResult; +use crate::status::health_check::NamedHealthCheckResult; use crate::util::clap::FromStrParser; use crate::util::Invoke; use crate::volume::data_dir; @@ -316,12 +318,16 @@ pub struct CheckDependenciesParam { #[ts(export)] pub struct CheckDependenciesResult { package_id: PackageId, - is_installed: bool, + #[ts(type = "string | null")] + title: Option, + #[ts(type = "string | null")] + installed_version: Option, + #[ts(type = "string[]")] + satisfies: BTreeSet, is_running: bool, config_satisfied: bool, - health_checks: BTreeMap, - #[ts(type = "string | null")] - version: Option, + #[ts(as = "BTreeMap::")] + health_checks: OrdMap, } pub async fn check_dependencies( context: EffectContext, @@ -347,36 +353,23 @@ pub async fn check_dependencies( let mut results = Vec::with_capacity(package_ids.len()); for (package_id, dependency_info) in package_ids { + let title = dependency_info.title.clone(); let Some(package) = db.as_public().as_package_data().as_idx(&package_id) else { results.push(CheckDependenciesResult { package_id, - is_installed: false, + title, + installed_version: None, + satisfies: BTreeSet::new(), is_running: false, config_satisfied: false, health_checks: Default::default(), - version: None, }); continue; }; let manifest = package.as_state_info().as_manifest(ManifestPreference::New); let installed_version = manifest.as_version().de()?.into_version(); let satisfies = manifest.as_satisfies().de()?; - let version = Some(installed_version.clone()); - if ![installed_version] - .into_iter() - .chain(satisfies.into_iter().map(|v| v.into_version())) - .any(|v| v.satisfies(&dependency_info.version_range)) - { - results.push(CheckDependenciesResult { - package_id, - is_installed: false, - is_running: false, - config_satisfied: false, - health_checks: Default::default(), - version, - }); - continue; - } + let installed_version = Some(installed_version.clone()); let is_installed = true; let status = package.as_status().as_main().de()?; let is_running = if is_installed { @@ -384,25 +377,15 @@ pub async fn check_dependencies( } else { false }; - let health_checks = - if let CurrentDependencyKind::Running { health_checks } = &dependency_info.kind { - status - .health() - .cloned() - .unwrap_or_default() - .into_iter() - .filter(|(id, _)| health_checks.contains(id)) - .collect() - } else { - Default::default() - }; + let health_checks = status.health().cloned().unwrap_or_default(); results.push(CheckDependenciesResult { package_id, - is_installed, + title, + installed_version, + satisfies, is_running, config_satisfied: dependency_info.config_satisfied, health_checks, - version, }); } Ok(results) diff --git a/core/startos/src/service/effects/health.rs b/core/startos/src/service/effects/health.rs index c8ef8fc4e..9bf756d60 100644 --- a/core/startos/src/service/effects/health.rs +++ b/core/startos/src/service/effects/health.rs @@ -1,7 +1,7 @@ use models::HealthCheckId; use crate::service::effects::prelude::*; -use crate::status::health_check::HealthCheckResult; +use crate::status::health_check::NamedHealthCheckResult; use crate::status::MainStatus; #[derive(Debug, Clone, Serialize, Deserialize, TS)] @@ -10,7 +10,7 @@ use crate::status::MainStatus; pub struct SetHealth { id: HealthCheckId, #[serde(flatten)] - result: HealthCheckResult, + result: NamedHealthCheckResult, } pub async fn set_health( context: EffectContext, @@ -32,8 +32,8 @@ pub async fn set_health( .as_main_mut() .mutate(|main| { match main { - &mut MainStatus::Running { ref mut health, .. } - | &mut MainStatus::BackingUp { ref mut health, .. } => { + MainStatus::Running { ref mut health, .. } + | MainStatus::Starting { ref mut health } => { health.insert(id, result); } _ => (), diff --git a/core/startos/src/service/effects/mod.rs b/core/startos/src/service/effects/mod.rs index 91a12a4d1..e85481e96 100644 --- a/core/startos/src/service/effects/mod.rs +++ b/core/startos/src/service/effects/mod.rs @@ -1,4 +1,4 @@ -use rpc_toolkit::{from_fn, from_fn_async, Context, HandlerExt, ParentHandler}; +use rpc_toolkit::{from_fn, from_fn_async, from_fn_blocking, Context, HandlerExt, ParentHandler}; use crate::echo; use crate::prelude::*; @@ -12,44 +12,44 @@ pub mod context; mod control; mod dependency; mod health; -mod image; mod net; mod prelude; mod store; +mod subcontainer; mod system; pub fn handler() -> ParentHandler { ParentHandler::new() - .subcommand("gitInfo", from_fn(|_: C| crate::version::git_info())) + .subcommand("git-info", from_fn(|_: C| crate::version::git_info())) .subcommand( "echo", from_fn(echo::).with_call_remote::(), ) // action .subcommand( - "executeAction", + "execute-action", from_fn_async(action::execute_action).no_cli(), ) .subcommand( - "exportAction", + "export-action", from_fn_async(action::export_action).no_cli(), ) .subcommand( - "clearActions", + "clear-actions", from_fn_async(action::clear_actions).no_cli(), ) // callbacks .subcommand( - "clearCallbacks", + "clear-callbacks", from_fn(callbacks::clear_callbacks).no_cli(), ) // config .subcommand( - "getConfigured", + "get-configured", from_fn_async(config::get_configured).no_cli(), ) .subcommand( - "setConfigured", + "set-configured", from_fn_async(config::set_configured) .no_display() .with_call_remote::(), @@ -68,105 +68,143 @@ pub fn handler() -> ParentHandler { .with_call_remote::(), ) .subcommand( - "setMainStatus", + "set-main-status", from_fn_async(control::set_main_status) .no_display() .with_call_remote::(), ) // dependency .subcommand( - "setDependencies", + "set-dependencies", from_fn_async(dependency::set_dependencies) .no_display() .with_call_remote::(), ) .subcommand( - "getDependencies", + "get-dependencies", from_fn_async(dependency::get_dependencies) .no_display() .with_call_remote::(), ) .subcommand( - "checkDependencies", + "check-dependencies", from_fn_async(dependency::check_dependencies) .no_display() .with_call_remote::(), ) .subcommand("mount", from_fn_async(dependency::mount).no_cli()) .subcommand( - "getInstalledPackages", + "get-installed-packages", from_fn_async(dependency::get_installed_packages).no_cli(), ) .subcommand( - "exposeForDependents", + "expose-for-dependents", from_fn_async(dependency::expose_for_dependents).no_cli(), ) // health - .subcommand("setHealth", from_fn_async(health::set_health).no_cli()) - // image + .subcommand("set-health", from_fn_async(health::set_health).no_cli()) + // subcontainer .subcommand( - "chroot", - from_fn(image::chroot::).no_display(), - ) - .subcommand( - "createOverlayedImage", - from_fn_async(image::create_overlayed_image) - .with_custom_display_fn(|_, (path, _)| Ok(println!("{}", path.display()))) - .with_call_remote::(), - ) - .subcommand( - "destroyOverlayedImage", - from_fn_async(image::destroy_overlayed_image).no_cli(), + "subcontainer", + ParentHandler::::new() + .subcommand( + "launch", + from_fn_blocking(subcontainer::launch).no_display(), + ) + .subcommand( + "launch-init", + from_fn_blocking(subcontainer::launch_init).no_display(), + ) + .subcommand("exec", from_fn_blocking(subcontainer::exec).no_display()) + .subcommand( + "exec-command", + from_fn_blocking(subcontainer::exec_command).no_display(), + ) + .subcommand( + "create-fs", + from_fn_async(subcontainer::create_subcontainer_fs) + .with_custom_display_fn(|_, (path, _)| Ok(println!("{}", path.display()))) + .with_call_remote::(), + ) + .subcommand( + "destroy-fs", + from_fn_async(subcontainer::destroy_subcontainer_fs) + .no_display() + .with_call_remote::(), + ), ) // net .subcommand("bind", from_fn_async(net::bind::bind).no_cli()) .subcommand( - "getServicePortForward", + "get-service-port-forward", from_fn_async(net::bind::get_service_port_forward).no_cli(), ) .subcommand( - "clearBindings", + "clear-bindings", from_fn_async(net::bind::clear_bindings).no_cli(), ) .subcommand( - "getHostInfo", + "get-host-info", from_fn_async(net::host::get_host_info).no_cli(), ) .subcommand( - "getPrimaryUrl", + "get-primary-url", from_fn_async(net::host::get_primary_url).no_cli(), ) .subcommand( - "getContainerIp", + "get-container-ip", from_fn_async(net::info::get_container_ip).no_cli(), ) .subcommand( - "exportServiceInterface", + "export-service-interface", from_fn_async(net::interface::export_service_interface).no_cli(), ) .subcommand( - "getServiceInterface", + "get-service-interface", from_fn_async(net::interface::get_service_interface).no_cli(), ) .subcommand( - "listServiceInterfaces", + "list-service-interfaces", from_fn_async(net::interface::list_service_interfaces).no_cli(), ) .subcommand( - "clearServiceInterfaces", + "clear-service-interfaces", from_fn_async(net::interface::clear_service_interfaces).no_cli(), ) .subcommand( - "getSslCertificate", + "get-ssl-certificate", from_fn_async(net::ssl::get_ssl_certificate).no_cli(), ) - .subcommand("getSslKey", from_fn_async(net::ssl::get_ssl_key).no_cli()) + .subcommand("get-ssl-key", from_fn_async(net::ssl::get_ssl_key).no_cli()) // store - .subcommand("getStore", from_fn_async(store::get_store).no_cli()) - .subcommand("setStore", from_fn_async(store::set_store).no_cli()) + .subcommand( + "store", + ParentHandler::::new() + .subcommand("get", from_fn_async(store::get_store).no_cli()) + .subcommand("set", from_fn_async(store::set_store).no_cli()), + ) + .subcommand( + "set-data-version", + from_fn_async(store::set_data_version) + .no_display() + .with_call_remote::(), + ) + .subcommand( + "get-data-version", + from_fn_async(store::get_data_version) + .with_custom_display_fn(|_, v| { + if let Some(v) = v { + println!("{v}") + } else { + println!("N/A") + } + Ok(()) + }) + .with_call_remote::(), + ) // system .subcommand( - "getSystemSmtp", + "get-system-smtp", from_fn_async(system::get_system_smtp).no_cli(), ) diff --git a/core/startos/src/service/effects/net/interface.rs b/core/startos/src/service/effects/net/interface.rs index e636e9b57..6cd4cd4c9 100644 --- a/core/startos/src/service/effects/net/interface.rs +++ b/core/startos/src/service/effects/net/interface.rs @@ -16,7 +16,6 @@ pub struct ExportServiceInterfaceParams { name: String, description: String, has_primary: bool, - disabled: bool, masked: bool, address_info: AddressInfo, r#type: ServiceInterfaceType, @@ -28,7 +27,6 @@ pub async fn export_service_interface( name, description, has_primary, - disabled, masked, address_info, r#type, @@ -42,7 +40,6 @@ pub async fn export_service_interface( name, description, has_primary, - disabled, masked, address_info, interface_type: r#type, diff --git a/core/startos/src/service/effects/store.rs b/core/startos/src/service/effects/store.rs index ab4484ab6..6c12b425e 100644 --- a/core/startos/src/service/effects/store.rs +++ b/core/startos/src/service/effects/store.rs @@ -1,6 +1,6 @@ use imbl::vector; use imbl_value::json; -use models::PackageId; +use models::{PackageId, VersionString}; use patch_db::json_ptr::JsonPointer; use crate::service::effects::callbacks::CallbackHandler; @@ -91,3 +91,50 @@ pub async fn set_store( Ok(()) } + +#[derive(Debug, Clone, Serialize, Deserialize, TS, Parser)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct SetDataVersionParams { + #[ts(type = "string")] + version: VersionString, +} +pub async fn set_data_version( + context: EffectContext, + SetDataVersionParams { version }: SetDataVersionParams, +) -> Result<(), Error> { + let context = context.deref()?; + let package_id = &context.seed.id; + context + .seed + .ctx + .db + .mutate(|db| { + db.as_public_mut() + .as_package_data_mut() + .as_idx_mut(package_id) + .or_not_found(package_id)? + .as_data_version_mut() + .ser(&Some(version)) + }) + .await?; + + Ok(()) +} + +pub async fn get_data_version(context: EffectContext) -> Result, Error> { + let context = context.deref()?; + let package_id = &context.seed.id; + context + .seed + .ctx + .db + .peek() + .await + .as_public() + .as_package_data() + .as_idx(package_id) + .or_not_found(package_id)? + .as_data_version() + .de() +} diff --git a/core/startos/src/service/effects/image.rs b/core/startos/src/service/effects/subcontainer/mod.rs similarity index 50% rename from core/startos/src/service/effects/image.rs rename to core/startos/src/service/effects/subcontainer/mod.rs index af62047ed..0375ef6c2 100644 --- a/core/startos/src/service/effects/image.rs +++ b/core/startos/src/service/effects/subcontainer/mod.rs @@ -1,9 +1,6 @@ -use std::ffi::OsString; -use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; use models::ImageId; -use rpc_toolkit::Context; use tokio::process::Command; use crate::disk::mount::filesystem::overlayfs::OverlayGuard; @@ -11,89 +8,39 @@ use crate::rpc_continuations::Guid; use crate::service::effects::prelude::*; use crate::util::Invoke; -#[derive(Debug, Clone, Serialize, Deserialize, Parser)] -pub struct ChrootParams { - #[arg(short = 'e', long = "env")] - env: Option, - #[arg(short = 'w', long = "workdir")] - workdir: Option, - #[arg(short = 'u', long = "user")] - user: Option, - path: PathBuf, - command: OsString, - args: Vec, -} -pub fn chroot( - _: C, - ChrootParams { - env, - workdir, - user, - path, - command, - args, - }: ChrootParams, -) -> Result<(), Error> { - let mut cmd = std::process::Command::new(command); - if let Some(env) = env { - for (k, v) in std::fs::read_to_string(env)? - .lines() - .map(|l| l.trim()) - .filter_map(|l| l.split_once("=")) - { - cmd.env(k, v); - } - } - nix::unistd::setsid().ok(); // https://stackoverflow.com/questions/25701333/os-setsid-operation-not-permitted - std::os::unix::fs::chroot(path)?; - if let Some(uid) = user.as_deref().and_then(|u| u.parse::().ok()) { - cmd.uid(uid); - } else if let Some(user) = user { - let (uid, gid) = std::fs::read_to_string("/etc/passwd")? - .lines() - .find_map(|l| { - let mut split = l.trim().split(":"); - if user != split.next()? { - return None; - } - split.next(); // throw away x - Some((split.next()?.parse().ok()?, split.next()?.parse().ok()?)) - // uid gid - }) - .or_not_found(lazy_format!("{user} in /etc/passwd"))?; - cmd.uid(uid); - cmd.gid(gid); - }; - if let Some(workdir) = workdir { - cmd.current_dir(workdir); - } - cmd.args(args); - Err(cmd.exec().into()) -} +#[cfg(feature = "container-runtime")] +mod sync; + +#[cfg(not(feature = "container-runtime"))] +mod sync_dummy; + +pub use sync::*; +#[cfg(not(feature = "container-runtime"))] +use sync_dummy as sync; #[derive(Debug, Deserialize, Serialize, Parser, TS)] #[serde(rename_all = "camelCase")] #[ts(export)] -pub struct DestroyOverlayedImageParams { +pub struct DestroySubcontainerFsParams { guid: Guid, } #[instrument(skip_all)] -pub async fn destroy_overlayed_image( +pub async fn destroy_subcontainer_fs( context: EffectContext, - DestroyOverlayedImageParams { guid }: DestroyOverlayedImageParams, + DestroySubcontainerFsParams { guid }: DestroySubcontainerFsParams, ) -> Result<(), Error> { let context = context.deref()?; if let Some(overlay) = context .seed .persistent_container - .overlays + .subcontainers .lock() .await .remove(&guid) { overlay.unmount(true).await?; } else { - tracing::warn!("Could not find a guard to remove on the destroy overlayed image; assumming that it already is removed and will be skipping"); + tracing::warn!("Could not find a subcontainer fs to destroy; assumming that it already is destroyed and will be skipping"); } Ok(()) } @@ -101,13 +48,13 @@ pub async fn destroy_overlayed_image( #[derive(Debug, Deserialize, Serialize, Parser, TS)] #[serde(rename_all = "camelCase")] #[ts(export)] -pub struct CreateOverlayedImageParams { +pub struct CreateSubcontainerFsParams { image_id: ImageId, } #[instrument(skip_all)] -pub async fn create_overlayed_image( +pub async fn create_subcontainer_fs( context: EffectContext, - CreateOverlayedImageParams { image_id }: CreateOverlayedImageParams, + CreateSubcontainerFsParams { image_id }: CreateSubcontainerFsParams, ) -> Result<(PathBuf, Guid), Error> { let context = context.deref()?; if let Some(image) = context @@ -131,7 +78,7 @@ pub async fn create_overlayed_image( })? .rootfs_dir(); let mountpoint = rootfs_dir - .join("media/startos/overlays") + .join("media/startos/subcontainers") .join(guid.as_ref()); tokio::fs::create_dir_all(&mountpoint).await?; let container_mountpoint = Path::new("/").join( @@ -150,7 +97,7 @@ pub async fn create_overlayed_image( context .seed .persistent_container - .overlays + .subcontainers .lock() .await .insert(guid.clone(), guard); diff --git a/core/startos/src/service/effects/subcontainer/sync.rs b/core/startos/src/service/effects/subcontainer/sync.rs new file mode 100644 index 000000000..e18586a54 --- /dev/null +++ b/core/startos/src/service/effects/subcontainer/sync.rs @@ -0,0 +1,392 @@ +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::ffi::{c_int, OsStr, OsString}; +use std::fs::File; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::{Command as StdCommand, Stdio}; + +use nix::sched::CloneFlags; +use nix::unistd::Pid; +use rpc_toolkit::Context; +use signal_hook::consts::signal::*; +use tokio::sync::oneshot; +use unshare::Command as NSCommand; + +use crate::service::effects::prelude::*; +use crate::service::effects::ContainerCliContext; + +const FWD_SIGNALS: &[c_int] = &[ + SIGABRT, SIGALRM, SIGCONT, SIGHUP, SIGINT, SIGIO, SIGPIPE, SIGPROF, SIGQUIT, SIGTERM, SIGTRAP, + SIGTSTP, SIGTTIN, SIGTTOU, SIGURG, SIGUSR1, SIGUSR2, SIGVTALRM, +]; + +struct NSPid(Vec); +impl procfs::FromBufRead for NSPid { + fn from_buf_read(r: R) -> procfs::ProcResult { + for line in r.lines() { + let line = line?; + if let Some(row) = line.trim().strip_prefix("NSpid") { + return Ok(Self( + row.split_ascii_whitespace() + .map(|pid| pid.parse::()) + .collect::, _>>()?, + )); + } + } + Err(procfs::ProcError::Incomplete(None)) + } +} + +fn open_file_read(path: impl AsRef) -> Result { + File::open(&path).with_ctx(|_| { + ( + ErrorKind::Filesystem, + lazy_format!("open r {}", path.as_ref().display()), + ) + }) +} + +#[derive(Debug, Clone, Serialize, Deserialize, Parser)] +pub struct ExecParams { + #[arg(short = 'e', long = "env")] + env: Option, + #[arg(short = 'w', long = "workdir")] + workdir: Option, + #[arg(short = 'u', long = "user")] + user: Option, + chroot: PathBuf, + #[arg(trailing_var_arg = true)] + command: Vec, +} +impl ExecParams { + fn exec(&self) -> Result<(), Error> { + let ExecParams { + env, + workdir, + user, + chroot, + command, + } = self; + let Some(([command], args)) = command.split_at_checked(1) else { + return Err(Error::new( + eyre!("command cannot be empty"), + ErrorKind::InvalidRequest, + )); + }; + let env_string = if let Some(env) = &env { + std::fs::read_to_string(env) + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("read {env:?}")))? + } else { + Default::default() + }; + let env = env_string + .lines() + .map(|l| l.trim()) + .filter_map(|l| l.split_once("=")) + .collect::>(); + std::os::unix::fs::chroot(chroot) + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("chroot {chroot:?}")))?; + let command = which::which_in( + command, + env.get("PATH") + .copied() + .map(Cow::Borrowed) + .or_else(|| std::env::var("PATH").ok().map(Cow::Owned)) + .as_deref(), + workdir.as_deref().unwrap_or(Path::new("/")), + ) + .with_kind(ErrorKind::Filesystem)?; + let mut cmd = StdCommand::new(command); + cmd.args(args); + for (k, v) in env { + cmd.env(k, v); + } + + if let Some(uid) = user.as_deref().and_then(|u| u.parse::().ok()) { + cmd.uid(uid); + } else if let Some(user) = user { + let (uid, gid) = std::fs::read_to_string("/etc/passwd") + .with_ctx(|_| (ErrorKind::Filesystem, "read /etc/passwd"))? + .lines() + .find_map(|l| { + let mut split = l.trim().split(":"); + if user != split.next()? { + return None; + } + split.next(); // throw away x + Some((split.next()?.parse().ok()?, split.next()?.parse().ok()?)) + // uid gid + }) + .or_not_found(lazy_format!("{user} in /etc/passwd"))?; + cmd.uid(uid); + cmd.gid(gid); + }; + if let Some(workdir) = workdir { + cmd.current_dir(workdir); + } else { + cmd.current_dir("/"); + } + Err(cmd.exec().into()) + } +} + +pub fn launch( + _: ContainerCliContext, + ExecParams { + env, + workdir, + user, + chroot, + command, + }: ExecParams, +) -> Result<(), Error> { + use unshare::{Namespace, Stdio}; + + use crate::service::cli::ContainerCliContext; + let mut sig = signal_hook::iterator::Signals::new(FWD_SIGNALS)?; + let mut cmd = NSCommand::new("/usr/bin/start-cli"); + cmd.arg("subcontainer").arg("launch-init"); + if let Some(env) = env { + cmd.arg("--env").arg(env); + } + if let Some(workdir) = workdir { + cmd.arg("--workdir").arg(workdir); + } + if let Some(user) = user { + cmd.arg("--user").arg(user); + } + cmd.arg(&chroot); + cmd.args(&command); + cmd.unshare(&[Namespace::Pid, Namespace::Cgroup, Namespace::Ipc]); + cmd.stdin(Stdio::piped()); + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + let (stdin_send, stdin_recv) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(mut stdin) = stdin_recv.blocking_recv() { + std::io::copy(&mut std::io::stdin(), &mut stdin).unwrap(); + } + }); + let (stdout_send, stdout_recv) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(mut stdout) = stdout_recv.blocking_recv() { + std::io::copy(&mut stdout, &mut std::io::stdout()).unwrap(); + } + }); + let (stderr_send, stderr_recv) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(mut stderr) = stderr_recv.blocking_recv() { + std::io::copy(&mut stderr, &mut std::io::stderr()).unwrap(); + } + }); + if chroot.join("proc/1").exists() { + let ns_id = procfs::process::Process::new_with_root(chroot.join("proc")) + .with_ctx(|_| (ErrorKind::Filesystem, "open subcontainer procfs"))? + .namespaces() + .with_ctx(|_| (ErrorKind::Filesystem, "read subcontainer pid 1 ns"))? + .0 + .get(OsStr::new("pid")) + .or_not_found("pid namespace")? + .identifier; + for proc in + procfs::process::all_processes().with_ctx(|_| (ErrorKind::Filesystem, "open procfs"))? + { + let proc = proc.with_ctx(|_| (ErrorKind::Filesystem, "read single process details"))?; + let pid = proc.pid(); + if proc + .namespaces() + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("read pid {} ns", pid)))? + .0 + .get(OsStr::new("pid")) + .map_or(false, |ns| ns.identifier == ns_id) + { + let pids = proc.read::("status").with_ctx(|_| { + ( + ErrorKind::Filesystem, + lazy_format!("read pid {} NSpid", pid), + ) + })?; + if pids.0.len() == 2 && pids.0[1] == 1 { + nix::sys::signal::kill(Pid::from_raw(pid), nix::sys::signal::SIGKILL) + .with_ctx(|_| { + ( + ErrorKind::Filesystem, + lazy_format!( + "kill pid {} (determined to be pid 1 in subcontainer)", + pid + ), + ) + })?; + } + } + } + nix::mount::umount(&chroot.join("proc")) + .with_ctx(|_| (ErrorKind::Filesystem, "unmounting subcontainer procfs"))?; + } + let mut child = cmd + .spawn() + .map_err(color_eyre::eyre::Report::msg) + .with_ctx(|_| (ErrorKind::Filesystem, "spawning child process"))?; + let pid = child.pid(); + std::thread::spawn(move || { + for sig in sig.forever() { + nix::sys::signal::kill( + Pid::from_raw(pid), + Some(nix::sys::signal::Signal::try_from(sig).unwrap()), + ) + .unwrap(); + } + }); + stdin_send + .send(child.stdin.take().unwrap()) + .unwrap_or_default(); + stdout_send + .send(child.stdout.take().unwrap()) + .unwrap_or_default(); + stderr_send + .send(child.stderr.take().unwrap()) + .unwrap_or_default(); + // TODO: subreaping, signal handling + let exit = child + .wait() + .with_ctx(|_| (ErrorKind::Filesystem, "waiting on child process"))?; + if let Some(code) = exit.code() { + std::process::exit(code); + } else { + if exit.success() { + Ok(()) + } else { + Err(Error::new( + color_eyre::eyre::Report::msg(exit), + ErrorKind::Unknown, + )) + } + } +} + +pub fn launch_init(_: ContainerCliContext, params: ExecParams) -> Result<(), Error> { + nix::mount::mount( + Some("proc"), + ¶ms.chroot.join("proc"), + Some("proc"), + nix::mount::MsFlags::empty(), + None::<&str>, + ) + .with_ctx(|_| (ErrorKind::Filesystem, "mount procfs"))?; + if params.command.is_empty() { + signal_hook::iterator::Signals::new(signal_hook::consts::TERM_SIGNALS)? + .forever() + .next(); + std::process::exit(0) + } else { + params.exec() + } +} + +pub fn exec( + _: ContainerCliContext, + ExecParams { + env, + workdir, + user, + chroot, + command, + }: ExecParams, +) -> Result<(), Error> { + let mut sig = signal_hook::iterator::Signals::new(FWD_SIGNALS)?; + let (send_pid, recv_pid) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(pid) = recv_pid.blocking_recv() { + for sig in sig.forever() { + nix::sys::signal::kill( + Pid::from_raw(pid), + Some(nix::sys::signal::Signal::try_from(sig).unwrap()), + ) + .unwrap(); + } + } + }); + let mut cmd = StdCommand::new("/usr/bin/start-cli"); + cmd.arg("subcontainer").arg("exec-command"); + if let Some(env) = env { + cmd.arg("--env").arg(env); + } + if let Some(workdir) = workdir { + cmd.arg("--workdir").arg(workdir); + } + if let Some(user) = user { + cmd.arg("--user").arg(user); + } + cmd.arg(&chroot); + cmd.args(&command); + cmd.stdin(Stdio::piped()); + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + let (stdin_send, stdin_recv) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(mut stdin) = stdin_recv.blocking_recv() { + std::io::copy(&mut std::io::stdin(), &mut stdin).unwrap(); + } + }); + let (stdout_send, stdout_recv) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(mut stdout) = stdout_recv.blocking_recv() { + std::io::copy(&mut stdout, &mut std::io::stdout()).unwrap(); + } + }); + let (stderr_send, stderr_recv) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(mut stderr) = stderr_recv.blocking_recv() { + std::io::copy(&mut stderr, &mut std::io::stderr()).unwrap(); + } + }); + nix::sched::setns( + open_file_read(chroot.join("proc/1/ns/pid"))?, + CloneFlags::CLONE_NEWPID, + ) + .with_ctx(|_| (ErrorKind::Filesystem, "set pid ns"))?; + nix::sched::setns( + open_file_read(chroot.join("proc/1/ns/cgroup"))?, + CloneFlags::CLONE_NEWCGROUP, + ) + .with_ctx(|_| (ErrorKind::Filesystem, "set cgroup ns"))?; + nix::sched::setns( + open_file_read(chroot.join("proc/1/ns/ipc"))?, + CloneFlags::CLONE_NEWIPC, + ) + .with_ctx(|_| (ErrorKind::Filesystem, "set ipc ns"))?; + let mut child = cmd + .spawn() + .map_err(color_eyre::eyre::Report::msg) + .with_ctx(|_| (ErrorKind::Filesystem, "spawning child process"))?; + send_pid.send(child.id() as i32).unwrap_or_default(); + stdin_send + .send(child.stdin.take().unwrap()) + .unwrap_or_default(); + stdout_send + .send(child.stdout.take().unwrap()) + .unwrap_or_default(); + stderr_send + .send(child.stderr.take().unwrap()) + .unwrap_or_default(); + let exit = child + .wait() + .with_ctx(|_| (ErrorKind::Filesystem, "waiting on child process"))?; + if let Some(code) = exit.code() { + std::process::exit(code); + } else { + if exit.success() { + Ok(()) + } else { + Err(Error::new( + color_eyre::eyre::Report::msg(exit), + ErrorKind::Unknown, + )) + } + } +} + +pub fn exec_command(_: ContainerCliContext, params: ExecParams) -> Result<(), Error> { + params.exec() +} diff --git a/core/startos/src/service/effects/subcontainer/sync_dummy.rs b/core/startos/src/service/effects/subcontainer/sync_dummy.rs new file mode 100644 index 000000000..285bdcbc1 --- /dev/null +++ b/core/startos/src/service/effects/subcontainer/sync_dummy.rs @@ -0,0 +1,30 @@ +use crate::service::effects::prelude::*; +use crate::service::effects::ContainerCliContext; + +pub fn launch(_: ContainerCliContext) -> Result<(), Error> { + Err(Error::new( + eyre!("requires feature container-runtime"), + ErrorKind::InvalidRequest, + )) +} + +pub fn launch_init(_: ContainerCliContext) -> Result<(), Error> { + Err(Error::new( + eyre!("requires feature container-runtime"), + ErrorKind::InvalidRequest, + )) +} + +pub fn exec(_: ContainerCliContext) -> Result<(), Error> { + Err(Error::new( + eyre!("requires feature container-runtime"), + ErrorKind::InvalidRequest, + )) +} + +pub fn exec_command(_: ContainerCliContext) -> Result<(), Error> { + Err(Error::new( + eyre!("requires feature container-runtime"), + ErrorKind::InvalidRequest, + )) +} diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index 2beb5c9fa..5eae62756 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -27,7 +27,7 @@ use crate::progress::{NamedProgress, Progress}; use crate::rpc_continuations::Guid; use crate::s9pk::S9pk; use crate::service::service_map::InstallProgressHandles; -use crate::status::health_check::HealthCheckResult; +use crate::status::health_check::NamedHealthCheckResult; use crate::util::actor::concurrent::ConcurrentActor; use crate::util::io::create_file; use crate::util::serde::{NoOutput, Pem}; @@ -45,7 +45,7 @@ mod properties; mod rpc; mod service_actor; pub mod service_map; -mod start_stop; +pub mod start_stop; mod transition; mod util; @@ -493,7 +493,6 @@ impl Service { #[derive(Debug, Clone)] pub struct RunningStatus { - health: OrdMap, started: DateTime, } @@ -516,7 +515,6 @@ impl ServiceActorSeed { .running_status .take() .unwrap_or_else(|| RunningStatus { - health: Default::default(), started: Utc::now(), }), ); diff --git a/core/startos/src/service/persistent_container.rs b/core/startos/src/service/persistent_container.rs index c81322719..dd7b5766d 100644 --- a/core/startos/src/service/persistent_container.rs +++ b/core/startos/src/service/persistent_container.rs @@ -98,7 +98,7 @@ pub struct PersistentContainer { volumes: BTreeMap, assets: BTreeMap, pub(super) images: BTreeMap>, - pub(super) overlays: Arc>>>>, + pub(super) subcontainers: Arc>>>>, pub(super) state: Arc>, pub(super) net_service: Mutex, destroyed: bool, @@ -273,7 +273,7 @@ impl PersistentContainer { volumes, assets, images, - overlays: Arc::new(Mutex::new(BTreeMap::new())), + subcontainers: Arc::new(Mutex::new(BTreeMap::new())), state: Arc::new(watch::channel(ServiceState::new(start)).0), net_service: Mutex::new(net_service), destroyed: false, @@ -388,7 +388,7 @@ impl PersistentContainer { let volumes = std::mem::take(&mut self.volumes); let assets = std::mem::take(&mut self.assets); let images = std::mem::take(&mut self.images); - let overlays = self.overlays.clone(); + let subcontainers = self.subcontainers.clone(); let lxc_container = self.lxc_container.take(); self.destroyed = true; Some(async move { @@ -404,7 +404,7 @@ impl PersistentContainer { for (_, assets) in assets { errs.handle(assets.unmount(true).await); } - for (_, overlay) in std::mem::take(&mut *overlays.lock().await) { + for (_, overlay) in std::mem::take(&mut *subcontainers.lock().await) { errs.handle(overlay.unmount(true).await); } for (_, images) in images { diff --git a/core/startos/src/service/service_actor.rs b/core/startos/src/service/service_actor.rs index e6578264c..0839afc0b 100644 --- a/core/startos/src/service/service_actor.rs +++ b/core/startos/src/service/service_actor.rs @@ -1,11 +1,10 @@ use std::sync::Arc; use std::time::Duration; -use imbl::OrdMap; - use super::start_stop::StartStop; use super::ServiceActorSeed; use crate::prelude::*; +use crate::service::persistent_container::ServiceStateKinds; use crate::service::transition::TransitionKind; use crate::service::SYNC_RETRY_COOLDOWN_SECONDS; use crate::status::MainStatus; @@ -46,96 +45,77 @@ async fn service_actor_loop( let id = &seed.id; let kinds = current.borrow().kinds(); if let Err(e) = async { - let main_status = match ( - kinds.transition_state, - kinds.desired_state, - kinds.running_status, - ) { - (Some(TransitionKind::Restarting), StartStop::Stop, Some(_)) => { - seed.persistent_container.stop().await?; - MainStatus::Restarting - } - (Some(TransitionKind::Restarting), StartStop::Start, _) => { - seed.persistent_container.start().await?; - MainStatus::Restarting - } - (Some(TransitionKind::Restarting), _, _) => MainStatus::Restarting, - (Some(TransitionKind::Restoring), _, _) => MainStatus::Restoring, - (Some(TransitionKind::BackingUp), StartStop::Stop, Some(status)) => { - seed.persistent_container.stop().await?; - MainStatus::BackingUp { - started: Some(status.started), - health: status.health.clone(), - } - } - (Some(TransitionKind::BackingUp), StartStop::Start, _) => { - seed.persistent_container.start().await?; - MainStatus::BackingUp { - started: None, - health: OrdMap::new(), - } - } - (Some(TransitionKind::BackingUp), _, _) => MainStatus::BackingUp { - started: None, - health: OrdMap::new(), - }, - (None, StartStop::Stop, None) => MainStatus::Stopped, - (None, StartStop::Stop, Some(_)) => { - let task_seed = seed.clone(); - seed.ctx - .db - .mutate(|d| { - if let Some(i) = d.as_public_mut().as_package_data_mut().as_idx_mut(&id) { - i.as_status_mut().as_main_mut().ser(&MainStatus::Stopping)?; - } - Ok(()) - }) - .await?; - task_seed.persistent_container.stop().await?; - MainStatus::Stopped - } - (None, StartStop::Start, Some(status)) => MainStatus::Running { - started: status.started, - health: status.health.clone(), - }, - (None, StartStop::Start, None) => { - seed.persistent_container.start().await?; - MainStatus::Starting - } - }; seed.ctx .db .mutate(|d| { if let Some(i) = d.as_public_mut().as_package_data_mut().as_idx_mut(&id) { let previous = i.as_status().as_main().de()?; - let previous_health = previous.health(); - let previous_started = previous.started(); - let mut main_status = main_status; - match &mut main_status { - &mut MainStatus::Running { ref mut health, .. } - | &mut MainStatus::BackingUp { ref mut health, .. } => { - *health = previous_health.unwrap_or(health).clone(); - } - _ => (), - }; - match &mut main_status { - MainStatus::Running { - ref mut started, .. - } => { - *started = previous_started.unwrap_or(*started); - } - MainStatus::BackingUp { - ref mut started, .. - } => { - *started = previous_started.map(Some).unwrap_or(*started); - } - _ => (), + let main_status = match &kinds { + ServiceStateKinds { + transition_state: Some(TransitionKind::Restarting), + .. + } => MainStatus::Restarting, + ServiceStateKinds { + transition_state: Some(TransitionKind::Restoring), + .. + } => MainStatus::Restoring, + ServiceStateKinds { + transition_state: Some(TransitionKind::BackingUp), + .. + } => previous.backing_up(), + ServiceStateKinds { + running_status: Some(status), + desired_state: StartStop::Start, + .. + } => MainStatus::Running { + started: status.started, + health: previous.health().cloned().unwrap_or_default(), + }, + ServiceStateKinds { + running_status: None, + desired_state: StartStop::Start, + .. + } => MainStatus::Starting { + health: previous.health().cloned().unwrap_or_default(), + }, + ServiceStateKinds { + running_status: Some(_), + desired_state: StartStop::Stop, + .. + } => MainStatus::Stopping, + ServiceStateKinds { + running_status: None, + desired_state: StartStop::Stop, + .. + } => MainStatus::Stopped, }; i.as_status_mut().as_main_mut().ser(&main_status)?; } Ok(()) }) .await?; + seed.synchronized.notify_waiters(); + + match kinds { + ServiceStateKinds { + running_status: None, + desired_state: StartStop::Start, + .. + } => { + seed.persistent_container.start().await?; + } + ServiceStateKinds { + running_status: Some(_), + desired_state: StartStop::Stop, + .. + } => { + seed.persistent_container.stop().await?; + seed.persistent_container + .state + .send_if_modified(|s| s.running_status.take().is_some()); + } + _ => (), + }; Ok::<_, Error>(()) } diff --git a/core/startos/src/service/service_map.rs b/core/startos/src/service/service_map.rs index 90223216c..0e6a959ae 100644 --- a/core/startos/src/service/service_map.rs +++ b/core/startos/src/service/service_map.rs @@ -173,6 +173,7 @@ impl ServiceMap { } else { PackageState::Installing(installing) }, + data_version: None, status: Status { configured: false, main: MainStatus::Stopped, diff --git a/core/startos/src/service/start_stop.rs b/core/startos/src/service/start_stop.rs index 178176023..64d4022d6 100644 --- a/core/startos/src/service/start_stop.rs +++ b/core/startos/src/service/start_stop.rs @@ -1,6 +1,10 @@ +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + use crate::status::MainStatus; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] pub enum StartStop { Start, Stop, @@ -11,23 +15,19 @@ impl StartStop { matches!(self, StartStop::Start) } } -impl From for StartStop { - fn from(value: MainStatus) -> Self { - match value { - MainStatus::Stopped => StartStop::Stop, - MainStatus::Restoring => StartStop::Stop, - MainStatus::Restarting => StartStop::Start, - MainStatus::Stopping { .. } => StartStop::Stop, - MainStatus::Starting => StartStop::Start, - MainStatus::Running { - started: _, - health: _, - } => StartStop::Start, - MainStatus::BackingUp { started, health: _ } if started.is_some() => StartStop::Start, - MainStatus::BackingUp { - started: _, - health: _, - } => StartStop::Stop, - } - } -} +// impl From for StartStop { +// fn from(value: MainStatus) -> Self { +// match value { +// MainStatus::Stopped => StartStop::Stop, +// MainStatus::Restoring => StartStop::Stop, +// MainStatus::Restarting => StartStop::Start, +// MainStatus::Stopping { .. } => StartStop::Stop, +// MainStatus::Starting => StartStop::Start, +// MainStatus::Running { +// started: _, +// health: _, +// } => StartStop::Start, +// MainStatus::BackingUp { on_complete } => on_complete, +// } +// } +// } diff --git a/core/startos/src/status/health_check.rs b/core/startos/src/status/health_check.rs index 90b20f8c5..1b1e2a7b6 100644 --- a/core/startos/src/status/health_check.rs +++ b/core/startos/src/status/health_check.rs @@ -9,25 +9,25 @@ use crate::util::clap::FromStrParser; #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, TS)] #[serde(rename_all = "camelCase")] -pub struct HealthCheckResult { +pub struct NamedHealthCheckResult { pub name: String, #[serde(flatten)] - pub kind: HealthCheckResultKind, + pub kind: NamedHealthCheckResultKind, } // healthCheckName:kind:message OR healthCheckName:kind -impl FromStr for HealthCheckResult { +impl FromStr for NamedHealthCheckResult { type Err = color_eyre::eyre::Report; fn from_str(s: &str) -> Result { let from_parts = |name: &str, kind: &str, message: Option<&str>| { let message = message.map(|x| x.to_string()); let kind = match kind { - "success" => HealthCheckResultKind::Success { message }, - "disabled" => HealthCheckResultKind::Disabled { message }, - "starting" => HealthCheckResultKind::Starting { message }, - "loading" => HealthCheckResultKind::Loading { + "success" => NamedHealthCheckResultKind::Success { message }, + "disabled" => NamedHealthCheckResultKind::Disabled { message }, + "starting" => NamedHealthCheckResultKind::Starting { message }, + "loading" => NamedHealthCheckResultKind::Loading { message: message.unwrap_or_default(), }, - "failure" => HealthCheckResultKind::Failure { + "failure" => NamedHealthCheckResultKind::Failure { message: message.unwrap_or_default(), }, _ => return Err(color_eyre::eyre::eyre!("Invalid health check kind")), @@ -47,7 +47,7 @@ impl FromStr for HealthCheckResult { } } } -impl ValueParserFactory for HealthCheckResult { +impl ValueParserFactory for NamedHealthCheckResult { type Parser = FromStrParser; fn value_parser() -> Self::Parser { FromStrParser::new() @@ -57,40 +57,44 @@ impl ValueParserFactory for HealthCheckResult { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, TS)] #[serde(rename_all = "camelCase")] #[serde(tag = "result")] -pub enum HealthCheckResultKind { +pub enum NamedHealthCheckResultKind { Success { message: Option }, Disabled { message: Option }, Starting { message: Option }, Loading { message: String }, Failure { message: String }, } -impl std::fmt::Display for HealthCheckResult { +impl std::fmt::Display for NamedHealthCheckResult { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let name = &self.name; match &self.kind { - HealthCheckResultKind::Success { message } => { + NamedHealthCheckResultKind::Success { message } => { if let Some(message) = message { write!(f, "{name}: Succeeded ({message})") } else { write!(f, "{name}: Succeeded") } } - HealthCheckResultKind::Disabled { message } => { + NamedHealthCheckResultKind::Disabled { message } => { if let Some(message) = message { write!(f, "{name}: Disabled ({message})") } else { write!(f, "{name}: Disabled") } } - HealthCheckResultKind::Starting { message } => { + NamedHealthCheckResultKind::Starting { message } => { if let Some(message) = message { write!(f, "{name}: Starting ({message})") } else { write!(f, "{name}: Starting") } } - HealthCheckResultKind::Loading { message } => write!(f, "{name}: Loading ({message})"), - HealthCheckResultKind::Failure { message } => write!(f, "{name}: Failed ({message})"), + NamedHealthCheckResultKind::Loading { message } => { + write!(f, "{name}: Loading ({message})") + } + NamedHealthCheckResultKind::Failure { message } => { + write!(f, "{name}: Failed ({message})") + } } } } diff --git a/core/startos/src/status/mod.rs b/core/startos/src/status/mod.rs index c1d3a36ad..c10a7b89f 100644 --- a/core/startos/src/status/mod.rs +++ b/core/startos/src/status/mod.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, sync::Arc}; +use std::collections::BTreeMap; use chrono::{DateTime, Utc}; use imbl::OrdMap; @@ -6,8 +6,9 @@ use serde::{Deserialize, Serialize}; use ts_rs::TS; use self::health_check::HealthCheckId; -use crate::status::health_check::HealthCheckResult; -use crate::{prelude::*, util::GeneralGuard}; +use crate::prelude::*; +use crate::service::start_stop::StartStop; +use crate::status::health_check::NamedHealthCheckResult; pub mod health_check; #[derive(Clone, Debug, Deserialize, Serialize, HasModel, TS)] @@ -22,25 +23,24 @@ pub struct Status { #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, TS)] #[serde(tag = "status")] #[serde(rename_all = "camelCase")] +#[serde(rename_all_fields = "camelCase")] pub enum MainStatus { Stopped, Restarting, Restoring, Stopping, - Starting, - #[serde(rename_all = "camelCase")] + Starting { + #[ts(as = "BTreeMap")] + health: OrdMap, + }, Running { #[ts(type = "string")] started: DateTime, - #[ts(as = "BTreeMap")] - health: OrdMap, + #[ts(as = "BTreeMap")] + health: OrdMap, }, - #[serde(rename_all = "camelCase")] BackingUp { - #[ts(type = "string | null")] - started: Option>, - #[ts(as = "BTreeMap")] - health: OrdMap, + on_complete: StartStop, }, } impl MainStatus { @@ -48,60 +48,37 @@ impl MainStatus { match self { MainStatus::Starting { .. } | MainStatus::Running { .. } + | MainStatus::Restarting | MainStatus::BackingUp { - started: Some(_), .. + on_complete: StartStop::Start, } => true, MainStatus::Stopped | MainStatus::Restoring | MainStatus::Stopping { .. } - | MainStatus::Restarting - | MainStatus::BackingUp { started: None, .. } => false, + | MainStatus::BackingUp { + on_complete: StartStop::Stop, + } => false, } } - // pub fn stop(&mut self) { - // match self { - // MainStatus::Starting { .. } | MainStatus::Running { .. } => { - // *self = MainStatus::Stopping; - // } - // MainStatus::BackingUp { started, .. } => { - // *started = None; - // } - // MainStatus::Stopped | MainStatus::Stopping | MainStatus::Restarting => (), - // } - // } - pub fn started(&self) -> Option> { - match self { - MainStatus::Running { started, .. } => Some(*started), - MainStatus::BackingUp { started, .. } => *started, - MainStatus::Stopped => None, - MainStatus::Restoring => None, - MainStatus::Restarting => None, - MainStatus::Stopping { .. } => None, - MainStatus::Starting { .. } => None, - } - } - pub fn backing_up(&self) -> Self { - let (started, health) = match self { - MainStatus::Starting { .. } => (Some(Utc::now()), Default::default()), - MainStatus::Running { started, health } => (Some(started.clone()), health.clone()), - MainStatus::Stopped - | MainStatus::Stopping { .. } - | MainStatus::Restoring - | MainStatus::Restarting => (None, Default::default()), - MainStatus::BackingUp { .. } => return self.clone(), - }; - MainStatus::BackingUp { started, health } - } - pub fn health(&self) -> Option<&OrdMap> { + pub fn backing_up(self) -> Self { + MainStatus::BackingUp { + on_complete: if self.running() { + StartStop::Start + } else { + StartStop::Stop + }, + } + } + + pub fn health(&self) -> Option<&OrdMap> { match self { - MainStatus::Running { health, .. } => Some(health), - MainStatus::BackingUp { health, .. } => Some(health), - MainStatus::Stopped + MainStatus::Running { health, .. } | MainStatus::Starting { health } => Some(health), + MainStatus::BackingUp { .. } + | MainStatus::Stopped | MainStatus::Restoring | MainStatus::Stopping { .. } | MainStatus::Restarting => None, - MainStatus::Starting { .. } => None, } } } diff --git a/core/startos/src/version/mod.rs b/core/startos/src/version/mod.rs index be5c8d9a5..18ff0e5b2 100644 --- a/core/startos/src/version/mod.rs +++ b/core/startos/src/version/mod.rs @@ -22,7 +22,7 @@ mod v0_3_6_alpha_5; mod v0_3_6_alpha_6; mod v0_3_6_alpha_7; -pub type Current = v0_3_6_alpha_3::Version; // VERSION_BUMP +pub type Current = v0_3_6_alpha_5::Version; // VERSION_BUMP #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] #[serde(untagged)] diff --git a/debian/postinst b/debian/postinst index cafa691e0..bbf61f344 100755 --- a/debian/postinst +++ b/debian/postinst @@ -79,6 +79,7 @@ sed -i '/\(^\|#\)SystemMaxUse=/c\SystemMaxUse=1G' /etc/systemd/journald.conf sed -i '/\(^\|#\)ForwardToSyslog=/c\ForwardToSyslog=no' /etc/systemd/journald.conf sed -i '/^\s*#\?\s*issue_discards\s*=\s*/c\issue_discards = 1' /etc/lvm/lvm.conf sed -i '/\(^\|#\)\s*unqualified-search-registries\s*=\s*/c\unqualified-search-registries = ["docker.io"]' /etc/containers/registries.conf +sed -i 's/\(#\|\^\)\s*\([^=]\+\)=\(suspend\|hibernate\)\s*$/\2=ignore/g' /etc/systemd/logind.conf mkdir -p /etc/nginx/ssl diff --git a/sdk/lib/StartSdk.ts b/sdk/lib/StartSdk.ts index 7cdedff90..658597bc2 100644 --- a/sdk/lib/StartSdk.ts +++ b/sdk/lib/StartSdk.ts @@ -30,17 +30,11 @@ import { healthCheck, HealthCheckParams } from "./health/HealthCheck" import { checkPortListening } from "./health/checkFns/checkPortListening" import { checkWebUrl, runHealthScript } from "./health/checkFns" import { List } from "./config/builder/list" -import { Migration } from "./inits/migrations/Migration" import { Install, InstallFn } from "./inits/setupInstall" import { setupActions } from "./actions/setupActions" import { setupDependencyConfig } from "./dependencies/setupDependencyConfig" import { SetupBackupsParams, setupBackups } from "./backup/setupBackups" import { setupInit } from "./inits/setupInit" -import { - EnsureUniqueId, - Migrations, - setupMigrations, -} from "./inits/migrations/setupMigrations" import { Uninstall, UninstallFn, setupUninstall } from "./inits/setupUninstall" import { setupMain } from "./mainFn" import { defaultTrigger } from "./trigger/defaultTrigger" @@ -67,7 +61,7 @@ import { } from "./util/getServiceInterface" import { getServiceInterfaces } from "./util/getServiceInterfaces" import { getStore } from "./store/getStore" -import { CommandOptions, MountOptions, Overlay } from "./util/Overlay" +import { CommandOptions, MountOptions, SubContainer } from "./util/SubContainer" import { splitCommand } from "./util/splitCommand" import { Mounts } from "./mainFn/Mounts" import { Dependency } from "./Dependency" @@ -75,9 +69,13 @@ import * as T from "./types" import { testTypeVersion, ValidateExVer } from "./exver" import { ExposedStorePaths } from "./store/setupExposeStore" import { PathBuilder, extractJsonPath, pathBuilder } from "./store/PathBuilder" -import { checkAllDependencies } from "./dependencies/dependencies" +import { + CheckDependencies, + checkDependencies, +} from "./dependencies/dependencies" import { health } from "." import { GetSslCertificate } from "./util/GetSslCertificate" +import { VersionGraph } from "./version" export const SDKVersion = testTypeVersion("0.3.6") @@ -141,8 +139,58 @@ export class StartSdk { }]?: Dependency } + type NestedEffects = "subcontainer" | "store" + type InterfaceEffects = + | "getServiceInterface" + | "listServiceInterfaces" + | "exportServiceInterface" + | "clearServiceInterfaces" + | "bind" + | "getHostInfo" + | "getPrimaryUrl" + type MainUsedEffects = "setMainStatus" | "setHealth" + type AlreadyExposed = "getSslCertificate" | "getSystemSmtp" + + // prettier-ignore + type StartSdkEffectWrapper = { + [K in keyof Omit]: (effects: Effects, ...args: Parameters) => ReturnType + } + const startSdkEffectWrapper: StartSdkEffectWrapper = { + executeAction: (effects, ...args) => effects.executeAction(...args), + exportAction: (effects, ...args) => effects.exportAction(...args), + clearActions: (effects, ...args) => effects.clearActions(...args), + getConfigured: (effects, ...args) => effects.getConfigured(...args), + setConfigured: (effects, ...args) => effects.setConfigured(...args), + restart: (effects, ...args) => effects.restart(...args), + setDependencies: (effects, ...args) => effects.setDependencies(...args), + checkDependencies: (effects, ...args) => + effects.checkDependencies(...args), + mount: (effects, ...args) => effects.mount(...args), + getInstalledPackages: (effects, ...args) => + effects.getInstalledPackages(...args), + exposeForDependents: (effects, ...args) => + effects.exposeForDependents(...args), + getServicePortForward: (effects, ...args) => + effects.getServicePortForward(...args), + clearBindings: (effects, ...args) => effects.clearBindings(...args), + getContainerIp: (effects, ...args) => effects.getContainerIp(...args), + getSslKey: (effects, ...args) => effects.getSslKey(...args), + setDataVersion: (effects, ...args) => effects.setDataVersion(...args), + getDataVersion: (effects, ...args) => effects.getDataVersion(...args), + shutdown: (effects, ...args) => effects.shutdown(...args), + getDependencies: (effects, ...args) => effects.getDependencies(...args), + } + return { - checkAllDependencies, + ...startSdkEffectWrapper, + + checkDependencies: checkDependencies as < + DependencyId extends keyof Manifest["dependencies"] & + PackageId = keyof Manifest["dependencies"] & PackageId, + >( + effects: Effects, + packageIds?: DependencyId[], + ) => Promise>, serviceInterface: { getOwn: (effects: E, id: ServiceInterfaceId) => removeCallbackTypes(effects)( @@ -247,7 +295,6 @@ export class StartSdk { id: string description: string hasPrimary: boolean - disabled: boolean type: ServiceInterfaceType username: null | string path: string @@ -293,8 +340,8 @@ export class StartSdk { ) }, HealthCheck: { - of(o: HealthCheckParams) { - return healthCheck(o) + of(o: HealthCheckParams) { + return healthCheck(o) }, }, Dependency: { @@ -311,7 +358,7 @@ export class StartSdk { setupActions: (...createdActions: CreatedAction[]) => setupActions(...createdActions), setupBackups: (...args: SetupBackupsParams) => - setupBackups(...args), + setupBackups(this.manifest, ...args), setupConfig: < ConfigType extends Config | Config, Type extends Record = ExtractConfigType, @@ -380,7 +427,7 @@ export class StartSdk { } }, setupInit: ( - migrations: Migrations, + versions: VersionGraph, install: Install, uninstall: Uninstall, setInterfaces: SetInterfaces, @@ -391,7 +438,7 @@ export class StartSdk { exposedStore: ExposedStorePaths, ) => setupInit( - migrations, + versions, install, uninstall, setInterfaces, @@ -412,15 +459,6 @@ export class StartSdk { started(onTerm: () => PromiseLike): PromiseLike }) => Promise>, ) => setupMain(fn), - setupMigrations: < - Migrations extends Array>, - >( - ...migrations: EnsureUniqueId - ) => - setupMigrations( - this.manifest, - ...migrations, - ), setupProperties: ( fn: (options: { effects: Effects }) => Promise, @@ -541,13 +579,6 @@ export class StartSdk { >, ) => List.dynamicText(getA), }, - Migration: { - of: (options: { - version: Version & ValidateExVer - up: (opts: { effects: Effects }) => Promise - down: (opts: { effects: Effects }) => Promise - }) => Migration.of(options), - }, StorePath: pathBuilder(), Value: { toggle: Value.toggle, @@ -747,15 +778,12 @@ export async function runCommand( }, ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> { const commands = splitCommand(command) - const overlay = await Overlay.of(effects, image) - try { - for (let mount of options.mounts || []) { - await overlay.mount(mount.options, mount.path) - } - return await overlay.exec(commands) - } finally { - await overlay.destroy() - } + return SubContainer.with( + effects, + image, + options.mounts || [], + (subcontainer) => subcontainer.exec(commands), + ) } function nullifyProperties(value: T.SdkPropertiesReturn): T.PropertiesReturn { return Object.fromEntries( diff --git a/sdk/lib/backup/Backups.ts b/sdk/lib/backup/Backups.ts index 6751b1910..031ac4e4c 100644 --- a/sdk/lib/backup/Backups.ts +++ b/sdk/lib/backup/Backups.ts @@ -2,6 +2,7 @@ import * as T from "../types" import * as child_process from "child_process" import { promises as fsPromises } from "fs" +import { asError } from "../util" export type BACKUP = "BACKUP" export const DEFAULT_OPTIONS: T.BackupOptions = { @@ -183,7 +184,7 @@ async function runRsync( }) spawned.stderr.on("data", (data: unknown) => { - console.error(String(data)) + console.error(`Backups.runAsync`, asError(data)) }) const id = async () => { diff --git a/sdk/lib/backup/setupBackups.ts b/sdk/lib/backup/setupBackups.ts index 40be01829..c12f1d2ed 100644 --- a/sdk/lib/backup/setupBackups.ts +++ b/sdk/lib/backup/setupBackups.ts @@ -8,6 +8,7 @@ export type SetupBackupsParams = Array< > export function setupBackups( + manifest: M, ...args: _> ) { const backups = Array>() @@ -36,6 +37,7 @@ export function setupBackups( for (const backup of backups) { await backup.build(options.pathMaker).restoreBackup(options) } + await options.effects.setDataVersion({ version: manifest.version }) }) as T.ExpectedExports.restoreBackup }, } diff --git a/sdk/lib/config/setupConfig.ts b/sdk/lib/config/setupConfig.ts index 8a1550d57..f354c81ed 100644 --- a/sdk/lib/config/setupConfig.ts +++ b/sdk/lib/config/setupConfig.ts @@ -57,7 +57,9 @@ export function setupConfig< return { setConfig: (async ({ effects, input }) => { if (!validator.test(input)) { - await console.error(String(validator.errorMessage(input))) + await console.error( + new Error(validator.errorMessage(input)?.toString()), + ) return { error: "Set config type error for config" } } await effects.clearBindings() diff --git a/sdk/lib/dependencies/dependencies.ts b/sdk/lib/dependencies/dependencies.ts index 28b04a07b..287f63b06 100644 --- a/sdk/lib/dependencies/dependencies.ts +++ b/sdk/lib/dependencies/dependencies.ts @@ -1,131 +1,206 @@ +import { ExtendedVersion, VersionRange } from "../exver" import { Effects, PackageId, DependencyRequirement, SetHealth, CheckDependenciesResult, + HealthCheckId, } from "../types" -export type CheckAllDependencies = { - notInstalled: () => Promise - notRunning: () => Promise - configNotSatisfied: () => Promise - healthErrors: () => Promise<{ [id: string]: SetHealth[] }> +export type CheckDependencies = { + installedSatisfied: (packageId: DependencyId) => boolean + installedVersionSatisfied: (packageId: DependencyId) => boolean + runningSatisfied: (packageId: DependencyId) => boolean + configSatisfied: (packageId: DependencyId) => boolean + healthCheckSatisfied: ( + packageId: DependencyId, + healthCheckId: HealthCheckId, + ) => boolean + satisfied: () => boolean - isValid: () => Promise - - throwIfNotRunning: () => Promise - throwIfNotInstalled: () => Promise - throwIfConfigNotSatisfied: () => Promise - throwIfHealthError: () => Promise - - throwIfNotValid: () => Promise + throwIfInstalledNotSatisfied: (packageId: DependencyId) => void + throwIfInstalledVersionNotSatisfied: (packageId: DependencyId) => void + throwIfRunningNotSatisfied: (packageId: DependencyId) => void + throwIfConfigNotSatisfied: (packageId: DependencyId) => void + throwIfHealthNotSatisfied: ( + packageId: DependencyId, + healthCheckId?: HealthCheckId, + ) => void + throwIfNotSatisfied: (packageId?: DependencyId) => void } -export function checkAllDependencies(effects: Effects): CheckAllDependencies { - const dependenciesPromise = effects.getDependencies() - const resultsPromise = dependenciesPromise.then((dependencies) => +export async function checkDependencies< + DependencyId extends PackageId = PackageId, +>( + effects: Effects, + packageIds?: DependencyId[], +): Promise> { + let [dependencies, results] = await Promise.all([ + effects.getDependencies(), effects.checkDependencies({ - packageIds: dependencies.map((dep) => dep.id), + packageIds, }), - ) - - const dependenciesByIdPromise = dependenciesPromise.then((d) => - d.reduce( - (acc, dep) => { - acc[dep.id] = dep - return acc - }, - {} as { [id: PackageId]: DependencyRequirement }, - ), - ) - - const healthErrors = async () => { - const results = await resultsPromise - const dependenciesById = await dependenciesByIdPromise - const answer: { [id: PackageId]: SetHealth[] } = {} - for (const result of results) { - const dependency = dependenciesById[result.packageId] - if (!dependency) continue - if (dependency.kind !== "running") continue - - const healthChecks = Object.entries(result.healthChecks) - .map(([id, hc]) => ({ ...hc, id })) - .filter((x) => !!x.message) - if (healthChecks.length === 0) continue - answer[result.packageId] = healthChecks - } - return answer - } - const configNotSatisfied = () => - resultsPromise.then((x) => x.filter((x) => !x.configSatisfied)) - const notInstalled = () => - resultsPromise.then((x) => x.filter((x) => !x.isInstalled)) - const notRunning = async () => { - const results = await resultsPromise - const dependenciesById = await dependenciesByIdPromise - return results.filter((x) => { - const dependency = dependenciesById[x.packageId] - if (!dependency) return false - if (dependency.kind !== "running") return false - return !x.isRunning - }) - } - const entries = (x: { [k: string]: B }) => Object.entries(x) - const first = (x: A[]): A | undefined => x[0] - const sinkVoid = (x: A) => void 0 - const throwIfHealthError = () => - healthErrors() - .then(entries) - .then(first) - .then((x) => { - if (!x) return - const [id, healthChecks] = x - if (healthChecks.length > 0) - throw `Package ${id} has the following errors: ${healthChecks.map((x) => x.message).join(", ")}` - }) - - const throwIfConfigNotSatisfied = () => - configNotSatisfied().then((results) => { - throw new Error( - `Package ${results[0].packageId} does not have a valid configuration`, - ) - }) - - const throwIfNotRunning = () => - notRunning().then((results) => { - if (results[0]) - throw new Error(`Package ${results[0].packageId} is not running`) - }) - - const throwIfNotInstalled = () => - notInstalled().then((results) => { - if (results[0]) - throw new Error(`Package ${results[0].packageId} is not installed`) - }) - const throwIfNotValid = async () => - Promise.all([ - throwIfNotRunning(), - throwIfNotInstalled(), - throwIfConfigNotSatisfied(), - throwIfHealthError(), - ]).then(sinkVoid) - - const isValid = () => - throwIfNotValid().then( - () => true, - () => false, + ]) + if (packageIds) { + dependencies = dependencies.filter((d) => + (packageIds as PackageId[]).includes(d.id), ) + } + + const find = (packageId: DependencyId) => { + const dependencyRequirement = dependencies.find((d) => d.id === packageId) + const dependencyResult = results.find((d) => d.packageId === packageId) + if (!dependencyRequirement || !dependencyResult) { + throw new Error(`Unknown DependencyId ${packageId}`) + } + return { requirement: dependencyRequirement, result: dependencyResult } + } + + const installedSatisfied = (packageId: DependencyId) => + !!find(packageId).result.installedVersion + const installedVersionSatisfied = (packageId: DependencyId) => { + const dep = find(packageId) + return ( + !!dep.result.installedVersion && + ExtendedVersion.parse(dep.result.installedVersion).satisfies( + VersionRange.parse(dep.requirement.versionRange), + ) + ) + } + const runningSatisfied = (packageId: DependencyId) => { + const dep = find(packageId) + return dep.requirement.kind !== "running" || dep.result.isRunning + } + const configSatisfied = (packageId: DependencyId) => + find(packageId).result.configSatisfied + const healthCheckSatisfied = ( + packageId: DependencyId, + healthCheckId?: HealthCheckId, + ) => { + const dep = find(packageId) + if ( + healthCheckId && + (dep.requirement.kind !== "running" || + !dep.requirement.healthChecks.includes(healthCheckId)) + ) { + throw new Error(`Unknown HealthCheckId ${healthCheckId}`) + } + const errors = Object.entries(dep.result.healthChecks) + .filter(([id, _]) => (healthCheckId ? id === healthCheckId : true)) + .filter(([_, res]) => res.result !== "success") + return errors.length === 0 + } + const pkgSatisfied = (packageId: DependencyId) => + installedSatisfied(packageId) && + installedVersionSatisfied(packageId) && + runningSatisfied(packageId) && + configSatisfied(packageId) && + healthCheckSatisfied(packageId) + const satisfied = (packageId?: DependencyId) => + packageId + ? pkgSatisfied(packageId) + : dependencies.every((d) => pkgSatisfied(d.id as DependencyId)) + + const throwIfInstalledNotSatisfied = (packageId: DependencyId) => { + const dep = find(packageId) + if (!dep.result.installedVersion) { + throw new Error(`${dep.result.title || packageId} is not installed`) + } + } + const throwIfInstalledVersionNotSatisfied = (packageId: DependencyId) => { + const dep = find(packageId) + if (!dep.result.installedVersion) { + throw new Error(`${dep.result.title || packageId} is not installed`) + } + if ( + ![dep.result.installedVersion, ...dep.result.satisfies].find((v) => + ExtendedVersion.parse(v).satisfies( + VersionRange.parse(dep.requirement.versionRange), + ), + ) + ) { + throw new Error( + `Installed version ${dep.result.installedVersion} of ${dep.result.title || packageId} does not match expected version range ${dep.requirement.versionRange}`, + ) + } + } + const throwIfRunningNotSatisfied = (packageId: DependencyId) => { + const dep = find(packageId) + if (dep.requirement.kind === "running" && !dep.result.isRunning) { + throw new Error(`${dep.result.title || packageId} is not running`) + } + } + const throwIfConfigNotSatisfied = (packageId: DependencyId) => { + const dep = find(packageId) + if (!dep.result.configSatisfied) { + throw new Error( + `${dep.result.title || packageId}'s configuration does not satisfy requirements`, + ) + } + } + const throwIfHealthNotSatisfied = ( + packageId: DependencyId, + healthCheckId?: HealthCheckId, + ) => { + const dep = find(packageId) + if ( + healthCheckId && + (dep.requirement.kind !== "running" || + !dep.requirement.healthChecks.includes(healthCheckId)) + ) { + throw new Error(`Unknown HealthCheckId ${healthCheckId}`) + } + const errors = Object.entries(dep.result.healthChecks) + .filter(([id, _]) => (healthCheckId ? id === healthCheckId : true)) + .filter(([_, res]) => res.result !== "success") + if (errors.length) { + throw new Error( + errors + .map( + ([_, e]) => + `Health Check ${e.name} of ${dep.result.title || packageId} failed with status ${e.result}${e.message ? `: ${e.message}` : ""}`, + ) + .join("; "), + ) + } + } + const throwIfPkgNotSatisfied = (packageId: DependencyId) => { + throwIfInstalledNotSatisfied(packageId) + throwIfInstalledVersionNotSatisfied(packageId) + throwIfRunningNotSatisfied(packageId) + throwIfConfigNotSatisfied(packageId) + throwIfHealthNotSatisfied(packageId) + } + const throwIfNotSatisfied = (packageId?: DependencyId) => + packageId + ? throwIfPkgNotSatisfied(packageId) + : (() => { + const err = dependencies.flatMap((d) => { + try { + throwIfPkgNotSatisfied(d.id as DependencyId) + } catch (e) { + if (e instanceof Error) return [e.message] + throw e + } + return [] + }) + if (err.length) { + throw new Error(err.join("; ")) + } + })() return { - notRunning, - notInstalled, - configNotSatisfied, - healthErrors, - throwIfNotRunning, + installedSatisfied, + installedVersionSatisfied, + runningSatisfied, + configSatisfied, + healthCheckSatisfied, + satisfied, + throwIfInstalledNotSatisfied, + throwIfInstalledVersionNotSatisfied, + throwIfRunningNotSatisfied, throwIfConfigNotSatisfied, - throwIfNotValid, - throwIfNotInstalled, - throwIfHealthError, - isValid, + throwIfHealthNotSatisfied, + throwIfNotSatisfied, } } diff --git a/sdk/lib/exver/index.ts b/sdk/lib/exver/index.ts index 913194875..331271c1a 100644 --- a/sdk/lib/exver/index.ts +++ b/sdk/lib/exver/index.ts @@ -3,7 +3,7 @@ import * as P from "./exver" // prettier-ignore export type ValidateVersion = T extends `-${infer A}` ? never : -T extends `${infer A}-${infer B}` ? ValidateVersion & ValidateVersion : +T extends `${infer A}-${string}` ? ValidateVersion : T extends `${bigint}` ? unknown : T extends `${bigint}.${infer A}` ? ValidateVersion : never @@ -16,9 +16,9 @@ export type ValidateExVer = // prettier-ignore export type ValidateExVers = - T extends [] ? unknown : + T extends [] ? unknown[] : T extends [infer A, ...infer B] ? ValidateExVer & ValidateExVers : - never + never[] type Anchor = { type: "Anchor" @@ -44,7 +44,7 @@ type Not = { } export class VersionRange { - private constructor(private atom: Anchor | And | Or | Not | P.Any | P.None) {} + private constructor(public atom: Anchor | And | Or | Not | P.Any | P.None) {} toString(): string { switch (this.atom.type) { @@ -63,67 +63,6 @@ export class VersionRange { } } - /** - * Returns a boolean indicating whether a given version satisfies the VersionRange - * !( >= 1:1 <= 2:2) || <=#bitcoin:1.2.0-alpha:0 - */ - satisfiedBy(version: ExtendedVersion): boolean { - switch (this.atom.type) { - case "Anchor": - const otherVersion = this.atom.version - switch (this.atom.operator) { - case "=": - return version.equals(otherVersion) - case ">": - return version.greaterThan(otherVersion) - case "<": - return version.lessThan(otherVersion) - case ">=": - return version.greaterThanOrEqual(otherVersion) - case "<=": - return version.lessThanOrEqual(otherVersion) - case "!=": - return !version.equals(otherVersion) - case "^": - const nextMajor = this.atom.version.incrementMajor() - if ( - version.greaterThanOrEqual(otherVersion) && - version.lessThan(nextMajor) - ) { - return true - } else { - return false - } - case "~": - const nextMinor = this.atom.version.incrementMinor() - if ( - version.greaterThanOrEqual(otherVersion) && - version.lessThan(nextMinor) - ) { - return true - } else { - return false - } - } - case "And": - return ( - this.atom.left.satisfiedBy(version) && - this.atom.right.satisfiedBy(version) - ) - case "Or": - return ( - this.atom.left.satisfiedBy(version) || - this.atom.right.satisfiedBy(version) - ) - case "Not": - return !this.atom.value.satisfiedBy(version) - case "Any": - return true - case "None": - return false - } - } - private static parseAtom(atom: P.VersionRangeAtom): VersionRange { switch (atom.type) { case "Not": @@ -207,6 +146,10 @@ export class VersionRange { static none() { return new VersionRange({ type: "None" }) } + + satisfiedBy(version: Version | ExtendedVersion) { + return version.satisfies(this) + } } export class Version { @@ -266,6 +209,12 @@ export class Version { const parsed = P.parse(version, { startRule: "Version" }) return new Version(parsed.number, parsed.prerelease) } + + satisfies(versionRange: VersionRange): boolean { + return new ExtendedVersion(null, this, new Version([0], [])).satisfies( + versionRange, + ) + } } // #flavor:0.1.2-beta.1:0 @@ -404,6 +353,67 @@ export class ExtendedVersion { updatedDownstream, ) } + + /** + * Returns a boolean indicating whether a given version satisfies the VersionRange + * !( >= 1:1 <= 2:2) || <=#bitcoin:1.2.0-alpha:0 + */ + satisfies(versionRange: VersionRange): boolean { + switch (versionRange.atom.type) { + case "Anchor": + const otherVersion = versionRange.atom.version + switch (versionRange.atom.operator) { + case "=": + return this.equals(otherVersion) + case ">": + return this.greaterThan(otherVersion) + case "<": + return this.lessThan(otherVersion) + case ">=": + return this.greaterThanOrEqual(otherVersion) + case "<=": + return this.lessThanOrEqual(otherVersion) + case "!=": + return !this.equals(otherVersion) + case "^": + const nextMajor = versionRange.atom.version.incrementMajor() + if ( + this.greaterThanOrEqual(otherVersion) && + this.lessThan(nextMajor) + ) { + return true + } else { + return false + } + case "~": + const nextMinor = versionRange.atom.version.incrementMinor() + if ( + this.greaterThanOrEqual(otherVersion) && + this.lessThan(nextMinor) + ) { + return true + } else { + return false + } + } + case "And": + return ( + this.satisfies(versionRange.atom.left) && + this.satisfies(versionRange.atom.right) + ) + case "Or": + return ( + this.satisfies(versionRange.atom.left) || + this.satisfies(versionRange.atom.right) + ) + case "Not": + return !this.satisfies(versionRange.atom.value) + case "Any": + return true + case "None": + return false + } + } } export const testTypeExVer = (t: T & ValidateExVer) => t @@ -416,6 +426,7 @@ function tests() { testTypeVersion("12.34.56") testTypeVersion("1.2-3") testTypeVersion("1-3") + testTypeVersion("1-alpha") // @ts-expect-error testTypeVersion("-3") // @ts-expect-error diff --git a/sdk/lib/health/HealthCheck.ts b/sdk/lib/health/HealthCheck.ts index 4b72dbf61..e007c4ea2 100644 --- a/sdk/lib/health/HealthCheck.ts +++ b/sdk/lib/health/HealthCheck.ts @@ -1,74 +1,61 @@ import { Effects } from "../types" -import { CheckResult } from "./checkFns/CheckResult" +import { HealthCheckResult } from "./checkFns/HealthCheckResult" import { HealthReceipt } from "./HealthReceipt" import { Trigger } from "../trigger" import { TriggerInput } from "../trigger/TriggerInput" import { defaultTrigger } from "../trigger/defaultTrigger" import { once } from "../util/once" -import { Overlay } from "../util/Overlay" +import { SubContainer } from "../util/SubContainer" import { object, unknown } from "ts-matches" import * as T from "../types" +import { asError } from "../util/asError" -export type HealthCheckParams = { +export type HealthCheckParams = { effects: Effects name: string - image: { - id: keyof Manifest["images"] & T.ImageId - sharedRun?: boolean - } trigger?: Trigger - fn(overlay: Overlay): Promise | CheckResult + fn(): Promise | HealthCheckResult onFirstSuccess?: () => unknown | Promise } -export function healthCheck( - o: HealthCheckParams, -) { +export function healthCheck(o: HealthCheckParams) { new Promise(async () => { - const overlay = await Overlay.of(o.effects, o.image) - try { - let currentValue: TriggerInput = { - hadSuccess: false, + let currentValue: TriggerInput = {} + const getCurrentValue = () => currentValue + const trigger = (o.trigger ?? defaultTrigger)(getCurrentValue) + const triggerFirstSuccess = once(() => + Promise.resolve( + "onFirstSuccess" in o && o.onFirstSuccess + ? o.onFirstSuccess() + : undefined, + ), + ) + for ( + let res = await trigger.next(); + !res.done; + res = await trigger.next() + ) { + try { + const { result, message } = await o.fn() + await o.effects.setHealth({ + name: o.name, + id: o.name, + result, + message: message || "", + }) + currentValue.lastResult = result + await triggerFirstSuccess().catch((err) => { + console.error(asError(err)) + }) + } catch (e) { + await o.effects.setHealth({ + name: o.name, + id: o.name, + result: "failure", + message: asMessage(e) || "", + }) + currentValue.lastResult = "failure" } - const getCurrentValue = () => currentValue - const trigger = (o.trigger ?? defaultTrigger)(getCurrentValue) - const triggerFirstSuccess = once(() => - Promise.resolve( - "onFirstSuccess" in o && o.onFirstSuccess - ? o.onFirstSuccess() - : undefined, - ), - ) - for ( - let res = await trigger.next(); - !res.done; - res = await trigger.next() - ) { - try { - const { status, message } = await o.fn(overlay) - await o.effects.setHealth({ - name: o.name, - id: o.name, - result: status, - message: message || "", - }) - currentValue.hadSuccess = true - currentValue.lastResult = "success" - await triggerFirstSuccess().catch((err) => { - console.error(err) - }) - } catch (e) { - await o.effects.setHealth({ - name: o.name, - id: o.name, - result: "failure", - message: asMessage(e) || "", - }) - currentValue.lastResult = "failure" - } - } - } finally { - await overlay.destroy() } }) return {} as HealthReceipt diff --git a/sdk/lib/health/checkFns/CheckResult.ts b/sdk/lib/health/checkFns/CheckResult.ts deleted file mode 100644 index 8b46ee5c4..000000000 --- a/sdk/lib/health/checkFns/CheckResult.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { HealthStatus } from "../../types" - -export type CheckResult = { - status: HealthStatus - message: string | null -} diff --git a/sdk/lib/health/checkFns/HealthCheckResult.ts b/sdk/lib/health/checkFns/HealthCheckResult.ts new file mode 100644 index 000000000..ba2468488 --- /dev/null +++ b/sdk/lib/health/checkFns/HealthCheckResult.ts @@ -0,0 +1,3 @@ +import { T } from "../.." + +export type HealthCheckResult = Omit diff --git a/sdk/lib/health/checkFns/checkPortListening.ts b/sdk/lib/health/checkFns/checkPortListening.ts index 4cc0738da..94d0becc0 100644 --- a/sdk/lib/health/checkFns/checkPortListening.ts +++ b/sdk/lib/health/checkFns/checkPortListening.ts @@ -1,6 +1,6 @@ import { Effects } from "../../types" import { stringFromStdErrOut } from "../../util/stringFromStdErrOut" -import { CheckResult } from "./CheckResult" +import { HealthCheckResult } from "./HealthCheckResult" import { promisify } from "node:util" import * as CP from "node:child_process" @@ -32,8 +32,8 @@ export async function checkPortListening( timeoutMessage?: string timeout?: number }, -): Promise { - return Promise.race([ +): Promise { + return Promise.race([ Promise.resolve().then(async () => { const hasAddress = containsAddress( @@ -45,10 +45,10 @@ export async function checkPortListening( port, ) if (hasAddress) { - return { status: "success", message: options.successMessage } + return { result: "success", message: options.successMessage } } return { - status: "failure", + result: "failure", message: options.errorMessage, } }), @@ -56,7 +56,7 @@ export async function checkPortListening( setTimeout( () => resolve({ - status: "failure", + result: "failure", message: options.timeoutMessage || `Timeout trying to check port ${port}`, }), diff --git a/sdk/lib/health/checkFns/checkWebUrl.ts b/sdk/lib/health/checkFns/checkWebUrl.ts index 8f61ae2ef..042115211 100644 --- a/sdk/lib/health/checkFns/checkWebUrl.ts +++ b/sdk/lib/health/checkFns/checkWebUrl.ts @@ -1,5 +1,6 @@ import { Effects } from "../../types" -import { CheckResult } from "./CheckResult" +import { asError } from "../../util/asError" +import { HealthCheckResult } from "./HealthCheckResult" import { timeoutPromise } from "./index" import "isomorphic-fetch" @@ -17,19 +18,19 @@ export const checkWebUrl = async ( successMessage = `Reached ${url}`, errorMessage = `Error while fetching URL: ${url}`, } = {}, -): Promise => { +): Promise => { return Promise.race([fetch(url), timeoutPromise(timeout)]) .then( (x) => ({ - status: "success", + result: "success", message: successMessage, }) as const, ) .catch((e) => { console.warn(`Error while fetching URL: ${url}`) console.error(JSON.stringify(e)) - console.error(e.toString()) - return { status: "failure" as const, message: errorMessage } + console.error(asError(e)) + return { result: "failure" as const, message: errorMessage } }) } diff --git a/sdk/lib/health/checkFns/index.ts b/sdk/lib/health/checkFns/index.ts index d33d5ad0d..2de37e38c 100644 --- a/sdk/lib/health/checkFns/index.ts +++ b/sdk/lib/health/checkFns/index.ts @@ -1,6 +1,6 @@ import { runHealthScript } from "./runHealthScript" export { checkPortListening } from "./checkPortListening" -export { CheckResult } from "./CheckResult" +export { HealthCheckResult } from "./HealthCheckResult" export { checkWebUrl } from "./checkWebUrl" export function timeoutPromise(ms: number, { message = "Timed out" } = {}) { diff --git a/sdk/lib/health/checkFns/runHealthScript.ts b/sdk/lib/health/checkFns/runHealthScript.ts index f0f41ee91..4bac211a9 100644 --- a/sdk/lib/health/checkFns/runHealthScript.ts +++ b/sdk/lib/health/checkFns/runHealthScript.ts @@ -1,7 +1,7 @@ import { Effects } from "../../types" -import { Overlay } from "../../util/Overlay" +import { SubContainer } from "../../util/SubContainer" import { stringFromStdErrOut } from "../../util/stringFromStdErrOut" -import { CheckResult } from "./CheckResult" +import { HealthCheckResult } from "./HealthCheckResult" import { timeoutPromise } from "./index" /** @@ -12,27 +12,26 @@ import { timeoutPromise } from "./index" * @returns */ export const runHealthScript = async ( - effects: Effects, runCommand: string[], - overlay: Overlay, + subcontainer: SubContainer, { timeout = 30000, errorMessage = `Error while running command: ${runCommand}`, message = (res: string) => `Have ran script ${runCommand} and the result: ${res}`, } = {}, -): Promise => { +): Promise => { const res = await Promise.race([ - overlay.exec(runCommand), + subcontainer.exec(runCommand), timeoutPromise(timeout), ]).catch((e) => { console.warn(errorMessage) console.warn(JSON.stringify(e)) console.warn(e.toString()) - throw { status: "failure", message: errorMessage } as CheckResult + throw { result: "failure", message: errorMessage } as HealthCheckResult }) return { - status: "success", + result: "success", message: message(res.stdout.toString()), - } as CheckResult + } as HealthCheckResult } diff --git a/sdk/lib/index.ts b/sdk/lib/index.ts index 935ffc023..a4caf9c01 100644 --- a/sdk/lib/index.ts +++ b/sdk/lib/index.ts @@ -1,5 +1,5 @@ export { Daemons } from "./mainFn/Daemons" -export { Overlay } from "./util/Overlay" +export { SubContainer } from "./util/SubContainer" export { StartSdk } from "./StartSdk" export { setupManifest } from "./manifest/setupManifest" export { FileHelper } from "./util/fileHelper" @@ -29,3 +29,4 @@ export * as utils from "./util" export * as matches from "ts-matches" export * as YAML from "yaml" export * as TOML from "@iarna/toml" +export * from "./version" diff --git a/sdk/lib/inits/migrations/Migration.ts b/sdk/lib/inits/migrations/Migration.ts deleted file mode 100644 index 16be93dbd..000000000 --- a/sdk/lib/inits/migrations/Migration.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { ValidateExVer } from "../../exver" -import * as T from "../../types" - -export class Migration< - Manifest extends T.Manifest, - Store, - Version extends string, -> { - constructor( - readonly options: { - version: Version & ValidateExVer - up: (opts: { effects: T.Effects }) => Promise - down: (opts: { effects: T.Effects }) => Promise - }, - ) {} - static of< - Manifest extends T.Manifest, - Store, - Version extends string, - >(options: { - version: Version & ValidateExVer - up: (opts: { effects: T.Effects }) => Promise - down: (opts: { effects: T.Effects }) => Promise - }) { - return new Migration(options) - } - - async up(opts: { effects: T.Effects }) { - this.up(opts) - } - - async down(opts: { effects: T.Effects }) { - this.down(opts) - } -} diff --git a/sdk/lib/inits/migrations/setupMigrations.ts b/sdk/lib/inits/migrations/setupMigrations.ts deleted file mode 100644 index 6d690b239..000000000 --- a/sdk/lib/inits/migrations/setupMigrations.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { ExtendedVersion } from "../../exver" - -import * as T from "../../types" -import { once } from "../../util/once" -import { Migration } from "./Migration" - -export class Migrations { - private constructor( - readonly manifest: T.Manifest, - readonly migrations: Array>, - ) {} - private sortedMigrations = once(() => { - const migrationsAsVersions = ( - this.migrations as Array> - ) - .map((x) => [ExtendedVersion.parse(x.options.version), x] as const) - .filter(([v, _]) => v.flavor === this.currentVersion().flavor) - migrationsAsVersions.sort((a, b) => a[0].compareForSort(b[0])) - return migrationsAsVersions - }) - private currentVersion = once(() => - ExtendedVersion.parse(this.manifest.version), - ) - static of< - Manifest extends T.Manifest, - Store, - Migrations extends Array>, - >(manifest: T.Manifest, ...migrations: EnsureUniqueId) { - return new Migrations( - manifest, - migrations as Array>, - ) - } - async init({ - effects, - previousVersion, - }: Parameters[0]) { - if (!!previousVersion) { - const previousVersionExVer = ExtendedVersion.parse(previousVersion) - for (const [_, migration] of this.sortedMigrations() - .filter((x) => x[0].greaterThan(previousVersionExVer)) - .filter((x) => x[0].lessThanOrEqual(this.currentVersion()))) { - await migration.up({ effects }) - } - } - } - async uninit({ - effects, - nextVersion, - }: Parameters[0]) { - if (!!nextVersion) { - const nextVersionExVer = ExtendedVersion.parse(nextVersion) - const reversed = [...this.sortedMigrations()].reverse() - for (const [_, migration] of reversed - .filter((x) => x[0].greaterThan(nextVersionExVer)) - .filter((x) => x[0].lessThanOrEqual(this.currentVersion()))) { - await migration.down({ effects }) - } - } - } -} - -export function setupMigrations< - Manifest extends T.Manifest, - Store, - Migrations extends Array>, ->(manifest: T.Manifest, ...migrations: EnsureUniqueId) { - return Migrations.of(manifest, ...migrations) -} - -// prettier-ignore -export type EnsureUniqueId = - B extends [] ? A : - B extends [Migration, ...infer Rest] ? ( - id extends ids ? "One of the ids are not unique"[] : - EnsureUniqueId - ) : "There exists a migration that is not a Migration"[] diff --git a/sdk/lib/inits/setupInit.ts b/sdk/lib/inits/setupInit.ts index 5718caa58..5fd1c481c 100644 --- a/sdk/lib/inits/setupInit.ts +++ b/sdk/lib/inits/setupInit.ts @@ -1,14 +1,15 @@ import { DependenciesReceipt } from "../config/setupConfig" +import { ExtendedVersion, VersionRange } from "../exver" import { SetInterfaces } from "../interfaces/setupInterfaces" import { ExposedStorePaths } from "../store/setupExposeStore" import * as T from "../types" -import { Migrations } from "./migrations/setupMigrations" +import { VersionGraph } from "../version/VersionGraph" import { Install } from "./setupInstall" import { Uninstall } from "./setupUninstall" export function setupInit( - migrations: Migrations, + versions: VersionGraph, install: Install, uninstall: Uninstall, setInterfaces: SetInterfaces, @@ -23,8 +24,19 @@ export function setupInit( } { return { init: async (opts) => { - await migrations.init(opts) - await install.init(opts) + const prev = await opts.effects.getDataVersion() + if (prev) { + await versions.migrate({ + effects: opts.effects, + from: ExtendedVersion.parse(prev), + to: versions.currentVersion(), + }) + } else { + await install.install(opts) + await opts.effects.setDataVersion({ + version: versions.current.options.version, + }) + } await setInterfaces({ ...opts, input: null, @@ -33,8 +45,18 @@ export function setupInit( await setDependencies({ effects: opts.effects, input: null }) }, uninit: async (opts) => { - await migrations.uninit(opts) - await uninstall.uninit(opts) + if (opts.nextVersion) { + const prev = await opts.effects.getDataVersion() + if (prev) { + await versions.migrate({ + effects: opts.effects, + from: ExtendedVersion.parse(prev), + to: ExtendedVersion.parse(opts.nextVersion), + }) + } + } else { + await uninstall.uninstall(opts) + } }, } } diff --git a/sdk/lib/inits/setupInstall.ts b/sdk/lib/inits/setupInstall.ts index 7b51a22ea..ab21380a0 100644 --- a/sdk/lib/inits/setupInstall.ts +++ b/sdk/lib/inits/setupInstall.ts @@ -11,14 +11,10 @@ export class Install { return new Install(fn) } - async init({ - effects, - previousVersion, - }: Parameters[0]) { - if (!previousVersion) - await this.fn({ - effects, - }) + async install({ effects }: Parameters[0]) { + await this.fn({ + effects, + }) } } diff --git a/sdk/lib/inits/setupUninstall.ts b/sdk/lib/inits/setupUninstall.ts index c8c3e490f..918f417e5 100644 --- a/sdk/lib/inits/setupUninstall.ts +++ b/sdk/lib/inits/setupUninstall.ts @@ -11,7 +11,7 @@ export class Uninstall { return new Uninstall(fn) } - async uninit({ + async uninstall({ effects, nextVersion, }: Parameters[0]) { diff --git a/sdk/lib/interfaces/Origin.ts b/sdk/lib/interfaces/Origin.ts index 52afe1ed3..cc84728ec 100644 --- a/sdk/lib/interfaces/Origin.ts +++ b/sdk/lib/interfaces/Origin.ts @@ -47,7 +47,6 @@ export class Origin { name, description, hasPrimary, - disabled, id, type, username, @@ -69,7 +68,6 @@ export class Origin { name, description, hasPrimary, - disabled, addressInfo, type, masked, diff --git a/sdk/lib/interfaces/ServiceInterfaceBuilder.ts b/sdk/lib/interfaces/ServiceInterfaceBuilder.ts index 14eaee1d3..49d8020d6 100644 --- a/sdk/lib/interfaces/ServiceInterfaceBuilder.ts +++ b/sdk/lib/interfaces/ServiceInterfaceBuilder.ts @@ -21,7 +21,6 @@ export class ServiceInterfaceBuilder { id: string description: string hasPrimary: boolean - disabled: boolean type: ServiceInterfaceType username: string | null path: string diff --git a/sdk/lib/mainFn/CommandController.ts b/sdk/lib/mainFn/CommandController.ts index 40f787f86..8a0505f68 100644 --- a/sdk/lib/mainFn/CommandController.ts +++ b/sdk/lib/mainFn/CommandController.ts @@ -2,30 +2,39 @@ import { DEFAULT_SIGTERM_TIMEOUT } from "." import { NO_TIMEOUT, SIGKILL, SIGTERM } from "../StartSdk" import * as T from "../types" -import { MountOptions, Overlay } from "../util/Overlay" +import { asError } from "../util/asError" +import { + ExecSpawnable, + MountOptions, + SubContainerHandle, + SubContainer, +} from "../util/SubContainer" import { splitCommand } from "../util/splitCommand" -import { cpExecFile, cpExec } from "./Daemons" +import * as cp from "child_process" export class CommandController { private constructor( readonly runningAnswer: Promise, - readonly overlay: Overlay, - readonly pid: number | undefined, + private state: { exited: boolean }, + private readonly subcontainer: SubContainer, + private process: cp.ChildProcessWithoutNullStreams, readonly sigtermTimeout: number = DEFAULT_SIGTERM_TIMEOUT, ) {} static of() { return async ( effects: T.Effects, - imageId: { - id: keyof Manifest["images"] & T.ImageId - sharedRun?: boolean - }, + subcontainer: + | { + id: keyof Manifest["images"] & T.ImageId + sharedRun?: boolean + } + | SubContainer, command: T.CommandType, options: { // Defaults to the DEFAULT_SIGTERM_TIMEOUT = 30_000ms sigtermTimeout?: number mounts?: { path: string; options: MountOptions }[] - overlay?: Overlay + runAsInit?: boolean env?: | { [variable: string]: string @@ -38,43 +47,62 @@ export class CommandController { }, ) => { const commands = splitCommand(command) - const overlay = options.overlay || (await Overlay.of(effects, imageId)) - for (let mount of options.mounts || []) { - await overlay.mount(mount.options, mount.path) + const subc = + subcontainer instanceof SubContainer + ? subcontainer + : await (async () => { + const subc = await SubContainer.of(effects, subcontainer) + for (let mount of options.mounts || []) { + await subc.mount(mount.options, mount.path) + } + return subc + })() + let childProcess: cp.ChildProcessWithoutNullStreams + if (options.runAsInit) { + childProcess = await subc.launch(commands, { + env: options.env, + }) + } else { + childProcess = await subc.spawn(commands, { + env: options.env, + }) } - const childProcess = await overlay.spawn(commands, { - env: options.env, - }) + const state = { exited: false } const answer = new Promise((resolve, reject) => { - childProcess.stdout.on( - "data", - options.onStdout ?? - ((data: any) => { - console.log(data.toString()) - }), - ) - childProcess.stderr.on( - "data", - options.onStderr ?? - ((data: any) => { - console.error(data.toString()) - }), - ) - - childProcess.on("exit", (code: any) => { - if (code === 0) { + childProcess.on("exit", (code) => { + state.exited = true + if ( + code === 0 || + code === 143 || + (code === null && childProcess.signalCode == "SIGTERM") + ) { return resolve(null) } - return reject(new Error(`${commands[0]} exited with code ${code}`)) + if (code) { + return reject(new Error(`${commands[0]} exited with code ${code}`)) + } else { + return reject( + new Error( + `${commands[0]} exited with signal ${childProcess.signalCode}`, + ), + ) + } }) }) - const pid = childProcess.pid - - return new CommandController(answer, overlay, pid, options.sigtermTimeout) + return new CommandController( + answer, + state, + subc, + childProcess, + options.sigtermTimeout, + ) } } - async wait(timeout: number = NO_TIMEOUT) { + get subContainerHandle() { + return new SubContainerHandle(this.subcontainer) + } + async wait({ timeout = NO_TIMEOUT } = {}) { if (timeout > 0) setTimeout(() => { this.term() @@ -82,75 +110,30 @@ export class CommandController { try { return await this.runningAnswer } finally { - if (this.pid !== undefined) { - await cpExecFile("pkill", ["-9", "-s", String(this.pid)]).catch( - (_) => {}, - ) + if (!this.state.exited) { + this.process.kill("SIGKILL") } - await this.overlay.destroy().catch((_) => {}) + await this.subcontainer.destroy?.().catch((_) => {}) } } async term({ signal = SIGTERM, timeout = this.sigtermTimeout } = {}) { - if (this.pid === undefined) return try { - await cpExecFile("pkill", [ - `-${signal.replace("SIG", "")}`, - "-s", - String(this.pid), - ]) - - const didTimeout = await waitSession(this.pid, timeout) - if (didTimeout) { - await cpExecFile("pkill", [`-9`, "-s", String(this.pid)]).catch( - (_) => {}, - ) + if (!this.state.exited) { + if (!this.process.kill(signal)) { + console.error( + `failed to send signal ${signal} to pid ${this.process.pid}`, + ) + } } - } finally { - await this.overlay.destroy() - } - } -} -function waitSession( - sid: number, - timeout = NO_TIMEOUT, - interval = 100, -): Promise { - let nextInterval = interval * 2 - if (timeout >= 0 && timeout < nextInterval) { - nextInterval = timeout - } - let nextTimeout = timeout - if (timeout > 0) { - if (timeout >= interval) { - nextTimeout -= interval - } else { - nextTimeout = 0 + if (signal !== "SIGKILL") { + setTimeout(() => { + this.process.kill("SIGKILL") + }, timeout) + } + await this.runningAnswer + } finally { + await this.subcontainer.destroy?.() } } - return new Promise((resolve, reject) => { - let next: NodeJS.Timeout | null = null - if (timeout !== 0) { - next = setTimeout(() => { - waitSession(sid, nextTimeout, nextInterval).then(resolve, reject) - }, interval) - } - cpExecFile("ps", [`--sid=${sid}`, "-o", "--pid="]).then( - (_) => { - if (timeout === 0) { - resolve(true) - } - }, - (e) => { - if (next) { - clearTimeout(next) - } - if (typeof e === "object" && e && "code" in e && e.code) { - resolve(false) - } else { - reject(e) - } - }, - ) - }) } diff --git a/sdk/lib/mainFn/Daemon.ts b/sdk/lib/mainFn/Daemon.ts index 6dceda951..87a7d705d 100644 --- a/sdk/lib/mainFn/Daemon.ts +++ b/sdk/lib/mainFn/Daemon.ts @@ -1,5 +1,6 @@ import * as T from "../types" -import { MountOptions, Overlay } from "../util/Overlay" +import { asError } from "../util/asError" +import { ExecSpawnable, MountOptions, SubContainer } from "../util/SubContainer" import { CommandController } from "./CommandController" const TIMEOUT_INCREMENT_MS = 1000 @@ -12,18 +13,22 @@ const MAX_TIMEOUT_MS = 30000 export class Daemon { private commandController: CommandController | null = null private shouldBeRunning = false - private constructor(private startCommand: () => Promise) {} + constructor(private startCommand: () => Promise) {} + get subContainerHandle(): undefined | ExecSpawnable { + return this.commandController?.subContainerHandle + } static of() { return async ( effects: T.Effects, - imageId: { - id: keyof Manifest["images"] & T.ImageId - sharedRun?: boolean - }, + subcontainer: + | { + id: keyof Manifest["images"] & T.ImageId + sharedRun?: boolean + } + | SubContainer, command: T.CommandType, options: { mounts?: { path: string; options: MountOptions }[] - overlay?: Overlay env?: | { [variable: string]: string @@ -37,11 +42,15 @@ export class Daemon { }, ) => { const startCommand = () => - CommandController.of()(effects, imageId, command, options) + CommandController.of()( + effects, + subcontainer, + command, + options, + ) return new Daemon(startCommand) } } - async start() { if (this.commandController) { return @@ -57,7 +66,7 @@ export class Daemon { timeoutCounter = Math.max(MAX_TIMEOUT_MS, timeoutCounter) } }).catch((err) => { - console.error(err) + console.error(asError(err)) }) } async term(termOptions?: { @@ -72,8 +81,8 @@ export class Daemon { }) { this.shouldBeRunning = false await this.commandController - ?.term(termOptions) - .catch((e) => console.error(e)) + ?.term({ ...termOptions }) + .catch((e) => console.error(asError(e))) this.commandController = null } } diff --git a/sdk/lib/mainFn/Daemons.ts b/sdk/lib/mainFn/Daemons.ts index c766e2f2e..1ecec28d3 100644 --- a/sdk/lib/mainFn/Daemons.ts +++ b/sdk/lib/mainFn/Daemons.ts @@ -1,13 +1,18 @@ import { NO_TIMEOUT, SIGKILL, SIGTERM, Signals } from "../StartSdk" import { HealthReceipt } from "../health/HealthReceipt" -import { CheckResult } from "../health/checkFns" +import { HealthCheckResult } from "../health/checkFns" import { Trigger } from "../trigger" import { TriggerInput } from "../trigger/TriggerInput" import { defaultTrigger } from "../trigger/defaultTrigger" import * as T from "../types" import { Mounts } from "./Mounts" -import { CommandOptions, MountOptions, Overlay } from "../util/Overlay" +import { + CommandOptions, + ExecSpawnable, + MountOptions, + SubContainer, +} from "../util/SubContainer" import { splitCommand } from "../util/splitCommand" import { promisify } from "node:util" @@ -23,7 +28,9 @@ export const cpExec = promisify(CP.exec) export const cpExecFile = promisify(CP.execFile) export type Ready = { display: string | null - fn: () => Promise | CheckResult + fn: ( + spawnable: ExecSpawnable, + ) => Promise | HealthCheckResult trigger?: Trigger } diff --git a/sdk/lib/mainFn/HealthDaemon.ts b/sdk/lib/mainFn/HealthDaemon.ts index 84865e59b..7ace3ed7b 100644 --- a/sdk/lib/mainFn/HealthDaemon.ts +++ b/sdk/lib/mainFn/HealthDaemon.ts @@ -1,9 +1,10 @@ -import { CheckResult } from "../health/checkFns" +import { HealthCheckResult } from "../health/checkFns" import { defaultTrigger } from "../trigger/defaultTrigger" import { Ready } from "./Daemons" import { Daemon } from "./Daemon" -import { Effects } from "../types" +import { Effects, SetHealth } from "../types" import { DEFAULT_SIGTERM_TIMEOUT } from "." +import { asError } from "../util/asError" const oncePromise = () => { let resolve: (value: T) => void @@ -21,14 +22,13 @@ const oncePromise = () => { * */ export class HealthDaemon { - #health: CheckResult = { status: "starting", message: null } - #healthWatchers: Array<() => unknown> = [] - #running = false - #hadSuccess = false + private _health: HealthCheckResult = { result: "starting", message: null } + private healthWatchers: Array<() => unknown> = [] + private running = false constructor( - readonly daemon: Promise, + private readonly daemon: Promise, readonly daemonIndex: number, - readonly dependencies: HealthDaemon[], + private readonly dependencies: HealthDaemon[], readonly id: string, readonly ids: string[], readonly ready: Ready, @@ -44,12 +44,12 @@ export class HealthDaemon { signal?: NodeJS.Signals | undefined timeout?: number | undefined }) { - this.#healthWatchers = [] - this.#running = false - this.#healthCheckCleanup?.() + this.healthWatchers = [] + this.running = false + this.healthCheckCleanup?.() await this.daemon.then((d) => - d.stop({ + d.term({ timeout: this.sigtermTimeout, ...termOptions, }), @@ -58,17 +58,17 @@ export class HealthDaemon { /** Want to add another notifier that the health might have changed */ addWatcher(watcher: () => unknown) { - this.#healthWatchers.push(watcher) + this.healthWatchers.push(watcher) } get health() { - return Object.freeze(this.#health) + return Object.freeze(this._health) } private async changeRunning(newStatus: boolean) { - if (this.#running === newStatus) return + if (this.running === newStatus) return - this.#running = newStatus + this.running = newStatus if (newStatus) { ;(await this.daemon).start() @@ -77,19 +77,18 @@ export class HealthDaemon { ;(await this.daemon).stop() this.turnOffHealthCheck() - this.setHealth({ status: "starting", message: null }) + this.setHealth({ result: "starting", message: null }) } } - #healthCheckCleanup: (() => void) | null = null + private healthCheckCleanup: (() => void) | null = null private turnOffHealthCheck() { - this.#healthCheckCleanup?.() + this.healthCheckCleanup?.() } private async setupHealthCheck() { - if (this.#healthCheckCleanup) return + if (this.healthCheckCleanup) return const trigger = (this.ready.trigger ?? defaultTrigger)(() => ({ - hadSuccess: this.#hadSuccess, - lastResult: this.#health.status, + lastResult: this._health.result, })) const { promise: status, resolve: setStatus } = oncePromise<{ @@ -101,59 +100,51 @@ export class HealthDaemon { !res.done; res = await Promise.race([status, trigger.next()]) ) { - const response: CheckResult = await Promise.resolve( - this.ready.fn(), - ).catch((err) => { - console.error(err) - return { - status: "failure", - message: "message" in err ? err.message : String(err), - } - }) - this.setHealth(response) - if (response.status === "success") { - this.#hadSuccess = true + const handle = (await this.daemon).subContainerHandle + + if (handle) { + const response: HealthCheckResult = await Promise.resolve( + this.ready.fn(handle), + ).catch((err) => { + console.error(asError(err)) + return { + result: "failure", + message: "message" in err ? err.message : String(err), + } + }) + await this.setHealth(response) + } else { + await this.setHealth({ + result: "failure", + message: "Daemon not running", + }) } } }).catch((err) => console.error(`Daemon ${this.id} failed: ${err}`)) - this.#healthCheckCleanup = () => { + this.healthCheckCleanup = () => { setStatus({ done: true }) - this.#healthCheckCleanup = null + this.healthCheckCleanup = null } } - private setHealth(health: CheckResult) { - this.#health = health - this.#healthWatchers.forEach((watcher) => watcher()) + private async setHealth(health: HealthCheckResult) { + this._health = health + this.healthWatchers.forEach((watcher) => watcher()) const display = this.ready.display - const status = health.status + const result = health.result if (!display) { return } - if ( - status === "success" || - status === "disabled" || - status === "starting" - ) { - this.effects.setHealth({ - result: status, - message: health.message, - id: this.id, - name: display, - }) - } else { - this.effects.setHealth({ - result: health.status, - message: health.message || "", - id: this.id, - name: display, - }) - } + await this.effects.setHealth({ + ...health, + id: this.id, + name: display, + } as SetHealth) } private async updateStatus() { - const healths = this.dependencies.map((d) => d.#health) - this.changeRunning(healths.every((x) => x.status === "success")) + const healths = this.dependencies.map((d) => d._health) + this.changeRunning(healths.every((x) => x.result === "success")) } } diff --git a/sdk/lib/mainFn/Mounts.ts b/sdk/lib/mainFn/Mounts.ts index 968b77b4e..bd947b759 100644 --- a/sdk/lib/mainFn/Mounts.ts +++ b/sdk/lib/mainFn/Mounts.ts @@ -1,5 +1,5 @@ import * as T from "../types" -import { MountOptions } from "../util/Overlay" +import { MountOptions } from "../util/SubContainer" type MountArray = { path: string; options: MountOptions }[] diff --git a/sdk/lib/manifest/ManifestTypes.ts b/sdk/lib/manifest/ManifestTypes.ts index ea349710f..cc564de2d 100644 --- a/sdk/lib/manifest/ManifestTypes.ts +++ b/sdk/lib/manifest/ManifestTypes.ts @@ -7,22 +7,11 @@ import { ImageSource, } from "../types" -export type SDKManifest< - Version extends string, - Satisfies extends string[] = [], -> = { +export type SDKManifest = { /** The package identifier used by the OS. This must be unique amongst all other known packages */ readonly id: string /** A human readable service title */ readonly title: string - /** Service version - accepts up to four digits, where the last confirms to revisions necessary for StartOs - * - see documentation: https://github.com/Start9Labs/emver-rs. This value will change with each release of - * the service - */ - readonly version: Version & ValidateExVer - readonly satisfies?: Satisfies & ValidateExVers - /** Release notes for the update - can be a string, paragraph or URL */ - readonly releaseNotes: string /** The type of license for the project. Include the LICENSE in the root of the project directory. A license is required for a Start9 package.*/ readonly license: string // name of license /** The Start9 wrapper repository URL for the package. This repo contains the manifest file (this), diff --git a/sdk/lib/manifest/setupManifest.ts b/sdk/lib/manifest/setupManifest.ts index e3b746874..10aaa03db 100644 --- a/sdk/lib/manifest/setupManifest.ts +++ b/sdk/lib/manifest/setupManifest.ts @@ -2,7 +2,13 @@ import * as T from "../types" import { ImageConfig, ImageId, VolumeId } from "../osBindings" import { SDKManifest, SDKImageConfig } from "./ManifestTypes" import { SDKVersion } from "../StartSdk" +import { VersionGraph } from "../version/VersionGraph" +/** + * This is an example of a function that takes a manifest and returns a new manifest with additional properties + * @param manifest Manifests are the description of the package + * @returns The manifest with additional properties + */ export function setupManifest< Id extends string, Version extends string, @@ -10,7 +16,7 @@ export function setupManifest< VolumesTypes extends VolumeId, AssetTypes extends VolumeId, ImagesTypes extends ImageId, - Manifest extends SDKManifest & { + Manifest extends { dependencies: Dependencies id: Id assets: AssetTypes[] @@ -18,7 +24,10 @@ export function setupManifest< volumes: VolumesTypes[] }, Satisfies extends string[] = [], ->(manifest: Manifest & { version: Version }): Manifest & T.Manifest { +>( + versions: VersionGraph, + manifest: SDKManifest & Manifest, +): Manifest & T.Manifest { const images = Object.entries(manifest.images).reduce( (images, [k, v]) => { v.arch = v.arch || ["aarch64", "x86_64"] @@ -33,7 +42,11 @@ export function setupManifest< ...manifest, gitHash: null, osVersion: SDKVersion, - satisfies: manifest.satisfies || [], + version: versions.current.options.version, + releaseNotes: versions.current.options.releaseNotes, + satisfies: versions.current.options.satisfies || [], + canMigrateTo: versions.canMigrateTo().toString(), + canMigrateFrom: versions.canMigrateFrom().toString(), images, alerts: { install: manifest.alerts?.install || null, diff --git a/sdk/lib/osBindings/CheckDependenciesResult.ts b/sdk/lib/osBindings/CheckDependenciesResult.ts index d349bdf18..a435ff87f 100644 --- a/sdk/lib/osBindings/CheckDependenciesResult.ts +++ b/sdk/lib/osBindings/CheckDependenciesResult.ts @@ -1,13 +1,14 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { HealthCheckId } from "./HealthCheckId" -import type { HealthCheckResult } from "./HealthCheckResult" +import type { NamedHealthCheckResult } from "./NamedHealthCheckResult" import type { PackageId } from "./PackageId" export type CheckDependenciesResult = { packageId: PackageId - isInstalled: boolean + title: string | null + installedVersion: string | null + satisfies: string[] isRunning: boolean configSatisfied: boolean - healthChecks: { [key: HealthCheckId]: HealthCheckResult } - version: string | null + healthChecks: { [key: HealthCheckId]: NamedHealthCheckResult } } diff --git a/sdk/lib/osBindings/CreateOverlayedImageParams.ts b/sdk/lib/osBindings/CreateSubcontainerFsParams.ts similarity index 70% rename from sdk/lib/osBindings/CreateOverlayedImageParams.ts rename to sdk/lib/osBindings/CreateSubcontainerFsParams.ts index aad94f01f..729ad4240 100644 --- a/sdk/lib/osBindings/CreateOverlayedImageParams.ts +++ b/sdk/lib/osBindings/CreateSubcontainerFsParams.ts @@ -1,4 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ImageId } from "./ImageId" -export type CreateOverlayedImageParams = { imageId: ImageId } +export type CreateSubcontainerFsParams = { imageId: ImageId } diff --git a/sdk/lib/osBindings/DestroyOverlayedImageParams.ts b/sdk/lib/osBindings/DestroySubcontainerFsParams.ts similarity index 71% rename from sdk/lib/osBindings/DestroyOverlayedImageParams.ts rename to sdk/lib/osBindings/DestroySubcontainerFsParams.ts index b5b7484a2..3f85d2217 100644 --- a/sdk/lib/osBindings/DestroyOverlayedImageParams.ts +++ b/sdk/lib/osBindings/DestroySubcontainerFsParams.ts @@ -1,4 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Guid } from "./Guid" -export type DestroyOverlayedImageParams = { guid: Guid } +export type DestroySubcontainerFsParams = { guid: Guid } diff --git a/sdk/lib/osBindings/ExportServiceInterfaceParams.ts b/sdk/lib/osBindings/ExportServiceInterfaceParams.ts index b93e83f7c..28ac89916 100644 --- a/sdk/lib/osBindings/ExportServiceInterfaceParams.ts +++ b/sdk/lib/osBindings/ExportServiceInterfaceParams.ts @@ -8,7 +8,6 @@ export type ExportServiceInterfaceParams = { name: string description: string hasPrimary: boolean - disabled: boolean masked: boolean addressInfo: AddressInfo type: ServiceInterfaceType diff --git a/sdk/lib/osBindings/HardwareRequirements.ts b/sdk/lib/osBindings/HardwareRequirements.ts index 3579e9524..e17568eec 100644 --- a/sdk/lib/osBindings/HardwareRequirements.ts +++ b/sdk/lib/osBindings/HardwareRequirements.ts @@ -1,7 +1,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. export type HardwareRequirements = { - device: { device?: string; processor?: string } + device: { display?: string; processor?: string } ram: number | null arch: string[] | null } diff --git a/sdk/lib/osBindings/MainStatus.ts b/sdk/lib/osBindings/MainStatus.ts index 6acdce14a..1f6b3babe 100644 --- a/sdk/lib/osBindings/MainStatus.ts +++ b/sdk/lib/osBindings/MainStatus.ts @@ -1,20 +1,20 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { HealthCheckId } from "./HealthCheckId" -import type { HealthCheckResult } from "./HealthCheckResult" +import type { NamedHealthCheckResult } from "./NamedHealthCheckResult" +import type { StartStop } from "./StartStop" export type MainStatus = | { status: "stopped" } | { status: "restarting" } | { status: "restoring" } | { status: "stopping" } - | { status: "starting" } + | { + status: "starting" + health: { [key: HealthCheckId]: NamedHealthCheckResult } + } | { status: "running" started: string - health: { [key: HealthCheckId]: HealthCheckResult } - } - | { - status: "backingUp" - started: string | null - health: { [key: HealthCheckId]: HealthCheckResult } + health: { [key: HealthCheckId]: NamedHealthCheckResult } } + | { status: "backingUp"; onComplete: StartStop } diff --git a/sdk/lib/osBindings/Manifest.ts b/sdk/lib/osBindings/Manifest.ts index 51f14935a..d40223236 100644 --- a/sdk/lib/osBindings/Manifest.ts +++ b/sdk/lib/osBindings/Manifest.ts @@ -15,6 +15,8 @@ export type Manifest = { version: Version satisfies: Array releaseNotes: string + canMigrateTo: string + canMigrateFrom: string license: string wrapperRepo: string upstreamRepo: string diff --git a/sdk/lib/osBindings/HealthCheckResult.ts b/sdk/lib/osBindings/NamedHealthCheckResult.ts similarity index 85% rename from sdk/lib/osBindings/HealthCheckResult.ts rename to sdk/lib/osBindings/NamedHealthCheckResult.ts index 6fa3d3f8c..c967e9b34 100644 --- a/sdk/lib/osBindings/HealthCheckResult.ts +++ b/sdk/lib/osBindings/NamedHealthCheckResult.ts @@ -1,6 +1,6 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type HealthCheckResult = { name: string } & ( +export type NamedHealthCheckResult = { name: string } & ( | { result: "success"; message: string | null } | { result: "disabled"; message: string | null } | { result: "starting"; message: string | null } diff --git a/sdk/lib/osBindings/PackageDataEntry.ts b/sdk/lib/osBindings/PackageDataEntry.ts index ef805741b..41bd98bba 100644 --- a/sdk/lib/osBindings/PackageDataEntry.ts +++ b/sdk/lib/osBindings/PackageDataEntry.ts @@ -8,9 +8,11 @@ import type { PackageState } from "./PackageState" import type { ServiceInterface } from "./ServiceInterface" import type { ServiceInterfaceId } from "./ServiceInterfaceId" import type { Status } from "./Status" +import type { Version } from "./Version" export type PackageDataEntry = { stateInfo: PackageState + dataVersion: Version | null status: Status registry: string | null developerKey: string diff --git a/sdk/lib/osBindings/ServiceInterface.ts b/sdk/lib/osBindings/ServiceInterface.ts index 91ac77515..9bcec0056 100644 --- a/sdk/lib/osBindings/ServiceInterface.ts +++ b/sdk/lib/osBindings/ServiceInterface.ts @@ -8,7 +8,6 @@ export type ServiceInterface = { name: string description: string hasPrimary: boolean - disabled: boolean masked: boolean addressInfo: AddressInfo type: ServiceInterfaceType diff --git a/sdk/lib/osBindings/SetDataVersionParams.ts b/sdk/lib/osBindings/SetDataVersionParams.ts new file mode 100644 index 000000000..3b577d2b1 --- /dev/null +++ b/sdk/lib/osBindings/SetDataVersionParams.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type SetDataVersionParams = { version: string } diff --git a/sdk/lib/osBindings/StartStop.ts b/sdk/lib/osBindings/StartStop.ts new file mode 100644 index 000000000..c8be35fb7 --- /dev/null +++ b/sdk/lib/osBindings/StartStop.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type StartStop = "start" | "stop" diff --git a/sdk/lib/osBindings/index.ts b/sdk/lib/osBindings/index.ts index 2708aef8c..9492fe796 100644 --- a/sdk/lib/osBindings/index.ts +++ b/sdk/lib/osBindings/index.ts @@ -31,7 +31,7 @@ export { CheckDependenciesParam } from "./CheckDependenciesParam" export { CheckDependenciesResult } from "./CheckDependenciesResult" export { Cifs } from "./Cifs" export { ContactInfo } from "./ContactInfo" -export { CreateOverlayedImageParams } from "./CreateOverlayedImageParams" +export { CreateSubcontainerFsParams } from "./CreateSubcontainerFsParams" export { CurrentDependencies } from "./CurrentDependencies" export { CurrentDependencyInfo } from "./CurrentDependencyInfo" export { DataUrl } from "./DataUrl" @@ -41,7 +41,7 @@ export { DependencyMetadata } from "./DependencyMetadata" export { DependencyRequirement } from "./DependencyRequirement" export { DepInfo } from "./DepInfo" export { Description } from "./Description" -export { DestroyOverlayedImageParams } from "./DestroyOverlayedImageParams" +export { DestroySubcontainerFsParams } from "./DestroySubcontainerFsParams" export { Duration } from "./Duration" export { EchoParams } from "./EchoParams" export { EncryptedWire } from "./EncryptedWire" @@ -69,7 +69,6 @@ export { Governor } from "./Governor" export { Guid } from "./Guid" export { HardwareRequirements } from "./HardwareRequirements" export { HealthCheckId } from "./HealthCheckId" -export { HealthCheckResult } from "./HealthCheckResult" export { HostAddress } from "./HostAddress" export { HostId } from "./HostId" export { HostKind } from "./HostKind" @@ -98,6 +97,7 @@ export { MaybeUtf8String } from "./MaybeUtf8String" export { MerkleArchiveCommitment } from "./MerkleArchiveCommitment" export { MountParams } from "./MountParams" export { MountTarget } from "./MountTarget" +export { NamedHealthCheckResult } from "./NamedHealthCheckResult" export { NamedProgress } from "./NamedProgress" export { OnionHostname } from "./OnionHostname" export { OsIndex } from "./OsIndex" @@ -132,6 +132,7 @@ export { SessionList } from "./SessionList" export { Sessions } from "./Sessions" export { Session } from "./Session" export { SetConfigured } from "./SetConfigured" +export { SetDataVersionParams } from "./SetDataVersionParams" export { SetDependenciesParams } from "./SetDependenciesParams" export { SetHealth } from "./SetHealth" export { SetMainStatusStatus } from "./SetMainStatusStatus" @@ -144,6 +145,7 @@ export { SetupStatusRes } from "./SetupStatusRes" export { SignAssetParams } from "./SignAssetParams" export { SignerInfo } from "./SignerInfo" export { SmtpValue } from "./SmtpValue" +export { StartStop } from "./StartStop" export { Status } from "./Status" export { UpdatingState } from "./UpdatingState" export { VerifyCifsParams } from "./VerifyCifsParams" diff --git a/sdk/lib/s9pk/merkleArchive/varint.ts b/sdk/lib/s9pk/merkleArchive/varint.ts index 2bf4793b1..016505307 100644 --- a/sdk/lib/s9pk/merkleArchive/varint.ts +++ b/sdk/lib/s9pk/merkleArchive/varint.ts @@ -1,3 +1,5 @@ +import { asError } from "../../util/asError" + const msb = 0x80 const dropMsb = 0x7f const maxSize = Math.floor((8 * 8 + 7) / 7) @@ -38,7 +40,7 @@ export class VarIntProcessor { if (success) { return result } else { - console.error(this.buf) + console.error(asError(this.buf)) return null } } diff --git a/sdk/lib/test/configBuilder.test.ts b/sdk/lib/test/configBuilder.test.ts index a413d76b8..5c65271a8 100644 --- a/sdk/lib/test/configBuilder.test.ts +++ b/sdk/lib/test/configBuilder.test.ts @@ -6,6 +6,8 @@ import { Variants } from "../config/builder/variants" import { ValueSpec } from "../config/configTypes" import { setupManifest } from "../manifest/setupManifest" import { StartSdk } from "../StartSdk" +import { VersionGraph } from "../version/VersionGraph" +import { VersionInfo } from "../version/VersionInfo" describe("builder tests", () => { test("text", async () => { @@ -366,42 +368,48 @@ describe("values", () => { test("datetime", async () => { const sdk = StartSdk.of() .withManifest( - setupManifest({ - id: "testOutput", - title: "", - version: "1.0.0:0", - releaseNotes: "", - license: "", - replaces: [], - wrapperRepo: "", - upstreamRepo: "", - supportSite: "", - marketingSite: "", - donationUrl: null, - description: { - short: "", - long: "", - }, - containers: {}, - images: {}, - volumes: [], - assets: [], - alerts: { - install: null, - update: null, - uninstall: null, - restore: null, - start: null, - stop: null, - }, - dependencies: { - "remote-test": { - description: "", - optional: true, - s9pk: "https://example.com/remote-test.s9pk", + setupManifest( + VersionGraph.of( + VersionInfo.of({ + version: "1.0.0:0", + releaseNotes: "", + migrations: {}, + }), + ), + { + id: "testOutput", + title: "", + license: "", + wrapperRepo: "", + upstreamRepo: "", + supportSite: "", + marketingSite: "", + donationUrl: null, + description: { + short: "", + long: "", + }, + containers: {}, + images: {}, + volumes: [], + assets: [], + alerts: { + install: null, + update: null, + uninstall: null, + restore: null, + start: null, + stop: null, + }, + dependencies: { + "remote-test": { + description: "", + optional: true, + s9pk: "https://example.com/remote-test.s9pk", + }, }, }, - }), + ), ) .withStore<{ test: "a" }>() .build(true) diff --git a/sdk/lib/test/graph.test.ts b/sdk/lib/test/graph.test.ts new file mode 100644 index 000000000..7f02adc2e --- /dev/null +++ b/sdk/lib/test/graph.test.ts @@ -0,0 +1,148 @@ +import { Graph } from "../util/graph" + +describe("graph", () => { + { + { + test("findVertex", () => { + const graph = new Graph() + const foo = graph.addVertex("foo", [], []) + const bar = graph.addVertex( + "bar", + [{ from: foo, metadata: "foo-bar" }], + [], + ) + const baz = graph.addVertex( + "baz", + [{ from: bar, metadata: "bar-baz" }], + [], + ) + const qux = graph.addVertex( + "qux", + [{ from: baz, metadata: "baz-qux" }], + [], + ) + const match = Array.from(graph.findVertex((v) => v.metadata === "qux")) + expect(match).toHaveLength(1) + expect(match[0]).toBe(qux) + }) + test("shortestPathA", () => { + const graph = new Graph() + const foo = graph.addVertex("foo", [], []) + const bar = graph.addVertex( + "bar", + [{ from: foo, metadata: "foo-bar" }], + [], + ) + const baz = graph.addVertex( + "baz", + [{ from: bar, metadata: "bar-baz" }], + [], + ) + const qux = graph.addVertex( + "qux", + [{ from: baz, metadata: "baz-qux" }], + [], + ) + graph.addEdge("foo-qux", foo, qux) + expect(graph.shortestPath(foo, qux) || []).toHaveLength(1) + }) + test("shortestPathB", () => { + const graph = new Graph() + const foo = graph.addVertex("foo", [], []) + const bar = graph.addVertex( + "bar", + [{ from: foo, metadata: "foo-bar" }], + [], + ) + const baz = graph.addVertex( + "baz", + [{ from: bar, metadata: "bar-baz" }], + [], + ) + const qux = graph.addVertex( + "qux", + [{ from: baz, metadata: "baz-qux" }], + [], + ) + graph.addEdge("bar-qux", bar, qux) + expect(graph.shortestPath(foo, qux) || []).toHaveLength(2) + }) + test("shortestPathC", () => { + const graph = new Graph() + const foo = graph.addVertex("foo", [], []) + const bar = graph.addVertex( + "bar", + [{ from: foo, metadata: "foo-bar" }], + [], + ) + const baz = graph.addVertex( + "baz", + [{ from: bar, metadata: "bar-baz" }], + [], + ) + const qux = graph.addVertex( + "qux", + [{ from: baz, metadata: "baz-qux" }], + [{ to: foo, metadata: "qux-foo" }], + ) + expect(graph.shortestPath(foo, qux) || []).toHaveLength(3) + }) + test("bfs", () => { + const graph = new Graph() + const foo = graph.addVertex("foo", [], []) + const bar = graph.addVertex( + "bar", + [{ from: foo, metadata: "foo-bar" }], + [], + ) + const baz = graph.addVertex( + "baz", + [{ from: bar, metadata: "bar-baz" }], + [], + ) + const qux = graph.addVertex( + "qux", + [ + { from: foo, metadata: "foo-qux" }, + { from: baz, metadata: "baz-qux" }, + ], + [], + ) + const bfs = Array.from(graph.breadthFirstSearch(foo)) + expect(bfs).toHaveLength(4) + expect(bfs[0]).toBe(foo) + expect(bfs[1]).toBe(bar) + expect(bfs[2]).toBe(qux) + expect(bfs[3]).toBe(baz) + }) + test("reverseBfs", () => { + const graph = new Graph() + const foo = graph.addVertex("foo", [], []) + const bar = graph.addVertex( + "bar", + [{ from: foo, metadata: "foo-bar" }], + [], + ) + const baz = graph.addVertex( + "baz", + [{ from: bar, metadata: "bar-baz" }], + [], + ) + const qux = graph.addVertex( + "qux", + [ + { from: foo, metadata: "foo-qux" }, + { from: baz, metadata: "baz-qux" }, + ], + [], + ) + const bfs = Array.from(graph.reverseBreadthFirstSearch(qux)) + expect(bfs).toHaveLength(4) + expect(bfs[0]).toBe(qux) + expect(bfs[1]).toBe(foo) + expect(bfs[2]).toBe(baz) + expect(bfs[3]).toBe(bar) + }) + } + } +}) diff --git a/sdk/lib/test/host.test.ts b/sdk/lib/test/host.test.ts index a8ae317ed..64d486a94 100644 --- a/sdk/lib/test/host.test.ts +++ b/sdk/lib/test/host.test.ts @@ -16,7 +16,6 @@ describe("host", () => { id: "foo", description: "A Foo", hasPrimary: false, - disabled: false, type: "ui", username: "bar", path: "/baz", diff --git a/sdk/lib/test/output.sdk.ts b/sdk/lib/test/output.sdk.ts index c56e05e60..4cdf85111 100644 --- a/sdk/lib/test/output.sdk.ts +++ b/sdk/lib/test/output.sdk.ts @@ -1,45 +1,56 @@ import { StartSdk } from "../StartSdk" import { setupManifest } from "../manifest/setupManifest" +import { VersionInfo } from "../version/VersionInfo" +import { VersionGraph } from "../version/VersionGraph" export type Manifest = any export const sdk = StartSdk.of() .withManifest( - setupManifest({ - id: "testOutput", - title: "", - version: "1.0:0", - releaseNotes: "", - license: "", - replaces: [], - wrapperRepo: "", - upstreamRepo: "", - supportSite: "", - marketingSite: "", - donationUrl: null, - description: { - short: "", - long: "", - }, - containers: {}, - images: {}, - volumes: [], - assets: [], - alerts: { - install: null, - update: null, - uninstall: null, - restore: null, - start: null, - stop: null, - }, - dependencies: { - "remote-test": { - description: "", - optional: false, - s9pk: "https://example.com/remote-test.s9pk", + setupManifest( + VersionGraph.of( + VersionInfo.of({ + version: "1.0.0:0", + releaseNotes: "", + migrations: {}, + }) + .satisfies("#other:1.0.0:0") + .satisfies("#other:2.0.0:0"), + ), + { + id: "testOutput", + title: "", + license: "", + replaces: [], + wrapperRepo: "", + upstreamRepo: "", + supportSite: "", + marketingSite: "", + donationUrl: null, + description: { + short: "", + long: "", + }, + containers: {}, + images: {}, + volumes: [], + assets: [], + alerts: { + install: null, + update: null, + uninstall: null, + restore: null, + start: null, + stop: null, + }, + dependencies: { + "remote-test": { + description: "", + optional: false, + s9pk: "https://example.com/remote-test.s9pk", + }, }, }, - }), + ), ) .withStore<{ storeRoot: { storeLeaf: "value" } }>() .build(true) diff --git a/sdk/lib/test/startosTypeValidation.test.ts b/sdk/lib/test/startosTypeValidation.test.ts index 846967c31..bcbc4b6ec 100644 --- a/sdk/lib/test/startosTypeValidation.test.ts +++ b/sdk/lib/test/startosTypeValidation.test.ts @@ -3,10 +3,13 @@ import { CheckDependenciesParam, ExecuteAction, GetConfiguredParams, + GetStoreParams, + SetDataVersionParams, SetMainStatus, + SetStoreParams, } from ".././osBindings" -import { CreateOverlayedImageParams } from ".././osBindings" -import { DestroyOverlayedImageParams } from ".././osBindings" +import { CreateSubcontainerFsParams } from ".././osBindings" +import { DestroySubcontainerFsParams } from ".././osBindings" import { BindParams } from ".././osBindings" import { GetHostInfoParams } from ".././osBindings" import { SetConfigured } from ".././osBindings" @@ -23,21 +26,28 @@ import { GetPrimaryUrlParams } from ".././osBindings" import { ListServiceInterfacesParams } from ".././osBindings" import { ExportActionParams } from ".././osBindings" import { MountParams } from ".././osBindings" +import { StringObject } from "../util" function typeEquality(_a: ExpectedType) {} type WithCallback = Omit & { callback: () => void } +type EffectsTypeChecker = { + [K in keyof T]: T[K] extends (args: infer A) => any + ? A + : T[K] extends StringObject + ? EffectsTypeChecker + : never +} + describe("startosTypeValidation ", () => { test(`checking the params match`, () => { const testInput: any = {} - typeEquality<{ - [K in keyof Effects]: Effects[K] extends (args: infer A) => any - ? A - : never - }>({ + typeEquality({ executeAction: {} as ExecuteAction, - createOverlayedImage: {} as CreateOverlayedImageParams, - destroyOverlayedImage: {} as DestroyOverlayedImageParams, + subcontainer: { + createFs: {} as CreateSubcontainerFsParams, + destroyFs: {} as DestroySubcontainerFsParams, + }, clearBindings: undefined, getInstalledPackages: undefined, bind: {} as BindParams, @@ -46,13 +56,18 @@ describe("startosTypeValidation ", () => { restart: undefined, shutdown: undefined, setConfigured: {} as SetConfigured, + setDataVersion: {} as SetDataVersionParams, + getDataVersion: undefined, setHealth: {} as SetHealth, exposeForDependents: {} as ExposeForDependentsParams, getSslCertificate: {} as WithCallback, getSslKey: {} as GetSslKeyParams, getServiceInterface: {} as WithCallback, setDependencies: {} as SetDependenciesParams, - store: {} as never, + store: { + get: {} as any, // as GetStoreParams, + set: {} as any, // as SetStoreParams, + }, getSystemSmtp: {} as WithCallback, getContainerIp: undefined, getServicePortForward: {} as GetServicePortForwardParams, diff --git a/sdk/lib/trigger/TriggerInput.ts b/sdk/lib/trigger/TriggerInput.ts index 9a52d8ca5..82fe79e07 100644 --- a/sdk/lib/trigger/TriggerInput.ts +++ b/sdk/lib/trigger/TriggerInput.ts @@ -2,5 +2,4 @@ import { HealthStatus } from "../types" export type TriggerInput = { lastResult?: HealthStatus - hadSuccess?: boolean } diff --git a/sdk/lib/trigger/changeOnFirstSuccess.ts b/sdk/lib/trigger/changeOnFirstSuccess.ts index 4c45afe31..3da7284df 100644 --- a/sdk/lib/trigger/changeOnFirstSuccess.ts +++ b/sdk/lib/trigger/changeOnFirstSuccess.ts @@ -5,10 +5,12 @@ export function changeOnFirstSuccess(o: { afterFirstSuccess: Trigger }): Trigger { return async function* (getInput) { - const beforeFirstSuccess = o.beforeFirstSuccess(getInput) - yield let currentValue = getInput() - beforeFirstSuccess.next() + while (!currentValue.lastResult) { + yield + currentValue = getInput() + } + const beforeFirstSuccess = o.beforeFirstSuccess(getInput) for ( let res = await beforeFirstSuccess.next(); currentValue?.lastResult !== "success" && !res.done; diff --git a/sdk/lib/trigger/defaultTrigger.ts b/sdk/lib/trigger/defaultTrigger.ts index bd52dc7cc..69cac2773 100644 --- a/sdk/lib/trigger/defaultTrigger.ts +++ b/sdk/lib/trigger/defaultTrigger.ts @@ -2,7 +2,7 @@ import { cooldownTrigger } from "./cooldownTrigger" import { changeOnFirstSuccess } from "./changeOnFirstSuccess" import { successFailure } from "./successFailure" -export const defaultTrigger = successFailure({ - duringSuccess: cooldownTrigger(0), - duringError: cooldownTrigger(30000), +export const defaultTrigger = changeOnFirstSuccess({ + beforeFirstSuccess: cooldownTrigger(1000), + afterFirstSuccess: cooldownTrigger(30000), }) diff --git a/sdk/lib/trigger/index.ts b/sdk/lib/trigger/index.ts index 6da034262..eb058437f 100644 --- a/sdk/lib/trigger/index.ts +++ b/sdk/lib/trigger/index.ts @@ -1,3 +1,4 @@ +import { ExecSpawnable } from "../util/SubContainer" import { TriggerInput } from "./TriggerInput" export { changeOnFirstSuccess } from "./changeOnFirstSuccess" export { cooldownTrigger } from "./cooldownTrigger" diff --git a/sdk/lib/trigger/lastStatus.ts b/sdk/lib/trigger/lastStatus.ts new file mode 100644 index 000000000..90b8c9851 --- /dev/null +++ b/sdk/lib/trigger/lastStatus.ts @@ -0,0 +1,33 @@ +import { Trigger } from "." +import { HealthStatus } from "../types" + +export type LastStatusTriggerParams = { [k in HealthStatus]?: Trigger } & { + default: Trigger +} + +export function lastStatus(o: LastStatusTriggerParams): Trigger { + return async function* (getInput) { + let trigger = o.default(getInput) + const triggers: { + [k in HealthStatus]?: AsyncIterator + } & { default: AsyncIterator } = { + default: trigger, + } + while (true) { + let currentValue = getInput() + let prev: HealthStatus | "default" | undefined = currentValue.lastResult + if (!prev) { + yield + continue + } + if (!(prev in o)) { + prev = "default" + } + if (!triggers[prev]) { + triggers[prev] = o[prev]!(getInput) + } + await triggers[prev]?.next() + yield + } + } +} diff --git a/sdk/lib/trigger/successFailure.ts b/sdk/lib/trigger/successFailure.ts index 1bab27289..7febcd356 100644 --- a/sdk/lib/trigger/successFailure.ts +++ b/sdk/lib/trigger/successFailure.ts @@ -1,32 +1,7 @@ import { Trigger } from "." +import { lastStatus } from "./lastStatus" -export function successFailure(o: { +export const successFailure = (o: { duringSuccess: Trigger duringError: Trigger -}): Trigger { - return async function* (getInput) { - while (true) { - const beforeSuccess = o.duringSuccess(getInput) - yield - let currentValue = getInput() - beforeSuccess.next() - for ( - let res = await beforeSuccess.next(); - currentValue?.lastResult !== "success" && !res.done; - res = await beforeSuccess.next() - ) { - yield - currentValue = getInput() - } - const duringError = o.duringError(getInput) - for ( - let res = await duringError.next(); - currentValue?.lastResult === "success" && !res.done; - res = await duringError.next() - ) { - yield - currentValue = getInput() - } - } - } -} +}) => lastStatus({ success: o.duringSuccess, default: o.duringError }) diff --git a/sdk/lib/types.ts b/sdk/lib/types.ts index 9a3157ed3..4820f419d 100644 --- a/sdk/lib/types.ts +++ b/sdk/lib/types.ts @@ -3,7 +3,7 @@ export * as configTypes from "./config/configTypes" import { DependencyRequirement, SetHealth, - HealthCheckResult, + NamedHealthCheckResult, SetMainStatus, ServiceInterface, Host, @@ -25,6 +25,7 @@ import { Daemons } from "./mainFn/Daemons" import { StorePath } from "./store/PathBuilder" import { ExposedStorePaths } from "./store/setupExposeStore" import { UrlString } from "./util/getServiceInterface" +import { StringObject, ToKebab } from "./util" export * from "./osBindings" export { SDKManifest } from "./manifest/ManifestTypes" export { HealthReceipt } from "./health/HealthReceipt" @@ -102,10 +103,7 @@ export namespace ExpectedExports { * Every time a package completes an install, this function is called before the main. * Can be used to do migration like things. */ - export type init = (options: { - effects: Effects - previousVersion: null | string - }) => Promise + export type init = (options: { effects: Effects }) => Promise /** This will be ran during any time a package is uninstalled, for example during a update * this will be called. */ @@ -174,7 +172,7 @@ export type Daemon = { [DaemonProof]: never } -export type HealthStatus = HealthCheckResult["result"] +export type HealthStatus = NamedHealthCheckResult["result"] export type SmtpValue = { server: string port: number @@ -249,15 +247,15 @@ export type SdkPropertiesValue = } | { type: "string" - /** Value */ + /** The value to display to the user */ value: string /** A human readable description or explanation of the value */ description?: string - /** (string/number only) Whether or not to mask the value, for example, when displaying a password */ + /** Whether or not to mask the value, for example, when displaying a password */ masked: boolean - /** (string/number only) Whether or not to include a button for copying the value to clipboard */ + /** Whether or not to include a button for copying the value to clipboard */ copyable?: boolean - /** (string/number only) Whether or not to include a button for displaying the value as a QR code */ + /** Whether or not to include a button for displaying the value as a QR code */ qr?: boolean } @@ -273,15 +271,15 @@ export type PropertiesValue = } | { type: "string" - /** Value */ + /** The value to display to the user */ value: string /** A human readable description or explanation of the value */ description: string | null - /** (string/number only) Whether or not to mask the value, for example, when displaying a password */ + /** Whether or not to mask the value, for example, when displaying a password */ masked: boolean - /** (string/number only) Whether or not to include a button for copying the value to clipboard */ + /** Whether or not to include a button for copying the value to clipboard */ copyable: boolean | null - /** (string/number only) Whether or not to include a button for displaying the value as a QR code */ + /** Whether or not to include a button for displaying the value as a QR code */ qr: boolean | null } @@ -289,6 +287,16 @@ export type PropertiesReturn = { [key: string]: PropertiesValue } +export type EffectMethod = { + [K in keyof T]-?: K extends string + ? T[K] extends Function + ? ToKebab + : T[K] extends StringObject + ? `${ToKebab}.${EffectMethod}` + : never + : never +}[keyof T] + /** Used to reach out from the pure js runtime */ export type Effects = { // action @@ -355,12 +363,13 @@ export type Effects = { /** sets the result of a health check */ setHealth(o: SetHealth): Promise - // image - - /** A low level api used by Overlay */ - createOverlayedImage(options: { imageId: string }): Promise<[string, string]> - /** A low level api used by Overlay */ - destroyOverlayedImage(options: { guid: string }): Promise + // subcontainer + subcontainer: { + /** A low level api used by SubContainer */ + createFs(options: { imageId: string }): Promise<[string, string]> + /** A low level api used by SubContainer */ + destroyFs(options: { guid: string }): Promise + } // net @@ -373,7 +382,7 @@ export type Effects = { hostId: HostId internalPort: number }): Promise - /** Removes all network bindings */ + /** Removes all network bindings, called in the setupConfig */ clearBindings(): Promise // host /** Returns information about the specified host, if it exists */ @@ -437,6 +446,10 @@ export type Effects = { value: ExtractStore }): Promise } + /** sets the version that this service's data has been migrated to */ + setDataVersion(options: { version: string }): Promise + /** returns the version that this service's data has been migrated to */ + getDataVersion(): Promise // system @@ -475,12 +488,11 @@ export type MigrationRes = { } export type ActionResult = { + version: "0" message: string - value: null | { - value: string - copyable: boolean - qr: boolean - } + value: string | null + copyable: boolean + qr: boolean } export type SetResult = { dependsOn: DependsOn diff --git a/sdk/lib/util/Overlay.ts b/sdk/lib/util/Overlay.ts deleted file mode 100644 index 93cb44238..000000000 --- a/sdk/lib/util/Overlay.ts +++ /dev/null @@ -1,240 +0,0 @@ -import * as fs from "fs/promises" -import * as T from "../types" -import * as cp from "child_process" -import { promisify } from "util" -import { Buffer } from "node:buffer" -export const execFile = promisify(cp.execFile) -const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/` -export class Overlay { - private constructor( - readonly effects: T.Effects, - readonly imageId: T.ImageId, - readonly rootfs: string, - readonly guid: T.Guid, - ) {} - static async of( - effects: T.Effects, - image: { id: T.ImageId; sharedRun?: boolean }, - ) { - const { id, sharedRun } = image - const [rootfs, guid] = await effects.createOverlayedImage({ - imageId: id as string, - }) - - const shared = ["dev", "sys", "proc"] - if (!!sharedRun) { - shared.push("run") - } - - fs.copyFile("/etc/resolv.conf", `${rootfs}/etc/resolv.conf`) - - for (const dirPart of shared) { - const from = `/${dirPart}` - const to = `${rootfs}/${dirPart}` - await fs.mkdir(from, { recursive: true }) - await fs.mkdir(to, { recursive: true }) - await execFile("mount", ["--rbind", from, to]) - } - - return new Overlay(effects, id, rootfs, guid) - } - - async mount(options: MountOptions, path: string): Promise { - path = path.startsWith("/") - ? `${this.rootfs}${path}` - : `${this.rootfs}/${path}` - if (options.type === "volume") { - const subpath = options.subpath - ? options.subpath.startsWith("/") - ? options.subpath - : `/${options.subpath}` - : "/" - const from = `/media/startos/volumes/${options.id}${subpath}` - - await fs.mkdir(from, { recursive: true }) - await fs.mkdir(path, { recursive: true }) - await await execFile("mount", ["--bind", from, path]) - } else if (options.type === "assets") { - const subpath = options.subpath - ? options.subpath.startsWith("/") - ? options.subpath - : `/${options.subpath}` - : "/" - const from = `/media/startos/assets/${options.id}${subpath}` - - await fs.mkdir(from, { recursive: true }) - await fs.mkdir(path, { recursive: true }) - await execFile("mount", ["--bind", from, path]) - } else if (options.type === "pointer") { - await this.effects.mount({ location: path, target: options }) - } else if (options.type === "backup") { - const subpath = options.subpath - ? options.subpath.startsWith("/") - ? options.subpath - : `/${options.subpath}` - : "/" - const from = `/media/startos/backup${subpath}` - - await fs.mkdir(from, { recursive: true }) - await fs.mkdir(path, { recursive: true }) - await execFile("mount", ["--bind", from, path]) - } else { - throw new Error(`unknown type ${(options as any).type}`) - } - return this - } - - async destroy() { - const imageId = this.imageId - const guid = this.guid - await this.effects.destroyOverlayedImage({ guid }) - } - - async exec( - command: string[], - options?: CommandOptions, - timeoutMs: number | null = 30000, - ): Promise<{ - exitCode: number | null - exitSignal: NodeJS.Signals | null - stdout: string | Buffer - stderr: string | Buffer - }> { - const imageMeta: T.ImageMetadata = await fs - .readFile(`/media/startos/images/${this.imageId}.json`, { - encoding: "utf8", - }) - .catch(() => "{}") - .then(JSON.parse) - let extra: string[] = [] - if (options?.user) { - extra.push(`--user=${options.user}`) - delete options.user - } - let workdir = imageMeta.workdir || "/" - if (options?.cwd) { - workdir = options.cwd - delete options.cwd - } - const child = cp.spawn( - "start-cli", - [ - "chroot", - `--env=/media/startos/images/${this.imageId}.env`, - `--workdir=${workdir}`, - ...extra, - this.rootfs, - ...command, - ], - options || {}, - ) - const pid = child.pid - const stdout = { data: "" as string | Buffer } - const stderr = { data: "" as string | Buffer } - const appendData = - (appendTo: { data: string | Buffer }) => - (chunk: string | Buffer | any) => { - if (typeof appendTo.data === "string" && typeof chunk === "string") { - appendTo.data += chunk - } else if (typeof chunk === "string" || chunk instanceof Buffer) { - appendTo.data = Buffer.concat([ - Buffer.from(appendTo.data), - Buffer.from(chunk), - ]) - } else { - console.error("received unexpected chunk", chunk) - } - } - return new Promise((resolve, reject) => { - child.on("error", reject) - if (timeoutMs !== null && pid) { - setTimeout( - () => execFile("pkill", ["-9", "-s", String(pid)]).catch((_) => {}), - timeoutMs, - ) - } - child.stdout.on("data", appendData(stdout)) - child.stderr.on("data", appendData(stderr)) - child.on("exit", (code, signal) => - resolve({ - exitCode: code, - exitSignal: signal, - stdout: stdout.data, - stderr: stderr.data, - }), - ) - }) - } - - async spawn( - command: string[], - options?: CommandOptions, - ): Promise { - const imageMeta: any = await fs - .readFile(`/media/startos/images/${this.imageId}.json`, { - encoding: "utf8", - }) - .catch(() => "{}") - .then(JSON.parse) - let extra: string[] = [] - if (options?.user) { - extra.push(`--user=${options.user}`) - delete options.user - } - let workdir = imageMeta.workdir || "/" - if (options?.cwd) { - workdir = options.cwd - delete options.cwd - } - return cp.spawn( - "start-cli", - [ - "chroot", - `--env=/media/startos/images/${this.imageId}.env`, - `--workdir=${workdir}`, - ...extra, - this.rootfs, - ...command, - ], - options, - ) - } -} - -export type CommandOptions = { - env?: { [variable: string]: string } - cwd?: string - user?: string -} - -export type MountOptions = - | MountOptionsVolume - | MountOptionsAssets - | MountOptionsPointer - | MountOptionsBackup - -export type MountOptionsVolume = { - type: "volume" - id: string - subpath: string | null - readonly: boolean -} - -export type MountOptionsAssets = { - type: "assets" - id: string - subpath: string | null -} - -export type MountOptionsPointer = { - type: "pointer" - packageId: string - volumeId: string - subpath: string | null - readonly: boolean -} - -export type MountOptionsBackup = { - type: "backup" - subpath: string | null -} diff --git a/sdk/lib/util/SubContainer.ts b/sdk/lib/util/SubContainer.ts new file mode 100644 index 000000000..0d5982886 --- /dev/null +++ b/sdk/lib/util/SubContainer.ts @@ -0,0 +1,434 @@ +import * as fs from "fs/promises" +import * as T from "../types" +import * as cp from "child_process" +import { promisify } from "util" +import { Buffer } from "node:buffer" +import { once } from "./once" +export const execFile = promisify(cp.execFile) +const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/` +const False = () => false +type ExecResults = { + exitCode: number | null + exitSignal: NodeJS.Signals | null + stdout: string | Buffer + stderr: string | Buffer +} + +export type ExecOptions = { + input?: string | Buffer +} + +const TIMES_TO_WAIT_FOR_PROC = 100 + +/** + * This is the type that is going to describe what an subcontainer could do. The main point of the + * subcontainer is to have commands that run in a chrooted environment. This is useful for running + * commands in a containerized environment. But, I wanted the destroy to sometimes be doable, for example the + * case where the subcontainer isn't owned by the process, the subcontainer shouldn't be destroyed. + */ +export interface ExecSpawnable { + get destroy(): undefined | (() => Promise) + exec( + command: string[], + options?: CommandOptions & ExecOptions, + timeoutMs?: number | null, + ): Promise + spawn( + command: string[], + options?: CommandOptions, + ): Promise +} +/** + * Want to limit what we can do in a container, so we want to launch a container with a specific image and the mounts. + * + * Implements: + * @see {@link ExecSpawnable} + */ +export class SubContainer implements ExecSpawnable { + private leader: cp.ChildProcess + private leaderExited: boolean = false + private waitProc: () => Promise + private constructor( + readonly effects: T.Effects, + readonly imageId: T.ImageId, + readonly rootfs: string, + readonly guid: T.Guid, + ) { + this.leaderExited = false + this.leader = cp.spawn("start-cli", ["subcontainer", "launch", rootfs], { + killSignal: "SIGKILL", + stdio: "ignore", + }) + this.leader.on("exit", () => { + this.leaderExited = true + }) + this.waitProc = once( + () => + new Promise(async (resolve, reject) => { + let count = 0 + while ( + !(await fs.stat(`${this.rootfs}/proc/1`).then((x) => !!x, False)) + ) { + if (count++ > TIMES_TO_WAIT_FOR_PROC) { + console.debug("Failed to start subcontainer", { + guid: this.guid, + imageId: this.imageId, + rootfs: this.rootfs, + }) + reject(new Error(`Failed to start subcontainer ${this.imageId}`)) + } + await wait(1) + } + resolve() + }), + ) + } + static async of( + effects: T.Effects, + image: { id: T.ImageId; sharedRun?: boolean }, + ) { + const { id, sharedRun } = image + const [rootfs, guid] = await effects.subcontainer.createFs({ + imageId: id as string, + }) + + const shared = ["dev", "sys"] + if (!!sharedRun) { + shared.push("run") + } + + fs.copyFile("/etc/resolv.conf", `${rootfs}/etc/resolv.conf`) + + for (const dirPart of shared) { + const from = `/${dirPart}` + const to = `${rootfs}/${dirPart}` + await fs.mkdir(from, { recursive: true }) + await fs.mkdir(to, { recursive: true }) + await execFile("mount", ["--rbind", from, to]) + } + + return new SubContainer(effects, id, rootfs, guid) + } + + static async with( + effects: T.Effects, + image: { id: T.ImageId; sharedRun?: boolean }, + mounts: { options: MountOptions; path: string }[], + fn: (subContainer: SubContainer) => Promise, + ): Promise { + const subContainer = await SubContainer.of(effects, image) + try { + for (let mount of mounts) { + await subContainer.mount(mount.options, mount.path) + } + return await fn(subContainer) + } finally { + await subContainer.destroy() + } + } + + async mount(options: MountOptions, path: string): Promise { + path = path.startsWith("/") + ? `${this.rootfs}${path}` + : `${this.rootfs}/${path}` + if (options.type === "volume") { + const subpath = options.subpath + ? options.subpath.startsWith("/") + ? options.subpath + : `/${options.subpath}` + : "/" + const from = `/media/startos/volumes/${options.id}${subpath}` + + await fs.mkdir(from, { recursive: true }) + await fs.mkdir(path, { recursive: true }) + await execFile("mount", ["--bind", from, path]) + } else if (options.type === "assets") { + const subpath = options.subpath + ? options.subpath.startsWith("/") + ? options.subpath + : `/${options.subpath}` + : "/" + const from = `/media/startos/assets/${options.id}${subpath}` + + await fs.mkdir(from, { recursive: true }) + await fs.mkdir(path, { recursive: true }) + await execFile("mount", ["--bind", from, path]) + } else if (options.type === "pointer") { + await this.effects.mount({ location: path, target: options }) + } else if (options.type === "backup") { + const subpath = options.subpath + ? options.subpath.startsWith("/") + ? options.subpath + : `/${options.subpath}` + : "/" + const from = `/media/startos/backup${subpath}` + + await fs.mkdir(from, { recursive: true }) + await fs.mkdir(path, { recursive: true }) + await execFile("mount", ["--bind", from, path]) + } else { + throw new Error(`unknown type ${(options as any).type}`) + } + return this + } + + private async killLeader() { + if (this.leaderExited) { + return + } + return new Promise((resolve, reject) => { + try { + this.leader.on("exit", () => { + resolve() + }) + if (!this.leader.kill("SIGKILL")) { + reject(new Error("kill(2) failed")) + } + } catch (e) { + reject(e) + } + }) + } + + get destroy() { + return async () => { + const guid = this.guid + await this.killLeader() + await this.effects.subcontainer.destroyFs({ guid }) + } + } + + async exec( + command: string[], + options?: CommandOptions & ExecOptions, + timeoutMs: number | null = 30000, + ): Promise<{ + exitCode: number | null + exitSignal: NodeJS.Signals | null + stdout: string | Buffer + stderr: string | Buffer + }> { + await this.waitProc() + const imageMeta: T.ImageMetadata = await fs + .readFile(`/media/startos/images/${this.imageId}.json`, { + encoding: "utf8", + }) + .catch(() => "{}") + .then(JSON.parse) + let extra: string[] = [] + if (options?.user) { + extra.push(`--user=${options.user}`) + delete options.user + } + let workdir = imageMeta.workdir || "/" + if (options?.cwd) { + workdir = options.cwd + delete options.cwd + } + const child = cp.spawn( + "start-cli", + [ + "subcontainer", + "exec", + `--env=/media/startos/images/${this.imageId}.env`, + `--workdir=${workdir}`, + ...extra, + this.rootfs, + ...command, + ], + options || {}, + ) + if (options?.input) { + await new Promise((resolve, reject) => + child.stdin.write(options.input, (e) => { + if (e) { + reject(e) + } else { + resolve() + } + }), + ) + await new Promise((resolve) => child.stdin.end(resolve)) + } + const pid = child.pid + const stdout = { data: "" as string | Buffer } + const stderr = { data: "" as string | Buffer } + const appendData = + (appendTo: { data: string | Buffer }) => + (chunk: string | Buffer | any) => { + if (typeof appendTo.data === "string" && typeof chunk === "string") { + appendTo.data += chunk + } else if (typeof chunk === "string" || chunk instanceof Buffer) { + appendTo.data = Buffer.concat([ + Buffer.from(appendTo.data), + Buffer.from(chunk), + ]) + } else { + console.error("received unexpected chunk", chunk) + } + } + return new Promise((resolve, reject) => { + child.on("error", reject) + let killTimeout: NodeJS.Timeout | undefined + if (timeoutMs !== null && child.pid) { + killTimeout = setTimeout(() => child.kill("SIGKILL"), timeoutMs) + } + child.stdout.on("data", appendData(stdout)) + child.stderr.on("data", appendData(stderr)) + child.on("exit", (code, signal) => { + clearTimeout(killTimeout) + resolve({ + exitCode: code, + exitSignal: signal, + stdout: stdout.data, + stderr: stderr.data, + }) + }) + }) + } + + async launch( + command: string[], + options?: CommandOptions, + ): Promise { + await this.waitProc() + const imageMeta: any = await fs + .readFile(`/media/startos/images/${this.imageId}.json`, { + encoding: "utf8", + }) + .catch(() => "{}") + .then(JSON.parse) + let extra: string[] = [] + if (options?.user) { + extra.push(`--user=${options.user}`) + delete options.user + } + let workdir = imageMeta.workdir || "/" + if (options?.cwd) { + workdir = options.cwd + delete options.cwd + } + await this.killLeader() + this.leaderExited = false + this.leader = cp.spawn( + "start-cli", + [ + "subcontainer", + "launch", + `--env=/media/startos/images/${this.imageId}.env`, + `--workdir=${workdir}`, + ...extra, + this.rootfs, + ...command, + ], + { ...options, stdio: "inherit" }, + ) + this.leader.on("exit", () => { + this.leaderExited = true + }) + return this.leader as cp.ChildProcessWithoutNullStreams + } + + async spawn( + command: string[], + options?: CommandOptions, + ): Promise { + await this.waitProc() + const imageMeta: any = await fs + .readFile(`/media/startos/images/${this.imageId}.json`, { + encoding: "utf8", + }) + .catch(() => "{}") + .then(JSON.parse) + let extra: string[] = [] + if (options?.user) { + extra.push(`--user=${options.user}`) + delete options.user + } + let workdir = imageMeta.workdir || "/" + if (options?.cwd) { + workdir = options.cwd + delete options.cwd + } + return cp.spawn( + "start-cli", + [ + "subcontainer", + "exec", + `--env=/media/startos/images/${this.imageId}.env`, + `--workdir=${workdir}`, + ...extra, + this.rootfs, + ...command, + ], + options, + ) + } +} + +/** + * Take an subcontainer but remove the ability to add the mounts and the destroy function. + * Lets other functions, like health checks, to not destroy the parents. + * + */ +export class SubContainerHandle implements ExecSpawnable { + constructor(private subContainer: ExecSpawnable) {} + get destroy() { + return undefined + } + + exec( + command: string[], + options?: CommandOptions, + timeoutMs?: number | null, + ): Promise { + return this.subContainer.exec(command, options, timeoutMs) + } + spawn( + command: string[], + options?: CommandOptions, + ): Promise { + return this.subContainer.spawn(command, options) + } +} + +export type CommandOptions = { + env?: { [variable: string]: string } + cwd?: string + user?: string +} + +export type MountOptions = + | MountOptionsVolume + | MountOptionsAssets + | MountOptionsPointer + | MountOptionsBackup + +export type MountOptionsVolume = { + type: "volume" + id: string + subpath: string | null + readonly: boolean +} + +export type MountOptionsAssets = { + type: "assets" + id: string + subpath: string | null +} + +export type MountOptionsPointer = { + type: "pointer" + packageId: string + volumeId: string + subpath: string | null + readonly: boolean +} + +export type MountOptionsBackup = { + type: "backup" + subpath: string | null +} +function wait(time: number) { + return new Promise((resolve) => setTimeout(resolve, time)) +} diff --git a/sdk/lib/util/asError.ts b/sdk/lib/util/asError.ts new file mode 100644 index 000000000..6e98afb6a --- /dev/null +++ b/sdk/lib/util/asError.ts @@ -0,0 +1,6 @@ +export const asError = (e: unknown) => { + if (e instanceof Error) { + return new Error(e as any) + } + return new Error(`${e}`) +} diff --git a/sdk/lib/util/getServiceInterface.ts b/sdk/lib/util/getServiceInterface.ts index 9148c8f9a..fd0fef779 100644 --- a/sdk/lib/util/getServiceInterface.ts +++ b/sdk/lib/util/getServiceInterface.ts @@ -50,8 +50,6 @@ export type ServiceInterfaceFilled = { description: string /** Whether or not the interface has a primary URL */ hasPrimary: boolean - /** Whether or not the interface disabled */ - disabled: boolean /** Whether or not to mask the URIs for this interface. Useful if the URIs contain sensitive information, such as a password, macaroon, or API key */ masked: boolean /** Information about the host for this binding */ diff --git a/sdk/lib/util/graph.ts b/sdk/lib/util/graph.ts new file mode 100644 index 000000000..5ad71a04d --- /dev/null +++ b/sdk/lib/util/graph.ts @@ -0,0 +1,244 @@ +import { boolean } from "ts-matches" + +export type Vertex = { + metadata: VMetadata + edges: Array> +} + +export type Edge = { + metadata: EMetadata + from: Vertex + to: Vertex +} + +export class Graph { + private readonly vertices: Array> = [] + constructor() {} + addVertex( + metadata: VMetadata, + fromEdges: Array, "to">>, + toEdges: Array, "from">>, + ): Vertex { + const vertex: Vertex = { + metadata, + edges: [], + } + for (let edge of fromEdges) { + const vEdge = { + metadata: edge.metadata, + from: edge.from, + to: vertex, + } + edge.from.edges.push(vEdge) + vertex.edges.push(vEdge) + } + for (let edge of toEdges) { + const vEdge = { + metadata: edge.metadata, + from: vertex, + to: edge.to, + } + edge.to.edges.push(vEdge) + vertex.edges.push(vEdge) + } + this.vertices.push(vertex) + return vertex + } + findVertex( + predicate: (vertex: Vertex) => boolean, + ): Generator, void> { + const veritces = this.vertices + function* gen() { + for (let vertex of veritces) { + if (predicate(vertex)) { + yield vertex + } + } + } + return gen() + } + addEdge( + metadata: EMetadata, + from: Vertex, + to: Vertex, + ): Edge { + const edge = { + metadata, + from, + to, + } + edge.from.edges.push(edge) + edge.to.edges.push(edge) + return edge + } + breadthFirstSearch( + from: + | Vertex + | ((vertex: Vertex) => boolean), + ): Generator, void> { + const visited: Array> = [] + function* rec( + vertex: Vertex, + ): Generator, void> { + if (visited.includes(vertex)) { + return + } + visited.push(vertex) + yield vertex + let generators = vertex.edges + .filter((e) => e.from === vertex) + .map((e) => rec(e.to)) + while (generators.length) { + let prev = generators + generators = [] + for (let gen of prev) { + const next = gen.next() + if (!next.done) { + generators.push(gen) + yield next.value + } + } + } + } + + if (from instanceof Function) { + let generators = this.vertices.filter(from).map(rec) + return (function* () { + while (generators.length) { + let prev = generators + generators = [] + for (let gen of prev) { + const next = gen.next() + if (!next.done) { + generators.push(gen) + yield next.value + } + } + } + })() + } else { + return rec(from) + } + } + reverseBreadthFirstSearch( + to: + | Vertex + | ((vertex: Vertex) => boolean), + ): Generator, void> { + const visited: Array> = [] + function* rec( + vertex: Vertex, + ): Generator, void> { + if (visited.includes(vertex)) { + return + } + visited.push(vertex) + yield vertex + let generators = vertex.edges + .filter((e) => e.to === vertex) + .map((e) => rec(e.from)) + while (generators.length) { + let prev = generators + generators = [] + for (let gen of prev) { + const next = gen.next() + if (!next.done) { + generators.push(gen) + yield next.value + } + } + } + } + + if (to instanceof Function) { + let generators = this.vertices.filter(to).map(rec) + return (function* () { + while (generators.length) { + let prev = generators + generators = [] + for (let gen of prev) { + const next = gen.next() + if (!next.done) { + generators.push(gen) + yield next.value + } + } + } + })() + } else { + return rec(to) + } + } + shortestPath( + from: + | Vertex + | ((vertex: Vertex) => boolean), + to: + | Vertex + | ((vertex: Vertex) => boolean), + ): Array> | void { + const isDone = + to instanceof Function + ? to + : (v: Vertex) => v === to + const path: Array> = [] + const visited: Array> = [] + function* check( + vertex: Vertex, + path: Array>, + ): Generator> | undefined> { + if (isDone(vertex)) { + return path + } + if (visited.includes(vertex)) { + return + } + visited.push(vertex) + yield + let generators = vertex.edges + .filter((e) => e.from === vertex) + .map((e) => check(e.to, [...path, e])) + while (generators.length) { + let prev = generators + generators = [] + for (let gen of prev) { + const next = gen.next() + if (next.done === true) { + if (next.value) { + return next.value + } + } else { + generators.push(gen) + yield + } + } + } + } + + if (from instanceof Function) { + let generators = this.vertices.filter(from).map((v) => check(v, [])) + while (generators.length) { + let prev = generators + generators = [] + for (let gen of prev) { + const next = gen.next() + if (next.done === true) { + if (next.value) { + return next.value + } + } else { + generators.push(gen) + } + } + } + } else { + const gen = check(from, []) + while (true) { + const next = gen.next() + if (next.done) { + return next.value + } + } + } + } +} diff --git a/sdk/lib/util/index.ts b/sdk/lib/util/index.ts index d4427adb0..9246cf791 100644 --- a/sdk/lib/util/index.ts +++ b/sdk/lib/util/index.ts @@ -3,10 +3,11 @@ import "./fileHelper" import "../store/getStore" import "./deepEqual" import "./deepMerge" -import "./Overlay" +import "./SubContainer" import "./once" export { GetServiceInterface, getServiceInterface } from "./getServiceInterface" +export { asError } from "./asError" export { getServiceInterfaces } from "./getServiceInterfaces" export { addressHostToUrl } from "./getServiceInterface" export { hostnameInfoToAddress } from "./Hostname" diff --git a/sdk/lib/util/typeHelpers.ts b/sdk/lib/util/typeHelpers.ts index f45a46f1e..d29d5c986 100644 --- a/sdk/lib/util/typeHelpers.ts +++ b/sdk/lib/util/typeHelpers.ts @@ -21,3 +21,96 @@ export type NoAny = NeverPossible extends A ? never : A : A + +type CapitalLetters = + | "A" + | "B" + | "C" + | "D" + | "E" + | "F" + | "G" + | "H" + | "I" + | "J" + | "K" + | "L" + | "M" + | "N" + | "O" + | "P" + | "Q" + | "R" + | "S" + | "T" + | "U" + | "V" + | "W" + | "X" + | "Y" + | "Z" + +type Numbers = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" + +type CapitalChars = CapitalLetters | Numbers + +export type ToKebab = S extends string + ? S extends `${infer Head}${CapitalChars}${infer Tail}` // string has a capital char somewhere + ? Head extends "" // there is a capital char in the first position + ? Tail extends "" + ? Lowercase /* 'A' */ + : S extends `${infer Caps}${Tail}` // tail exists, has capital characters + ? Caps extends CapitalChars + ? Tail extends CapitalLetters + ? `${Lowercase}-${Lowercase}` /* 'AB' */ + : Tail extends `${CapitalLetters}${string}` + ? `${ToKebab}-${ToKebab}` /* first tail char is upper? 'ABcd' */ + : `${ToKebab}${ToKebab}` /* 'AbCD','AbcD', */ /* TODO: if tail is only numbers, append without underscore */ + : never /* never reached, used for inference of caps */ + : never + : Tail extends "" /* 'aB' 'abCD' 'ABCD' 'AB' */ + ? S extends `${Head}${infer Caps}` + ? Caps extends CapitalChars + ? Head extends Lowercase /* 'abcD' */ + ? Caps extends Numbers + ? // Head exists and is lowercase, tail does not, Caps is a number, we may be in a sub-select + // if head ends with number, don't split head an Caps, keep contiguous numbers together + Head extends `${string}${Numbers}` + ? never + : // head does not end in number, safe to split. 'abc2' -> 'abc-2' + `${ToKebab}-${Caps}` + : `${ToKebab}-${ToKebab}` /* 'abcD' 'abc25' */ + : never /* stop union type forming */ + : never + : never /* never reached, used for inference of caps */ + : S extends `${Head}${infer Caps}${Tail}` /* 'abCd' 'ABCD' 'AbCd' 'ABcD' */ + ? Caps extends CapitalChars + ? Head extends Lowercase /* is 'abCd' 'abCD' ? */ + ? Tail extends CapitalLetters /* is 'abCD' where Caps = 'C' */ + ? `${ToKebab}-${ToKebab}-${Lowercase}` /* aBCD Tail = 'D', Head = 'aB' */ + : Tail extends `${CapitalLetters}${string}` /* is 'aBCd' where Caps = 'B' */ + ? Head extends Numbers + ? never /* stop union type forming */ + : Head extends `${string}${Numbers}` + ? never /* stop union type forming */ + : `${Head}-${ToKebab}-${ToKebab}` /* 'aBCd' => `${'a'}-${Lowercase<'B'>}-${ToSnake<'Cd'>}` */ + : `${ToKebab}-${Lowercase}${ToKebab}` /* 'aBcD' where Caps = 'B' tail starts as lowercase */ + : never + : never + : never + : S /* 'abc' */ + : never + +export type StringObject = Record + +function test() { + // prettier-ignore + const t = (a: ( + A extends B ? ( + B extends A ? null : never + ) : never + )) =>{ } + t<"foo-bar", ToKebab<"FooBar">>(null) + // @ts-expect-error + t<"foo-3ar", ToKebab<"FooBar">>(null) +} diff --git a/sdk/lib/version/VersionGraph.ts b/sdk/lib/version/VersionGraph.ts new file mode 100644 index 000000000..1b6b49bb9 --- /dev/null +++ b/sdk/lib/version/VersionGraph.ts @@ -0,0 +1,200 @@ +import { ExtendedVersion, VersionRange } from "../exver" + +import * as T from "../types" +import { Graph, Vertex } from "../util/graph" +import { once } from "../util/once" +import { IMPOSSIBLE, VersionInfo } from "./VersionInfo" + +export class VersionGraph { + private readonly graph: () => Graph< + ExtendedVersion | VersionRange, + ((opts: { effects: T.Effects }) => Promise) | undefined + > + private constructor( + readonly current: VersionInfo, + versions: Array>, + ) { + this.graph = once(() => { + const graph = new Graph< + ExtendedVersion | VersionRange, + ((opts: { effects: T.Effects }) => Promise) | undefined + >() + const flavorMap: Record< + string, + [ + ExtendedVersion, + VersionInfo, + Vertex< + ExtendedVersion | VersionRange, + ((opts: { effects: T.Effects }) => Promise) | undefined + >, + ][] + > = {} + for (let version of [current, ...versions]) { + const v = ExtendedVersion.parse(version.options.version) + const vertex = graph.addVertex(v, [], []) + const flavor = v.flavor || "" + if (!flavorMap[flavor]) { + flavorMap[flavor] = [] + } + flavorMap[flavor].push([v, version, vertex]) + } + for (let flavor in flavorMap) { + flavorMap[flavor].sort((a, b) => a[0].compareForSort(b[0])) + let prev: + | [ + ExtendedVersion, + VersionInfo, + Vertex< + ExtendedVersion | VersionRange, + (opts: { effects: T.Effects }) => Promise + >, + ] + | undefined = undefined + for (let [v, version, vertex] of flavorMap[flavor]) { + if (version.options.migrations.up !== IMPOSSIBLE) { + let range + if (prev) { + graph.addEdge(version.options.migrations.up, prev[2], vertex) + range = VersionRange.anchor(">=", prev[0]).and( + VersionRange.anchor("<", v), + ) + } else { + range = VersionRange.anchor("<", v) + } + const vRange = graph.addVertex(range, [], []) + graph.addEdge(version.options.migrations.up, vRange, vertex) + } + + if (version.options.migrations.down !== IMPOSSIBLE) { + let range + if (prev) { + graph.addEdge(version.options.migrations.down, vertex, prev[2]) + range = VersionRange.anchor(">=", prev[0]).and( + VersionRange.anchor("<", v), + ) + } else { + range = VersionRange.anchor("<", v) + } + const vRange = graph.addVertex(range, [], []) + graph.addEdge(version.options.migrations.down, vertex, vRange) + } + + if (version.options.migrations.other) { + for (let rangeStr in version.options.migrations.other) { + const range = VersionRange.parse(rangeStr) + const vRange = graph.addVertex(range, [], []) + graph.addEdge( + version.options.migrations.other[rangeStr], + vRange, + vertex, + ) + for (let matching of graph.findVertex( + (v) => + v.metadata instanceof ExtendedVersion && + v.metadata.satisfies(range), + )) { + graph.addEdge( + version.options.migrations.other[rangeStr], + matching, + vertex, + ) + } + } + } + } + } + return graph + }) + } + currentVersion = once(() => + ExtendedVersion.parse(this.current.options.version), + ) + static of< + CurrentVersion extends string, + OtherVersions extends Array>, + >( + currentVersion: VersionInfo, + ...other: EnsureUniqueId + ) { + return new VersionGraph(currentVersion, other as Array>) + } + async migrate({ + effects, + from, + to, + }: { + effects: T.Effects + from: ExtendedVersion + to: ExtendedVersion + }) { + const graph = this.graph() + if (from && to) { + const path = graph.shortestPath( + (v) => + (v.metadata instanceof VersionRange && + v.metadata.satisfiedBy(from)) || + (v.metadata instanceof ExtendedVersion && v.metadata.equals(from)), + (v) => + (v.metadata instanceof VersionRange && v.metadata.satisfiedBy(to)) || + (v.metadata instanceof ExtendedVersion && v.metadata.equals(to)), + ) + if (path) { + for (let edge of path) { + if (edge.metadata) { + await edge.metadata({ effects }) + } + await effects.setDataVersion({ version: edge.to.metadata.toString() }) + } + return + } + } + throw new Error() + } + canMigrateFrom = once(() => + Array.from( + this.graph().reverseBreadthFirstSearch( + (v) => + (v.metadata instanceof VersionRange && + v.metadata.satisfiedBy(this.currentVersion())) || + (v.metadata instanceof ExtendedVersion && + v.metadata.equals(this.currentVersion())), + ), + ).reduce( + (acc, x) => + acc.or( + x.metadata instanceof VersionRange + ? x.metadata + : VersionRange.anchor("=", x.metadata), + ), + VersionRange.none(), + ), + ) + canMigrateTo = once(() => + Array.from( + this.graph().breadthFirstSearch( + (v) => + (v.metadata instanceof VersionRange && + v.metadata.satisfiedBy(this.currentVersion())) || + (v.metadata instanceof ExtendedVersion && + v.metadata.equals(this.currentVersion())), + ), + ).reduce( + (acc, x) => + acc.or( + x.metadata instanceof VersionRange + ? x.metadata + : VersionRange.anchor("=", x.metadata), + ), + VersionRange.none(), + ), + ) +} + +// prettier-ignore +export type EnsureUniqueId = + B extends [] ? A : + B extends [VersionInfo, ...infer Rest] ? ( + Version extends OtherVersions ? "One or more versions are not unique"[] : + EnsureUniqueId + ) : "There exists a migration that is not a Migration"[] diff --git a/sdk/lib/version/VersionInfo.ts b/sdk/lib/version/VersionInfo.ts new file mode 100644 index 000000000..beea16019 --- /dev/null +++ b/sdk/lib/version/VersionInfo.ts @@ -0,0 +1,78 @@ +import { ValidateExVer } from "../exver" +import * as T from "../types" + +export const IMPOSSIBLE = Symbol("IMPOSSIBLE") + +export type VersionOptions = { + /** The version being described */ + version: Version & ValidateExVer + /** The release notes for this version */ + releaseNotes: string + /** Data migrations for this version */ + migrations: { + /** + * A migration from the previous version + * Leave blank to indicate no migration is necessary + * Set to `IMPOSSIBLE` to indicate migrating from the previous version is not possible + */ + up?: ((opts: { effects: T.Effects }) => Promise) | typeof IMPOSSIBLE + /** + * A migration to the previous version + * Leave blank to indicate no migration is necessary + * Set to `IMPOSSIBLE` to indicate downgrades are prohibited + */ + down?: ((opts: { effects: T.Effects }) => Promise) | typeof IMPOSSIBLE + /** + * Additional migrations, such as fast-forward migrations, or migrations from other flavors + */ + other?: Record Promise> + } +} + +export class VersionInfo { + private _version: null | Version = null + private constructor( + readonly options: VersionOptions & { satisfies: string[] }, + ) {} + static of(options: VersionOptions) { + return new VersionInfo({ ...options, satisfies: [] }) + } + /** Specify a version that this version is 100% backwards compatible to */ + satisfies( + version: V & ValidateExVer, + ): VersionInfo { + return new VersionInfo({ + ...this.options, + satisfies: [...this.options.satisfies, version], + }) + } +} + +function __type_tests() { + const version: VersionInfo<"1.0.0:0"> = VersionInfo.of({ + version: "1.0.0:0", + releaseNotes: "", + migrations: {}, + }) + .satisfies("#other:1.0.0:0") + .satisfies("#other:2.0.0:0") + // @ts-expect-error + .satisfies("#other:2.f.0:0") + + let a: VersionInfo<"1.0.0:0"> = version + // @ts-expect-error + let b: VersionInfo<"1.0.0:3"> = version + + VersionInfo.of({ + // @ts-expect-error + version: "test", + releaseNotes: "", + migrations: {}, + }) + VersionInfo.of({ + // @ts-expect-error + version: "test" as string, + releaseNotes: "", + migrations: {}, + }) +} diff --git a/sdk/lib/version/index.ts b/sdk/lib/version/index.ts new file mode 100644 index 000000000..c7a47fc38 --- /dev/null +++ b/sdk/lib/version/index.ts @@ -0,0 +1,2 @@ +export * from "./VersionGraph" +export * from "./VersionInfo" diff --git a/sdk/package-lock.json b/sdk/package-lock.json index 9d3ea5a38..fe4fee235 100644 --- a/sdk/package-lock.json +++ b/sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "@start9labs/start-sdk", - "version": "0.3.6-alpha6", + "version": "0.3.6-alpha8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@start9labs/start-sdk", - "version": "0.3.6-alpha6", + "version": "0.3.6-alpha8", "license": "MIT", "dependencies": { "@iarna/toml": "^2.2.5", diff --git a/sdk/package.json b/sdk/package.json index ffe62a063..038db0244 100644 --- a/sdk/package.json +++ b/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@start9labs/start-sdk", - "version": "0.3.6-alpha6", + "version": "0.3.6-alpha8", "description": "Software development kit to facilitate packaging services for StartOS", "main": "./cjs/lib/index.js", "types": "./cjs/lib/index.d.ts", diff --git a/web/package-lock.json b/web/package-lock.json index 8cacf6e07..f86534510 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,12 +1,12 @@ { "name": "startos-ui", - "version": "0.3.6-alpha.3", + "version": "0.3.6-alpha.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "startos-ui", - "version": "0.3.6-alpha.3", + "version": "0.3.6-alpha.5", "license": "MIT", "dependencies": { "@angular/animations": "^17.3.1", @@ -4922,7 +4922,6 @@ }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", - "dev": true, "license": "MIT", "optional": true, "engines": { @@ -7547,7 +7546,6 @@ }, "node_modules/encoding": { "version": "0.1.13", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -7556,7 +7554,6 @@ }, "node_modules/encoding/node_modules/iconv-lite": { "version": "0.6.3", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -8155,7 +8152,6 @@ }, "node_modules/fsevents": { "version": "2.3.3", - "dev": true, "license": "MIT", "optional": true, "os": [ diff --git a/web/package.json b/web/package.json index 74a8af251..2f91927e6 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "startos-ui", - "version": "0.3.6-alpha.3", + "version": "0.3.6-alpha.5", "author": "Start9 Labs, Inc", "homepage": "https://start9.com/", "license": "MIT", diff --git a/web/patchdb-ui-seed.json b/web/patchdb-ui-seed.json index 0e15cd2cf..221b80a3b 100644 --- a/web/patchdb-ui-seed.json +++ b/web/patchdb-ui-seed.json @@ -21,5 +21,5 @@ "ackInstructions": {}, "theme": "Dark", "widgets": [], - "ack-welcome": "0.3.6-alpha.3" + "ack-welcome": "0.3.6-alpha.5" } diff --git a/web/projects/ui/src/app/routes/portal/routes/service/components/health-check.component.ts b/web/projects/ui/src/app/routes/portal/routes/service/components/health-check.component.ts index 90f60a9c6..5293b4670 100644 --- a/web/projects/ui/src/app/routes/portal/routes/service/components/health-check.component.ts +++ b/web/projects/ui/src/app/routes/portal/routes/service/components/health-check.component.ts @@ -42,7 +42,7 @@ import { TuiIcon, TuiLoader } from '@taiga-ui/core' }) export class ServiceHealthCheckComponent { @Input({ required: true }) - check!: T.HealthCheckResult + check!: T.NamedHealthCheckResult @Input() connected = false diff --git a/web/projects/ui/src/app/routes/portal/routes/service/components/health-checks.component.ts b/web/projects/ui/src/app/routes/portal/routes/service/components/health-checks.component.ts index e559e8a28..c306db896 100644 --- a/web/projects/ui/src/app/routes/portal/routes/service/components/health-checks.component.ts +++ b/web/projects/ui/src/app/routes/portal/routes/service/components/health-checks.component.ts @@ -31,7 +31,7 @@ import { ConnectionService } from 'src/app/services/connection.service' }) export class ServiceHealthChecksComponent { @Input({ required: true }) - checks: readonly T.HealthCheckResult[] = [] + checks: readonly T.NamedHealthCheckResult[] = [] readonly connected$ = inject(ConnectionService) } diff --git a/web/projects/ui/src/app/routes/portal/routes/service/routes/service.component.ts b/web/projects/ui/src/app/routes/portal/routes/service/routes/service.component.ts index ddacc3cab..c87fed068 100644 --- a/web/projects/ui/src/app/routes/portal/routes/service/routes/service.component.ts +++ b/web/projects/ui/src/app/routes/portal/routes/service/routes/service.component.ts @@ -329,7 +329,7 @@ export class ServiceRoute { } } -function toHealthCheck(main: T.MainStatus): T.HealthCheckResult[] | null { +function toHealthCheck(main: T.MainStatus): T.NamedHealthCheckResult[] | null { return main.status !== 'running' || isEmptyObject(main.health) ? null : Object.values(main.health) diff --git a/web/projects/ui/src/app/routes/portal/routes/system/settings/routes/interfaces/ui.component.ts b/web/projects/ui/src/app/routes/portal/routes/system/settings/routes/interfaces/ui.component.ts index 9368421ad..188c1118c 100644 --- a/web/projects/ui/src/app/routes/portal/routes/system/settings/routes/interfaces/ui.component.ts +++ b/web/projects/ui/src/app/routes/portal/routes/system/settings/routes/interfaces/ui.component.ts @@ -35,7 +35,6 @@ export class StartOsUiComponent { description: 'The primary web user interface for StartOS', type: 'ui', hasPrimary: false, - disabled: false, masked: false, addressInfo: { hostId: '', diff --git a/web/projects/ui/src/app/services/api/api.fixures.ts b/web/projects/ui/src/app/services/api/api.fixures.ts index 14f890c98..ad88f8749 100644 --- a/web/projects/ui/src/app/services/api/api.fixures.ts +++ b/web/projects/ui/src/app/services/api/api.fixures.ts @@ -89,6 +89,8 @@ export module Mock { title: 'Bitcoin Core', version: '0.21.0:0', satisfies: [], + canMigrateTo: '!', + canMigrateFrom: '*', gitHash: 'abcdefgh', description: { short: 'A Bitcoin full node by Bitcoin Core.', @@ -133,6 +135,8 @@ export module Mock { title: 'Lightning Network Daemon', version: '0.11.1:0', satisfies: [], + canMigrateTo: '!', + canMigrateFrom: '*', gitHash: 'abcdefgh', description: { short: 'A bolt spec compliant client.', @@ -189,6 +193,8 @@ export module Mock { title: 'Bitcoin Proxy', version: '0.2.2:0', satisfies: [], + canMigrateTo: '!', + canMigrateFrom: '*', gitHash: 'lmnopqrx', description: { short: 'A super charger for your Bitcoin node.', @@ -1693,6 +1699,7 @@ export module Mock { state: 'installed', manifest: MockManifestBitcoind, }, + dataVersion: MockManifestBitcoind.version, icon: '/assets/img/service-icons/bitcoind.svg', installedAt: new Date().toISOString(), lastBackup: null, @@ -1710,7 +1717,6 @@ export module Mock { ui: { id: 'ui', hasPrimary: false, - disabled: false, masked: false, name: 'Web UI', description: @@ -1728,7 +1734,6 @@ export module Mock { rpc: { id: 'rpc', hasPrimary: false, - disabled: false, masked: false, name: 'RPC', description: @@ -1746,7 +1751,6 @@ export module Mock { p2p: { id: 'p2p', hasPrimary: true, - disabled: false, masked: false, name: 'P2P', description: @@ -1875,6 +1879,7 @@ export module Mock { state: 'installed', manifest: MockManifestBitcoinProxy, }, + dataVersion: MockManifestBitcoinProxy.version, icon: '/assets/img/service-icons/btc-rpc-proxy.png', installedAt: new Date().toISOString(), lastBackup: null, @@ -1890,7 +1895,6 @@ export module Mock { ui: { id: 'ui', hasPrimary: false, - disabled: false, masked: false, name: 'Web UI', description: 'A launchable web app for Bitcoin Proxy', @@ -1927,6 +1931,7 @@ export module Mock { state: 'installed', manifest: MockManifestLnd, }, + dataVersion: MockManifestLnd.version, icon: '/assets/img/service-icons/lnd.png', installedAt: new Date().toISOString(), lastBackup: null, @@ -1942,7 +1947,6 @@ export module Mock { grpc: { id: 'grpc', hasPrimary: false, - disabled: false, masked: false, name: 'GRPC', description: @@ -1960,7 +1964,6 @@ export module Mock { lndconnect: { id: 'lndconnect', hasPrimary: false, - disabled: false, masked: true, name: 'LND Connect', description: @@ -1978,7 +1981,6 @@ export module Mock { p2p: { id: 'p2p', hasPrimary: true, - disabled: false, masked: false, name: 'P2P', description: diff --git a/web/projects/ui/src/app/services/api/api.types.ts b/web/projects/ui/src/app/services/api/api.types.ts index e9ed0f781..6deed4cd1 100644 --- a/web/projects/ui/src/app/services/api/api.types.ts +++ b/web/projects/ui/src/app/services/api/api.types.ts @@ -691,7 +691,7 @@ export type DependencyErrorConfigUnsatisfied = { export type DependencyErrorHealthChecksFailed = { type: 'healthChecksFailed' - check: T.HealthCheckResult + check: T.NamedHealthCheckResult } export type DependencyErrorTransitive = { diff --git a/web/projects/ui/src/app/services/api/mock-patch.ts b/web/projects/ui/src/app/services/api/mock-patch.ts index bcda83c6a..afdca04b9 100644 --- a/web/projects/ui/src/app/services/api/mock-patch.ts +++ b/web/projects/ui/src/app/services/api/mock-patch.ts @@ -148,6 +148,7 @@ export const mockPatchData: DataModel = { version: '0.20.0:0', }, }, + dataVersion: '0.20.0:0', icon: '/assets/img/service-icons/bitcoind.svg', installedAt: new Date().toISOString(), lastBackup: new Date(new Date().valueOf() - 604800001).toISOString(), @@ -191,7 +192,6 @@ export const mockPatchData: DataModel = { ui: { id: 'ui', hasPrimary: false, - disabled: false, masked: false, name: 'Web UI', description: @@ -209,7 +209,6 @@ export const mockPatchData: DataModel = { rpc: { id: 'rpc', hasPrimary: false, - disabled: false, masked: false, name: 'RPC', description: @@ -227,7 +226,6 @@ export const mockPatchData: DataModel = { p2p: { id: 'p2p', hasPrimary: true, - disabled: false, masked: false, name: 'P2P', description: @@ -358,6 +356,7 @@ export const mockPatchData: DataModel = { version: '0.11.0:0.0.1', }, }, + dataVersion: '0.11.0:0.0.1', icon: '/assets/img/service-icons/lnd.png', installedAt: new Date().toISOString(), lastBackup: null, @@ -373,7 +372,6 @@ export const mockPatchData: DataModel = { grpc: { id: 'grpc', hasPrimary: false, - disabled: false, masked: false, name: 'GRPC', description: @@ -391,7 +389,6 @@ export const mockPatchData: DataModel = { lndconnect: { id: 'lndconnect', hasPrimary: false, - disabled: false, masked: true, name: 'LND Connect', description: @@ -409,7 +406,6 @@ export const mockPatchData: DataModel = { p2p: { id: 'p2p', hasPrimary: true, - disabled: false, masked: false, name: 'P2P', description: