From 4defec194f974005a1a10a0da1d57b3bbcf878b1 Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Thu, 22 Aug 2024 21:45:54 -0600 Subject: [PATCH] Feature/subcontainers (#2720) * wip: subcontainers * wip: subcontainer infra * rename NonDestroyableOverlay to SubContainerHandle * chore: Changes to the container and other things * wip: * wip: fixes * fix launch & exec Co-authored-by: Jade * tweak apis * misc fixes * don't treat sigterm as error * handle health check set during starting --------- Co-authored-by: J H Co-authored-by: Jade --- .../src/Adapters/EffectCreator.ts | 82 ++-- .../DockerProcedureContainer.ts | 61 +-- .../Systems/SystemForEmbassy/MainLoop.ts | 34 +- .../Systems/SystemForEmbassy/index.ts | 6 + .../SystemForEmbassy/polyfillEffects.ts | 10 +- core/Cargo.lock | 106 ++++- core/startos/Cargo.toml | 14 +- core/startos/src/lxc/mod.rs | 7 +- core/startos/src/service/effects/health.rs | 4 +- core/startos/src/service/effects/mod.rs | 113 +++-- .../effects/{image.rs => subcontainer/mod.rs} | 85 +--- .../src/service/effects/subcontainer/sync.rs | 389 ++++++++++++++++++ core/startos/src/service/mod.rs | 4 +- .../src/service/persistent_container.rs | 8 +- core/startos/src/service/service_actor.rs | 142 +++---- core/startos/src/service/start_stop.rs | 42 +- core/startos/src/status/mod.rs | 71 ++-- sdk/lib/StartSdk.ts | 9 +- sdk/lib/health/HealthCheck.ts | 2 +- sdk/lib/health/checkFns/runHealthScript.ts | 6 +- sdk/lib/index.ts | 2 +- sdk/lib/mainFn/CommandController.ts | 185 ++++----- sdk/lib/mainFn/Daemon.ts | 24 +- sdk/lib/mainFn/Daemons.ts | 11 +- sdk/lib/mainFn/HealthDaemon.ts | 27 +- sdk/lib/mainFn/Mounts.ts | 2 +- ...arams.ts => CreateSubcontainerFsParams.ts} | 2 +- ...rams.ts => DestroySubcontainerFsParams.ts} | 2 +- sdk/lib/osBindings/MainStatus.ts | 12 +- sdk/lib/osBindings/StartStop.ts | 3 + sdk/lib/osBindings/index.ts | 5 +- sdk/lib/test/startosTypeValidation.test.ts | 32 +- sdk/lib/trigger/index.ts | 1 + sdk/lib/types.ts | 24 +- sdk/lib/util/{Overlay.ts => SubContainer.ts} | 156 +++++-- sdk/lib/util/index.ts | 2 +- sdk/lib/util/typeHelpers.ts | 93 +++++ 37 files changed, 1212 insertions(+), 566 deletions(-) rename core/startos/src/service/effects/{image.rs => subcontainer/mod.rs} (50%) create mode 100644 core/startos/src/service/effects/subcontainer/sync.rs rename sdk/lib/osBindings/{CreateOverlayedImageParams.ts => CreateSubcontainerFsParams.ts} (70%) rename sdk/lib/osBindings/{DestroyOverlayedImageParams.ts => DestroySubcontainerFsParams.ts} (71%) create mode 100644 sdk/lib/osBindings/StartStop.ts rename sdk/lib/util/{Overlay.ts => SubContainer.ts} (66%) diff --git a/container-runtime/src/Adapters/EffectCreator.ts b/container-runtime/src/Adapters/EffectCreator.ts index 1c2954cb2..e0390b1e1 100644 --- a/container-runtime/src/Adapters/EffectCreator.ts +++ b/container-runtime/src/Adapters/EffectCreator.ts @@ -40,7 +40,7 @@ export type EffectContext = { const rpcRoundFor = (procedureId: string | null) => - ( + ( method: K, params: Record, ) => { @@ -110,65 +110,65 @@ function makeEffects(context: EffectContext): Effects { }) as ReturnType }, clearBindings(...[]: Parameters) { - return rpcRound("clearBindings", {}) as ReturnType< + return rpcRound("clear-bindings", {}) as ReturnType< T.Effects["clearBindings"] > }, clearServiceInterfaces( ...[]: Parameters ) { - return rpcRound("clearServiceInterfaces", {}) as ReturnType< + return rpcRound("clear-service-interfaces", {}) as ReturnType< T.Effects["clearServiceInterfaces"] > }, getInstalledPackages(...[]: Parameters) { - return rpcRound("getInstalledPackages", {}) as ReturnType< + return rpcRound("get-installed-packages", {}) as ReturnType< T.Effects["getInstalledPackages"] > }, - createOverlayedImage(options: { - imageId: string - }): Promise<[string, string]> { - return rpcRound("createOverlayedImage", options) as ReturnType< - T.Effects["createOverlayedImage"] - > - }, - destroyOverlayedImage(options: { guid: string }): Promise { - return rpcRound("destroyOverlayedImage", options) as ReturnType< - T.Effects["destroyOverlayedImage"] - > + subcontainer: { + createFs(options: { imageId: string }) { + return rpcRound("subcontainer.create-fs", options) as ReturnType< + T.Effects["subcontainer"]["createFs"] + > + }, + destroyFs(options: { guid: string }): Promise { + return rpcRound("subcontainer.destroy-fs", options) as ReturnType< + T.Effects["subcontainer"]["destroyFs"] + > + }, }, executeAction(...[options]: Parameters) { - return rpcRound("executeAction", options) as ReturnType< + return rpcRound("execute-action", options) as ReturnType< T.Effects["executeAction"] > }, exportAction(...[options]: Parameters) { - return rpcRound("exportAction", options) as ReturnType< + return rpcRound("export-action", options) as ReturnType< T.Effects["exportAction"] > }, exportServiceInterface: (( ...[options]: Parameters ) => { - return rpcRound("exportServiceInterface", options) as ReturnType< + return rpcRound("export-service-interface", options) as ReturnType< T.Effects["exportServiceInterface"] > }) as Effects["exportServiceInterface"], exposeForDependents( ...[options]: Parameters ) { - return rpcRound("exposeForDependents", options) as ReturnType< + return rpcRound("expose-for-dependents", options) as ReturnType< T.Effects["exposeForDependents"] > }, getConfigured(...[]: Parameters) { - return rpcRound("getConfigured", {}) as ReturnType< + return rpcRound("get-configured", {}) as ReturnType< T.Effects["getConfigured"] > }, getContainerIp(...[]: Parameters) { - return rpcRound("getContainerIp", {}) as ReturnType< + return rpcRound("get-container-ip", {}) as ReturnType< T.Effects["getContainerIp"] > }, @@ -177,21 +177,21 @@ function makeEffects(context: EffectContext): Effects { ...allOptions, callback: context.callbacks?.addCallback(allOptions.callback) || null, } - return rpcRound("getHostInfo", options) as ReturnType< + return rpcRound("get-host-info", options) as ReturnType< T.Effects["getHostInfo"] > as any }) as Effects["getHostInfo"], getServiceInterface( ...[options]: Parameters ) { - return rpcRound("getServiceInterface", { + return rpcRound("get-service-interface", { ...options, callback: context.callbacks?.addCallback(options.callback) || null, }) as ReturnType }, getPrimaryUrl(...[options]: Parameters) { - return rpcRound("getPrimaryUrl", { + return rpcRound("get-primary-url", { ...options, callback: context.callbacks?.addCallback(options.callback) || null, }) as ReturnType @@ -199,22 +199,22 @@ function makeEffects(context: EffectContext): Effects { getServicePortForward( ...[options]: Parameters ) { - return rpcRound("getServicePortForward", options) as ReturnType< + return rpcRound("get-service-port-forward", options) as ReturnType< T.Effects["getServicePortForward"] > }, getSslCertificate(options: Parameters[0]) { - return rpcRound("getSslCertificate", options) as ReturnType< + return rpcRound("get-ssl-certificate", options) as ReturnType< T.Effects["getSslCertificate"] > }, getSslKey(options: Parameters[0]) { - return rpcRound("getSslKey", options) as ReturnType< + return rpcRound("get-ssl-key", options) as ReturnType< T.Effects["getSslKey"] > }, getSystemSmtp(...[options]: Parameters) { - return rpcRound("getSystemSmtp", { + return rpcRound("get-system-smtp", { ...options, callback: context.callbacks?.addCallback(options.callback) || null, }) as ReturnType @@ -222,7 +222,7 @@ function makeEffects(context: EffectContext): Effects { listServiceInterfaces( ...[options]: Parameters ) { - return rpcRound("listServiceInterfaces", { + return rpcRound("list-service-interfaces", { ...options, callback: context.callbacks?.addCallback(options.callback) || null, }) as ReturnType @@ -231,7 +231,7 @@ function makeEffects(context: EffectContext): Effects { return rpcRound("mount", options) as ReturnType }, clearActions(...[]: Parameters) { - return rpcRound("clearActions", {}) as ReturnType< + return rpcRound("clear-actions", {}) as ReturnType< T.Effects["clearActions"] > }, @@ -239,37 +239,39 @@ function makeEffects(context: EffectContext): Effects { return rpcRound("restart", {}) as ReturnType }, setConfigured(...[configured]: Parameters) { - return rpcRound("setConfigured", { configured }) as ReturnType< + return rpcRound("set-configured", { configured }) as ReturnType< T.Effects["setConfigured"] > }, setDependencies( dependencies: Parameters[0], ): ReturnType { - return rpcRound("setDependencies", dependencies) as ReturnType< + return rpcRound("set-dependencies", dependencies) as ReturnType< T.Effects["setDependencies"] > }, checkDependencies( options: Parameters[0], ): ReturnType { - return rpcRound("checkDependencies", options) as ReturnType< + return rpcRound("check-dependencies", options) as ReturnType< T.Effects["checkDependencies"] > }, getDependencies(): ReturnType { - return rpcRound("getDependencies", {}) as ReturnType< + return rpcRound("get-dependencies", {}) as ReturnType< T.Effects["getDependencies"] > }, setHealth(...[options]: Parameters) { - return rpcRound("setHealth", options) as ReturnType< + return rpcRound("set-health", options) as ReturnType< T.Effects["setHealth"] > }, setMainStatus(o: { status: "running" | "stopped" }): Promise { - return rpcRound("setMainStatus", o) as ReturnType + return rpcRound("set-main-status", o) as ReturnType< + T.Effects["setHealth"] + > }, shutdown(...[]: Parameters) { @@ -277,20 +279,20 @@ function makeEffects(context: EffectContext): Effects { }, store: { get: async (options: any) => - rpcRound("getStore", { + rpcRound("store.get", { ...options, callback: context.callbacks?.addCallback(options.callback) || null, }) as any, set: async (options: any) => - rpcRound("setStore", options) as ReturnType, + rpcRound("store.set", options) as ReturnType, } as T.Effects["store"], getDataVersion() { - return rpcRound("getDataVersion", {}) as ReturnType< + return rpcRound("get-data-version", {}) as ReturnType< T.Effects["getDataVersion"] > }, setDataVersion(...[options]: Parameters) { - return rpcRound("setDataVersion", options) as ReturnType< + return rpcRound("set-data-version", options) as ReturnType< T.Effects["setDataVersion"] > }, diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts index 47325170d..805f9b531 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts @@ -1,56 +1,60 @@ import * as fs from "fs/promises" import * as cp from "child_process" -import { Overlay, types as T } from "@start9labs/start-sdk" +import { SubContainer, types as T } from "@start9labs/start-sdk" import { promisify } from "util" import { DockerProcedure, VolumeId } from "../../../Models/DockerProcedure" import { Volume } from "./matchVolume" -import { ExecSpawnable } from "@start9labs/start-sdk/cjs/lib/util/Overlay" +import { + CommandOptions, + ExecOptions, + ExecSpawnable, +} from "@start9labs/start-sdk/cjs/lib/util/SubContainer" export const exec = promisify(cp.exec) export const execFile = promisify(cp.execFile) export class DockerProcedureContainer { - private constructor(private readonly overlay: ExecSpawnable) {} + private constructor(private readonly subcontainer: ExecSpawnable) {} static async of( effects: T.Effects, packageId: string, data: DockerProcedure, volumes: { [id: VolumeId]: Volume }, - options: { overlay?: ExecSpawnable } = {}, + options: { subcontainer?: ExecSpawnable } = {}, ) { - const overlay = - options?.overlay ?? - (await DockerProcedureContainer.createOverlay( + const subcontainer = + options?.subcontainer ?? + (await DockerProcedureContainer.createSubContainer( effects, packageId, data, volumes, )) - return new DockerProcedureContainer(overlay) + return new DockerProcedureContainer(subcontainer) } - static async createOverlay( + static async createSubContainer( effects: T.Effects, packageId: string, data: DockerProcedure, volumes: { [id: VolumeId]: Volume }, ) { - const overlay = await Overlay.of(effects, { id: data.image }) + const subcontainer = await SubContainer.of(effects, { id: data.image }) if (data.mounts) { const mounts = data.mounts for (const mount in mounts) { const path = mounts[mount].startsWith("/") - ? `${overlay.rootfs}${mounts[mount]}` - : `${overlay.rootfs}/${mounts[mount]}` + ? `${subcontainer.rootfs}${mounts[mount]}` + : `${subcontainer.rootfs}/${mounts[mount]}` await fs.mkdir(path, { recursive: true }) const volumeMount = volumes[mount] if (volumeMount.type === "data") { - await overlay.mount( + await subcontainer.mount( { type: "volume", id: mount, subpath: null, readonly: false }, mounts[mount], ) } else if (volumeMount.type === "assets") { - await overlay.mount( + await subcontainer.mount( { type: "assets", id: mount, subpath: null }, mounts[mount], ) @@ -96,24 +100,35 @@ export class DockerProcedureContainer { }) .catch(console.warn) } else if (volumeMount.type === "backup") { - await overlay.mount({ type: "backup", subpath: null }, mounts[mount]) + await subcontainer.mount( + { type: "backup", subpath: null }, + mounts[mount], + ) } } } - return overlay + return subcontainer } - async exec(commands: string[], {} = {}) { + async exec( + commands: string[], + options?: CommandOptions & ExecOptions, + timeoutMs?: number | null, + ) { try { - return await this.overlay.exec(commands) + return await this.subcontainer.exec(commands, options, timeoutMs) } finally { - await this.overlay.destroy?.() + await this.subcontainer.destroy?.() } } - async execFail(commands: string[], timeoutMs: number | null, {} = {}) { + async execFail( + commands: string[], + timeoutMs: number | null, + options?: CommandOptions & ExecOptions, + ) { try { - const res = await this.overlay.exec(commands, {}, timeoutMs) + const res = await this.subcontainer.exec(commands, options, timeoutMs) if (res.exitCode !== 0) { const codeOrSignal = res.exitCode !== null @@ -125,11 +140,11 @@ export class DockerProcedureContainer { } return res } finally { - await this.overlay.destroy?.() + await this.subcontainer.destroy?.() } } async spawn(commands: string[]): Promise { - return await this.overlay.spawn(commands) + return await this.subcontainer.spawn(commands) } } diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts index 79f197091..e5aaacfdb 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts @@ -6,6 +6,7 @@ import { Daemon } from "@start9labs/start-sdk/cjs/lib/mainFn/Daemon" import { Effects } from "../../../Models/Effects" import { off } from "node:process" import { CommandController } from "@start9labs/start-sdk/cjs/lib/mainFn/CommandController" +import { asError } from "@start9labs/start-sdk/cjs/lib/util" const EMBASSY_HEALTH_INTERVAL = 15 * 1000 const EMBASSY_PROPERTIES_LOOP = 30 * 1000 @@ -15,8 +16,8 @@ const EMBASSY_PROPERTIES_LOOP = 30 * 1000 * Also, this has an ability to clean itself up too if need be. */ export class MainLoop { - get mainOverlay() { - return this.mainEvent?.daemon?.overlay + get mainSubContainerHandle() { + return this.mainEvent?.daemon?.subContainerHandle } private healthLoops?: { name: string @@ -56,7 +57,7 @@ export class MainLoop { throw new Error("Unreachable") } const daemon = new Daemon(async () => { - const overlay = await DockerProcedureContainer.createOverlay( + const subcontainer = await DockerProcedureContainer.createSubContainer( effects, this.system.manifest.id, this.system.manifest.main, @@ -64,11 +65,10 @@ export class MainLoop { ) return CommandController.of()( this.effects, - - { id: this.system.manifest.main.image }, + subcontainer, currentCommand, { - overlay, + runAsInit: true, env: { TINI_SUBREAPER: "true", }, @@ -147,12 +147,20 @@ export class MainLoop { const start = Date.now() return Object.entries(manifest["health-checks"]).map( ([healthId, value]) => { + effects + .setHealth({ + id: healthId, + name: value.name, + result: "starting", + message: null, + }) + .catch((e) => console.error(asError(e))) const interval = setInterval(async () => { const actionProcedure = value const timeChanged = Date.now() - start if (actionProcedure.type === "docker") { - const overlay = actionProcedure.inject - ? this.mainOverlay + const subcontainer = actionProcedure.inject + ? this.mainSubContainerHandle : undefined // prettier-ignore const container = @@ -162,16 +170,12 @@ export class MainLoop { actionProcedure, manifest.volumes, { - overlay, + subcontainer, } ) const executed = await container.exec( - [ - actionProcedure.entrypoint, - ...actionProcedure.args, - JSON.stringify(timeChanged), - ], - {}, + [actionProcedure.entrypoint, ...actionProcedure.args], + { input: JSON.stringify(timeChanged) }, ) if (executed.exitCode === 0) { await effects.setHealth({ diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index cee873c21..1e0c34189 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -747,11 +747,17 @@ export class SystemForEmbassy implements System { }) if (!actionProcedure) throw Error("Action not found") if (actionProcedure.type === "docker") { + const subcontainer = actionProcedure.inject + ? this.currentRunning?.mainSubContainerHandle + : undefined const container = await DockerProcedureContainer.of( effects, this.manifest.id, actionProcedure, this.manifest.volumes, + { + subcontainer, + }, ) return toActionResult( JSON.parse( diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts index 03af30c90..c212722e6 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts @@ -124,20 +124,18 @@ export const polyfillEffects = ( wait(): Promise> term(): Promise } { - const promiseOverlay = DockerProcedureContainer.createOverlay( + const promiseSubcontainer = DockerProcedureContainer.createSubContainer( effects, manifest.id, manifest.main, manifest.volumes, ) - const daemon = promiseOverlay.then((overlay) => + const daemon = promiseSubcontainer.then((subcontainer) => daemons.runCommand()( effects, - { id: manifest.main.image }, + subcontainer, [input.command, ...(input.args || [])], - { - overlay, - }, + {}, ), ) return { diff --git a/core/Cargo.lock b/core/Cargo.lock index 951e31916..f81170753 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -533,7 +533,7 @@ dependencies = [ "rustc-hash", "shlex", "syn 2.0.74", - "which", + "which 4.4.2", ] [[package]] @@ -1580,6 +1580,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + [[package]] name = "errno" version = "0.3.9" @@ -1590,6 +1601,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -3034,6 +3055,18 @@ dependencies = [ "unicase", ] +[[package]] +name = "nix" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa9b4819da1bc61c0ea48b63b7bc8604064dd43013e7cc325df098d49cd7c18a" +dependencies = [ + "bitflags 1.3.2", + "cc", + "cfg-if", + "libc", +] + [[package]] name = "nix" version = "0.24.3" @@ -3687,6 +3720,32 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" +dependencies = [ + "bitflags 2.6.0", + "chrono", + "flate2", + "hex", + "lazy_static", + "procfs-core", + "rustix", +] + +[[package]] +name = "procfs-core" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" +dependencies = [ + "bitflags 2.6.0", + "chrono", + "hex", +] + [[package]] name = "proptest" version = "1.5.0" @@ -4174,7 +4233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.6.0", - "errno", + "errno 0.3.9", "libc", "linux-raw-sys", "windows-sys 0.52.0", @@ -5040,6 +5099,7 @@ dependencies = [ "pin-project", "pkcs8", "prettytable-rs", + "procfs", "proptest", "proptest-derive", "rand 0.8.5", @@ -5058,6 +5118,7 @@ dependencies = [ "serde_yml", "sha2 0.10.8", "shell-words", + "signal-hook", "simple-logging", "socket2", "sqlx", @@ -5084,9 +5145,12 @@ dependencies = [ "trust-dns-server", "ts-rs", "typed-builder", + "unix-named-pipe", + "unshare", "url", "urlencoding", "uuid", + "which 6.0.3", "zeroize", ] @@ -5965,6 +6029,26 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "unix-named-pipe" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ad653da8f36ac5825ba06642b5a3cce14a4e52c6a5fab4a8928d53f4426dae2" +dependencies = [ + "errno 0.2.8", + "libc", +] + +[[package]] +name = "unshare" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ceda295552a1eda89f8a748237654ad76b9c87e383fc07af5c4e423eb8e7b9b" +dependencies = [ + "libc", + "nix 0.20.0", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -6182,6 +6266,18 @@ dependencies = [ "rustix", ] +[[package]] +name = "which" +version = "6.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ee928febd44d98f2f459a4a79bd4d928591333a494a10a868418ac1b39cf1f" +dependencies = [ + "either", + "home", + "rustix", + "winsafe", +] + [[package]] name = "whoami" version = "1.5.1" @@ -6408,6 +6504,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winsafe" +version = "0.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" + [[package]] name = "wyz" version = "0.2.0" diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index 3bab5c6f3..41effd672 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -130,7 +130,14 @@ log = "0.4.20" mbrman = "0.5.2" models = { version = "*", path = "../models" } new_mime_guess = "4" -nix = { version = "0.29.0", features = ["user", "process", "signal", "fs"] } +nix = { version = "0.29.0", features = [ + "fs", + "mount", + "process", + "sched", + "signal", + "user", +] } nom = "7.1.3" num = "0.4.1" num_enum = "0.7.0" @@ -146,6 +153,7 @@ pbkdf2 = "0.12.2" pin-project = "1.1.3" pkcs8 = { version = "0.10.2", features = ["std"] } prettytable-rs = "0.10.0" +procfs = "0.16.0" proptest = "1.3.1" proptest-derive = "0.5.0" rand = { version = "0.8.5", features = ["std"] } @@ -166,6 +174,7 @@ serde_with = { version = "3.4.0", features = ["macros", "json"] } serde_yaml = { package = "serde_yml", version = "0.0.10" } sha2 = "0.10.2" shell-words = "1" +signal-hook = "0.3.17" simple-logging = "2.0.2" socket2 = "0.5.7" sqlx = { version = "0.7.2", features = [ @@ -197,6 +206,9 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } trust-dns-server = "0.23.1" ts-rs = { git = "https://github.com/dr-bonez/ts-rs.git", branch = "feature/top-level-as" } # "8.1.0" typed-builder = "0.18.0" +which = "6.0.3" +unix-named-pipe = "0.2.0" +unshare = "0.7.0" url = { version = "2.4.1", features = ["serde"] } urlencoding = "2.1.3" uuid = { version = "1.4.1", features = ["v4"] } diff --git a/core/startos/src/lxc/mod.rs b/core/startos/src/lxc/mod.rs index b8ce9c703..480f7a24c 100644 --- a/core/startos/src/lxc/mod.rs +++ b/core/startos/src/lxc/mod.rs @@ -268,9 +268,10 @@ impl LxcContainer { .invoke(ErrorKind::Docker) .await?, )?; - let out_str = output.trim(); - if !out_str.is_empty() { - return Ok(out_str.parse()?); + for line in output.lines() { + if let Ok(ip) = line.trim().parse() { + return Ok(ip); + } } if start.elapsed() > CONTAINER_DHCP_TIMEOUT { return Err(Error::new( diff --git a/core/startos/src/service/effects/health.rs b/core/startos/src/service/effects/health.rs index aad06a004..9bf756d60 100644 --- a/core/startos/src/service/effects/health.rs +++ b/core/startos/src/service/effects/health.rs @@ -32,8 +32,8 @@ pub async fn set_health( .as_main_mut() .mutate(|main| { match main { - &mut MainStatus::Running { ref mut health, .. } - | &mut MainStatus::BackingUp { ref mut health, .. } => { + MainStatus::Running { ref mut health, .. } + | MainStatus::Starting { ref mut health } => { health.insert(id, result); } _ => (), diff --git a/core/startos/src/service/effects/mod.rs b/core/startos/src/service/effects/mod.rs index a7ee6fb4d..3ec77ef21 100644 --- a/core/startos/src/service/effects/mod.rs +++ b/core/startos/src/service/effects/mod.rs @@ -1,4 +1,4 @@ -use rpc_toolkit::{from_fn, from_fn_async, Context, HandlerExt, ParentHandler}; +use rpc_toolkit::{from_fn, from_fn_async, from_fn_blocking, Context, HandlerExt, ParentHandler}; use crate::echo; use crate::prelude::*; @@ -12,44 +12,44 @@ pub mod context; mod control; mod dependency; mod health; -mod image; mod net; mod prelude; mod store; +mod subcontainer; mod system; pub fn handler() -> ParentHandler { ParentHandler::new() - .subcommand("gitInfo", from_fn(|_: C| crate::version::git_info())) + .subcommand("git-info", from_fn(|_: C| crate::version::git_info())) .subcommand( "echo", from_fn(echo::).with_call_remote::(), ) // action .subcommand( - "executeAction", + "execute-action", from_fn_async(action::execute_action).no_cli(), ) .subcommand( - "exportAction", + "export-action", from_fn_async(action::export_action).no_cli(), ) .subcommand( - "clearActions", + "clear-actions", from_fn_async(action::clear_actions).no_cli(), ) // callbacks .subcommand( - "clearCallbacks", + "clear-callbacks", from_fn(callbacks::clear_callbacks).no_cli(), ) // config .subcommand( - "getConfigured", + "get-configured", from_fn_async(config::get_configured).no_cli(), ) .subcommand( - "setConfigured", + "set-configured", from_fn_async(config::set_configured) .no_display() .with_call_remote::(), @@ -68,110 +68,133 @@ pub fn handler() -> ParentHandler { .with_call_remote::(), ) .subcommand( - "setMainStatus", + "set-main-status", from_fn_async(control::set_main_status) .no_display() .with_call_remote::(), ) // dependency .subcommand( - "setDependencies", + "set-dependencies", from_fn_async(dependency::set_dependencies) .no_display() .with_call_remote::(), ) .subcommand( - "getDependencies", + "get-dependencies", from_fn_async(dependency::get_dependencies) .no_display() .with_call_remote::(), ) .subcommand( - "checkDependencies", + "check-dependencies", from_fn_async(dependency::check_dependencies) .no_display() .with_call_remote::(), ) .subcommand("mount", from_fn_async(dependency::mount).no_cli()) .subcommand( - "getInstalledPackages", + "get-installed-packages", from_fn_async(dependency::get_installed_packages).no_cli(), ) .subcommand( - "exposeForDependents", + "expose-for-dependents", from_fn_async(dependency::expose_for_dependents).no_cli(), ) // health - .subcommand("setHealth", from_fn_async(health::set_health).no_cli()) - // image + .subcommand("set-health", from_fn_async(health::set_health).no_cli()) + // subcontainer .subcommand( - "chroot", - from_fn(image::chroot::).no_display(), - ) - .subcommand( - "createOverlayedImage", - from_fn_async(image::create_overlayed_image) - .with_custom_display_fn(|_, (path, _)| Ok(println!("{}", path.display()))) - .with_call_remote::(), - ) - .subcommand( - "destroyOverlayedImage", - from_fn_async(image::destroy_overlayed_image).no_cli(), + "subcontainer", + ParentHandler::::new() + .subcommand( + "launch", + from_fn_blocking(subcontainer::launch::).no_display(), + ) + .subcommand( + "launch-init", + from_fn_blocking(subcontainer::launch_init::).no_display(), + ) + .subcommand( + "exec", + from_fn_blocking(subcontainer::exec::).no_display(), + ) + .subcommand( + "exec-command", + from_fn_blocking(subcontainer::exec_command::) + .no_display(), + ) + .subcommand( + "create-fs", + from_fn_async(subcontainer::create_subcontainer_fs) + .with_custom_display_fn(|_, (path, _)| Ok(println!("{}", path.display()))) + .with_call_remote::(), + ) + .subcommand( + "destroy-fs", + from_fn_async(subcontainer::destroy_subcontainer_fs) + .no_display() + .with_call_remote::(), + ), ) // net .subcommand("bind", from_fn_async(net::bind::bind).no_cli()) .subcommand( - "getServicePortForward", + "get-service-port-forward", from_fn_async(net::bind::get_service_port_forward).no_cli(), ) .subcommand( - "clearBindings", + "clear-bindings", from_fn_async(net::bind::clear_bindings).no_cli(), ) .subcommand( - "getHostInfo", + "get-host-info", from_fn_async(net::host::get_host_info).no_cli(), ) .subcommand( - "getPrimaryUrl", + "get-primary-url", from_fn_async(net::host::get_primary_url).no_cli(), ) .subcommand( - "getContainerIp", + "get-container-ip", from_fn_async(net::info::get_container_ip).no_cli(), ) .subcommand( - "exportServiceInterface", + "export-service-interface", from_fn_async(net::interface::export_service_interface).no_cli(), ) .subcommand( - "getServiceInterface", + "get-service-interface", from_fn_async(net::interface::get_service_interface).no_cli(), ) .subcommand( - "listServiceInterfaces", + "list-service-interfaces", from_fn_async(net::interface::list_service_interfaces).no_cli(), ) .subcommand( - "clearServiceInterfaces", + "clear-service-interfaces", from_fn_async(net::interface::clear_service_interfaces).no_cli(), ) .subcommand( - "getSslCertificate", + "get-ssl-certificate", from_fn_async(net::ssl::get_ssl_certificate).no_cli(), ) - .subcommand("getSslKey", from_fn_async(net::ssl::get_ssl_key).no_cli()) + .subcommand("get-ssl-key", from_fn_async(net::ssl::get_ssl_key).no_cli()) // store - .subcommand("getStore", from_fn_async(store::get_store).no_cli()) - .subcommand("setStore", from_fn_async(store::set_store).no_cli()) .subcommand( - "setDataVersion", + "store", + ParentHandler::::new() + .subcommand("get", from_fn_async(store::get_store).no_cli()) + .subcommand("set", from_fn_async(store::set_store).no_cli()), + ) + .subcommand( + "set-data-version", from_fn_async(store::set_data_version) .no_display() .with_call_remote::(), ) .subcommand( - "getDataVersion", + "get-data-version", from_fn_async(store::get_data_version) .with_custom_display_fn(|_, v| { if let Some(v) = v { @@ -185,7 +208,7 @@ pub fn handler() -> ParentHandler { ) // system .subcommand( - "getSystemSmtp", + "get-system-smtp", from_fn_async(system::get_system_smtp).no_cli(), ) diff --git a/core/startos/src/service/effects/image.rs b/core/startos/src/service/effects/subcontainer/mod.rs similarity index 50% rename from core/startos/src/service/effects/image.rs rename to core/startos/src/service/effects/subcontainer/mod.rs index 4b5293506..86e77e196 100644 --- a/core/startos/src/service/effects/image.rs +++ b/core/startos/src/service/effects/subcontainer/mod.rs @@ -1,9 +1,6 @@ -use std::ffi::OsString; -use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; use models::ImageId; -use rpc_toolkit::Context; use tokio::process::Command; use crate::disk::mount::filesystem::overlayfs::OverlayGuard; @@ -11,89 +8,33 @@ use crate::rpc_continuations::Guid; use crate::service::effects::prelude::*; use crate::util::Invoke; -#[derive(Debug, Clone, Serialize, Deserialize, Parser)] -pub struct ChrootParams { - #[arg(short = 'e', long = "env")] - env: Option, - #[arg(short = 'w', long = "workdir")] - workdir: Option, - #[arg(short = 'u', long = "user")] - user: Option, - path: PathBuf, - command: OsString, - args: Vec, -} -pub fn chroot( - _: C, - ChrootParams { - env, - workdir, - user, - path, - command, - args, - }: ChrootParams, -) -> Result<(), Error> { - let mut cmd: std::process::Command = std::process::Command::new(command); - if let Some(env) = env { - for (k, v) in std::fs::read_to_string(env)? - .lines() - .map(|l| l.trim()) - .filter_map(|l| l.split_once("=")) - { - cmd.env(k, v); - } - } - nix::unistd::setsid().ok(); // https://stackoverflow.com/questions/25701333/os-setsid-operation-not-permitted - std::os::unix::fs::chroot(path)?; - if let Some(uid) = user.as_deref().and_then(|u| u.parse::().ok()) { - cmd.uid(uid); - } else if let Some(user) = user { - let (uid, gid) = std::fs::read_to_string("/etc/passwd")? - .lines() - .find_map(|l| { - let mut split = l.trim().split(":"); - if user != split.next()? { - return None; - } - split.next(); // throw away x - Some((split.next()?.parse().ok()?, split.next()?.parse().ok()?)) - // uid gid - }) - .or_not_found(lazy_format!("{user} in /etc/passwd"))?; - cmd.uid(uid); - cmd.gid(gid); - }; - if let Some(workdir) = workdir { - cmd.current_dir(workdir); - } - cmd.args(args); - Err(cmd.exec().into()) -} +mod sync; + +pub use sync::*; #[derive(Debug, Deserialize, Serialize, Parser, TS)] #[serde(rename_all = "camelCase")] #[ts(export)] -pub struct DestroyOverlayedImageParams { +pub struct DestroySubcontainerFsParams { guid: Guid, } #[instrument(skip_all)] -pub async fn destroy_overlayed_image( +pub async fn destroy_subcontainer_fs( context: EffectContext, - DestroyOverlayedImageParams { guid }: DestroyOverlayedImageParams, + DestroySubcontainerFsParams { guid }: DestroySubcontainerFsParams, ) -> Result<(), Error> { let context = context.deref()?; if let Some(overlay) = context .seed .persistent_container - .overlays + .subcontainers .lock() .await .remove(&guid) { overlay.unmount(true).await?; } else { - tracing::warn!("Could not find a guard to remove on the destroy overlayed image; assumming that it already is removed and will be skipping"); + tracing::warn!("Could not find a subcontainer fs to destroy; assumming that it already is destroyed and will be skipping"); } Ok(()) } @@ -101,13 +42,13 @@ pub async fn destroy_overlayed_image( #[derive(Debug, Deserialize, Serialize, Parser, TS)] #[serde(rename_all = "camelCase")] #[ts(export)] -pub struct CreateOverlayedImageParams { +pub struct CreateSubcontainerFsParams { image_id: ImageId, } #[instrument(skip_all)] -pub async fn create_overlayed_image( +pub async fn create_subcontainer_fs( context: EffectContext, - CreateOverlayedImageParams { image_id }: CreateOverlayedImageParams, + CreateSubcontainerFsParams { image_id }: CreateSubcontainerFsParams, ) -> Result<(PathBuf, Guid), Error> { let context = context.deref()?; if let Some(image) = context @@ -131,7 +72,7 @@ pub async fn create_overlayed_image( })? .rootfs_dir(); let mountpoint = rootfs_dir - .join("media/startos/overlays") + .join("media/startos/subcontainers") .join(guid.as_ref()); tokio::fs::create_dir_all(&mountpoint).await?; let container_mountpoint = Path::new("/").join( @@ -150,7 +91,7 @@ pub async fn create_overlayed_image( context .seed .persistent_container - .overlays + .subcontainers .lock() .await .insert(guid.clone(), guard); diff --git a/core/startos/src/service/effects/subcontainer/sync.rs b/core/startos/src/service/effects/subcontainer/sync.rs new file mode 100644 index 000000000..513d0b1e5 --- /dev/null +++ b/core/startos/src/service/effects/subcontainer/sync.rs @@ -0,0 +1,389 @@ +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::ffi::{c_int, OsStr, OsString}; +use std::fs::File; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::{Command as StdCommand, Stdio}; + +use nix::sched::CloneFlags; +use nix::unistd::Pid; +use rpc_toolkit::Context; +use signal_hook::consts::signal::*; +use tokio::sync::oneshot; +use unshare::Command as NSCommand; + +use crate::service::effects::prelude::*; + +const FWD_SIGNALS: &[c_int] = &[ + SIGABRT, SIGALRM, SIGCONT, SIGHUP, SIGINT, SIGIO, SIGPIPE, SIGPROF, SIGQUIT, SIGTERM, SIGTRAP, + SIGTSTP, SIGTTIN, SIGTTOU, SIGURG, SIGUSR1, SIGUSR2, SIGVTALRM, +]; + +struct NSPid(Vec); +impl procfs::FromBufRead for NSPid { + fn from_buf_read(r: R) -> procfs::ProcResult { + for line in r.lines() { + let line = line?; + if let Some(row) = line.trim().strip_prefix("NSpid") { + return Ok(Self( + row.split_ascii_whitespace() + .map(|pid| pid.parse::()) + .collect::, _>>()?, + )); + } + } + Err(procfs::ProcError::Incomplete(None)) + } +} + +fn open_file_read(path: impl AsRef) -> Result { + File::open(&path).with_ctx(|_| { + ( + ErrorKind::Filesystem, + lazy_format!("open r {}", path.as_ref().display()), + ) + }) +} + +#[derive(Debug, Clone, Serialize, Deserialize, Parser)] +pub struct ExecParams { + #[arg(short = 'e', long = "env")] + env: Option, + #[arg(short = 'w', long = "workdir")] + workdir: Option, + #[arg(short = 'u', long = "user")] + user: Option, + chroot: PathBuf, + #[arg(trailing_var_arg = true)] + command: Vec, +} +impl ExecParams { + fn exec(&self) -> Result<(), Error> { + let ExecParams { + env, + workdir, + user, + chroot, + command, + } = self; + let Some(([command], args)) = command.split_at_checked(1) else { + return Err(Error::new( + eyre!("command cannot be empty"), + ErrorKind::InvalidRequest, + )); + }; + let env_string = if let Some(env) = &env { + std::fs::read_to_string(env) + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("read {env:?}")))? + } else { + Default::default() + }; + let env = env_string + .lines() + .map(|l| l.trim()) + .filter_map(|l| l.split_once("=")) + .collect::>(); + std::os::unix::fs::chroot(chroot) + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("chroot {chroot:?}")))?; + let command = which::which_in( + command, + env.get("PATH") + .copied() + .map(Cow::Borrowed) + .or_else(|| std::env::var("PATH").ok().map(Cow::Owned)) + .as_deref(), + workdir.as_deref().unwrap_or(Path::new("/")), + ) + .with_kind(ErrorKind::Filesystem)?; + let mut cmd = StdCommand::new(command); + cmd.args(args); + for (k, v) in env { + cmd.env(k, v); + } + + if let Some(uid) = user.as_deref().and_then(|u| u.parse::().ok()) { + cmd.uid(uid); + } else if let Some(user) = user { + let (uid, gid) = std::fs::read_to_string("/etc/passwd") + .with_ctx(|_| (ErrorKind::Filesystem, "read /etc/passwd"))? + .lines() + .find_map(|l| { + let mut split = l.trim().split(":"); + if user != split.next()? { + return None; + } + split.next(); // throw away x + Some((split.next()?.parse().ok()?, split.next()?.parse().ok()?)) + // uid gid + }) + .or_not_found(lazy_format!("{user} in /etc/passwd"))?; + cmd.uid(uid); + cmd.gid(gid); + }; + if let Some(workdir) = workdir { + cmd.current_dir(workdir); + } else { + cmd.current_dir("/"); + } + Err(cmd.exec().into()) + } +} + +pub fn launch( + _: C, + ExecParams { + env, + workdir, + user, + chroot, + command, + }: ExecParams, +) -> Result<(), Error> { + use unshare::{Namespace, Stdio}; + let mut sig = signal_hook::iterator::Signals::new(FWD_SIGNALS)?; + let mut cmd = NSCommand::new("/usr/bin/start-cli"); + cmd.arg("subcontainer").arg("launch-init"); + if let Some(env) = env { + cmd.arg("--env").arg(env); + } + if let Some(workdir) = workdir { + cmd.arg("--workdir").arg(workdir); + } + if let Some(user) = user { + cmd.arg("--user").arg(user); + } + cmd.arg(&chroot); + cmd.args(&command); + cmd.unshare(&[Namespace::Pid, Namespace::Cgroup, Namespace::Ipc]); + cmd.stdin(Stdio::piped()); + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + let (stdin_send, stdin_recv) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(mut stdin) = stdin_recv.blocking_recv() { + std::io::copy(&mut std::io::stdin(), &mut stdin).unwrap(); + } + }); + let (stdout_send, stdout_recv) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(mut stdout) = stdout_recv.blocking_recv() { + std::io::copy(&mut stdout, &mut std::io::stdout()).unwrap(); + } + }); + let (stderr_send, stderr_recv) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(mut stderr) = stderr_recv.blocking_recv() { + std::io::copy(&mut stderr, &mut std::io::stderr()).unwrap(); + } + }); + if chroot.join("proc/1").exists() { + let ns_id = procfs::process::Process::new_with_root(chroot.join("proc")) + .with_ctx(|_| (ErrorKind::Filesystem, "open subcontainer procfs"))? + .namespaces() + .with_ctx(|_| (ErrorKind::Filesystem, "read subcontainer pid 1 ns"))? + .0 + .get(OsStr::new("pid")) + .or_not_found("pid namespace")? + .identifier; + for proc in + procfs::process::all_processes().with_ctx(|_| (ErrorKind::Filesystem, "open procfs"))? + { + let proc = proc.with_ctx(|_| (ErrorKind::Filesystem, "read single process details"))?; + let pid = proc.pid(); + if proc + .namespaces() + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("read pid {} ns", pid)))? + .0 + .get(OsStr::new("pid")) + .map_or(false, |ns| ns.identifier == ns_id) + { + let pids = proc.read::("status").with_ctx(|_| { + ( + ErrorKind::Filesystem, + lazy_format!("read pid {} NSpid", pid), + ) + })?; + if pids.0.len() == 2 && pids.0[1] == 1 { + nix::sys::signal::kill(Pid::from_raw(pid), nix::sys::signal::SIGKILL) + .with_ctx(|_| { + ( + ErrorKind::Filesystem, + lazy_format!( + "kill pid {} (determined to be pid 1 in subcontainer)", + pid + ), + ) + })?; + } + } + } + nix::mount::umount(&chroot.join("proc")) + .with_ctx(|_| (ErrorKind::Filesystem, "unmounting subcontainer procfs"))?; + } + let mut child = cmd + .spawn() + .map_err(color_eyre::eyre::Report::msg) + .with_ctx(|_| (ErrorKind::Filesystem, "spawning child process"))?; + let pid = child.pid(); + std::thread::spawn(move || { + for sig in sig.forever() { + nix::sys::signal::kill( + Pid::from_raw(pid), + Some(nix::sys::signal::Signal::try_from(sig).unwrap()), + ) + .unwrap(); + } + }); + stdin_send + .send(child.stdin.take().unwrap()) + .unwrap_or_default(); + stdout_send + .send(child.stdout.take().unwrap()) + .unwrap_or_default(); + stderr_send + .send(child.stderr.take().unwrap()) + .unwrap_or_default(); + // TODO: subreaping, signal handling + let exit = child + .wait() + .with_ctx(|_| (ErrorKind::Filesystem, "waiting on child process"))?; + if let Some(code) = exit.code() { + std::process::exit(code); + } else { + if exit.success() { + Ok(()) + } else { + Err(Error::new( + color_eyre::eyre::Report::msg(exit), + ErrorKind::Unknown, + )) + } + } +} + +pub fn launch_init(_: C, params: ExecParams) -> Result<(), Error> { + nix::mount::mount( + Some("proc"), + ¶ms.chroot.join("proc"), + Some("proc"), + nix::mount::MsFlags::empty(), + None::<&str>, + ) + .with_ctx(|_| (ErrorKind::Filesystem, "mount procfs"))?; + if params.command.is_empty() { + signal_hook::iterator::Signals::new(signal_hook::consts::TERM_SIGNALS)? + .forever() + .next(); + std::process::exit(0) + } else { + params.exec() + } +} + +pub fn exec( + _: C, + ExecParams { + env, + workdir, + user, + chroot, + command, + }: ExecParams, +) -> Result<(), Error> { + let mut sig = signal_hook::iterator::Signals::new(FWD_SIGNALS)?; + let (send_pid, recv_pid) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(pid) = recv_pid.blocking_recv() { + for sig in sig.forever() { + nix::sys::signal::kill( + Pid::from_raw(pid), + Some(nix::sys::signal::Signal::try_from(sig).unwrap()), + ) + .unwrap(); + } + } + }); + let mut cmd = StdCommand::new("/usr/bin/start-cli"); + cmd.arg("subcontainer").arg("exec-command"); + if let Some(env) = env { + cmd.arg("--env").arg(env); + } + if let Some(workdir) = workdir { + cmd.arg("--workdir").arg(workdir); + } + if let Some(user) = user { + cmd.arg("--user").arg(user); + } + cmd.arg(&chroot); + cmd.args(&command); + cmd.stdin(Stdio::piped()); + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + let (stdin_send, stdin_recv) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(mut stdin) = stdin_recv.blocking_recv() { + std::io::copy(&mut std::io::stdin(), &mut stdin).unwrap(); + } + }); + let (stdout_send, stdout_recv) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(mut stdout) = stdout_recv.blocking_recv() { + std::io::copy(&mut stdout, &mut std::io::stdout()).unwrap(); + } + }); + let (stderr_send, stderr_recv) = oneshot::channel(); + std::thread::spawn(move || { + if let Ok(mut stderr) = stderr_recv.blocking_recv() { + std::io::copy(&mut stderr, &mut std::io::stderr()).unwrap(); + } + }); + nix::sched::setns( + open_file_read(chroot.join("proc/1/ns/pid"))?, + CloneFlags::CLONE_NEWPID, + ) + .with_ctx(|_| (ErrorKind::Filesystem, "set pid ns"))?; + nix::sched::setns( + open_file_read(chroot.join("proc/1/ns/cgroup"))?, + CloneFlags::CLONE_NEWCGROUP, + ) + .with_ctx(|_| (ErrorKind::Filesystem, "set cgroup ns"))?; + nix::sched::setns( + open_file_read(chroot.join("proc/1/ns/ipc"))?, + CloneFlags::CLONE_NEWIPC, + ) + .with_ctx(|_| (ErrorKind::Filesystem, "set ipc ns"))?; + let mut child = cmd + .spawn() + .map_err(color_eyre::eyre::Report::msg) + .with_ctx(|_| (ErrorKind::Filesystem, "spawning child process"))?; + send_pid.send(child.id() as i32).unwrap_or_default(); + stdin_send + .send(child.stdin.take().unwrap()) + .unwrap_or_default(); + stdout_send + .send(child.stdout.take().unwrap()) + .unwrap_or_default(); + stderr_send + .send(child.stderr.take().unwrap()) + .unwrap_or_default(); + let exit = child + .wait() + .with_ctx(|_| (ErrorKind::Filesystem, "waiting on child process"))?; + if let Some(code) = exit.code() { + std::process::exit(code); + } else { + if exit.success() { + Ok(()) + } else { + Err(Error::new( + color_eyre::eyre::Report::msg(exit), + ErrorKind::Unknown, + )) + } + } +} + +pub fn exec_command(_: C, params: ExecParams) -> Result<(), Error> { + params.exec() +} diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index 6a99841d6..5eae62756 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -45,7 +45,7 @@ mod properties; mod rpc; mod service_actor; pub mod service_map; -mod start_stop; +pub mod start_stop; mod transition; mod util; @@ -493,7 +493,6 @@ impl Service { #[derive(Debug, Clone)] pub struct RunningStatus { - health: OrdMap, started: DateTime, } @@ -516,7 +515,6 @@ impl ServiceActorSeed { .running_status .take() .unwrap_or_else(|| RunningStatus { - health: Default::default(), started: Utc::now(), }), ); diff --git a/core/startos/src/service/persistent_container.rs b/core/startos/src/service/persistent_container.rs index c81322719..dd7b5766d 100644 --- a/core/startos/src/service/persistent_container.rs +++ b/core/startos/src/service/persistent_container.rs @@ -98,7 +98,7 @@ pub struct PersistentContainer { volumes: BTreeMap, assets: BTreeMap, pub(super) images: BTreeMap>, - pub(super) overlays: Arc>>>>, + pub(super) subcontainers: Arc>>>>, pub(super) state: Arc>, pub(super) net_service: Mutex, destroyed: bool, @@ -273,7 +273,7 @@ impl PersistentContainer { volumes, assets, images, - overlays: Arc::new(Mutex::new(BTreeMap::new())), + subcontainers: Arc::new(Mutex::new(BTreeMap::new())), state: Arc::new(watch::channel(ServiceState::new(start)).0), net_service: Mutex::new(net_service), destroyed: false, @@ -388,7 +388,7 @@ impl PersistentContainer { let volumes = std::mem::take(&mut self.volumes); let assets = std::mem::take(&mut self.assets); let images = std::mem::take(&mut self.images); - let overlays = self.overlays.clone(); + let subcontainers = self.subcontainers.clone(); let lxc_container = self.lxc_container.take(); self.destroyed = true; Some(async move { @@ -404,7 +404,7 @@ impl PersistentContainer { for (_, assets) in assets { errs.handle(assets.unmount(true).await); } - for (_, overlay) in std::mem::take(&mut *overlays.lock().await) { + for (_, overlay) in std::mem::take(&mut *subcontainers.lock().await) { errs.handle(overlay.unmount(true).await); } for (_, images) in images { diff --git a/core/startos/src/service/service_actor.rs b/core/startos/src/service/service_actor.rs index e6578264c..0839afc0b 100644 --- a/core/startos/src/service/service_actor.rs +++ b/core/startos/src/service/service_actor.rs @@ -1,11 +1,10 @@ use std::sync::Arc; use std::time::Duration; -use imbl::OrdMap; - use super::start_stop::StartStop; use super::ServiceActorSeed; use crate::prelude::*; +use crate::service::persistent_container::ServiceStateKinds; use crate::service::transition::TransitionKind; use crate::service::SYNC_RETRY_COOLDOWN_SECONDS; use crate::status::MainStatus; @@ -46,96 +45,77 @@ async fn service_actor_loop( let id = &seed.id; let kinds = current.borrow().kinds(); if let Err(e) = async { - let main_status = match ( - kinds.transition_state, - kinds.desired_state, - kinds.running_status, - ) { - (Some(TransitionKind::Restarting), StartStop::Stop, Some(_)) => { - seed.persistent_container.stop().await?; - MainStatus::Restarting - } - (Some(TransitionKind::Restarting), StartStop::Start, _) => { - seed.persistent_container.start().await?; - MainStatus::Restarting - } - (Some(TransitionKind::Restarting), _, _) => MainStatus::Restarting, - (Some(TransitionKind::Restoring), _, _) => MainStatus::Restoring, - (Some(TransitionKind::BackingUp), StartStop::Stop, Some(status)) => { - seed.persistent_container.stop().await?; - MainStatus::BackingUp { - started: Some(status.started), - health: status.health.clone(), - } - } - (Some(TransitionKind::BackingUp), StartStop::Start, _) => { - seed.persistent_container.start().await?; - MainStatus::BackingUp { - started: None, - health: OrdMap::new(), - } - } - (Some(TransitionKind::BackingUp), _, _) => MainStatus::BackingUp { - started: None, - health: OrdMap::new(), - }, - (None, StartStop::Stop, None) => MainStatus::Stopped, - (None, StartStop::Stop, Some(_)) => { - let task_seed = seed.clone(); - seed.ctx - .db - .mutate(|d| { - if let Some(i) = d.as_public_mut().as_package_data_mut().as_idx_mut(&id) { - i.as_status_mut().as_main_mut().ser(&MainStatus::Stopping)?; - } - Ok(()) - }) - .await?; - task_seed.persistent_container.stop().await?; - MainStatus::Stopped - } - (None, StartStop::Start, Some(status)) => MainStatus::Running { - started: status.started, - health: status.health.clone(), - }, - (None, StartStop::Start, None) => { - seed.persistent_container.start().await?; - MainStatus::Starting - } - }; seed.ctx .db .mutate(|d| { if let Some(i) = d.as_public_mut().as_package_data_mut().as_idx_mut(&id) { let previous = i.as_status().as_main().de()?; - let previous_health = previous.health(); - let previous_started = previous.started(); - let mut main_status = main_status; - match &mut main_status { - &mut MainStatus::Running { ref mut health, .. } - | &mut MainStatus::BackingUp { ref mut health, .. } => { - *health = previous_health.unwrap_or(health).clone(); - } - _ => (), - }; - match &mut main_status { - MainStatus::Running { - ref mut started, .. - } => { - *started = previous_started.unwrap_or(*started); - } - MainStatus::BackingUp { - ref mut started, .. - } => { - *started = previous_started.map(Some).unwrap_or(*started); - } - _ => (), + let main_status = match &kinds { + ServiceStateKinds { + transition_state: Some(TransitionKind::Restarting), + .. + } => MainStatus::Restarting, + ServiceStateKinds { + transition_state: Some(TransitionKind::Restoring), + .. + } => MainStatus::Restoring, + ServiceStateKinds { + transition_state: Some(TransitionKind::BackingUp), + .. + } => previous.backing_up(), + ServiceStateKinds { + running_status: Some(status), + desired_state: StartStop::Start, + .. + } => MainStatus::Running { + started: status.started, + health: previous.health().cloned().unwrap_or_default(), + }, + ServiceStateKinds { + running_status: None, + desired_state: StartStop::Start, + .. + } => MainStatus::Starting { + health: previous.health().cloned().unwrap_or_default(), + }, + ServiceStateKinds { + running_status: Some(_), + desired_state: StartStop::Stop, + .. + } => MainStatus::Stopping, + ServiceStateKinds { + running_status: None, + desired_state: StartStop::Stop, + .. + } => MainStatus::Stopped, }; i.as_status_mut().as_main_mut().ser(&main_status)?; } Ok(()) }) .await?; + seed.synchronized.notify_waiters(); + + match kinds { + ServiceStateKinds { + running_status: None, + desired_state: StartStop::Start, + .. + } => { + seed.persistent_container.start().await?; + } + ServiceStateKinds { + running_status: Some(_), + desired_state: StartStop::Stop, + .. + } => { + seed.persistent_container.stop().await?; + seed.persistent_container + .state + .send_if_modified(|s| s.running_status.take().is_some()); + } + _ => (), + }; Ok::<_, Error>(()) } diff --git a/core/startos/src/service/start_stop.rs b/core/startos/src/service/start_stop.rs index 178176023..64d4022d6 100644 --- a/core/startos/src/service/start_stop.rs +++ b/core/startos/src/service/start_stop.rs @@ -1,6 +1,10 @@ +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + use crate::status::MainStatus; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] pub enum StartStop { Start, Stop, @@ -11,23 +15,19 @@ impl StartStop { matches!(self, StartStop::Start) } } -impl From for StartStop { - fn from(value: MainStatus) -> Self { - match value { - MainStatus::Stopped => StartStop::Stop, - MainStatus::Restoring => StartStop::Stop, - MainStatus::Restarting => StartStop::Start, - MainStatus::Stopping { .. } => StartStop::Stop, - MainStatus::Starting => StartStop::Start, - MainStatus::Running { - started: _, - health: _, - } => StartStop::Start, - MainStatus::BackingUp { started, health: _ } if started.is_some() => StartStop::Start, - MainStatus::BackingUp { - started: _, - health: _, - } => StartStop::Stop, - } - } -} +// impl From for StartStop { +// fn from(value: MainStatus) -> Self { +// match value { +// MainStatus::Stopped => StartStop::Stop, +// MainStatus::Restoring => StartStop::Stop, +// MainStatus::Restarting => StartStop::Start, +// MainStatus::Stopping { .. } => StartStop::Stop, +// MainStatus::Starting => StartStop::Start, +// MainStatus::Running { +// started: _, +// health: _, +// } => StartStop::Start, +// MainStatus::BackingUp { on_complete } => on_complete, +// } +// } +// } diff --git a/core/startos/src/status/mod.rs b/core/startos/src/status/mod.rs index 1701a965e..c10a7b89f 100644 --- a/core/startos/src/status/mod.rs +++ b/core/startos/src/status/mod.rs @@ -1,5 +1,4 @@ use std::collections::BTreeMap; -use std::sync::Arc; use chrono::{DateTime, Utc}; use imbl::OrdMap; @@ -8,8 +7,8 @@ use ts_rs::TS; use self::health_check::HealthCheckId; use crate::prelude::*; +use crate::service::start_stop::StartStop; use crate::status::health_check::NamedHealthCheckResult; -use crate::util::GeneralGuard; pub mod health_check; #[derive(Clone, Debug, Deserialize, Serialize, HasModel, TS)] @@ -24,25 +23,24 @@ pub struct Status { #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, TS)] #[serde(tag = "status")] #[serde(rename_all = "camelCase")] +#[serde(rename_all_fields = "camelCase")] pub enum MainStatus { Stopped, Restarting, Restoring, Stopping, - Starting, - #[serde(rename_all = "camelCase")] + Starting { + #[ts(as = "BTreeMap")] + health: OrdMap, + }, Running { #[ts(type = "string")] started: DateTime, #[ts(as = "BTreeMap")] health: OrdMap, }, - #[serde(rename_all = "camelCase")] BackingUp { - #[ts(type = "string | null")] - started: Option>, - #[ts(as = "BTreeMap")] - health: OrdMap, + on_complete: StartStop, }, } impl MainStatus { @@ -50,60 +48,37 @@ impl MainStatus { match self { MainStatus::Starting { .. } | MainStatus::Running { .. } + | MainStatus::Restarting | MainStatus::BackingUp { - started: Some(_), .. + on_complete: StartStop::Start, } => true, MainStatus::Stopped | MainStatus::Restoring | MainStatus::Stopping { .. } - | MainStatus::Restarting - | MainStatus::BackingUp { started: None, .. } => false, + | MainStatus::BackingUp { + on_complete: StartStop::Stop, + } => false, } } - // pub fn stop(&mut self) { - // match self { - // MainStatus::Starting { .. } | MainStatus::Running { .. } => { - // *self = MainStatus::Stopping; - // } - // MainStatus::BackingUp { started, .. } => { - // *started = None; - // } - // MainStatus::Stopped | MainStatus::Stopping | MainStatus::Restarting => (), - // } - // } - pub fn started(&self) -> Option> { - match self { - MainStatus::Running { started, .. } => Some(*started), - MainStatus::BackingUp { started, .. } => *started, - MainStatus::Stopped => None, - MainStatus::Restoring => None, - MainStatus::Restarting => None, - MainStatus::Stopping { .. } => None, - MainStatus::Starting { .. } => None, + + pub fn backing_up(self) -> Self { + MainStatus::BackingUp { + on_complete: if self.running() { + StartStop::Start + } else { + StartStop::Stop + }, } } - pub fn backing_up(&self) -> Self { - let (started, health) = match self { - MainStatus::Starting { .. } => (Some(Utc::now()), Default::default()), - MainStatus::Running { started, health } => (Some(started.clone()), health.clone()), - MainStatus::Stopped - | MainStatus::Stopping { .. } - | MainStatus::Restoring - | MainStatus::Restarting => (None, Default::default()), - MainStatus::BackingUp { .. } => return self.clone(), - }; - MainStatus::BackingUp { started, health } - } pub fn health(&self) -> Option<&OrdMap> { match self { - MainStatus::Running { health, .. } => Some(health), - MainStatus::BackingUp { health, .. } => Some(health), - MainStatus::Stopped + MainStatus::Running { health, .. } | MainStatus::Starting { health } => Some(health), + MainStatus::BackingUp { .. } + | MainStatus::Stopped | MainStatus::Restoring | MainStatus::Stopping { .. } | MainStatus::Restarting => None, - MainStatus::Starting { .. } => None, } } } diff --git a/sdk/lib/StartSdk.ts b/sdk/lib/StartSdk.ts index 569d83b16..390e8e87f 100644 --- a/sdk/lib/StartSdk.ts +++ b/sdk/lib/StartSdk.ts @@ -61,7 +61,7 @@ import { } from "./util/getServiceInterface" import { getServiceInterfaces } from "./util/getServiceInterfaces" import { getStore } from "./store/getStore" -import { CommandOptions, MountOptions, Overlay } from "./util/Overlay" +import { CommandOptions, MountOptions, SubContainer } from "./util/SubContainer" import { splitCommand } from "./util/splitCommand" import { Mounts } from "./mainFn/Mounts" import { Dependency } from "./Dependency" @@ -734,8 +734,11 @@ export async function runCommand( }, ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> { const commands = splitCommand(command) - return Overlay.with(effects, image, options.mounts || [], (overlay) => - overlay.exec(commands), + return SubContainer.with( + effects, + image, + options.mounts || [], + (subcontainer) => subcontainer.exec(commands), ) } function nullifyProperties(value: T.SdkPropertiesReturn): T.PropertiesReturn { diff --git a/sdk/lib/health/HealthCheck.ts b/sdk/lib/health/HealthCheck.ts index 8186a3cce..e007c4ea2 100644 --- a/sdk/lib/health/HealthCheck.ts +++ b/sdk/lib/health/HealthCheck.ts @@ -5,7 +5,7 @@ import { Trigger } from "../trigger" import { TriggerInput } from "../trigger/TriggerInput" import { defaultTrigger } from "../trigger/defaultTrigger" import { once } from "../util/once" -import { Overlay } from "../util/Overlay" +import { SubContainer } from "../util/SubContainer" import { object, unknown } from "ts-matches" import * as T from "../types" import { asError } from "../util/asError" diff --git a/sdk/lib/health/checkFns/runHealthScript.ts b/sdk/lib/health/checkFns/runHealthScript.ts index 87fc6c69c..4bac211a9 100644 --- a/sdk/lib/health/checkFns/runHealthScript.ts +++ b/sdk/lib/health/checkFns/runHealthScript.ts @@ -1,5 +1,5 @@ import { Effects } from "../../types" -import { Overlay } from "../../util/Overlay" +import { SubContainer } from "../../util/SubContainer" import { stringFromStdErrOut } from "../../util/stringFromStdErrOut" import { HealthCheckResult } from "./HealthCheckResult" import { timeoutPromise } from "./index" @@ -13,7 +13,7 @@ import { timeoutPromise } from "./index" */ export const runHealthScript = async ( runCommand: string[], - overlay: Overlay, + subcontainer: SubContainer, { timeout = 30000, errorMessage = `Error while running command: ${runCommand}`, @@ -22,7 +22,7 @@ export const runHealthScript = async ( } = {}, ): Promise => { const res = await Promise.race([ - overlay.exec(runCommand), + subcontainer.exec(runCommand), timeoutPromise(timeout), ]).catch((e) => { console.warn(errorMessage) diff --git a/sdk/lib/index.ts b/sdk/lib/index.ts index a023c4e81..a4caf9c01 100644 --- a/sdk/lib/index.ts +++ b/sdk/lib/index.ts @@ -1,5 +1,5 @@ export { Daemons } from "./mainFn/Daemons" -export { Overlay } from "./util/Overlay" +export { SubContainer } from "./util/SubContainer" export { StartSdk } from "./StartSdk" export { setupManifest } from "./manifest/setupManifest" export { FileHelper } from "./util/fileHelper" diff --git a/sdk/lib/mainFn/CommandController.ts b/sdk/lib/mainFn/CommandController.ts index 8635f76e6..8a0505f68 100644 --- a/sdk/lib/mainFn/CommandController.ts +++ b/sdk/lib/mainFn/CommandController.ts @@ -6,32 +6,35 @@ import { asError } from "../util/asError" import { ExecSpawnable, MountOptions, - NonDestroyableOverlay, - Overlay, -} from "../util/Overlay" + SubContainerHandle, + SubContainer, +} from "../util/SubContainer" import { splitCommand } from "../util/splitCommand" -import { cpExecFile, cpExec } from "./Daemons" +import * as cp from "child_process" export class CommandController { private constructor( readonly runningAnswer: Promise, - private readonly overlay: ExecSpawnable, - readonly pid: number | undefined, + private state: { exited: boolean }, + private readonly subcontainer: SubContainer, + private process: cp.ChildProcessWithoutNullStreams, readonly sigtermTimeout: number = DEFAULT_SIGTERM_TIMEOUT, ) {} static of() { return async ( effects: T.Effects, - imageId: { - id: keyof Manifest["images"] & T.ImageId - sharedRun?: boolean - }, + subcontainer: + | { + id: keyof Manifest["images"] & T.ImageId + sharedRun?: boolean + } + | SubContainer, command: T.CommandType, options: { // Defaults to the DEFAULT_SIGTERM_TIMEOUT = 30_000ms sigtermTimeout?: number mounts?: { path: string; options: MountOptions }[] - overlay?: ExecSpawnable + runAsInit?: boolean env?: | { [variable: string]: string @@ -44,49 +47,60 @@ export class CommandController { }, ) => { const commands = splitCommand(command) - const overlay = - options.overlay || - (await (async () => { - const overlay = await Overlay.of(effects, imageId) - for (let mount of options.mounts || []) { - await overlay.mount(mount.options, mount.path) - } - return overlay - })()) - const childProcess = await overlay.spawn(commands, { - env: options.env, - }) + const subc = + subcontainer instanceof SubContainer + ? subcontainer + : await (async () => { + const subc = await SubContainer.of(effects, subcontainer) + for (let mount of options.mounts || []) { + await subc.mount(mount.options, mount.path) + } + return subc + })() + let childProcess: cp.ChildProcessWithoutNullStreams + if (options.runAsInit) { + childProcess = await subc.launch(commands, { + env: options.env, + }) + } else { + childProcess = await subc.spawn(commands, { + env: options.env, + }) + } + const state = { exited: false } const answer = new Promise((resolve, reject) => { - childProcess.stdout.on( - "data", - options.onStdout ?? - ((data: any) => { - console.log(data.toString()) - }), - ) - childProcess.stderr.on( - "data", - options.onStderr ?? - ((data: any) => { - console.error(asError(data)) - }), - ) - - childProcess.on("exit", (code: any) => { - if (code === 0) { + childProcess.on("exit", (code) => { + state.exited = true + if ( + code === 0 || + code === 143 || + (code === null && childProcess.signalCode == "SIGTERM") + ) { return resolve(null) } - return reject(new Error(`${commands[0]} exited with code ${code}`)) + if (code) { + return reject(new Error(`${commands[0]} exited with code ${code}`)) + } else { + return reject( + new Error( + `${commands[0]} exited with signal ${childProcess.signalCode}`, + ), + ) + } }) }) - const pid = childProcess.pid - - return new CommandController(answer, overlay, pid, options.sigtermTimeout) + return new CommandController( + answer, + state, + subc, + childProcess, + options.sigtermTimeout, + ) } } - get nonDestroyableOverlay() { - return new NonDestroyableOverlay(this.overlay) + get subContainerHandle() { + return new SubContainerHandle(this.subcontainer) } async wait({ timeout = NO_TIMEOUT } = {}) { if (timeout > 0) @@ -96,75 +110,30 @@ export class CommandController { try { return await this.runningAnswer } finally { - if (this.pid !== undefined) { - await cpExecFile("pkill", ["-9", "-s", String(this.pid)]).catch( - (_) => {}, - ) + if (!this.state.exited) { + this.process.kill("SIGKILL") } - await this.overlay.destroy?.().catch((_) => {}) + await this.subcontainer.destroy?.().catch((_) => {}) } } async term({ signal = SIGTERM, timeout = this.sigtermTimeout } = {}) { - if (this.pid === undefined) return try { - await cpExecFile("pkill", [ - `-${signal.replace("SIG", "")}`, - "-s", - String(this.pid), - ]) - - const didTimeout = await waitSession(this.pid, timeout) - if (didTimeout) { - await cpExecFile("pkill", [`-9`, "-s", String(this.pid)]).catch( - (_) => {}, - ) + if (!this.state.exited) { + if (!this.process.kill(signal)) { + console.error( + `failed to send signal ${signal} to pid ${this.process.pid}`, + ) + } } - } finally { - await this.overlay.destroy?.() - } - } -} -function waitSession( - sid: number, - timeout = NO_TIMEOUT, - interval = 100, -): Promise { - let nextInterval = interval * 2 - if (timeout >= 0 && timeout < nextInterval) { - nextInterval = timeout - } - let nextTimeout = timeout - if (timeout > 0) { - if (timeout >= interval) { - nextTimeout -= interval - } else { - nextTimeout = 0 + if (signal !== "SIGKILL") { + setTimeout(() => { + this.process.kill("SIGKILL") + }, timeout) + } + await this.runningAnswer + } finally { + await this.subcontainer.destroy?.() } } - return new Promise((resolve, reject) => { - let next: NodeJS.Timeout | null = null - if (timeout !== 0) { - next = setTimeout(() => { - waitSession(sid, nextTimeout, nextInterval).then(resolve, reject) - }, interval) - } - cpExecFile("ps", [`--sid=${sid}`, "-o", "--pid="]).then( - (_) => { - if (timeout === 0) { - resolve(true) - } - }, - (e) => { - if (next) { - clearTimeout(next) - } - if (typeof e === "object" && e && "code" in e && e.code) { - resolve(false) - } else { - reject(e) - } - }, - ) - }) } diff --git a/sdk/lib/mainFn/Daemon.ts b/sdk/lib/mainFn/Daemon.ts index 20a067ff6..87a7d705d 100644 --- a/sdk/lib/mainFn/Daemon.ts +++ b/sdk/lib/mainFn/Daemon.ts @@ -1,6 +1,6 @@ import * as T from "../types" import { asError } from "../util/asError" -import { ExecSpawnable, MountOptions, Overlay } from "../util/Overlay" +import { ExecSpawnable, MountOptions, SubContainer } from "../util/SubContainer" import { CommandController } from "./CommandController" const TIMEOUT_INCREMENT_MS = 1000 @@ -14,20 +14,21 @@ export class Daemon { private commandController: CommandController | null = null private shouldBeRunning = false constructor(private startCommand: () => Promise) {} - get overlay(): undefined | ExecSpawnable { - return this.commandController?.nonDestroyableOverlay + get subContainerHandle(): undefined | ExecSpawnable { + return this.commandController?.subContainerHandle } static of() { return async ( effects: T.Effects, - imageId: { - id: keyof Manifest["images"] & T.ImageId - sharedRun?: boolean - }, + subcontainer: + | { + id: keyof Manifest["images"] & T.ImageId + sharedRun?: boolean + } + | SubContainer, command: T.CommandType, options: { mounts?: { path: string; options: MountOptions }[] - overlay?: Overlay env?: | { [variable: string]: string @@ -41,7 +42,12 @@ export class Daemon { }, ) => { const startCommand = () => - CommandController.of()(effects, imageId, command, options) + CommandController.of()( + effects, + subcontainer, + command, + options, + ) return new Daemon(startCommand) } } diff --git a/sdk/lib/mainFn/Daemons.ts b/sdk/lib/mainFn/Daemons.ts index 3134d0459..1ecec28d3 100644 --- a/sdk/lib/mainFn/Daemons.ts +++ b/sdk/lib/mainFn/Daemons.ts @@ -7,7 +7,12 @@ import { TriggerInput } from "../trigger/TriggerInput" import { defaultTrigger } from "../trigger/defaultTrigger" import * as T from "../types" import { Mounts } from "./Mounts" -import { CommandOptions, MountOptions, Overlay } from "../util/Overlay" +import { + CommandOptions, + ExecSpawnable, + MountOptions, + SubContainer, +} from "../util/SubContainer" import { splitCommand } from "../util/splitCommand" import { promisify } from "node:util" @@ -23,7 +28,9 @@ export const cpExec = promisify(CP.exec) export const cpExecFile = promisify(CP.execFile) export type Ready = { display: string | null - fn: () => Promise | HealthCheckResult + fn: ( + spawnable: ExecSpawnable, + ) => Promise | HealthCheckResult trigger?: Trigger } diff --git a/sdk/lib/mainFn/HealthDaemon.ts b/sdk/lib/mainFn/HealthDaemon.ts index 1a036532f..7ace3ed7b 100644 --- a/sdk/lib/mainFn/HealthDaemon.ts +++ b/sdk/lib/mainFn/HealthDaemon.ts @@ -100,16 +100,25 @@ export class HealthDaemon { !res.done; res = await Promise.race([status, trigger.next()]) ) { - const response: HealthCheckResult = await Promise.resolve( - this.ready.fn(), - ).catch((err) => { - console.error(asError(err)) - return { + const handle = (await this.daemon).subContainerHandle + + if (handle) { + const response: HealthCheckResult = await Promise.resolve( + this.ready.fn(handle), + ).catch((err) => { + console.error(asError(err)) + return { + result: "failure", + message: "message" in err ? err.message : String(err), + } + }) + await this.setHealth(response) + } else { + await this.setHealth({ result: "failure", - message: "message" in err ? err.message : String(err), - } - }) - await this.setHealth(response) + message: "Daemon not running", + }) + } } }).catch((err) => console.error(`Daemon ${this.id} failed: ${err}`)) diff --git a/sdk/lib/mainFn/Mounts.ts b/sdk/lib/mainFn/Mounts.ts index 968b77b4e..bd947b759 100644 --- a/sdk/lib/mainFn/Mounts.ts +++ b/sdk/lib/mainFn/Mounts.ts @@ -1,5 +1,5 @@ import * as T from "../types" -import { MountOptions } from "../util/Overlay" +import { MountOptions } from "../util/SubContainer" type MountArray = { path: string; options: MountOptions }[] diff --git a/sdk/lib/osBindings/CreateOverlayedImageParams.ts b/sdk/lib/osBindings/CreateSubcontainerFsParams.ts similarity index 70% rename from sdk/lib/osBindings/CreateOverlayedImageParams.ts rename to sdk/lib/osBindings/CreateSubcontainerFsParams.ts index aad94f01f..729ad4240 100644 --- a/sdk/lib/osBindings/CreateOverlayedImageParams.ts +++ b/sdk/lib/osBindings/CreateSubcontainerFsParams.ts @@ -1,4 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ImageId } from "./ImageId" -export type CreateOverlayedImageParams = { imageId: ImageId } +export type CreateSubcontainerFsParams = { imageId: ImageId } diff --git a/sdk/lib/osBindings/DestroyOverlayedImageParams.ts b/sdk/lib/osBindings/DestroySubcontainerFsParams.ts similarity index 71% rename from sdk/lib/osBindings/DestroyOverlayedImageParams.ts rename to sdk/lib/osBindings/DestroySubcontainerFsParams.ts index b5b7484a2..3f85d2217 100644 --- a/sdk/lib/osBindings/DestroyOverlayedImageParams.ts +++ b/sdk/lib/osBindings/DestroySubcontainerFsParams.ts @@ -1,4 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Guid } from "./Guid" -export type DestroyOverlayedImageParams = { guid: Guid } +export type DestroySubcontainerFsParams = { guid: Guid } diff --git a/sdk/lib/osBindings/MainStatus.ts b/sdk/lib/osBindings/MainStatus.ts index a528aa187..1f6b3babe 100644 --- a/sdk/lib/osBindings/MainStatus.ts +++ b/sdk/lib/osBindings/MainStatus.ts @@ -1,20 +1,20 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { HealthCheckId } from "./HealthCheckId" import type { NamedHealthCheckResult } from "./NamedHealthCheckResult" +import type { StartStop } from "./StartStop" export type MainStatus = | { status: "stopped" } | { status: "restarting" } | { status: "restoring" } | { status: "stopping" } - | { status: "starting" } + | { + status: "starting" + health: { [key: HealthCheckId]: NamedHealthCheckResult } + } | { status: "running" started: string health: { [key: HealthCheckId]: NamedHealthCheckResult } } - | { - status: "backingUp" - started: string | null - health: { [key: HealthCheckId]: NamedHealthCheckResult } - } + | { status: "backingUp"; onComplete: StartStop } diff --git a/sdk/lib/osBindings/StartStop.ts b/sdk/lib/osBindings/StartStop.ts new file mode 100644 index 000000000..c8be35fb7 --- /dev/null +++ b/sdk/lib/osBindings/StartStop.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type StartStop = "start" | "stop" diff --git a/sdk/lib/osBindings/index.ts b/sdk/lib/osBindings/index.ts index 59cbb7b13..9492fe796 100644 --- a/sdk/lib/osBindings/index.ts +++ b/sdk/lib/osBindings/index.ts @@ -31,7 +31,7 @@ export { CheckDependenciesParam } from "./CheckDependenciesParam" export { CheckDependenciesResult } from "./CheckDependenciesResult" export { Cifs } from "./Cifs" export { ContactInfo } from "./ContactInfo" -export { CreateOverlayedImageParams } from "./CreateOverlayedImageParams" +export { CreateSubcontainerFsParams } from "./CreateSubcontainerFsParams" export { CurrentDependencies } from "./CurrentDependencies" export { CurrentDependencyInfo } from "./CurrentDependencyInfo" export { DataUrl } from "./DataUrl" @@ -41,7 +41,7 @@ export { DependencyMetadata } from "./DependencyMetadata" export { DependencyRequirement } from "./DependencyRequirement" export { DepInfo } from "./DepInfo" export { Description } from "./Description" -export { DestroyOverlayedImageParams } from "./DestroyOverlayedImageParams" +export { DestroySubcontainerFsParams } from "./DestroySubcontainerFsParams" export { Duration } from "./Duration" export { EchoParams } from "./EchoParams" export { EncryptedWire } from "./EncryptedWire" @@ -145,6 +145,7 @@ export { SetupStatusRes } from "./SetupStatusRes" export { SignAssetParams } from "./SignAssetParams" export { SignerInfo } from "./SignerInfo" export { SmtpValue } from "./SmtpValue" +export { StartStop } from "./StartStop" export { Status } from "./Status" export { UpdatingState } from "./UpdatingState" export { VerifyCifsParams } from "./VerifyCifsParams" diff --git a/sdk/lib/test/startosTypeValidation.test.ts b/sdk/lib/test/startosTypeValidation.test.ts index 661f21079..bcbc4b6ec 100644 --- a/sdk/lib/test/startosTypeValidation.test.ts +++ b/sdk/lib/test/startosTypeValidation.test.ts @@ -3,11 +3,13 @@ import { CheckDependenciesParam, ExecuteAction, GetConfiguredParams, + GetStoreParams, SetDataVersionParams, SetMainStatus, + SetStoreParams, } from ".././osBindings" -import { CreateOverlayedImageParams } from ".././osBindings" -import { DestroyOverlayedImageParams } from ".././osBindings" +import { CreateSubcontainerFsParams } from ".././osBindings" +import { DestroySubcontainerFsParams } from ".././osBindings" import { BindParams } from ".././osBindings" import { GetHostInfoParams } from ".././osBindings" import { SetConfigured } from ".././osBindings" @@ -24,21 +26,28 @@ import { GetPrimaryUrlParams } from ".././osBindings" import { ListServiceInterfacesParams } from ".././osBindings" import { ExportActionParams } from ".././osBindings" import { MountParams } from ".././osBindings" +import { StringObject } from "../util" function typeEquality(_a: ExpectedType) {} type WithCallback = Omit & { callback: () => void } +type EffectsTypeChecker = { + [K in keyof T]: T[K] extends (args: infer A) => any + ? A + : T[K] extends StringObject + ? EffectsTypeChecker + : never +} + describe("startosTypeValidation ", () => { test(`checking the params match`, () => { const testInput: any = {} - typeEquality<{ - [K in keyof Effects]: Effects[K] extends (args: infer A) => any - ? A - : never - }>({ + typeEquality({ executeAction: {} as ExecuteAction, - createOverlayedImage: {} as CreateOverlayedImageParams, - destroyOverlayedImage: {} as DestroyOverlayedImageParams, + subcontainer: { + createFs: {} as CreateSubcontainerFsParams, + destroyFs: {} as DestroySubcontainerFsParams, + }, clearBindings: undefined, getInstalledPackages: undefined, bind: {} as BindParams, @@ -55,7 +64,10 @@ describe("startosTypeValidation ", () => { getSslKey: {} as GetSslKeyParams, getServiceInterface: {} as WithCallback, setDependencies: {} as SetDependenciesParams, - store: {} as never, + store: { + get: {} as any, // as GetStoreParams, + set: {} as any, // as SetStoreParams, + }, getSystemSmtp: {} as WithCallback, getContainerIp: undefined, getServicePortForward: {} as GetServicePortForwardParams, diff --git a/sdk/lib/trigger/index.ts b/sdk/lib/trigger/index.ts index 6da034262..eb058437f 100644 --- a/sdk/lib/trigger/index.ts +++ b/sdk/lib/trigger/index.ts @@ -1,3 +1,4 @@ +import { ExecSpawnable } from "../util/SubContainer" import { TriggerInput } from "./TriggerInput" export { changeOnFirstSuccess } from "./changeOnFirstSuccess" export { cooldownTrigger } from "./cooldownTrigger" diff --git a/sdk/lib/types.ts b/sdk/lib/types.ts index d448b2255..772609ea0 100644 --- a/sdk/lib/types.ts +++ b/sdk/lib/types.ts @@ -25,6 +25,7 @@ import { Daemons } from "./mainFn/Daemons" import { StorePath } from "./store/PathBuilder" import { ExposedStorePaths } from "./store/setupExposeStore" import { UrlString } from "./util/getServiceInterface" +import { StringObject, ToKebab } from "./util" export * from "./osBindings" export { SDKManifest } from "./manifest/ManifestTypes" export { HealthReceipt } from "./health/HealthReceipt" @@ -286,6 +287,16 @@ export type PropertiesReturn = { [key: string]: PropertiesValue } +export type EffectMethod = { + [K in keyof T]-?: K extends string + ? T[K] extends Function + ? ToKebab + : T[K] extends StringObject + ? `${ToKebab}.${EffectMethod}` + : never + : never +}[keyof T] + /** Used to reach out from the pure js runtime */ export type Effects = { // action @@ -352,12 +363,13 @@ export type Effects = { /** sets the result of a health check */ setHealth(o: SetHealth): Promise - // image - - /** A low level api used by Overlay */ - createOverlayedImage(options: { imageId: string }): Promise<[string, string]> - /** A low level api used by Overlay */ - destroyOverlayedImage(options: { guid: string }): Promise + // subcontainer + subcontainer: { + /** A low level api used by SubContainer */ + createFs(options: { imageId: string }): Promise<[string, string]> + /** A low level api used by SubContainer */ + destroyFs(options: { guid: string }): Promise + } // net diff --git a/sdk/lib/util/Overlay.ts b/sdk/lib/util/SubContainer.ts similarity index 66% rename from sdk/lib/util/Overlay.ts rename to sdk/lib/util/SubContainer.ts index 14908b90f..1cf9484bc 100644 --- a/sdk/lib/util/Overlay.ts +++ b/sdk/lib/util/SubContainer.ts @@ -13,17 +13,21 @@ type ExecResults = { stderr: string | Buffer } +export type ExecOptions = { + input?: string | Buffer +} + /** - * This is the type that is going to describe what an overlay could do. The main point of the - * overlay is to have commands that run in a chrooted environment. This is useful for running + * This is the type that is going to describe what an subcontainer could do. The main point of the + * subcontainer is to have commands that run in a chrooted environment. This is useful for running * commands in a containerized environment. But, I wanted the destroy to sometimes be doable, for example the - * case where the overlay isn't owned by the process, the overlay shouldn't be destroyed. + * case where the subcontainer isn't owned by the process, the subcontainer shouldn't be destroyed. */ export interface ExecSpawnable { get destroy(): undefined | (() => Promise) exec( command: string[], - options?: CommandOptions, + options?: CommandOptions & ExecOptions, timeoutMs?: number | null, ): Promise spawn( @@ -37,24 +41,34 @@ export interface ExecSpawnable { * Implements: * @see {@link ExecSpawnable} */ -export class Overlay implements ExecSpawnable { - private destroyed = false +export class SubContainer implements ExecSpawnable { + private leader: cp.ChildProcess + private leaderExited: boolean = false private constructor( readonly effects: T.Effects, readonly imageId: T.ImageId, readonly rootfs: string, readonly guid: T.Guid, - ) {} + ) { + this.leaderExited = false + this.leader = cp.spawn("start-cli", ["subcontainer", "launch", rootfs], { + killSignal: "SIGKILL", + stdio: "ignore", + }) + this.leader.on("exit", () => { + this.leaderExited = true + }) + } static async of( effects: T.Effects, image: { id: T.ImageId; sharedRun?: boolean }, ) { const { id, sharedRun } = image - const [rootfs, guid] = await effects.createOverlayedImage({ + const [rootfs, guid] = await effects.subcontainer.createFs({ imageId: id as string, }) - const shared = ["dev", "sys", "proc"] + const shared = ["dev", "sys"] if (!!sharedRun) { shared.push("run") } @@ -69,27 +83,27 @@ export class Overlay implements ExecSpawnable { await execFile("mount", ["--rbind", from, to]) } - return new Overlay(effects, id, rootfs, guid) + return new SubContainer(effects, id, rootfs, guid) } static async with( effects: T.Effects, image: { id: T.ImageId; sharedRun?: boolean }, mounts: { options: MountOptions; path: string }[], - fn: (overlay: Overlay) => Promise, + fn: (subContainer: SubContainer) => Promise, ): Promise { - const overlay = await Overlay.of(effects, image) + const subContainer = await SubContainer.of(effects, image) try { for (let mount of mounts) { - await overlay.mount(mount.options, mount.path) + await subContainer.mount(mount.options, mount.path) } - return await fn(overlay) + return await fn(subContainer) } finally { - await overlay.destroy() + await subContainer.destroy() } } - async mount(options: MountOptions, path: string): Promise { + async mount(options: MountOptions, path: string): Promise { path = path.startsWith("/") ? `${this.rootfs}${path}` : `${this.rootfs}/${path}` @@ -134,19 +148,35 @@ export class Overlay implements ExecSpawnable { return this } + private async killLeader() { + if (this.leaderExited) { + return + } + return new Promise((resolve, reject) => { + try { + this.leader.on("exit", () => { + resolve() + }) + if (!this.leader.kill("SIGKILL")) { + reject(new Error("kill(2) failed")) + } + } catch (e) { + reject(e) + } + }) + } + get destroy() { return async () => { - if (this.destroyed) return - this.destroyed = true - const imageId = this.imageId const guid = this.guid - await this.effects.destroyOverlayedImage({ guid }) + await this.killLeader() + await this.effects.subcontainer.destroyFs({ guid }) } } async exec( command: string[], - options?: CommandOptions, + options?: CommandOptions & ExecOptions, timeoutMs: number | null = 30000, ): Promise<{ exitCode: number | null @@ -173,7 +203,8 @@ export class Overlay implements ExecSpawnable { const child = cp.spawn( "start-cli", [ - "chroot", + "subcontainer", + "exec", `--env=/media/startos/images/${this.imageId}.env`, `--workdir=${workdir}`, ...extra, @@ -182,6 +213,18 @@ export class Overlay implements ExecSpawnable { ], options || {}, ) + if (options?.input) { + await new Promise((resolve, reject) => + child.stdin.write(options.input, (e) => { + if (e) { + reject(e) + } else { + resolve() + } + }), + ) + await new Promise((resolve) => child.stdin.end(resolve)) + } const pid = child.pid const stdout = { data: "" as string | Buffer } const stderr = { data: "" as string | Buffer } @@ -201,25 +244,65 @@ export class Overlay implements ExecSpawnable { } return new Promise((resolve, reject) => { child.on("error", reject) - if (timeoutMs !== null && pid) { - setTimeout( - () => execFile("pkill", ["-9", "-s", String(pid)]).catch((_) => {}), - timeoutMs, - ) + let killTimeout: NodeJS.Timeout | undefined + if (timeoutMs !== null && child.pid) { + killTimeout = setTimeout(() => child.kill("SIGKILL"), timeoutMs) } child.stdout.on("data", appendData(stdout)) child.stderr.on("data", appendData(stderr)) - child.on("exit", (code, signal) => + child.on("exit", (code, signal) => { + clearTimeout(killTimeout) resolve({ exitCode: code, exitSignal: signal, stdout: stdout.data, stderr: stderr.data, - }), - ) + }) + }) }) } + async launch( + command: string[], + options?: CommandOptions, + ): Promise { + const imageMeta: any = await fs + .readFile(`/media/startos/images/${this.imageId}.json`, { + encoding: "utf8", + }) + .catch(() => "{}") + .then(JSON.parse) + let extra: string[] = [] + if (options?.user) { + extra.push(`--user=${options.user}`) + delete options.user + } + let workdir = imageMeta.workdir || "/" + if (options?.cwd) { + workdir = options.cwd + delete options.cwd + } + await this.killLeader() + this.leaderExited = false + this.leader = cp.spawn( + "start-cli", + [ + "subcontainer", + "launch", + `--env=/media/startos/images/${this.imageId}.env`, + `--workdir=${workdir}`, + ...extra, + this.rootfs, + ...command, + ], + { ...options, stdio: "inherit" }, + ) + this.leader.on("exit", () => { + this.leaderExited = true + }) + return this.leader as cp.ChildProcessWithoutNullStreams + } + async spawn( command: string[], options?: CommandOptions, @@ -243,7 +326,8 @@ export class Overlay implements ExecSpawnable { return cp.spawn( "start-cli", [ - "chroot", + "subcontainer", + "exec", `--env=/media/startos/images/${this.imageId}.env`, `--workdir=${workdir}`, ...extra, @@ -256,12 +340,12 @@ export class Overlay implements ExecSpawnable { } /** - * Take an overlay but remove the ability to add the mounts and the destroy function. + * Take an subcontainer but remove the ability to add the mounts and the destroy function. * Lets other functions, like health checks, to not destroy the parents. * */ -export class NonDestroyableOverlay implements ExecSpawnable { - constructor(private overlay: ExecSpawnable) {} +export class SubContainerHandle implements ExecSpawnable { + constructor(private subContainer: ExecSpawnable) {} get destroy() { return undefined } @@ -271,13 +355,13 @@ export class NonDestroyableOverlay implements ExecSpawnable { options?: CommandOptions, timeoutMs?: number | null, ): Promise { - return this.overlay.exec(command, options, timeoutMs) + return this.subContainer.exec(command, options, timeoutMs) } spawn( command: string[], options?: CommandOptions, ): Promise { - return this.overlay.spawn(command, options) + return this.subContainer.spawn(command, options) } } diff --git a/sdk/lib/util/index.ts b/sdk/lib/util/index.ts index d7606d5d0..9246cf791 100644 --- a/sdk/lib/util/index.ts +++ b/sdk/lib/util/index.ts @@ -3,7 +3,7 @@ import "./fileHelper" import "../store/getStore" import "./deepEqual" import "./deepMerge" -import "./Overlay" +import "./SubContainer" import "./once" export { GetServiceInterface, getServiceInterface } from "./getServiceInterface" diff --git a/sdk/lib/util/typeHelpers.ts b/sdk/lib/util/typeHelpers.ts index f45a46f1e..d29d5c986 100644 --- a/sdk/lib/util/typeHelpers.ts +++ b/sdk/lib/util/typeHelpers.ts @@ -21,3 +21,96 @@ export type NoAny = NeverPossible extends A ? never : A : A + +type CapitalLetters = + | "A" + | "B" + | "C" + | "D" + | "E" + | "F" + | "G" + | "H" + | "I" + | "J" + | "K" + | "L" + | "M" + | "N" + | "O" + | "P" + | "Q" + | "R" + | "S" + | "T" + | "U" + | "V" + | "W" + | "X" + | "Y" + | "Z" + +type Numbers = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" + +type CapitalChars = CapitalLetters | Numbers + +export type ToKebab = S extends string + ? S extends `${infer Head}${CapitalChars}${infer Tail}` // string has a capital char somewhere + ? Head extends "" // there is a capital char in the first position + ? Tail extends "" + ? Lowercase /* 'A' */ + : S extends `${infer Caps}${Tail}` // tail exists, has capital characters + ? Caps extends CapitalChars + ? Tail extends CapitalLetters + ? `${Lowercase}-${Lowercase}` /* 'AB' */ + : Tail extends `${CapitalLetters}${string}` + ? `${ToKebab}-${ToKebab}` /* first tail char is upper? 'ABcd' */ + : `${ToKebab}${ToKebab}` /* 'AbCD','AbcD', */ /* TODO: if tail is only numbers, append without underscore */ + : never /* never reached, used for inference of caps */ + : never + : Tail extends "" /* 'aB' 'abCD' 'ABCD' 'AB' */ + ? S extends `${Head}${infer Caps}` + ? Caps extends CapitalChars + ? Head extends Lowercase /* 'abcD' */ + ? Caps extends Numbers + ? // Head exists and is lowercase, tail does not, Caps is a number, we may be in a sub-select + // if head ends with number, don't split head an Caps, keep contiguous numbers together + Head extends `${string}${Numbers}` + ? never + : // head does not end in number, safe to split. 'abc2' -> 'abc-2' + `${ToKebab}-${Caps}` + : `${ToKebab}-${ToKebab}` /* 'abcD' 'abc25' */ + : never /* stop union type forming */ + : never + : never /* never reached, used for inference of caps */ + : S extends `${Head}${infer Caps}${Tail}` /* 'abCd' 'ABCD' 'AbCd' 'ABcD' */ + ? Caps extends CapitalChars + ? Head extends Lowercase /* is 'abCd' 'abCD' ? */ + ? Tail extends CapitalLetters /* is 'abCD' where Caps = 'C' */ + ? `${ToKebab}-${ToKebab}-${Lowercase}` /* aBCD Tail = 'D', Head = 'aB' */ + : Tail extends `${CapitalLetters}${string}` /* is 'aBCd' where Caps = 'B' */ + ? Head extends Numbers + ? never /* stop union type forming */ + : Head extends `${string}${Numbers}` + ? never /* stop union type forming */ + : `${Head}-${ToKebab}-${ToKebab}` /* 'aBCd' => `${'a'}-${Lowercase<'B'>}-${ToSnake<'Cd'>}` */ + : `${ToKebab}-${Lowercase}${ToKebab}` /* 'aBcD' where Caps = 'B' tail starts as lowercase */ + : never + : never + : never + : S /* 'abc' */ + : never + +export type StringObject = Record + +function test() { + // prettier-ignore + const t = (a: ( + A extends B ? ( + B extends A ? null : never + ) : never + )) =>{ } + t<"foo-bar", ToKebab<"FooBar">>(null) + // @ts-expect-error + t<"foo-3ar", ToKebab<"FooBar">>(null) +}