Merge branch 'next/minor' of github.com:Start9Labs/start-os into next/major

sdk/lib/StartSdk.ts

@@ -30,17 +30,11 @@ import { healthCheck, HealthCheckParams } from "./health/HealthCheck"
 import { checkPortListening } from "./health/checkFns/checkPortListening"
 import { checkWebUrl, runHealthScript } from "./health/checkFns"
 import { List } from "./config/builder/list"
-import { Migration } from "./inits/migrations/Migration"
 import { Install, InstallFn } from "./inits/setupInstall"
 import { setupActions } from "./actions/setupActions"
 import { setupDependencyConfig } from "./dependencies/setupDependencyConfig"
 import { SetupBackupsParams, setupBackups } from "./backup/setupBackups"
 import { setupInit } from "./inits/setupInit"
-import {
-  EnsureUniqueId,
-  Migrations,
-  setupMigrations,
-} from "./inits/migrations/setupMigrations"
 import { Uninstall, UninstallFn, setupUninstall } from "./inits/setupUninstall"
 import { setupMain } from "./mainFn"
 import { defaultTrigger } from "./trigger/defaultTrigger"
@@ -67,7 +61,7 @@ import {
 } from "./util/getServiceInterface"
 import { getServiceInterfaces } from "./util/getServiceInterfaces"
 import { getStore } from "./store/getStore"
-import { CommandOptions, MountOptions, Overlay } from "./util/Overlay"
+import { CommandOptions, MountOptions, SubContainer } from "./util/SubContainer"
 import { splitCommand } from "./util/splitCommand"
 import { Mounts } from "./mainFn/Mounts"
 import { Dependency } from "./Dependency"
@@ -75,9 +69,13 @@ import * as T from "./types"
 import { testTypeVersion, ValidateExVer } from "./exver"
 import { ExposedStorePaths } from "./store/setupExposeStore"
 import { PathBuilder, extractJsonPath, pathBuilder } from "./store/PathBuilder"
-import { checkAllDependencies } from "./dependencies/dependencies"
+import {
+  CheckDependencies,
+  checkDependencies,
+} from "./dependencies/dependencies"
 import { health } from "."
 import { GetSslCertificate } from "./util/GetSslCertificate"
+import { VersionGraph } from "./version"
 
 export const SDKVersion = testTypeVersion("0.3.6")
 
@@ -141,8 +139,58 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
       }]?: Dependency
     }
 
+    type NestedEffects = "subcontainer" | "store"
+    type InterfaceEffects =
+      | "getServiceInterface"
+      | "listServiceInterfaces"
+      | "exportServiceInterface"
+      | "clearServiceInterfaces"
+      | "bind"
+      | "getHostInfo"
+      | "getPrimaryUrl"
+    type MainUsedEffects = "setMainStatus" | "setHealth"
+    type AlreadyExposed = "getSslCertificate" | "getSystemSmtp"
+
+    // prettier-ignore
+    type StartSdkEffectWrapper = {
+      [K in keyof Omit<Effects, NestedEffects | InterfaceEffects | MainUsedEffects| AlreadyExposed>]: (effects: Effects, ...args: Parameters<Effects[K]>) => ReturnType<Effects[K]>
+    }
+    const startSdkEffectWrapper: StartSdkEffectWrapper = {
+      executeAction: (effects, ...args) => effects.executeAction(...args),
+      exportAction: (effects, ...args) => effects.exportAction(...args),
+      clearActions: (effects, ...args) => effects.clearActions(...args),
+      getConfigured: (effects, ...args) => effects.getConfigured(...args),
+      setConfigured: (effects, ...args) => effects.setConfigured(...args),
+      restart: (effects, ...args) => effects.restart(...args),
+      setDependencies: (effects, ...args) => effects.setDependencies(...args),
+      checkDependencies: (effects, ...args) =>
+        effects.checkDependencies(...args),
+      mount: (effects, ...args) => effects.mount(...args),
+      getInstalledPackages: (effects, ...args) =>
+        effects.getInstalledPackages(...args),
+      exposeForDependents: (effects, ...args) =>
+        effects.exposeForDependents(...args),
+      getServicePortForward: (effects, ...args) =>
+        effects.getServicePortForward(...args),
+      clearBindings: (effects, ...args) => effects.clearBindings(...args),
+      getContainerIp: (effects, ...args) => effects.getContainerIp(...args),
+      getSslKey: (effects, ...args) => effects.getSslKey(...args),
+      setDataVersion: (effects, ...args) => effects.setDataVersion(...args),
+      getDataVersion: (effects, ...args) => effects.getDataVersion(...args),
+      shutdown: (effects, ...args) => effects.shutdown(...args),
+      getDependencies: (effects, ...args) => effects.getDependencies(...args),
+    }
+
     return {
-      checkAllDependencies,
+      ...startSdkEffectWrapper,
+
+      checkDependencies: checkDependencies as <
+        DependencyId extends keyof Manifest["dependencies"] &
+          PackageId = keyof Manifest["dependencies"] & PackageId,
+      >(
+        effects: Effects,
+        packageIds?: DependencyId[],
+      ) => Promise<CheckDependencies<DependencyId>>,
       serviceInterface: {
         getOwn: <E extends Effects>(effects: E, id: ServiceInterfaceId) =>
           removeCallbackTypes<E>(effects)(
@@ -247,7 +295,6 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
           id: string
           description: string
           hasPrimary: boolean
-          disabled: boolean
           type: ServiceInterfaceType
           username: null | string
           path: string
@@ -293,8 +340,8 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
         )
       },
       HealthCheck: {
-        of(o: HealthCheckParams<Manifest>) {
-          return healthCheck<Manifest>(o)
+        of(o: HealthCheckParams) {
+          return healthCheck(o)
         },
       },
       Dependency: {
@@ -311,7 +358,7 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
       setupActions: (...createdActions: CreatedAction<any, any, any>[]) =>
         setupActions<Manifest, Store>(...createdActions),
       setupBackups: (...args: SetupBackupsParams<Manifest>) =>
-        setupBackups<Manifest>(...args),
+        setupBackups<Manifest>(this.manifest, ...args),
       setupConfig: <
         ConfigType extends Config<any, Store> | Config<any, never>,
         Type extends Record<string, any> = ExtractConfigType<ConfigType>,
@@ -380,7 +427,7 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
         }
       },
       setupInit: (
-        migrations: Migrations<Manifest, Store>,
+        versions: VersionGraph<Manifest["version"]>,
         install: Install<Manifest, Store>,
         uninstall: Uninstall<Manifest, Store>,
         setInterfaces: SetInterfaces<Manifest, Store, any, any>,
@@ -391,7 +438,7 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
         exposedStore: ExposedStorePaths,
       ) =>
         setupInit<Manifest, Store>(
-          migrations,
+          versions,
           install,
           uninstall,
           setInterfaces,
@@ -412,15 +459,6 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
           started(onTerm: () => PromiseLike<void>): PromiseLike<void>
         }) => Promise<Daemons<Manifest, any>>,
       ) => setupMain<Manifest, Store>(fn),
-      setupMigrations: <
-        Migrations extends Array<Migration<Manifest, Store, any>>,
-      >(
-        ...migrations: EnsureUniqueId<Migrations>
-      ) =>
-        setupMigrations<Manifest, Store, Migrations>(
-          this.manifest,
-          ...migrations,
-        ),
       setupProperties:
         (
           fn: (options: { effects: Effects }) => Promise<T.SdkPropertiesReturn>,
@@ -541,13 +579,6 @@ export class StartSdk<Manifest extends T.Manifest, Store> {
           >,
         ) => List.dynamicText<Store>(getA),
       },
-      Migration: {
-        of: <Version extends string>(options: {
-          version: Version & ValidateExVer<Version>
-          up: (opts: { effects: Effects }) => Promise<void>
-          down: (opts: { effects: Effects }) => Promise<void>
-        }) => Migration.of<Manifest, Store, Version>(options),
-      },
       StorePath: pathBuilder<Store>(),
       Value: {
         toggle: Value.toggle,
@@ -747,15 +778,12 @@ export async function runCommand<Manifest extends T.Manifest>(
   },
 ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> {
   const commands = splitCommand(command)
-  const overlay = await Overlay.of(effects, image)
-  try {
-    for (let mount of options.mounts || []) {
-      await overlay.mount(mount.options, mount.path)
-    }
-    return await overlay.exec(commands)
-  } finally {
-    await overlay.destroy()
-  }
+  return SubContainer.with(
+    effects,
+    image,
+    options.mounts || [],
+    (subcontainer) => subcontainer.exec(commands),
+  )
 }
 function nullifyProperties(value: T.SdkPropertiesReturn): T.PropertiesReturn {
   return Object.fromEntries(

sdk/lib/backup/Backups.ts

@@ -2,6 +2,7 @@ import * as T from "../types"
 
 import * as child_process from "child_process"
 import { promises as fsPromises } from "fs"
+import { asError } from "../util"
 
 export type BACKUP = "BACKUP"
 export const DEFAULT_OPTIONS: T.BackupOptions = {
@@ -183,7 +184,7 @@ async function runRsync(
   })
 
   spawned.stderr.on("data", (data: unknown) => {
-    console.error(String(data))
+    console.error(`Backups.runAsync`, asError(data))
   })
 
   const id = async () => {

sdk/lib/backup/setupBackups.ts

@@ -8,6 +8,7 @@ export type SetupBackupsParams<M extends T.Manifest> = Array<
 >
 
 export function setupBackups<M extends T.Manifest>(
+  manifest: M,
   ...args: _<SetupBackupsParams<M>>
 ) {
   const backups = Array<Backups<M>>()
@@ -36,6 +37,7 @@ export function setupBackups<M extends T.Manifest>(
         for (const backup of backups) {
           await backup.build(options.pathMaker).restoreBackup(options)
         }
+        await options.effects.setDataVersion({ version: manifest.version })
       }) as T.ExpectedExports.restoreBackup
     },
   }

sdk/lib/config/setupConfig.ts

@@ -57,7 +57,9 @@ export function setupConfig<
   return {
     setConfig: (async ({ effects, input }) => {
       if (!validator.test(input)) {
-        await console.error(String(validator.errorMessage(input)))
+        await console.error(
+          new Error(validator.errorMessage(input)?.toString()),
+        )
         return { error: "Set config type error for config" }
       }
       await effects.clearBindings()

sdk/lib/dependencies/dependencies.ts

@@ -1,131 +1,206 @@
+import { ExtendedVersion, VersionRange } from "../exver"
 import {
   Effects,
   PackageId,
   DependencyRequirement,
   SetHealth,
   CheckDependenciesResult,
+  HealthCheckId,
 } from "../types"
 
-export type CheckAllDependencies = {
-  notInstalled: () => Promise<CheckDependenciesResult[]>
-  notRunning: () => Promise<CheckDependenciesResult[]>
-  configNotSatisfied: () => Promise<CheckDependenciesResult[]>
-  healthErrors: () => Promise<{ [id: string]: SetHealth[] }>
+export type CheckDependencies<DependencyId extends PackageId = PackageId> = {
+  installedSatisfied: (packageId: DependencyId) => boolean
+  installedVersionSatisfied: (packageId: DependencyId) => boolean
+  runningSatisfied: (packageId: DependencyId) => boolean
+  configSatisfied: (packageId: DependencyId) => boolean
+  healthCheckSatisfied: (
+    packageId: DependencyId,
+    healthCheckId: HealthCheckId,
+  ) => boolean
+  satisfied: () => boolean
 
-  isValid: () => Promise<boolean>
-
-  throwIfNotRunning: () => Promise<void>
-  throwIfNotInstalled: () => Promise<void>
-  throwIfConfigNotSatisfied: () => Promise<void>
-  throwIfHealthError: () => Promise<void>
-
-  throwIfNotValid: () => Promise<void>
+  throwIfInstalledNotSatisfied: (packageId: DependencyId) => void
+  throwIfInstalledVersionNotSatisfied: (packageId: DependencyId) => void
+  throwIfRunningNotSatisfied: (packageId: DependencyId) => void
+  throwIfConfigNotSatisfied: (packageId: DependencyId) => void
+  throwIfHealthNotSatisfied: (
+    packageId: DependencyId,
+    healthCheckId?: HealthCheckId,
+  ) => void
+  throwIfNotSatisfied: (packageId?: DependencyId) => void
 }
-export function checkAllDependencies(effects: Effects): CheckAllDependencies {
-  const dependenciesPromise = effects.getDependencies()
-  const resultsPromise = dependenciesPromise.then((dependencies) =>
+export async function checkDependencies<
+  DependencyId extends PackageId = PackageId,
+>(
+  effects: Effects,
+  packageIds?: DependencyId[],
+): Promise<CheckDependencies<DependencyId>> {
+  let [dependencies, results] = await Promise.all([
+    effects.getDependencies(),
     effects.checkDependencies({
-      packageIds: dependencies.map((dep) => dep.id),
+      packageIds,
     }),
-  )
-
-  const dependenciesByIdPromise = dependenciesPromise.then((d) =>
-    d.reduce(
-      (acc, dep) => {
-        acc[dep.id] = dep
-        return acc
-      },
-      {} as { [id: PackageId]: DependencyRequirement },
-    ),
-  )
-
-  const healthErrors = async () => {
-    const results = await resultsPromise
-    const dependenciesById = await dependenciesByIdPromise
-    const answer: { [id: PackageId]: SetHealth[] } = {}
-    for (const result of results) {
-      const dependency = dependenciesById[result.packageId]
-      if (!dependency) continue
-      if (dependency.kind !== "running") continue
-
-      const healthChecks = Object.entries(result.healthChecks)
-        .map(([id, hc]) => ({ ...hc, id }))
-        .filter((x) => !!x.message)
-      if (healthChecks.length === 0) continue
-      answer[result.packageId] = healthChecks
-    }
-    return answer
-  }
-  const configNotSatisfied = () =>
-    resultsPromise.then((x) => x.filter((x) => !x.configSatisfied))
-  const notInstalled = () =>
-    resultsPromise.then((x) => x.filter((x) => !x.isInstalled))
-  const notRunning = async () => {
-    const results = await resultsPromise
-    const dependenciesById = await dependenciesByIdPromise
-    return results.filter((x) => {
-      const dependency = dependenciesById[x.packageId]
-      if (!dependency) return false
-      if (dependency.kind !== "running") return false
-      return !x.isRunning
-    })
-  }
-  const entries = <B>(x: { [k: string]: B }) => Object.entries(x)
-  const first = <A>(x: A[]): A | undefined => x[0]
-  const sinkVoid = <A>(x: A) => void 0
-  const throwIfHealthError = () =>
-    healthErrors()
-      .then(entries)
-      .then(first)
-      .then((x) => {
-        if (!x) return
-        const [id, healthChecks] = x
-        if (healthChecks.length > 0)
-          throw `Package ${id} has the following errors: ${healthChecks.map((x) => x.message).join(", ")}`
-      })
-
-  const throwIfConfigNotSatisfied = () =>
-    configNotSatisfied().then((results) => {
-      throw new Error(
-        `Package ${results[0].packageId} does not have a valid configuration`,
-      )
-    })
-
-  const throwIfNotRunning = () =>
-    notRunning().then((results) => {
-      if (results[0])
-        throw new Error(`Package ${results[0].packageId} is not running`)
-    })
-
-  const throwIfNotInstalled = () =>
-    notInstalled().then((results) => {
-      if (results[0])
-        throw new Error(`Package ${results[0].packageId} is not installed`)
-    })
-  const throwIfNotValid = async () =>
-    Promise.all([
-      throwIfNotRunning(),
-      throwIfNotInstalled(),
-      throwIfConfigNotSatisfied(),
-      throwIfHealthError(),
-    ]).then(sinkVoid)
-
-  const isValid = () =>
-    throwIfNotValid().then(
-      () => true,
-      () => false,
+  ])
+  if (packageIds) {
+    dependencies = dependencies.filter((d) =>
+      (packageIds as PackageId[]).includes(d.id),
     )
+  }
+
+  const find = (packageId: DependencyId) => {
+    const dependencyRequirement = dependencies.find((d) => d.id === packageId)
+    const dependencyResult = results.find((d) => d.packageId === packageId)
+    if (!dependencyRequirement || !dependencyResult) {
+      throw new Error(`Unknown DependencyId ${packageId}`)
+    }
+    return { requirement: dependencyRequirement, result: dependencyResult }
+  }
+
+  const installedSatisfied = (packageId: DependencyId) =>
+    !!find(packageId).result.installedVersion
+  const installedVersionSatisfied = (packageId: DependencyId) => {
+    const dep = find(packageId)
+    return (
+      !!dep.result.installedVersion &&
+      ExtendedVersion.parse(dep.result.installedVersion).satisfies(
+        VersionRange.parse(dep.requirement.versionRange),
+      )
+    )
+  }
+  const runningSatisfied = (packageId: DependencyId) => {
+    const dep = find(packageId)
+    return dep.requirement.kind !== "running" || dep.result.isRunning
+  }
+  const configSatisfied = (packageId: DependencyId) =>
+    find(packageId).result.configSatisfied
+  const healthCheckSatisfied = (
+    packageId: DependencyId,
+    healthCheckId?: HealthCheckId,
+  ) => {
+    const dep = find(packageId)
+    if (
+      healthCheckId &&
+      (dep.requirement.kind !== "running" ||
+        !dep.requirement.healthChecks.includes(healthCheckId))
+    ) {
+      throw new Error(`Unknown HealthCheckId ${healthCheckId}`)
+    }
+    const errors = Object.entries(dep.result.healthChecks)
+      .filter(([id, _]) => (healthCheckId ? id === healthCheckId : true))
+      .filter(([_, res]) => res.result !== "success")
+    return errors.length === 0
+  }
+  const pkgSatisfied = (packageId: DependencyId) =>
+    installedSatisfied(packageId) &&
+    installedVersionSatisfied(packageId) &&
+    runningSatisfied(packageId) &&
+    configSatisfied(packageId) &&
+    healthCheckSatisfied(packageId)
+  const satisfied = (packageId?: DependencyId) =>
+    packageId
+      ? pkgSatisfied(packageId)
+      : dependencies.every((d) => pkgSatisfied(d.id as DependencyId))
+
+  const throwIfInstalledNotSatisfied = (packageId: DependencyId) => {
+    const dep = find(packageId)
+    if (!dep.result.installedVersion) {
+      throw new Error(`${dep.result.title || packageId} is not installed`)
+    }
+  }
+  const throwIfInstalledVersionNotSatisfied = (packageId: DependencyId) => {
+    const dep = find(packageId)
+    if (!dep.result.installedVersion) {
+      throw new Error(`${dep.result.title || packageId} is not installed`)
+    }
+    if (
+      ![dep.result.installedVersion, ...dep.result.satisfies].find((v) =>
+        ExtendedVersion.parse(v).satisfies(
+          VersionRange.parse(dep.requirement.versionRange),
+        ),
+      )
+    ) {
+      throw new Error(
+        `Installed version ${dep.result.installedVersion} of ${dep.result.title || packageId} does not match expected version range ${dep.requirement.versionRange}`,
+      )
+    }
+  }
+  const throwIfRunningNotSatisfied = (packageId: DependencyId) => {
+    const dep = find(packageId)
+    if (dep.requirement.kind === "running" && !dep.result.isRunning) {
+      throw new Error(`${dep.result.title || packageId} is not running`)
+    }
+  }
+  const throwIfConfigNotSatisfied = (packageId: DependencyId) => {
+    const dep = find(packageId)
+    if (!dep.result.configSatisfied) {
+      throw new Error(
+        `${dep.result.title || packageId}'s configuration does not satisfy requirements`,
+      )
+    }
+  }
+  const throwIfHealthNotSatisfied = (
+    packageId: DependencyId,
+    healthCheckId?: HealthCheckId,
+  ) => {
+    const dep = find(packageId)
+    if (
+      healthCheckId &&
+      (dep.requirement.kind !== "running" ||
+        !dep.requirement.healthChecks.includes(healthCheckId))
+    ) {
+      throw new Error(`Unknown HealthCheckId ${healthCheckId}`)
+    }
+    const errors = Object.entries(dep.result.healthChecks)
+      .filter(([id, _]) => (healthCheckId ? id === healthCheckId : true))
+      .filter(([_, res]) => res.result !== "success")
+    if (errors.length) {
+      throw new Error(
+        errors
+          .map(
+            ([_, e]) =>
+              `Health Check ${e.name} of ${dep.result.title || packageId} failed with status ${e.result}${e.message ? `: ${e.message}` : ""}`,
+          )
+          .join("; "),
+      )
+    }
+  }
+  const throwIfPkgNotSatisfied = (packageId: DependencyId) => {
+    throwIfInstalledNotSatisfied(packageId)
+    throwIfInstalledVersionNotSatisfied(packageId)
+    throwIfRunningNotSatisfied(packageId)
+    throwIfConfigNotSatisfied(packageId)
+    throwIfHealthNotSatisfied(packageId)
+  }
+  const throwIfNotSatisfied = (packageId?: DependencyId) =>
+    packageId
+      ? throwIfPkgNotSatisfied(packageId)
+      : (() => {
+          const err = dependencies.flatMap((d) => {
+            try {
+              throwIfPkgNotSatisfied(d.id as DependencyId)
+            } catch (e) {
+              if (e instanceof Error) return [e.message]
+              throw e
+            }
+            return []
+          })
+          if (err.length) {
+            throw new Error(err.join("; "))
+          }
+        })()
 
   return {
-    notRunning,
-    notInstalled,
-    configNotSatisfied,
-    healthErrors,
-    throwIfNotRunning,
+    installedSatisfied,
+    installedVersionSatisfied,
+    runningSatisfied,
+    configSatisfied,
+    healthCheckSatisfied,
+    satisfied,
+    throwIfInstalledNotSatisfied,
+    throwIfInstalledVersionNotSatisfied,
+    throwIfRunningNotSatisfied,
     throwIfConfigNotSatisfied,
-    throwIfNotValid,
-    throwIfNotInstalled,
-    throwIfHealthError,
-    isValid,
+    throwIfHealthNotSatisfied,
+    throwIfNotSatisfied,
   }
 }

sdk/lib/exver/index.ts

@@ -3,7 +3,7 @@ import * as P from "./exver"
 // prettier-ignore
 export type ValidateVersion<T extends String> = 
 T extends `-${infer A}` ? never  :
-T extends `${infer A}-${infer B}` ? ValidateVersion<A> & ValidateVersion<B> :
+T extends `${infer A}-${string}` ? ValidateVersion<A> :
   T extends `${bigint}` ? unknown :
   T extends `${bigint}.${infer A}` ? ValidateVersion<A> :
   never
@@ -16,9 +16,9 @@ export type ValidateExVer<T extends string> =
 
 // prettier-ignore
 export type ValidateExVers<T> =
-  T extends [] ? unknown :
+  T extends [] ? unknown[] :
   T extends [infer A, ...infer B] ? ValidateExVer<A & string> & ValidateExVers<B> :
-  never
+  never[]
 
 type Anchor = {
   type: "Anchor"
@@ -44,7 +44,7 @@ type Not = {
 }
 
 export class VersionRange {
-  private constructor(private atom: Anchor | And | Or | Not | P.Any | P.None) {}
+  private constructor(public atom: Anchor | And | Or | Not | P.Any | P.None) {}
 
   toString(): string {
     switch (this.atom.type) {
@@ -63,67 +63,6 @@ export class VersionRange {
     }
   }
 
-  /**
-   * Returns a boolean indicating whether a given version satisfies the VersionRange
-   * !( >= 1:1 <= 2:2) || <=#bitcoin:1.2.0-alpha:0
-   */
-  satisfiedBy(version: ExtendedVersion): boolean {
-    switch (this.atom.type) {
-      case "Anchor":
-        const otherVersion = this.atom.version
-        switch (this.atom.operator) {
-          case "=":
-            return version.equals(otherVersion)
-          case ">":
-            return version.greaterThan(otherVersion)
-          case "<":
-            return version.lessThan(otherVersion)
-          case ">=":
-            return version.greaterThanOrEqual(otherVersion)
-          case "<=":
-            return version.lessThanOrEqual(otherVersion)
-          case "!=":
-            return !version.equals(otherVersion)
-          case "^":
-            const nextMajor = this.atom.version.incrementMajor()
-            if (
-              version.greaterThanOrEqual(otherVersion) &&
-              version.lessThan(nextMajor)
-            ) {
-              return true
-            } else {
-              return false
-            }
-          case "~":
-            const nextMinor = this.atom.version.incrementMinor()
-            if (
-              version.greaterThanOrEqual(otherVersion) &&
-              version.lessThan(nextMinor)
-            ) {
-              return true
-            } else {
-              return false
-            }
-        }
-      case "And":
-        return (
-          this.atom.left.satisfiedBy(version) &&
-          this.atom.right.satisfiedBy(version)
-        )
-      case "Or":
-        return (
-          this.atom.left.satisfiedBy(version) ||
-          this.atom.right.satisfiedBy(version)
-        )
-      case "Not":
-        return !this.atom.value.satisfiedBy(version)
-      case "Any":
-        return true
-      case "None":
-        return false
-    }
-  }
-
   private static parseAtom(atom: P.VersionRangeAtom): VersionRange {
     switch (atom.type) {
       case "Not":
@@ -207,6 +146,10 @@ export class VersionRange {
   static none() {
     return new VersionRange({ type: "None" })
   }
+
+  satisfiedBy(version: Version | ExtendedVersion) {
+    return version.satisfies(this)
+  }
 }
 
 export class Version {
@@ -266,6 +209,12 @@ export class Version {
     const parsed = P.parse(version, { startRule: "Version" })
     return new Version(parsed.number, parsed.prerelease)
   }
+
+  satisfies(versionRange: VersionRange): boolean {
+    return new ExtendedVersion(null, this, new Version([0], [])).satisfies(
+      versionRange,
+    )
+  }
 }
 
 // #flavor:0.1.2-beta.1:0
@@ -404,6 +353,67 @@ export class ExtendedVersion {
       updatedDownstream,
     )
   }
+
+  /**
+   * Returns a boolean indicating whether a given version satisfies the VersionRange
+   * !( >= 1:1 <= 2:2) || <=#bitcoin:1.2.0-alpha:0
+   */
+  satisfies(versionRange: VersionRange): boolean {
+    switch (versionRange.atom.type) {
+      case "Anchor":
+        const otherVersion = versionRange.atom.version
+        switch (versionRange.atom.operator) {
+          case "=":
+            return this.equals(otherVersion)
+          case ">":
+            return this.greaterThan(otherVersion)
+          case "<":
+            return this.lessThan(otherVersion)
+          case ">=":
+            return this.greaterThanOrEqual(otherVersion)
+          case "<=":
+            return this.lessThanOrEqual(otherVersion)
+          case "!=":
+            return !this.equals(otherVersion)
+          case "^":
+            const nextMajor = versionRange.atom.version.incrementMajor()
+            if (
+              this.greaterThanOrEqual(otherVersion) &&
+              this.lessThan(nextMajor)
+            ) {
+              return true
+            } else {
+              return false
+            }
+          case "~":
+            const nextMinor = versionRange.atom.version.incrementMinor()
+            if (
+              this.greaterThanOrEqual(otherVersion) &&
+              this.lessThan(nextMinor)
+            ) {
+              return true
+            } else {
+              return false
+            }
+        }
+      case "And":
+        return (
+          this.satisfies(versionRange.atom.left) &&
+          this.satisfies(versionRange.atom.right)
+        )
+      case "Or":
+        return (
+          this.satisfies(versionRange.atom.left) ||
+          this.satisfies(versionRange.atom.right)
+        )
+      case "Not":
+        return !this.satisfies(versionRange.atom.value)
+      case "Any":
+        return true
+      case "None":
+        return false
+    }
+  }
 }
 
 export const testTypeExVer = <T extends string>(t: T & ValidateExVer<T>) => t
@@ -416,6 +426,7 @@ function tests() {
   testTypeVersion("12.34.56")
   testTypeVersion("1.2-3")
   testTypeVersion("1-3")
+  testTypeVersion("1-alpha")
   // @ts-expect-error
   testTypeVersion("-3")
   // @ts-expect-error

sdk/lib/health/HealthCheck.ts

@@ -1,74 +1,61 @@
 import { Effects } from "../types"
-import { CheckResult } from "./checkFns/CheckResult"
+import { HealthCheckResult } from "./checkFns/HealthCheckResult"
 import { HealthReceipt } from "./HealthReceipt"
 import { Trigger } from "../trigger"
 import { TriggerInput } from "../trigger/TriggerInput"
 import { defaultTrigger } from "../trigger/defaultTrigger"
 import { once } from "../util/once"
-import { Overlay } from "../util/Overlay"
+import { SubContainer } from "../util/SubContainer"
 import { object, unknown } from "ts-matches"
 import * as T from "../types"
+import { asError } from "../util/asError"
 
-export type HealthCheckParams<Manifest extends T.Manifest> = {
+export type HealthCheckParams = {
   effects: Effects
   name: string
-  image: {
-    id: keyof Manifest["images"] & T.ImageId
-    sharedRun?: boolean
-  }
   trigger?: Trigger
-  fn(overlay: Overlay): Promise<CheckResult> | CheckResult
+  fn(): Promise<HealthCheckResult> | HealthCheckResult
   onFirstSuccess?: () => unknown | Promise<unknown>
 }
 
-export function healthCheck<Manifest extends T.Manifest>(
-  o: HealthCheckParams<Manifest>,
-) {
+export function healthCheck(o: HealthCheckParams) {
   new Promise(async () => {
-    const overlay = await Overlay.of(o.effects, o.image)
-    try {
-      let currentValue: TriggerInput = {
-        hadSuccess: false,
+    let currentValue: TriggerInput = {}
+    const getCurrentValue = () => currentValue
+    const trigger = (o.trigger ?? defaultTrigger)(getCurrentValue)
+    const triggerFirstSuccess = once(() =>
+      Promise.resolve(
+        "onFirstSuccess" in o && o.onFirstSuccess
+          ? o.onFirstSuccess()
+          : undefined,
+      ),
+    )
+    for (
+      let res = await trigger.next();
+      !res.done;
+      res = await trigger.next()
+    ) {
+      try {
+        const { result, message } = await o.fn()
+        await o.effects.setHealth({
+          name: o.name,
+          id: o.name,
+          result,
+          message: message || "",
+        })
+        currentValue.lastResult = result
+        await triggerFirstSuccess().catch((err) => {
+          console.error(asError(err))
+        })
+      } catch (e) {
+        await o.effects.setHealth({
+          name: o.name,
+          id: o.name,
+          result: "failure",
+          message: asMessage(e) || "",
+        })
+        currentValue.lastResult = "failure"
       }
-      const getCurrentValue = () => currentValue
-      const trigger = (o.trigger ?? defaultTrigger)(getCurrentValue)
-      const triggerFirstSuccess = once(() =>
-        Promise.resolve(
-          "onFirstSuccess" in o && o.onFirstSuccess
-            ? o.onFirstSuccess()
-            : undefined,
-        ),
-      )
-      for (
-        let res = await trigger.next();
-        !res.done;
-        res = await trigger.next()
-      ) {
-        try {
-          const { status, message } = await o.fn(overlay)
-          await o.effects.setHealth({
-            name: o.name,
-            id: o.name,
-            result: status,
-            message: message || "",
-          })
-          currentValue.hadSuccess = true
-          currentValue.lastResult = "success"
-          await triggerFirstSuccess().catch((err) => {
-            console.error(err)
-          })
-        } catch (e) {
-          await o.effects.setHealth({
-            name: o.name,
-            id: o.name,
-            result: "failure",
-            message: asMessage(e) || "",
-          })
-          currentValue.lastResult = "failure"
-        }
-      }
-    } finally {
-      await overlay.destroy()
     }
   })
   return {} as HealthReceipt

sdk/lib/health/checkFns/CheckResult.ts

@@ -1,6 +0,0 @@
-import { HealthStatus } from "../../types"
-
-export type CheckResult = {
-  status: HealthStatus
-  message: string | null
-}

sdk/lib/health/checkFns/HealthCheckResult.ts

@@ -0,0 +1,3 @@
+import { T } from "../.."
+
+export type HealthCheckResult = Omit<T.NamedHealthCheckResult, "name">

sdk/lib/health/checkFns/checkPortListening.ts

@@ -1,6 +1,6 @@
 import { Effects } from "../../types"
 import { stringFromStdErrOut } from "../../util/stringFromStdErrOut"
-import { CheckResult } from "./CheckResult"
+import { HealthCheckResult } from "./HealthCheckResult"
 
 import { promisify } from "node:util"
 import * as CP from "node:child_process"
@@ -32,8 +32,8 @@ export async function checkPortListening(
     timeoutMessage?: string
     timeout?: number
   },
-): Promise<CheckResult> {
-  return Promise.race<CheckResult>([
+): Promise<HealthCheckResult> {
+  return Promise.race<HealthCheckResult>([
     Promise.resolve().then(async () => {
       const hasAddress =
         containsAddress(
@@ -45,10 +45,10 @@ export async function checkPortListening(
           port,
         )
       if (hasAddress) {
-        return { status: "success", message: options.successMessage }
+        return { result: "success", message: options.successMessage }
       }
       return {
-        status: "failure",
+        result: "failure",
         message: options.errorMessage,
       }
     }),
@@ -56,7 +56,7 @@ export async function checkPortListening(
       setTimeout(
         () =>
           resolve({
-            status: "failure",
+            result: "failure",
             message:
               options.timeoutMessage || `Timeout trying to check port ${port}`,
           }),

sdk/lib/health/checkFns/checkWebUrl.ts

@@ -1,5 +1,6 @@
 import { Effects } from "../../types"
-import { CheckResult } from "./CheckResult"
+import { asError } from "../../util/asError"
+import { HealthCheckResult } from "./HealthCheckResult"
 import { timeoutPromise } from "./index"
 import "isomorphic-fetch"
 
@@ -17,19 +18,19 @@ export const checkWebUrl = async (
     successMessage = `Reached ${url}`,
     errorMessage = `Error while fetching URL: ${url}`,
   } = {},
-): Promise<CheckResult> => {
+): Promise<HealthCheckResult> => {
   return Promise.race([fetch(url), timeoutPromise(timeout)])
     .then(
       (x) =>
         ({
-          status: "success",
+          result: "success",
           message: successMessage,
         }) as const,
     )
     .catch((e) => {
       console.warn(`Error while fetching URL: ${url}`)
       console.error(JSON.stringify(e))
-      console.error(e.toString())
-      return { status: "failure" as const, message: errorMessage }
+      console.error(asError(e))
+      return { result: "failure" as const, message: errorMessage }
     })
 }

sdk/lib/health/checkFns/index.ts

@@ -1,6 +1,6 @@
 import { runHealthScript } from "./runHealthScript"
 export { checkPortListening } from "./checkPortListening"
-export { CheckResult } from "./CheckResult"
+export { HealthCheckResult } from "./HealthCheckResult"
 export { checkWebUrl } from "./checkWebUrl"
 
 export function timeoutPromise(ms: number, { message = "Timed out" } = {}) {

sdk/lib/health/checkFns/runHealthScript.ts

@@ -1,7 +1,7 @@
 import { Effects } from "../../types"
-import { Overlay } from "../../util/Overlay"
+import { SubContainer } from "../../util/SubContainer"
 import { stringFromStdErrOut } from "../../util/stringFromStdErrOut"
-import { CheckResult } from "./CheckResult"
+import { HealthCheckResult } from "./HealthCheckResult"
 import { timeoutPromise } from "./index"
 
 /**
@@ -12,27 +12,26 @@ import { timeoutPromise } from "./index"
  * @returns
  */
 export const runHealthScript = async (
-  effects: Effects,
   runCommand: string[],
-  overlay: Overlay,
+  subcontainer: SubContainer,
   {
     timeout = 30000,
     errorMessage = `Error while running command: ${runCommand}`,
     message = (res: string) =>
       `Have ran script ${runCommand} and the result: ${res}`,
   } = {},
-): Promise<CheckResult> => {
+): Promise<HealthCheckResult> => {
   const res = await Promise.race([
-    overlay.exec(runCommand),
+    subcontainer.exec(runCommand),
     timeoutPromise(timeout),
   ]).catch((e) => {
     console.warn(errorMessage)
     console.warn(JSON.stringify(e))
     console.warn(e.toString())
-    throw { status: "failure", message: errorMessage } as CheckResult
+    throw { result: "failure", message: errorMessage } as HealthCheckResult
   })
   return {
-    status: "success",
+    result: "success",
     message: message(res.stdout.toString()),
-  } as CheckResult
+  } as HealthCheckResult
 }

sdk/lib/index.ts

@@ -1,5 +1,5 @@
 export { Daemons } from "./mainFn/Daemons"
-export { Overlay } from "./util/Overlay"
+export { SubContainer } from "./util/SubContainer"
 export { StartSdk } from "./StartSdk"
 export { setupManifest } from "./manifest/setupManifest"
 export { FileHelper } from "./util/fileHelper"
@@ -29,3 +29,4 @@ export * as utils from "./util"
 export * as matches from "ts-matches"
 export * as YAML from "yaml"
 export * as TOML from "@iarna/toml"
+export * from "./version"

sdk/lib/inits/migrations/Migration.ts

@@ -1,35 +0,0 @@
-import { ValidateExVer } from "../../exver"
-import * as T from "../../types"
-
-export class Migration<
-  Manifest extends T.Manifest,
-  Store,
-  Version extends string,
-> {
-  constructor(
-    readonly options: {
-      version: Version & ValidateExVer<Version>
-      up: (opts: { effects: T.Effects }) => Promise<void>
-      down: (opts: { effects: T.Effects }) => Promise<void>
-    },
-  ) {}
-  static of<
-    Manifest extends T.Manifest,
-    Store,
-    Version extends string,
-  >(options: {
-    version: Version & ValidateExVer<Version>
-    up: (opts: { effects: T.Effects }) => Promise<void>
-    down: (opts: { effects: T.Effects }) => Promise<void>
-  }) {
-    return new Migration<Manifest, Store, Version>(options)
-  }
-
-  async up(opts: { effects: T.Effects }) {
-    this.up(opts)
-  }
-
-  async down(opts: { effects: T.Effects }) {
-    this.down(opts)
-  }
-}

sdk/lib/inits/migrations/setupMigrations.ts

@@ -1,77 +0,0 @@
-import { ExtendedVersion } from "../../exver"
-
-import * as T from "../../types"
-import { once } from "../../util/once"
-import { Migration } from "./Migration"
-
-export class Migrations<Manifest extends T.Manifest, Store> {
-  private constructor(
-    readonly manifest: T.Manifest,
-    readonly migrations: Array<Migration<Manifest, Store, any>>,
-  ) {}
-  private sortedMigrations = once(() => {
-    const migrationsAsVersions = (
-      this.migrations as Array<Migration<Manifest, Store, any>>
-    )
-      .map((x) => [ExtendedVersion.parse(x.options.version), x] as const)
-      .filter(([v, _]) => v.flavor === this.currentVersion().flavor)
-    migrationsAsVersions.sort((a, b) => a[0].compareForSort(b[0]))
-    return migrationsAsVersions
-  })
-  private currentVersion = once(() =>
-    ExtendedVersion.parse(this.manifest.version),
-  )
-  static of<
-    Manifest extends T.Manifest,
-    Store,
-    Migrations extends Array<Migration<Manifest, Store, any>>,
-  >(manifest: T.Manifest, ...migrations: EnsureUniqueId<Migrations>) {
-    return new Migrations(
-      manifest,
-      migrations as Array<Migration<Manifest, Store, any>>,
-    )
-  }
-  async init({
-    effects,
-    previousVersion,
-  }: Parameters<T.ExpectedExports.init>[0]) {
-    if (!!previousVersion) {
-      const previousVersionExVer = ExtendedVersion.parse(previousVersion)
-      for (const [_, migration] of this.sortedMigrations()
-        .filter((x) => x[0].greaterThan(previousVersionExVer))
-        .filter((x) => x[0].lessThanOrEqual(this.currentVersion()))) {
-        await migration.up({ effects })
-      }
-    }
-  }
-  async uninit({
-    effects,
-    nextVersion,
-  }: Parameters<T.ExpectedExports.uninit>[0]) {
-    if (!!nextVersion) {
-      const nextVersionExVer = ExtendedVersion.parse(nextVersion)
-      const reversed = [...this.sortedMigrations()].reverse()
-      for (const [_, migration] of reversed
-        .filter((x) => x[0].greaterThan(nextVersionExVer))
-        .filter((x) => x[0].lessThanOrEqual(this.currentVersion()))) {
-        await migration.down({ effects })
-      }
-    }
-  }
-}
-
-export function setupMigrations<
-  Manifest extends T.Manifest,
-  Store,
-  Migrations extends Array<Migration<Manifest, Store, any>>,
->(manifest: T.Manifest, ...migrations: EnsureUniqueId<Migrations>) {
-  return Migrations.of<Manifest, Store, Migrations>(manifest, ...migrations)
-}
-
-// prettier-ignore
-export type EnsureUniqueId<A, B = A, ids = never> =
-  B extends [] ? A : 
-  B extends [Migration<any, any, infer id>, ...infer Rest] ? (
-    id extends ids ? "One of the ids are not unique"[] :
-    EnsureUniqueId<A, Rest, id | ids>
-  ) : "There exists a migration that is not a Migration"[]

sdk/lib/inits/setupInit.ts

@@ -1,14 +1,15 @@
 import { DependenciesReceipt } from "../config/setupConfig"
+import { ExtendedVersion, VersionRange } from "../exver"
 import { SetInterfaces } from "../interfaces/setupInterfaces"
 
 import { ExposedStorePaths } from "../store/setupExposeStore"
 import * as T from "../types"
-import { Migrations } from "./migrations/setupMigrations"
+import { VersionGraph } from "../version/VersionGraph"
 import { Install } from "./setupInstall"
 import { Uninstall } from "./setupUninstall"
 
 export function setupInit<Manifest extends T.Manifest, Store>(
-  migrations: Migrations<Manifest, Store>,
+  versions: VersionGraph<Manifest["version"]>,
   install: Install<Manifest, Store>,
   uninstall: Uninstall<Manifest, Store>,
   setInterfaces: SetInterfaces<Manifest, Store, any, any>,
@@ -23,8 +24,19 @@ export function setupInit<Manifest extends T.Manifest, Store>(
 } {
   return {
     init: async (opts) => {
-      await migrations.init(opts)
-      await install.init(opts)
+      const prev = await opts.effects.getDataVersion()
+      if (prev) {
+        await versions.migrate({
+          effects: opts.effects,
+          from: ExtendedVersion.parse(prev),
+          to: versions.currentVersion(),
+        })
+      } else {
+        await install.install(opts)
+        await opts.effects.setDataVersion({
+          version: versions.current.options.version,
+        })
+      }
       await setInterfaces({
         ...opts,
         input: null,
@@ -33,8 +45,18 @@ export function setupInit<Manifest extends T.Manifest, Store>(
       await setDependencies({ effects: opts.effects, input: null })
     },
     uninit: async (opts) => {
-      await migrations.uninit(opts)
-      await uninstall.uninit(opts)
+      if (opts.nextVersion) {
+        const prev = await opts.effects.getDataVersion()
+        if (prev) {
+          await versions.migrate({
+            effects: opts.effects,
+            from: ExtendedVersion.parse(prev),
+            to: ExtendedVersion.parse(opts.nextVersion),
+          })
+        }
+      } else {
+        await uninstall.uninstall(opts)
+      }
     },
   }
 }

sdk/lib/inits/setupInstall.ts

@@ -11,14 +11,10 @@ export class Install<Manifest extends T.Manifest, Store> {
     return new Install(fn)
   }
 
-  async init({
-    effects,
-    previousVersion,
-  }: Parameters<T.ExpectedExports.init>[0]) {
-    if (!previousVersion)
-      await this.fn({
-        effects,
-      })
+  async install({ effects }: Parameters<T.ExpectedExports.init>[0]) {
+    await this.fn({
+      effects,
+    })
   }
 }
 

sdk/lib/inits/setupUninstall.ts

@@ -11,7 +11,7 @@ export class Uninstall<Manifest extends T.Manifest, Store> {
     return new Uninstall(fn)
   }
 
-  async uninit({
+  async uninstall({
     effects,
     nextVersion,
   }: Parameters<T.ExpectedExports.uninit>[0]) {

sdk/lib/interfaces/Origin.ts

@@ -47,7 +47,6 @@ export class Origin<T extends Host> {
         name,
         description,
         hasPrimary,
-        disabled,
         id,
         type,
         username,
@@ -69,7 +68,6 @@ export class Origin<T extends Host> {
         name,
         description,
         hasPrimary,
-        disabled,
         addressInfo,
         type,
         masked,

sdk/lib/interfaces/ServiceInterfaceBuilder.ts

@@ -21,7 +21,6 @@ export class ServiceInterfaceBuilder {
       id: string
       description: string
       hasPrimary: boolean
-      disabled: boolean
       type: ServiceInterfaceType
       username: string | null
       path: string

sdk/lib/mainFn/CommandController.ts

@@ -2,30 +2,39 @@ import { DEFAULT_SIGTERM_TIMEOUT } from "."
 import { NO_TIMEOUT, SIGKILL, SIGTERM } from "../StartSdk"
 
 import * as T from "../types"
-import { MountOptions, Overlay } from "../util/Overlay"
+import { asError } from "../util/asError"
+import {
+  ExecSpawnable,
+  MountOptions,
+  SubContainerHandle,
+  SubContainer,
+} from "../util/SubContainer"
 import { splitCommand } from "../util/splitCommand"
-import { cpExecFile, cpExec } from "./Daemons"
+import * as cp from "child_process"
 
 export class CommandController {
   private constructor(
     readonly runningAnswer: Promise<unknown>,
-    readonly overlay: Overlay,
-    readonly pid: number | undefined,
+    private state: { exited: boolean },
+    private readonly subcontainer: SubContainer,
+    private process: cp.ChildProcessWithoutNullStreams,
     readonly sigtermTimeout: number = DEFAULT_SIGTERM_TIMEOUT,
   ) {}
   static of<Manifest extends T.Manifest>() {
     return async <A extends string>(
       effects: T.Effects,
-      imageId: {
-        id: keyof Manifest["images"] & T.ImageId
-        sharedRun?: boolean
-      },
+      subcontainer:
+        | {
+            id: keyof Manifest["images"] & T.ImageId
+            sharedRun?: boolean
+          }
+        | SubContainer,
       command: T.CommandType,
       options: {
         // Defaults to the DEFAULT_SIGTERM_TIMEOUT = 30_000ms
         sigtermTimeout?: number
         mounts?: { path: string; options: MountOptions }[]
-        overlay?: Overlay
+        runAsInit?: boolean
         env?:
           | {
               [variable: string]: string
@@ -38,43 +47,62 @@ export class CommandController {
       },
     ) => {
       const commands = splitCommand(command)
-      const overlay = options.overlay || (await Overlay.of(effects, imageId))
-      for (let mount of options.mounts || []) {
-        await overlay.mount(mount.options, mount.path)
+      const subc =
+        subcontainer instanceof SubContainer
+          ? subcontainer
+          : await (async () => {
+              const subc = await SubContainer.of(effects, subcontainer)
+              for (let mount of options.mounts || []) {
+                await subc.mount(mount.options, mount.path)
+              }
+              return subc
+            })()
+      let childProcess: cp.ChildProcessWithoutNullStreams
+      if (options.runAsInit) {
+        childProcess = await subc.launch(commands, {
+          env: options.env,
+        })
+      } else {
+        childProcess = await subc.spawn(commands, {
+          env: options.env,
+        })
       }
-      const childProcess = await overlay.spawn(commands, {
-        env: options.env,
-      })
+      const state = { exited: false }
       const answer = new Promise<null>((resolve, reject) => {
-        childProcess.stdout.on(
-          "data",
-          options.onStdout ??
-            ((data: any) => {
-              console.log(data.toString())
-            }),
-        )
-        childProcess.stderr.on(
-          "data",
-          options.onStderr ??
-            ((data: any) => {
-              console.error(data.toString())
-            }),
-        )
-
-        childProcess.on("exit", (code: any) => {
-          if (code === 0) {
+        childProcess.on("exit", (code) => {
+          state.exited = true
+          if (
+            code === 0 ||
+            code === 143 ||
+            (code === null && childProcess.signalCode == "SIGTERM")
+          ) {
             return resolve(null)
           }
-          return reject(new Error(`${commands[0]} exited with code ${code}`))
+          if (code) {
+            return reject(new Error(`${commands[0]} exited with code ${code}`))
+          } else {
+            return reject(
+              new Error(
+                `${commands[0]} exited with signal ${childProcess.signalCode}`,
+              ),
+            )
+          }
         })
       })
 
-      const pid = childProcess.pid
-
-      return new CommandController(answer, overlay, pid, options.sigtermTimeout)
+      return new CommandController(
+        answer,
+        state,
+        subc,
+        childProcess,
+        options.sigtermTimeout,
+      )
     }
   }
-  async wait(timeout: number = NO_TIMEOUT) {
+  get subContainerHandle() {
+    return new SubContainerHandle(this.subcontainer)
+  }
+  async wait({ timeout = NO_TIMEOUT } = {}) {
     if (timeout > 0)
       setTimeout(() => {
         this.term()
@@ -82,75 +110,30 @@ export class CommandController {
     try {
       return await this.runningAnswer
     } finally {
-      if (this.pid !== undefined) {
-        await cpExecFile("pkill", ["-9", "-s", String(this.pid)]).catch(
-          (_) => {},
-        )
+      if (!this.state.exited) {
+        this.process.kill("SIGKILL")
       }
-      await this.overlay.destroy().catch((_) => {})
+      await this.subcontainer.destroy?.().catch((_) => {})
     }
   }
   async term({ signal = SIGTERM, timeout = this.sigtermTimeout } = {}) {
-    if (this.pid === undefined) return
     try {
-      await cpExecFile("pkill", [
-        `-${signal.replace("SIG", "")}`,
-        "-s",
-        String(this.pid),
-      ])
-
-      const didTimeout = await waitSession(this.pid, timeout)
-      if (didTimeout) {
-        await cpExecFile("pkill", [`-9`, "-s", String(this.pid)]).catch(
-          (_) => {},
-        )
+      if (!this.state.exited) {
+        if (!this.process.kill(signal)) {
+          console.error(
+            `failed to send signal ${signal} to pid ${this.process.pid}`,
+          )
+        }
       }
-    } finally {
-      await this.overlay.destroy()
-    }
-  }
-}
 
-function waitSession(
-  sid: number,
-  timeout = NO_TIMEOUT,
-  interval = 100,
-): Promise<boolean> {
-  let nextInterval = interval * 2
-  if (timeout >= 0 && timeout < nextInterval) {
-    nextInterval = timeout
-  }
-  let nextTimeout = timeout
-  if (timeout > 0) {
-    if (timeout >= interval) {
-      nextTimeout -= interval
-    } else {
-      nextTimeout = 0
+      if (signal !== "SIGKILL") {
+        setTimeout(() => {
+          this.process.kill("SIGKILL")
+        }, timeout)
+      }
+      await this.runningAnswer
+    } finally {
+      await this.subcontainer.destroy?.()
     }
   }
-  return new Promise((resolve, reject) => {
-    let next: NodeJS.Timeout | null = null
-    if (timeout !== 0) {
-      next = setTimeout(() => {
-        waitSession(sid, nextTimeout, nextInterval).then(resolve, reject)
-      }, interval)
-    }
-    cpExecFile("ps", [`--sid=${sid}`, "-o", "--pid="]).then(
-      (_) => {
-        if (timeout === 0) {
-          resolve(true)
-        }
-      },
-      (e) => {
-        if (next) {
-          clearTimeout(next)
-        }
-        if (typeof e === "object" && e && "code" in e && e.code) {
-          resolve(false)
-        } else {
-          reject(e)
-        }
-      },
-    )
-  })
 }

sdk/lib/mainFn/Daemon.ts

@@ -1,5 +1,6 @@
 import * as T from "../types"
-import { MountOptions, Overlay } from "../util/Overlay"
+import { asError } from "../util/asError"
+import { ExecSpawnable, MountOptions, SubContainer } from "../util/SubContainer"
 import { CommandController } from "./CommandController"
 
 const TIMEOUT_INCREMENT_MS = 1000
@@ -12,18 +13,22 @@ const MAX_TIMEOUT_MS = 30000
 export class Daemon {
   private commandController: CommandController | null = null
   private shouldBeRunning = false
-  private constructor(private startCommand: () => Promise<CommandController>) {}
+  constructor(private startCommand: () => Promise<CommandController>) {}
+  get subContainerHandle(): undefined | ExecSpawnable {
+    return this.commandController?.subContainerHandle
+  }
   static of<Manifest extends T.Manifest>() {
     return async <A extends string>(
       effects: T.Effects,
-      imageId: {
-        id: keyof Manifest["images"] & T.ImageId
-        sharedRun?: boolean
-      },
+      subcontainer:
+        | {
+            id: keyof Manifest["images"] & T.ImageId
+            sharedRun?: boolean
+          }
+        | SubContainer,
       command: T.CommandType,
       options: {
         mounts?: { path: string; options: MountOptions }[]
-        overlay?: Overlay
         env?:
           | {
               [variable: string]: string
@@ -37,11 +42,15 @@ export class Daemon {
       },
     ) => {
       const startCommand = () =>
-        CommandController.of<Manifest>()(effects, imageId, command, options)
+        CommandController.of<Manifest>()(
+          effects,
+          subcontainer,
+          command,
+          options,
+        )
       return new Daemon(startCommand)
     }
   }
-
   async start() {
     if (this.commandController) {
       return
@@ -57,7 +66,7 @@ export class Daemon {
         timeoutCounter = Math.max(MAX_TIMEOUT_MS, timeoutCounter)
       }
     }).catch((err) => {
-      console.error(err)
+      console.error(asError(err))
     })
   }
   async term(termOptions?: {
@@ -72,8 +81,8 @@ export class Daemon {
   }) {
     this.shouldBeRunning = false
     await this.commandController
-      ?.term(termOptions)
-      .catch((e) => console.error(e))
+      ?.term({ ...termOptions })
+      .catch((e) => console.error(asError(e)))
     this.commandController = null
   }
 }

sdk/lib/mainFn/Daemons.ts

@@ -1,13 +1,18 @@
 import { NO_TIMEOUT, SIGKILL, SIGTERM, Signals } from "../StartSdk"
 import { HealthReceipt } from "../health/HealthReceipt"
-import { CheckResult } from "../health/checkFns"
+import { HealthCheckResult } from "../health/checkFns"
 
 import { Trigger } from "../trigger"
 import { TriggerInput } from "../trigger/TriggerInput"
 import { defaultTrigger } from "../trigger/defaultTrigger"
 import * as T from "../types"
 import { Mounts } from "./Mounts"
-import { CommandOptions, MountOptions, Overlay } from "../util/Overlay"
+import {
+  CommandOptions,
+  ExecSpawnable,
+  MountOptions,
+  SubContainer,
+} from "../util/SubContainer"
 import { splitCommand } from "../util/splitCommand"
 
 import { promisify } from "node:util"
@@ -23,7 +28,9 @@ export const cpExec = promisify(CP.exec)
 export const cpExecFile = promisify(CP.execFile)
 export type Ready = {
   display: string | null
-  fn: () => Promise<CheckResult> | CheckResult
+  fn: (
+    spawnable: ExecSpawnable,
+  ) => Promise<HealthCheckResult> | HealthCheckResult
   trigger?: Trigger
 }
 

sdk/lib/mainFn/HealthDaemon.ts

@@ -1,9 +1,10 @@
-import { CheckResult } from "../health/checkFns"
+import { HealthCheckResult } from "../health/checkFns"
 import { defaultTrigger } from "../trigger/defaultTrigger"
 import { Ready } from "./Daemons"
 import { Daemon } from "./Daemon"
-import { Effects } from "../types"
+import { Effects, SetHealth } from "../types"
 import { DEFAULT_SIGTERM_TIMEOUT } from "."
+import { asError } from "../util/asError"
 
 const oncePromise = <T>() => {
   let resolve: (value: T) => void
@@ -21,14 +22,13 @@ const oncePromise = <T>() => {
  *
  */
 export class HealthDaemon {
-  #health: CheckResult = { status: "starting", message: null }
-  #healthWatchers: Array<() => unknown> = []
-  #running = false
-  #hadSuccess = false
+  private _health: HealthCheckResult = { result: "starting", message: null }
+  private healthWatchers: Array<() => unknown> = []
+  private running = false
   constructor(
-    readonly daemon: Promise<Daemon>,
+    private readonly daemon: Promise<Daemon>,
     readonly daemonIndex: number,
-    readonly dependencies: HealthDaemon[],
+    private readonly dependencies: HealthDaemon[],
     readonly id: string,
     readonly ids: string[],
     readonly ready: Ready,
@@ -44,12 +44,12 @@ export class HealthDaemon {
     signal?: NodeJS.Signals | undefined
     timeout?: number | undefined
   }) {
-    this.#healthWatchers = []
-    this.#running = false
-    this.#healthCheckCleanup?.()
+    this.healthWatchers = []
+    this.running = false
+    this.healthCheckCleanup?.()
 
     await this.daemon.then((d) =>
-      d.stop({
+      d.term({
         timeout: this.sigtermTimeout,
         ...termOptions,
       }),
@@ -58,17 +58,17 @@ export class HealthDaemon {
 
   /** Want to add another notifier that the health might have changed */
   addWatcher(watcher: () => unknown) {
-    this.#healthWatchers.push(watcher)
+    this.healthWatchers.push(watcher)
   }
 
   get health() {
-    return Object.freeze(this.#health)
+    return Object.freeze(this._health)
   }
 
   private async changeRunning(newStatus: boolean) {
-    if (this.#running === newStatus) return
+    if (this.running === newStatus) return
 
-    this.#running = newStatus
+    this.running = newStatus
 
     if (newStatus) {
       ;(await this.daemon).start()
@@ -77,19 +77,18 @@ export class HealthDaemon {
       ;(await this.daemon).stop()
       this.turnOffHealthCheck()
 
-      this.setHealth({ status: "starting", message: null })
+      this.setHealth({ result: "starting", message: null })
     }
   }
 
-  #healthCheckCleanup: (() => void) | null = null
+  private healthCheckCleanup: (() => void) | null = null
   private turnOffHealthCheck() {
-    this.#healthCheckCleanup?.()
+    this.healthCheckCleanup?.()
   }
   private async setupHealthCheck() {
-    if (this.#healthCheckCleanup) return
+    if (this.healthCheckCleanup) return
     const trigger = (this.ready.trigger ?? defaultTrigger)(() => ({
-      hadSuccess: this.#hadSuccess,
-      lastResult: this.#health.status,
+      lastResult: this._health.result,
     }))
 
     const { promise: status, resolve: setStatus } = oncePromise<{
@@ -101,59 +100,51 @@ export class HealthDaemon {
         !res.done;
         res = await Promise.race([status, trigger.next()])
       ) {
-        const response: CheckResult = await Promise.resolve(
-          this.ready.fn(),
-        ).catch((err) => {
-          console.error(err)
-          return {
-            status: "failure",
-            message: "message" in err ? err.message : String(err),
-          }
-        })
-        this.setHealth(response)
-        if (response.status === "success") {
-          this.#hadSuccess = true
+        const handle = (await this.daemon).subContainerHandle
+
+        if (handle) {
+          const response: HealthCheckResult = await Promise.resolve(
+            this.ready.fn(handle),
+          ).catch((err) => {
+            console.error(asError(err))
+            return {
+              result: "failure",
+              message: "message" in err ? err.message : String(err),
+            }
+          })
+          await this.setHealth(response)
+        } else {
+          await this.setHealth({
+            result: "failure",
+            message: "Daemon not running",
+          })
         }
       }
     }).catch((err) => console.error(`Daemon ${this.id} failed: ${err}`))
 
-    this.#healthCheckCleanup = () => {
+    this.healthCheckCleanup = () => {
       setStatus({ done: true })
-      this.#healthCheckCleanup = null
+      this.healthCheckCleanup = null
     }
   }
 
-  private setHealth(health: CheckResult) {
-    this.#health = health
-    this.#healthWatchers.forEach((watcher) => watcher())
+  private async setHealth(health: HealthCheckResult) {
+    this._health = health
+    this.healthWatchers.forEach((watcher) => watcher())
     const display = this.ready.display
-    const status = health.status
+    const result = health.result
     if (!display) {
       return
     }
-    if (
-      status === "success" ||
-      status === "disabled" ||
-      status === "starting"
-    ) {
-      this.effects.setHealth({
-        result: status,
-        message: health.message,
-        id: this.id,
-        name: display,
-      })
-    } else {
-      this.effects.setHealth({
-        result: health.status,
-        message: health.message || "",
-        id: this.id,
-        name: display,
-      })
-    }
+    await this.effects.setHealth({
+      ...health,
+      id: this.id,
+      name: display,
+    } as SetHealth)
   }
 
   private async updateStatus() {
-    const healths = this.dependencies.map((d) => d.#health)
-    this.changeRunning(healths.every((x) => x.status === "success"))
+    const healths = this.dependencies.map((d) => d._health)
+    this.changeRunning(healths.every((x) => x.result === "success"))
   }
 }

sdk/lib/mainFn/Mounts.ts

@@ -1,5 +1,5 @@
 import * as T from "../types"
-import { MountOptions } from "../util/Overlay"
+import { MountOptions } from "../util/SubContainer"
 
 type MountArray = { path: string; options: MountOptions }[]
 

sdk/lib/manifest/ManifestTypes.ts

@@ -7,22 +7,11 @@ import {
   ImageSource,
 } from "../types"
 
-export type SDKManifest<
-  Version extends string,
-  Satisfies extends string[] = [],
-> = {
+export type SDKManifest = {
   /**  The package identifier used by the OS. This must be unique amongst all other known packages */
   readonly id: string
   /** A human readable service title */
   readonly title: string
-  /** Service version - accepts up to four digits, where the last confirms to revisions necessary for StartOs
-   * - see documentation: https://github.com/Start9Labs/emver-rs. This value will change with each release of
-   * the service
-   */
-  readonly version: Version & ValidateExVer<Version>
-  readonly satisfies?: Satisfies & ValidateExVers<Satisfies>
-  /** Release notes for the update - can be a string, paragraph or URL */
-  readonly releaseNotes: string
   /** The type of license for the project. Include the LICENSE in the root of the project directory. A license is required for a Start9 package.*/
   readonly license: string // name of license
   /** The Start9 wrapper repository URL for the package. This repo contains the manifest file (this),

sdk/lib/manifest/setupManifest.ts

@@ -2,7 +2,13 @@ import * as T from "../types"
 import { ImageConfig, ImageId, VolumeId } from "../osBindings"
 import { SDKManifest, SDKImageConfig } from "./ManifestTypes"
 import { SDKVersion } from "../StartSdk"
+import { VersionGraph } from "../version/VersionGraph"
 
+/**
+ * This is an example of a function that takes a manifest and returns a new manifest with additional properties
+ * @param manifest Manifests are the description of the package
+ * @returns The manifest with additional properties
+ */
 export function setupManifest<
   Id extends string,
   Version extends string,
@@ -10,7 +16,7 @@ export function setupManifest<
   VolumesTypes extends VolumeId,
   AssetTypes extends VolumeId,
   ImagesTypes extends ImageId,
-  Manifest extends SDKManifest<Version, Satisfies> & {
+  Manifest extends {
     dependencies: Dependencies
     id: Id
     assets: AssetTypes[]
@@ -18,7 +24,10 @@ export function setupManifest<
     volumes: VolumesTypes[]
   },
   Satisfies extends string[] = [],
->(manifest: Manifest & { version: Version }): Manifest & T.Manifest {
+>(
+  versions: VersionGraph<Version>,
+  manifest: SDKManifest & Manifest,
+): Manifest & T.Manifest {
   const images = Object.entries(manifest.images).reduce(
     (images, [k, v]) => {
       v.arch = v.arch || ["aarch64", "x86_64"]
@@ -33,7 +42,11 @@ export function setupManifest<
     ...manifest,
     gitHash: null,
     osVersion: SDKVersion,
-    satisfies: manifest.satisfies || [],
+    version: versions.current.options.version,
+    releaseNotes: versions.current.options.releaseNotes,
+    satisfies: versions.current.options.satisfies || [],
+    canMigrateTo: versions.canMigrateTo().toString(),
+    canMigrateFrom: versions.canMigrateFrom().toString(),
     images,
     alerts: {
       install: manifest.alerts?.install || null,

sdk/lib/osBindings/CheckDependenciesResult.ts

@@ -1,13 +1,14 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { HealthCheckId } from "./HealthCheckId"
-import type { HealthCheckResult } from "./HealthCheckResult"
+import type { NamedHealthCheckResult } from "./NamedHealthCheckResult"
 import type { PackageId } from "./PackageId"
 
 export type CheckDependenciesResult = {
   packageId: PackageId
-  isInstalled: boolean
+  title: string | null
+  installedVersion: string | null
+  satisfies: string[]
   isRunning: boolean
   configSatisfied: boolean
-  healthChecks: { [key: HealthCheckId]: HealthCheckResult }
-  version: string | null
+  healthChecks: { [key: HealthCheckId]: NamedHealthCheckResult }
 }

sdk/lib/osBindings/CreateSubcontainerFsParams.ts

@@ -1,4 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { ImageId } from "./ImageId"
 
-export type CreateOverlayedImageParams = { imageId: ImageId }
+export type CreateSubcontainerFsParams = { imageId: ImageId }

sdk/lib/osBindings/DestroySubcontainerFsParams.ts

@@ -1,4 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { Guid } from "./Guid"
 
-export type DestroyOverlayedImageParams = { guid: Guid }
+export type DestroySubcontainerFsParams = { guid: Guid }

sdk/lib/osBindings/ExportServiceInterfaceParams.ts

@@ -8,7 +8,6 @@ export type ExportServiceInterfaceParams = {
   name: string
   description: string
   hasPrimary: boolean
-  disabled: boolean
   masked: boolean
   addressInfo: AddressInfo
   type: ServiceInterfaceType

sdk/lib/osBindings/HardwareRequirements.ts

@@ -1,7 +1,7 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
 export type HardwareRequirements = {
-  device: { device?: string; processor?: string }
+  device: { display?: string; processor?: string }
   ram: number | null
   arch: string[] | null
 }

sdk/lib/osBindings/MainStatus.ts

@@ -1,20 +1,20 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { HealthCheckId } from "./HealthCheckId"
-import type { HealthCheckResult } from "./HealthCheckResult"
+import type { NamedHealthCheckResult } from "./NamedHealthCheckResult"
+import type { StartStop } from "./StartStop"
 
 export type MainStatus =
   | { status: "stopped" }
   | { status: "restarting" }
   | { status: "restoring" }
   | { status: "stopping" }
-  | { status: "starting" }
+  | {
+      status: "starting"
+      health: { [key: HealthCheckId]: NamedHealthCheckResult }
+    }
   | {
       status: "running"
       started: string
-      health: { [key: HealthCheckId]: HealthCheckResult }
-    }
-  | {
-      status: "backingUp"
-      started: string | null
-      health: { [key: HealthCheckId]: HealthCheckResult }
+      health: { [key: HealthCheckId]: NamedHealthCheckResult }
     }
+  | { status: "backingUp"; onComplete: StartStop }

sdk/lib/osBindings/Manifest.ts

@@ -15,6 +15,8 @@ export type Manifest = {
   version: Version
   satisfies: Array<Version>
   releaseNotes: string
+  canMigrateTo: string
+  canMigrateFrom: string
   license: string
   wrapperRepo: string
   upstreamRepo: string

sdk/lib/osBindings/NamedHealthCheckResult.ts

@@ -1,6 +1,6 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export type HealthCheckResult = { name: string } & (
+export type NamedHealthCheckResult = { name: string } & (
   | { result: "success"; message: string | null }
   | { result: "disabled"; message: string | null }
   | { result: "starting"; message: string | null }

sdk/lib/osBindings/PackageDataEntry.ts

@@ -8,9 +8,11 @@ import type { PackageState } from "./PackageState"
 import type { ServiceInterface } from "./ServiceInterface"
 import type { ServiceInterfaceId } from "./ServiceInterfaceId"
 import type { Status } from "./Status"
+import type { Version } from "./Version"
 
 export type PackageDataEntry = {
   stateInfo: PackageState
+  dataVersion: Version | null
   status: Status
   registry: string | null
   developerKey: string

sdk/lib/osBindings/ServiceInterface.ts

@@ -8,7 +8,6 @@ export type ServiceInterface = {
   name: string
   description: string
   hasPrimary: boolean
-  disabled: boolean
   masked: boolean
   addressInfo: AddressInfo
   type: ServiceInterfaceType

sdk/lib/osBindings/SetDataVersionParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type SetDataVersionParams = { version: string }

sdk/lib/osBindings/StartStop.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type StartStop = "start" | "stop"

sdk/lib/osBindings/index.ts

@@ -31,7 +31,7 @@ export { CheckDependenciesParam } from "./CheckDependenciesParam"
 export { CheckDependenciesResult } from "./CheckDependenciesResult"
 export { Cifs } from "./Cifs"
 export { ContactInfo } from "./ContactInfo"
-export { CreateOverlayedImageParams } from "./CreateOverlayedImageParams"
+export { CreateSubcontainerFsParams } from "./CreateSubcontainerFsParams"
 export { CurrentDependencies } from "./CurrentDependencies"
 export { CurrentDependencyInfo } from "./CurrentDependencyInfo"
 export { DataUrl } from "./DataUrl"
@@ -41,7 +41,7 @@ export { DependencyMetadata } from "./DependencyMetadata"
 export { DependencyRequirement } from "./DependencyRequirement"
 export { DepInfo } from "./DepInfo"
 export { Description } from "./Description"
-export { DestroyOverlayedImageParams } from "./DestroyOverlayedImageParams"
+export { DestroySubcontainerFsParams } from "./DestroySubcontainerFsParams"
 export { Duration } from "./Duration"
 export { EchoParams } from "./EchoParams"
 export { EncryptedWire } from "./EncryptedWire"
@@ -69,7 +69,6 @@ export { Governor } from "./Governor"
 export { Guid } from "./Guid"
 export { HardwareRequirements } from "./HardwareRequirements"
 export { HealthCheckId } from "./HealthCheckId"
-export { HealthCheckResult } from "./HealthCheckResult"
 export { HostAddress } from "./HostAddress"
 export { HostId } from "./HostId"
 export { HostKind } from "./HostKind"
@@ -98,6 +97,7 @@ export { MaybeUtf8String } from "./MaybeUtf8String"
 export { MerkleArchiveCommitment } from "./MerkleArchiveCommitment"
 export { MountParams } from "./MountParams"
 export { MountTarget } from "./MountTarget"
+export { NamedHealthCheckResult } from "./NamedHealthCheckResult"
 export { NamedProgress } from "./NamedProgress"
 export { OnionHostname } from "./OnionHostname"
 export { OsIndex } from "./OsIndex"
@@ -132,6 +132,7 @@ export { SessionList } from "./SessionList"
 export { Sessions } from "./Sessions"
 export { Session } from "./Session"
 export { SetConfigured } from "./SetConfigured"
+export { SetDataVersionParams } from "./SetDataVersionParams"
 export { SetDependenciesParams } from "./SetDependenciesParams"
 export { SetHealth } from "./SetHealth"
 export { SetMainStatusStatus } from "./SetMainStatusStatus"
@@ -144,6 +145,7 @@ export { SetupStatusRes } from "./SetupStatusRes"
 export { SignAssetParams } from "./SignAssetParams"
 export { SignerInfo } from "./SignerInfo"
 export { SmtpValue } from "./SmtpValue"
+export { StartStop } from "./StartStop"
 export { Status } from "./Status"
 export { UpdatingState } from "./UpdatingState"
 export { VerifyCifsParams } from "./VerifyCifsParams"

sdk/lib/s9pk/merkleArchive/varint.ts

@@ -1,3 +1,5 @@
+import { asError } from "../../util/asError"
+
 const msb = 0x80
 const dropMsb = 0x7f
 const maxSize = Math.floor((8 * 8 + 7) / 7)
@@ -38,7 +40,7 @@ export class VarIntProcessor {
     if (success) {
       return result
     } else {
-      console.error(this.buf)
+      console.error(asError(this.buf))
       return null
     }
   }

sdk/lib/test/configBuilder.test.ts

@@ -6,6 +6,8 @@ import { Variants } from "../config/builder/variants"
 import { ValueSpec } from "../config/configTypes"
 import { setupManifest } from "../manifest/setupManifest"
 import { StartSdk } from "../StartSdk"
+import { VersionGraph } from "../version/VersionGraph"
+import { VersionInfo } from "../version/VersionInfo"
 
 describe("builder tests", () => {
   test("text", async () => {
@@ -366,42 +368,48 @@ describe("values", () => {
     test("datetime", async () => {
       const sdk = StartSdk.of()
         .withManifest(
-          setupManifest({
-            id: "testOutput",
-            title: "",
-            version: "1.0.0:0",
-            releaseNotes: "",
-            license: "",
-            replaces: [],
-            wrapperRepo: "",
-            upstreamRepo: "",
-            supportSite: "",
-            marketingSite: "",
-            donationUrl: null,
-            description: {
-              short: "",
-              long: "",
-            },
-            containers: {},
-            images: {},
-            volumes: [],
-            assets: [],
-            alerts: {
-              install: null,
-              update: null,
-              uninstall: null,
-              restore: null,
-              start: null,
-              stop: null,
-            },
-            dependencies: {
-              "remote-test": {
-                description: "",
-                optional: true,
-                s9pk: "https://example.com/remote-test.s9pk",
+          setupManifest(
+            VersionGraph.of(
+              VersionInfo.of({
+                version: "1.0.0:0",
+                releaseNotes: "",
+                migrations: {},
+              }),
+            ),
+            {
+              id: "testOutput",
+              title: "",
+              license: "",
+              wrapperRepo: "",
+              upstreamRepo: "",
+              supportSite: "",
+              marketingSite: "",
+              donationUrl: null,
+              description: {
+                short: "",
+                long: "",
+              },
+              containers: {},
+              images: {},
+              volumes: [],
+              assets: [],
+              alerts: {
+                install: null,
+                update: null,
+                uninstall: null,
+                restore: null,
+                start: null,
+                stop: null,
+              },
+              dependencies: {
+                "remote-test": {
+                  description: "",
+                  optional: true,
+                  s9pk: "https://example.com/remote-test.s9pk",
+                },
               },
             },
-          }),
+          ),
         )
         .withStore<{ test: "a" }>()
         .build(true)

sdk/lib/test/graph.test.ts

@@ -0,0 +1,148 @@
+import { Graph } from "../util/graph"
+
+describe("graph", () => {
+  {
+    {
+      test("findVertex", () => {
+        const graph = new Graph<string, string>()
+        const foo = graph.addVertex("foo", [], [])
+        const bar = graph.addVertex(
+          "bar",
+          [{ from: foo, metadata: "foo-bar" }],
+          [],
+        )
+        const baz = graph.addVertex(
+          "baz",
+          [{ from: bar, metadata: "bar-baz" }],
+          [],
+        )
+        const qux = graph.addVertex(
+          "qux",
+          [{ from: baz, metadata: "baz-qux" }],
+          [],
+        )
+        const match = Array.from(graph.findVertex((v) => v.metadata === "qux"))
+        expect(match).toHaveLength(1)
+        expect(match[0]).toBe(qux)
+      })
+      test("shortestPathA", () => {
+        const graph = new Graph<string, string>()
+        const foo = graph.addVertex("foo", [], [])
+        const bar = graph.addVertex(
+          "bar",
+          [{ from: foo, metadata: "foo-bar" }],
+          [],
+        )
+        const baz = graph.addVertex(
+          "baz",
+          [{ from: bar, metadata: "bar-baz" }],
+          [],
+        )
+        const qux = graph.addVertex(
+          "qux",
+          [{ from: baz, metadata: "baz-qux" }],
+          [],
+        )
+        graph.addEdge("foo-qux", foo, qux)
+        expect(graph.shortestPath(foo, qux) || []).toHaveLength(1)
+      })
+      test("shortestPathB", () => {
+        const graph = new Graph<string, string>()
+        const foo = graph.addVertex("foo", [], [])
+        const bar = graph.addVertex(
+          "bar",
+          [{ from: foo, metadata: "foo-bar" }],
+          [],
+        )
+        const baz = graph.addVertex(
+          "baz",
+          [{ from: bar, metadata: "bar-baz" }],
+          [],
+        )
+        const qux = graph.addVertex(
+          "qux",
+          [{ from: baz, metadata: "baz-qux" }],
+          [],
+        )
+        graph.addEdge("bar-qux", bar, qux)
+        expect(graph.shortestPath(foo, qux) || []).toHaveLength(2)
+      })
+      test("shortestPathC", () => {
+        const graph = new Graph<string, string>()
+        const foo = graph.addVertex("foo", [], [])
+        const bar = graph.addVertex(
+          "bar",
+          [{ from: foo, metadata: "foo-bar" }],
+          [],
+        )
+        const baz = graph.addVertex(
+          "baz",
+          [{ from: bar, metadata: "bar-baz" }],
+          [],
+        )
+        const qux = graph.addVertex(
+          "qux",
+          [{ from: baz, metadata: "baz-qux" }],
+          [{ to: foo, metadata: "qux-foo" }],
+        )
+        expect(graph.shortestPath(foo, qux) || []).toHaveLength(3)
+      })
+      test("bfs", () => {
+        const graph = new Graph<string, string>()
+        const foo = graph.addVertex("foo", [], [])
+        const bar = graph.addVertex(
+          "bar",
+          [{ from: foo, metadata: "foo-bar" }],
+          [],
+        )
+        const baz = graph.addVertex(
+          "baz",
+          [{ from: bar, metadata: "bar-baz" }],
+          [],
+        )
+        const qux = graph.addVertex(
+          "qux",
+          [
+            { from: foo, metadata: "foo-qux" },
+            { from: baz, metadata: "baz-qux" },
+          ],
+          [],
+        )
+        const bfs = Array.from(graph.breadthFirstSearch(foo))
+        expect(bfs).toHaveLength(4)
+        expect(bfs[0]).toBe(foo)
+        expect(bfs[1]).toBe(bar)
+        expect(bfs[2]).toBe(qux)
+        expect(bfs[3]).toBe(baz)
+      })
+      test("reverseBfs", () => {
+        const graph = new Graph<string, string>()
+        const foo = graph.addVertex("foo", [], [])
+        const bar = graph.addVertex(
+          "bar",
+          [{ from: foo, metadata: "foo-bar" }],
+          [],
+        )
+        const baz = graph.addVertex(
+          "baz",
+          [{ from: bar, metadata: "bar-baz" }],
+          [],
+        )
+        const qux = graph.addVertex(
+          "qux",
+          [
+            { from: foo, metadata: "foo-qux" },
+            { from: baz, metadata: "baz-qux" },
+          ],
+          [],
+        )
+        const bfs = Array.from(graph.reverseBreadthFirstSearch(qux))
+        expect(bfs).toHaveLength(4)
+        expect(bfs[0]).toBe(qux)
+        expect(bfs[1]).toBe(foo)
+        expect(bfs[2]).toBe(baz)
+        expect(bfs[3]).toBe(bar)
+      })
+    }
+  }
+})

sdk/lib/test/host.test.ts

@@ -16,7 +16,6 @@ describe("host", () => {
         id: "foo",
         description: "A Foo",
         hasPrimary: false,
-        disabled: false,
         type: "ui",
         username: "bar",
         path: "/baz",

sdk/lib/test/output.sdk.ts

@@ -1,45 +1,56 @@
 import { StartSdk } from "../StartSdk"
 import { setupManifest } from "../manifest/setupManifest"
+import { VersionInfo } from "../version/VersionInfo"
+import { VersionGraph } from "../version/VersionGraph"
 
 export type Manifest = any
 export const sdk = StartSdk.of()
   .withManifest(
-    setupManifest({
-      id: "testOutput",
-      title: "",
-      version: "1.0:0",
-      releaseNotes: "",
-      license: "",
-      replaces: [],
-      wrapperRepo: "",
-      upstreamRepo: "",
-      supportSite: "",
-      marketingSite: "",
-      donationUrl: null,
-      description: {
-        short: "",
-        long: "",
-      },
-      containers: {},
-      images: {},
-      volumes: [],
-      assets: [],
-      alerts: {
-        install: null,
-        update: null,
-        uninstall: null,
-        restore: null,
-        start: null,
-        stop: null,
-      },
-      dependencies: {
-        "remote-test": {
-          description: "",
-          optional: false,
-          s9pk: "https://example.com/remote-test.s9pk",
+    setupManifest(
+      VersionGraph.of(
+        VersionInfo.of({
+          version: "1.0.0:0",
+          releaseNotes: "",
+          migrations: {},
+        })
+          .satisfies("#other:1.0.0:0")
+          .satisfies("#other:2.0.0:0"),
+      ),
+      {
+        id: "testOutput",
+        title: "",
+        license: "",
+        replaces: [],
+        wrapperRepo: "",
+        upstreamRepo: "",
+        supportSite: "",
+        marketingSite: "",
+        donationUrl: null,
+        description: {
+          short: "",
+          long: "",
+        },
+        containers: {},
+        images: {},
+        volumes: [],
+        assets: [],
+        alerts: {
+          install: null,
+          update: null,
+          uninstall: null,
+          restore: null,
+          start: null,
+          stop: null,
+        },
+        dependencies: {
+          "remote-test": {
+            description: "",
+            optional: false,
+            s9pk: "https://example.com/remote-test.s9pk",
+          },
         },
       },
-    }),
+    ),
   )
   .withStore<{ storeRoot: { storeLeaf: "value" } }>()
   .build(true)

sdk/lib/test/startosTypeValidation.test.ts

@@ -3,10 +3,13 @@ import {
   CheckDependenciesParam,
   ExecuteAction,
   GetConfiguredParams,
+  GetStoreParams,
+  SetDataVersionParams,
   SetMainStatus,
+  SetStoreParams,
 } from ".././osBindings"
-import { CreateOverlayedImageParams } from ".././osBindings"
-import { DestroyOverlayedImageParams } from ".././osBindings"
+import { CreateSubcontainerFsParams } from ".././osBindings"
+import { DestroySubcontainerFsParams } from ".././osBindings"
 import { BindParams } from ".././osBindings"
 import { GetHostInfoParams } from ".././osBindings"
 import { SetConfigured } from ".././osBindings"
@@ -23,21 +26,28 @@ import { GetPrimaryUrlParams } from ".././osBindings"
 import { ListServiceInterfacesParams } from ".././osBindings"
 import { ExportActionParams } from ".././osBindings"
 import { MountParams } from ".././osBindings"
+import { StringObject } from "../util"
 function typeEquality<ExpectedType>(_a: ExpectedType) {}
 
 type WithCallback<T> = Omit<T, "callback"> & { callback: () => void }
 
+type EffectsTypeChecker<T extends StringObject = Effects> = {
+  [K in keyof T]: T[K] extends (args: infer A) => any
+    ? A
+    : T[K] extends StringObject
+      ? EffectsTypeChecker<T[K]>
+      : never
+}
+
 describe("startosTypeValidation ", () => {
   test(`checking the params match`, () => {
     const testInput: any = {}
-    typeEquality<{
-      [K in keyof Effects]: Effects[K] extends (args: infer A) => any
-        ? A
-        : never
-    }>({
+    typeEquality<EffectsTypeChecker>({
       executeAction: {} as ExecuteAction,
-      createOverlayedImage: {} as CreateOverlayedImageParams,
-      destroyOverlayedImage: {} as DestroyOverlayedImageParams,
+      subcontainer: {
+        createFs: {} as CreateSubcontainerFsParams,
+        destroyFs: {} as DestroySubcontainerFsParams,
+      },
       clearBindings: undefined,
       getInstalledPackages: undefined,
       bind: {} as BindParams,
@@ -46,13 +56,18 @@ describe("startosTypeValidation ", () => {
       restart: undefined,
       shutdown: undefined,
       setConfigured: {} as SetConfigured,
+      setDataVersion: {} as SetDataVersionParams,
+      getDataVersion: undefined,
       setHealth: {} as SetHealth,
       exposeForDependents: {} as ExposeForDependentsParams,
       getSslCertificate: {} as WithCallback<GetSslCertificateParams>,
       getSslKey: {} as GetSslKeyParams,
       getServiceInterface: {} as WithCallback<GetServiceInterfaceParams>,
       setDependencies: {} as SetDependenciesParams,
-      store: {} as never,
+      store: {
+        get: {} as any, // as GetStoreParams,
+        set: {} as any, // as SetStoreParams,
+      },
       getSystemSmtp: {} as WithCallback<GetSystemSmtpParams>,
       getContainerIp: undefined,
       getServicePortForward: {} as GetServicePortForwardParams,

sdk/lib/trigger/TriggerInput.ts

@@ -2,5 +2,4 @@ import { HealthStatus } from "../types"
 
 export type TriggerInput = {
   lastResult?: HealthStatus
-  hadSuccess?: boolean
 }

sdk/lib/trigger/changeOnFirstSuccess.ts

@@ -5,10 +5,12 @@ export function changeOnFirstSuccess(o: {
   afterFirstSuccess: Trigger
 }): Trigger {
   return async function* (getInput) {
-    const beforeFirstSuccess = o.beforeFirstSuccess(getInput)
-    yield
     let currentValue = getInput()
-    beforeFirstSuccess.next()
+    while (!currentValue.lastResult) {
+      yield
+      currentValue = getInput()
+    }
+    const beforeFirstSuccess = o.beforeFirstSuccess(getInput)
     for (
       let res = await beforeFirstSuccess.next();
       currentValue?.lastResult !== "success" && !res.done;

sdk/lib/trigger/defaultTrigger.ts

@@ -2,7 +2,7 @@ import { cooldownTrigger } from "./cooldownTrigger"
 import { changeOnFirstSuccess } from "./changeOnFirstSuccess"
 import { successFailure } from "./successFailure"
 
-export const defaultTrigger = successFailure({
-  duringSuccess: cooldownTrigger(0),
-  duringError: cooldownTrigger(30000),
+export const defaultTrigger = changeOnFirstSuccess({
+  beforeFirstSuccess: cooldownTrigger(1000),
+  afterFirstSuccess: cooldownTrigger(30000),
 })

sdk/lib/trigger/index.ts

@@ -1,3 +1,4 @@
+import { ExecSpawnable } from "../util/SubContainer"
 import { TriggerInput } from "./TriggerInput"
 export { changeOnFirstSuccess } from "./changeOnFirstSuccess"
 export { cooldownTrigger } from "./cooldownTrigger"

sdk/lib/trigger/lastStatus.ts

@@ -0,0 +1,33 @@
+import { Trigger } from "."
+import { HealthStatus } from "../types"
+
+export type LastStatusTriggerParams = { [k in HealthStatus]?: Trigger } & {
+  default: Trigger
+}
+
+export function lastStatus(o: LastStatusTriggerParams): Trigger {
+  return async function* (getInput) {
+    let trigger = o.default(getInput)
+    const triggers: {
+      [k in HealthStatus]?: AsyncIterator<unknown, unknown, never>
+    } & { default: AsyncIterator<unknown, unknown, never> } = {
+      default: trigger,
+    }
+    while (true) {
+      let currentValue = getInput()
+      let prev: HealthStatus | "default" | undefined = currentValue.lastResult
+      if (!prev) {
+        yield
+        continue
+      }
+      if (!(prev in o)) {
+        prev = "default"
+      }
+      if (!triggers[prev]) {
+        triggers[prev] = o[prev]!(getInput)
+      }
+      await triggers[prev]?.next()
+      yield
+    }
+  }
+}

sdk/lib/trigger/successFailure.ts

@@ -1,32 +1,7 @@
 import { Trigger } from "."
+import { lastStatus } from "./lastStatus"
 
-export function successFailure(o: {
+export const successFailure = (o: {
   duringSuccess: Trigger
   duringError: Trigger
-}): Trigger {
-  return async function* (getInput) {
-    while (true) {
-      const beforeSuccess = o.duringSuccess(getInput)
-      yield
-      let currentValue = getInput()
-      beforeSuccess.next()
-      for (
-        let res = await beforeSuccess.next();
-        currentValue?.lastResult !== "success" && !res.done;
-        res = await beforeSuccess.next()
-      ) {
-        yield
-        currentValue = getInput()
-      }
-      const duringError = o.duringError(getInput)
-      for (
-        let res = await duringError.next();
-        currentValue?.lastResult === "success" && !res.done;
-        res = await duringError.next()
-      ) {
-        yield
-        currentValue = getInput()
-      }
-    }
-  }
-}
+}) => lastStatus({ success: o.duringSuccess, default: o.duringError })

sdk/lib/types.ts

@@ -3,7 +3,7 @@ export * as configTypes from "./config/configTypes"
 import {
   DependencyRequirement,
   SetHealth,
-  HealthCheckResult,
+  NamedHealthCheckResult,
   SetMainStatus,
   ServiceInterface,
   Host,
@@ -25,6 +25,7 @@ import { Daemons } from "./mainFn/Daemons"
 import { StorePath } from "./store/PathBuilder"
 import { ExposedStorePaths } from "./store/setupExposeStore"
 import { UrlString } from "./util/getServiceInterface"
+import { StringObject, ToKebab } from "./util"
 export * from "./osBindings"
 export { SDKManifest } from "./manifest/ManifestTypes"
 export { HealthReceipt } from "./health/HealthReceipt"
@@ -102,10 +103,7 @@ export namespace ExpectedExports {
    * Every time a package completes an install, this function is called before the main.
    * Can be used to do migration like things.
    */
-  export type init = (options: {
-    effects: Effects
-    previousVersion: null | string
-  }) => Promise<unknown>
+  export type init = (options: { effects: Effects }) => Promise<unknown>
   /** This will be ran during any time a package is uninstalled, for example during a update
    * this will be called.
    */
@@ -174,7 +172,7 @@ export type Daemon = {
   [DaemonProof]: never
 }
 
-export type HealthStatus = HealthCheckResult["result"]
+export type HealthStatus = NamedHealthCheckResult["result"]
 export type SmtpValue = {
   server: string
   port: number
@@ -249,15 +247,15 @@ export type SdkPropertiesValue =
     }
   | {
       type: "string"
-      /** Value  */
+      /** The value to display to the user */
       value: string
       /** A human readable description or explanation of the value */
       description?: string
-      /** (string/number only) Whether or not to mask the value, for example, when displaying a password */
+      /** Whether or not to mask the value, for example, when displaying a password */
       masked: boolean
-      /** (string/number only) Whether or not to include a button for copying the value to clipboard */
+      /** Whether or not to include a button for copying the value to clipboard */
       copyable?: boolean
-      /** (string/number only) Whether or not to include a button for displaying the value as a QR code */
+      /** Whether or not to include a button for displaying the value as a QR code */
       qr?: boolean
     }
 
@@ -273,15 +271,15 @@ export type PropertiesValue =
     }
   | {
       type: "string"
-      /** Value  */
+      /** The value to display to the user */
       value: string
       /** A human readable description or explanation of the value */
       description: string | null
-      /** (string/number only) Whether or not to mask the value, for example, when displaying a password */
+      /** Whether or not to mask the value, for example, when displaying a password */
       masked: boolean
-      /** (string/number only) Whether or not to include a button for copying the value to clipboard */
+      /** Whether or not to include a button for copying the value to clipboard */
       copyable: boolean | null
-      /** (string/number only) Whether or not to include a button for displaying the value as a QR code */
+      /** Whether or not to include a button for displaying the value as a QR code */
       qr: boolean | null
     }
 
@@ -289,6 +287,16 @@ export type PropertiesReturn = {
   [key: string]: PropertiesValue
 }
 
+export type EffectMethod<T extends StringObject = Effects> = {
+  [K in keyof T]-?: K extends string
+    ? T[K] extends Function
+      ? ToKebab<K>
+      : T[K] extends StringObject
+        ? `${ToKebab<K>}.${EffectMethod<T[K]>}`
+        : never
+    : never
+}[keyof T]
+
 /** Used to reach out from the pure js runtime */
 export type Effects = {
   // action
@@ -355,12 +363,13 @@ export type Effects = {
   /** sets the result of a health check */
   setHealth(o: SetHealth): Promise<void>
 
-  // image
-
-  /** A low level api used by Overlay */
-  createOverlayedImage(options: { imageId: string }): Promise<[string, string]>
-  /** A low level api used by Overlay */
-  destroyOverlayedImage(options: { guid: string }): Promise<void>
+  // subcontainer
+  subcontainer: {
+    /** A low level api used by SubContainer */
+    createFs(options: { imageId: string }): Promise<[string, string]>
+    /** A low level api used by SubContainer */
+    destroyFs(options: { guid: string }): Promise<void>
+  }
 
   // net
 
@@ -373,7 +382,7 @@ export type Effects = {
     hostId: HostId
     internalPort: number
   }): Promise<LanInfo>
-  /** Removes all network bindings */
+  /** Removes all network bindings, called in the setupConfig */
   clearBindings(): Promise<void>
   // host
   /** Returns information about the specified host, if it exists */
@@ -437,6 +446,10 @@ export type Effects = {
       value: ExtractStore
     }): Promise<void>
   }
+  /** sets the version that this service's data has been migrated to */
+  setDataVersion(options: { version: string }): Promise<void>
+  /** returns the version that this service's data has been migrated to */
+  getDataVersion(): Promise<string | null>
 
   // system
 
@@ -475,12 +488,11 @@ export type MigrationRes = {
 }
 
 export type ActionResult = {
+  version: "0"
   message: string
-  value: null | {
-    value: string
-    copyable: boolean
-    qr: boolean
-  }
+  value: string | null
+  copyable: boolean
+  qr: boolean
 }
 export type SetResult = {
   dependsOn: DependsOn

sdk/lib/util/Overlay.ts

@@ -1,240 +0,0 @@
-import * as fs from "fs/promises"
-import * as T from "../types"
-import * as cp from "child_process"
-import { promisify } from "util"
-import { Buffer } from "node:buffer"
-export const execFile = promisify(cp.execFile)
-const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/`
-export class Overlay {
-  private constructor(
-    readonly effects: T.Effects,
-    readonly imageId: T.ImageId,
-    readonly rootfs: string,
-    readonly guid: T.Guid,
-  ) {}
-  static async of(
-    effects: T.Effects,
-    image: { id: T.ImageId; sharedRun?: boolean },
-  ) {
-    const { id, sharedRun } = image
-    const [rootfs, guid] = await effects.createOverlayedImage({
-      imageId: id as string,
-    })
-
-    const shared = ["dev", "sys", "proc"]
-    if (!!sharedRun) {
-      shared.push("run")
-    }
-
-    fs.copyFile("/etc/resolv.conf", `${rootfs}/etc/resolv.conf`)
-
-    for (const dirPart of shared) {
-      const from = `/${dirPart}`
-      const to = `${rootfs}/${dirPart}`
-      await fs.mkdir(from, { recursive: true })
-      await fs.mkdir(to, { recursive: true })
-      await execFile("mount", ["--rbind", from, to])
-    }
-
-    return new Overlay(effects, id, rootfs, guid)
-  }
-
-  async mount(options: MountOptions, path: string): Promise<Overlay> {
-    path = path.startsWith("/")
-      ? `${this.rootfs}${path}`
-      : `${this.rootfs}/${path}`
-    if (options.type === "volume") {
-      const subpath = options.subpath
-        ? options.subpath.startsWith("/")
-          ? options.subpath
-          : `/${options.subpath}`
-        : "/"
-      const from = `/media/startos/volumes/${options.id}${subpath}`
-
-      await fs.mkdir(from, { recursive: true })
-      await fs.mkdir(path, { recursive: true })
-      await await execFile("mount", ["--bind", from, path])
-    } else if (options.type === "assets") {
-      const subpath = options.subpath
-        ? options.subpath.startsWith("/")
-          ? options.subpath
-          : `/${options.subpath}`
-        : "/"
-      const from = `/media/startos/assets/${options.id}${subpath}`
-
-      await fs.mkdir(from, { recursive: true })
-      await fs.mkdir(path, { recursive: true })
-      await execFile("mount", ["--bind", from, path])
-    } else if (options.type === "pointer") {
-      await this.effects.mount({ location: path, target: options })
-    } else if (options.type === "backup") {
-      const subpath = options.subpath
-        ? options.subpath.startsWith("/")
-          ? options.subpath
-          : `/${options.subpath}`
-        : "/"
-      const from = `/media/startos/backup${subpath}`
-
-      await fs.mkdir(from, { recursive: true })
-      await fs.mkdir(path, { recursive: true })
-      await execFile("mount", ["--bind", from, path])
-    } else {
-      throw new Error(`unknown type ${(options as any).type}`)
-    }
-    return this
-  }
-
-  async destroy() {
-    const imageId = this.imageId
-    const guid = this.guid
-    await this.effects.destroyOverlayedImage({ guid })
-  }
-
-  async exec(
-    command: string[],
-    options?: CommandOptions,
-    timeoutMs: number | null = 30000,
-  ): Promise<{
-    exitCode: number | null
-    exitSignal: NodeJS.Signals | null
-    stdout: string | Buffer
-    stderr: string | Buffer
-  }> {
-    const imageMeta: T.ImageMetadata = await fs
-      .readFile(`/media/startos/images/${this.imageId}.json`, {
-        encoding: "utf8",
-      })
-      .catch(() => "{}")
-      .then(JSON.parse)
-    let extra: string[] = []
-    if (options?.user) {
-      extra.push(`--user=${options.user}`)
-      delete options.user
-    }
-    let workdir = imageMeta.workdir || "/"
-    if (options?.cwd) {
-      workdir = options.cwd
-      delete options.cwd
-    }
-    const child = cp.spawn(
-      "start-cli",
-      [
-        "chroot",
-        `--env=/media/startos/images/${this.imageId}.env`,
-        `--workdir=${workdir}`,
-        ...extra,
-        this.rootfs,
-        ...command,
-      ],
-      options || {},
-    )
-    const pid = child.pid
-    const stdout = { data: "" as string | Buffer }
-    const stderr = { data: "" as string | Buffer }
-    const appendData =
-      (appendTo: { data: string | Buffer }) =>
-      (chunk: string | Buffer | any) => {
-        if (typeof appendTo.data === "string" && typeof chunk === "string") {
-          appendTo.data += chunk
-        } else if (typeof chunk === "string" || chunk instanceof Buffer) {
-          appendTo.data = Buffer.concat([
-            Buffer.from(appendTo.data),
-            Buffer.from(chunk),
-          ])
-        } else {
-          console.error("received unexpected chunk", chunk)
-        }
-      }
-    return new Promise((resolve, reject) => {
-      child.on("error", reject)
-      if (timeoutMs !== null && pid) {
-        setTimeout(
-          () => execFile("pkill", ["-9", "-s", String(pid)]).catch((_) => {}),
-          timeoutMs,
-        )
-      }
-      child.stdout.on("data", appendData(stdout))
-      child.stderr.on("data", appendData(stderr))
-      child.on("exit", (code, signal) =>
-        resolve({
-          exitCode: code,
-          exitSignal: signal,
-          stdout: stdout.data,
-          stderr: stderr.data,
-        }),
-      )
-    })
-  }
-
-  async spawn(
-    command: string[],
-    options?: CommandOptions,
-  ): Promise<cp.ChildProcessWithoutNullStreams> {
-    const imageMeta: any = await fs
-      .readFile(`/media/startos/images/${this.imageId}.json`, {
-        encoding: "utf8",
-      })
-      .catch(() => "{}")
-      .then(JSON.parse)
-    let extra: string[] = []
-    if (options?.user) {
-      extra.push(`--user=${options.user}`)
-      delete options.user
-    }
-    let workdir = imageMeta.workdir || "/"
-    if (options?.cwd) {
-      workdir = options.cwd
-      delete options.cwd
-    }
-    return cp.spawn(
-      "start-cli",
-      [
-        "chroot",
-        `--env=/media/startos/images/${this.imageId}.env`,
-        `--workdir=${workdir}`,
-        ...extra,
-        this.rootfs,
-        ...command,
-      ],
-      options,
-    )
-  }
-}
-
-export type CommandOptions = {
-  env?: { [variable: string]: string }
-  cwd?: string
-  user?: string
-}
-
-export type MountOptions =
-  | MountOptionsVolume
-  | MountOptionsAssets
-  | MountOptionsPointer
-  | MountOptionsBackup
-
-export type MountOptionsVolume = {
-  type: "volume"
-  id: string
-  subpath: string | null
-  readonly: boolean
-}
-
-export type MountOptionsAssets = {
-  type: "assets"
-  id: string
-  subpath: string | null
-}
-
-export type MountOptionsPointer = {
-  type: "pointer"
-  packageId: string
-  volumeId: string
-  subpath: string | null
-  readonly: boolean
-}
-
-export type MountOptionsBackup = {
-  type: "backup"
-  subpath: string | null
-}

sdk/lib/util/SubContainer.ts

@@ -0,0 +1,434 @@
+import * as fs from "fs/promises"
+import * as T from "../types"
+import * as cp from "child_process"
+import { promisify } from "util"
+import { Buffer } from "node:buffer"
+import { once } from "./once"
+export const execFile = promisify(cp.execFile)
+const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/`
+const False = () => false
+type ExecResults = {
+  exitCode: number | null
+  exitSignal: NodeJS.Signals | null
+  stdout: string | Buffer
+  stderr: string | Buffer
+}
+
+export type ExecOptions = {
+  input?: string | Buffer
+}
+
+const TIMES_TO_WAIT_FOR_PROC = 100
+
+/**
+ * This is the type that is going to describe what an subcontainer could do. The main point of the
+ * subcontainer is to have commands that run in a chrooted environment. This is useful for running
+ * commands in a containerized environment. But, I wanted the destroy to sometimes be doable, for example the
+ * case where the subcontainer isn't owned by the process, the subcontainer shouldn't be destroyed.
+ */
+export interface ExecSpawnable {
+  get destroy(): undefined | (() => Promise<void>)
+  exec(
+    command: string[],
+    options?: CommandOptions & ExecOptions,
+    timeoutMs?: number | null,
+  ): Promise<ExecResults>
+  spawn(
+    command: string[],
+    options?: CommandOptions,
+  ): Promise<cp.ChildProcessWithoutNullStreams>
+}
+/**
+ * Want to limit what we can do in a container, so we want to launch a container with a specific image and the mounts.
+ *
+ * Implements:
+ * @see {@link ExecSpawnable}
+ */
+export class SubContainer implements ExecSpawnable {
+  private leader: cp.ChildProcess
+  private leaderExited: boolean = false
+  private waitProc: () => Promise<void>
+  private constructor(
+    readonly effects: T.Effects,
+    readonly imageId: T.ImageId,
+    readonly rootfs: string,
+    readonly guid: T.Guid,
+  ) {
+    this.leaderExited = false
+    this.leader = cp.spawn("start-cli", ["subcontainer", "launch", rootfs], {
+      killSignal: "SIGKILL",
+      stdio: "ignore",
+    })
+    this.leader.on("exit", () => {
+      this.leaderExited = true
+    })
+    this.waitProc = once(
+      () =>
+        new Promise(async (resolve, reject) => {
+          let count = 0
+          while (
+            !(await fs.stat(`${this.rootfs}/proc/1`).then((x) => !!x, False))
+          ) {
+            if (count++ > TIMES_TO_WAIT_FOR_PROC) {
+              console.debug("Failed to start subcontainer", {
+                guid: this.guid,
+                imageId: this.imageId,
+                rootfs: this.rootfs,
+              })
+              reject(new Error(`Failed to start subcontainer ${this.imageId}`))
+            }
+            await wait(1)
+          }
+          resolve()
+        }),
+    )
+  }
+  static async of(
+    effects: T.Effects,
+    image: { id: T.ImageId; sharedRun?: boolean },
+  ) {
+    const { id, sharedRun } = image
+    const [rootfs, guid] = await effects.subcontainer.createFs({
+      imageId: id as string,
+    })
+
+    const shared = ["dev", "sys"]
+    if (!!sharedRun) {
+      shared.push("run")
+    }
+
+    fs.copyFile("/etc/resolv.conf", `${rootfs}/etc/resolv.conf`)
+
+    for (const dirPart of shared) {
+      const from = `/${dirPart}`
+      const to = `${rootfs}/${dirPart}`
+      await fs.mkdir(from, { recursive: true })
+      await fs.mkdir(to, { recursive: true })
+      await execFile("mount", ["--rbind", from, to])
+    }
+
+    return new SubContainer(effects, id, rootfs, guid)
+  }
+
+  static async with<T>(
+    effects: T.Effects,
+    image: { id: T.ImageId; sharedRun?: boolean },
+    mounts: { options: MountOptions; path: string }[],
+    fn: (subContainer: SubContainer) => Promise<T>,
+  ): Promise<T> {
+    const subContainer = await SubContainer.of(effects, image)
+    try {
+      for (let mount of mounts) {
+        await subContainer.mount(mount.options, mount.path)
+      }
+      return await fn(subContainer)
+    } finally {
+      await subContainer.destroy()
+    }
+  }
+
+  async mount(options: MountOptions, path: string): Promise<SubContainer> {
+    path = path.startsWith("/")
+      ? `${this.rootfs}${path}`
+      : `${this.rootfs}/${path}`
+    if (options.type === "volume") {
+      const subpath = options.subpath
+        ? options.subpath.startsWith("/")
+          ? options.subpath
+          : `/${options.subpath}`
+        : "/"
+      const from = `/media/startos/volumes/${options.id}${subpath}`
+
+      await fs.mkdir(from, { recursive: true })
+      await fs.mkdir(path, { recursive: true })
+      await execFile("mount", ["--bind", from, path])
+    } else if (options.type === "assets") {
+      const subpath = options.subpath
+        ? options.subpath.startsWith("/")
+          ? options.subpath
+          : `/${options.subpath}`
+        : "/"
+      const from = `/media/startos/assets/${options.id}${subpath}`
+
+      await fs.mkdir(from, { recursive: true })
+      await fs.mkdir(path, { recursive: true })
+      await execFile("mount", ["--bind", from, path])
+    } else if (options.type === "pointer") {
+      await this.effects.mount({ location: path, target: options })
+    } else if (options.type === "backup") {
+      const subpath = options.subpath
+        ? options.subpath.startsWith("/")
+          ? options.subpath
+          : `/${options.subpath}`
+        : "/"
+      const from = `/media/startos/backup${subpath}`
+
+      await fs.mkdir(from, { recursive: true })
+      await fs.mkdir(path, { recursive: true })
+      await execFile("mount", ["--bind", from, path])
+    } else {
+      throw new Error(`unknown type ${(options as any).type}`)
+    }
+    return this
+  }
+
+  private async killLeader() {
+    if (this.leaderExited) {
+      return
+    }
+    return new Promise<void>((resolve, reject) => {
+      try {
+        this.leader.on("exit", () => {
+          resolve()
+        })
+        if (!this.leader.kill("SIGKILL")) {
+          reject(new Error("kill(2) failed"))
+        }
+      } catch (e) {
+        reject(e)
+      }
+    })
+  }
+
+  get destroy() {
+    return async () => {
+      const guid = this.guid
+      await this.killLeader()
+      await this.effects.subcontainer.destroyFs({ guid })
+    }
+  }
+
+  async exec(
+    command: string[],
+    options?: CommandOptions & ExecOptions,
+    timeoutMs: number | null = 30000,
+  ): Promise<{
+    exitCode: number | null
+    exitSignal: NodeJS.Signals | null
+    stdout: string | Buffer
+    stderr: string | Buffer
+  }> {
+    await this.waitProc()
+    const imageMeta: T.ImageMetadata = await fs
+      .readFile(`/media/startos/images/${this.imageId}.json`, {
+        encoding: "utf8",
+      })
+      .catch(() => "{}")
+      .then(JSON.parse)
+    let extra: string[] = []
+    if (options?.user) {
+      extra.push(`--user=${options.user}`)
+      delete options.user
+    }
+    let workdir = imageMeta.workdir || "/"
+    if (options?.cwd) {
+      workdir = options.cwd
+      delete options.cwd
+    }
+    const child = cp.spawn(
+      "start-cli",
+      [
+        "subcontainer",
+        "exec",
+        `--env=/media/startos/images/${this.imageId}.env`,
+        `--workdir=${workdir}`,
+        ...extra,
+        this.rootfs,
+        ...command,
+      ],
+      options || {},
+    )
+    if (options?.input) {
+      await new Promise<void>((resolve, reject) =>
+        child.stdin.write(options.input, (e) => {
+          if (e) {
+            reject(e)
+          } else {
+            resolve()
+          }
+        }),
+      )
+      await new Promise<void>((resolve) => child.stdin.end(resolve))
+    }
+    const pid = child.pid
+    const stdout = { data: "" as string | Buffer }
+    const stderr = { data: "" as string | Buffer }
+    const appendData =
+      (appendTo: { data: string | Buffer }) =>
+      (chunk: string | Buffer | any) => {
+        if (typeof appendTo.data === "string" && typeof chunk === "string") {
+          appendTo.data += chunk
+        } else if (typeof chunk === "string" || chunk instanceof Buffer) {
+          appendTo.data = Buffer.concat([
+            Buffer.from(appendTo.data),
+            Buffer.from(chunk),
+          ])
+        } else {
+          console.error("received unexpected chunk", chunk)
+        }
+      }
+    return new Promise((resolve, reject) => {
+      child.on("error", reject)
+      let killTimeout: NodeJS.Timeout | undefined
+      if (timeoutMs !== null && child.pid) {
+        killTimeout = setTimeout(() => child.kill("SIGKILL"), timeoutMs)
+      }
+      child.stdout.on("data", appendData(stdout))
+      child.stderr.on("data", appendData(stderr))
+      child.on("exit", (code, signal) => {
+        clearTimeout(killTimeout)
+        resolve({
+          exitCode: code,
+          exitSignal: signal,
+          stdout: stdout.data,
+          stderr: stderr.data,
+        })
+      })
+    })
+  }
+
+  async launch(
+    command: string[],
+    options?: CommandOptions,
+  ): Promise<cp.ChildProcessWithoutNullStreams> {
+    await this.waitProc()
+    const imageMeta: any = await fs
+      .readFile(`/media/startos/images/${this.imageId}.json`, {
+        encoding: "utf8",
+      })
+      .catch(() => "{}")
+      .then(JSON.parse)
+    let extra: string[] = []
+    if (options?.user) {
+      extra.push(`--user=${options.user}`)
+      delete options.user
+    }
+    let workdir = imageMeta.workdir || "/"
+    if (options?.cwd) {
+      workdir = options.cwd
+      delete options.cwd
+    }
+    await this.killLeader()
+    this.leaderExited = false
+    this.leader = cp.spawn(
+      "start-cli",
+      [
+        "subcontainer",
+        "launch",
+        `--env=/media/startos/images/${this.imageId}.env`,
+        `--workdir=${workdir}`,
+        ...extra,
+        this.rootfs,
+        ...command,
+      ],
+      { ...options, stdio: "inherit" },
+    )
+    this.leader.on("exit", () => {
+      this.leaderExited = true
+    })
+    return this.leader as cp.ChildProcessWithoutNullStreams
+  }
+
+  async spawn(
+    command: string[],
+    options?: CommandOptions,
+  ): Promise<cp.ChildProcessWithoutNullStreams> {
+    await this.waitProc()
+    const imageMeta: any = await fs
+      .readFile(`/media/startos/images/${this.imageId}.json`, {
+        encoding: "utf8",
+      })
+      .catch(() => "{}")
+      .then(JSON.parse)
+    let extra: string[] = []
+    if (options?.user) {
+      extra.push(`--user=${options.user}`)
+      delete options.user
+    }
+    let workdir = imageMeta.workdir || "/"
+    if (options?.cwd) {
+      workdir = options.cwd
+      delete options.cwd
+    }
+    return cp.spawn(
+      "start-cli",
+      [
+        "subcontainer",
+        "exec",
+        `--env=/media/startos/images/${this.imageId}.env`,
+        `--workdir=${workdir}`,
+        ...extra,
+        this.rootfs,
+        ...command,
+      ],
+      options,
+    )
+  }
+}
+
+/**
+ * Take an subcontainer but remove the ability to add the mounts and the destroy function.
+ * Lets other functions, like health checks, to not destroy the parents.
+ *
+ */
+export class SubContainerHandle implements ExecSpawnable {
+  constructor(private subContainer: ExecSpawnable) {}
+  get destroy() {
+    return undefined
+  }
+
+  exec(
+    command: string[],
+    options?: CommandOptions,
+    timeoutMs?: number | null,
+  ): Promise<ExecResults> {
+    return this.subContainer.exec(command, options, timeoutMs)
+  }
+  spawn(
+    command: string[],
+    options?: CommandOptions,
+  ): Promise<cp.ChildProcessWithoutNullStreams> {
+    return this.subContainer.spawn(command, options)
+  }
+}
+
+export type CommandOptions = {
+  env?: { [variable: string]: string }
+  cwd?: string
+  user?: string
+}
+
+export type MountOptions =
+  | MountOptionsVolume
+  | MountOptionsAssets
+  | MountOptionsPointer
+  | MountOptionsBackup
+
+export type MountOptionsVolume = {
+  type: "volume"
+  id: string
+  subpath: string | null
+  readonly: boolean
+}
+
+export type MountOptionsAssets = {
+  type: "assets"
+  id: string
+  subpath: string | null
+}
+
+export type MountOptionsPointer = {
+  type: "pointer"
+  packageId: string
+  volumeId: string
+  subpath: string | null
+  readonly: boolean
+}
+
+export type MountOptionsBackup = {
+  type: "backup"
+  subpath: string | null
+}
+function wait(time: number) {
+  return new Promise((resolve) => setTimeout(resolve, time))
+}

sdk/lib/util/asError.ts

@@ -0,0 +1,6 @@
+export const asError = (e: unknown) => {
+  if (e instanceof Error) {
+    return new Error(e as any)
+  }
+  return new Error(`${e}`)
+}

sdk/lib/util/getServiceInterface.ts

@@ -50,8 +50,6 @@ export type ServiceInterfaceFilled = {
   description: string
   /** Whether or not the interface has a primary URL */
   hasPrimary: boolean
-  /** Whether or not the interface disabled */
-  disabled: boolean
   /** Whether or not to mask the URIs for this interface. Useful if the URIs contain sensitive information, such as a password, macaroon, or API key */
   masked: boolean
   /** Information about the host for this binding */

sdk/lib/util/graph.ts

@@ -0,0 +1,244 @@
+import { boolean } from "ts-matches"
+
+export type Vertex<VMetadata = void, EMetadata = void> = {
+  metadata: VMetadata
+  edges: Array<Edge<EMetadata, VMetadata>>
+}
+
+export type Edge<EMetadata = void, VMetadata = void> = {
+  metadata: EMetadata
+  from: Vertex<VMetadata, EMetadata>
+  to: Vertex<VMetadata, EMetadata>
+}
+
+export class Graph<VMetadata = void, EMetadata = void> {
+  private readonly vertices: Array<Vertex<VMetadata, EMetadata>> = []
+  constructor() {}
+  addVertex(
+    metadata: VMetadata,
+    fromEdges: Array<Omit<Edge<EMetadata, VMetadata>, "to">>,
+    toEdges: Array<Omit<Edge<EMetadata, VMetadata>, "from">>,
+  ): Vertex<VMetadata, EMetadata> {
+    const vertex: Vertex<VMetadata, EMetadata> = {
+      metadata,
+      edges: [],
+    }
+    for (let edge of fromEdges) {
+      const vEdge = {
+        metadata: edge.metadata,
+        from: edge.from,
+        to: vertex,
+      }
+      edge.from.edges.push(vEdge)
+      vertex.edges.push(vEdge)
+    }
+    for (let edge of toEdges) {
+      const vEdge = {
+        metadata: edge.metadata,
+        from: vertex,
+        to: edge.to,
+      }
+      edge.to.edges.push(vEdge)
+      vertex.edges.push(vEdge)
+    }
+    this.vertices.push(vertex)
+    return vertex
+  }
+  findVertex(
+    predicate: (vertex: Vertex<VMetadata, EMetadata>) => boolean,
+  ): Generator<Vertex<VMetadata, EMetadata>, void> {
+    const veritces = this.vertices
+    function* gen() {
+      for (let vertex of veritces) {
+        if (predicate(vertex)) {
+          yield vertex
+        }
+      }
+    }
+    return gen()
+  }
+  addEdge(
+    metadata: EMetadata,
+    from: Vertex<VMetadata, EMetadata>,
+    to: Vertex<VMetadata, EMetadata>,
+  ): Edge<EMetadata, VMetadata> {
+    const edge = {
+      metadata,
+      from,
+      to,
+    }
+    edge.from.edges.push(edge)
+    edge.to.edges.push(edge)
+    return edge
+  }
+  breadthFirstSearch(
+    from:
+      | Vertex<VMetadata, EMetadata>
+      | ((vertex: Vertex<VMetadata, EMetadata>) => boolean),
+  ): Generator<Vertex<VMetadata, EMetadata>, void> {
+    const visited: Array<Vertex<VMetadata, EMetadata>> = []
+    function* rec(
+      vertex: Vertex<VMetadata, EMetadata>,
+    ): Generator<Vertex<VMetadata, EMetadata>, void> {
+      if (visited.includes(vertex)) {
+        return
+      }
+      visited.push(vertex)
+      yield vertex
+      let generators = vertex.edges
+        .filter((e) => e.from === vertex)
+        .map((e) => rec(e.to))
+      while (generators.length) {
+        let prev = generators
+        generators = []
+        for (let gen of prev) {
+          const next = gen.next()
+          if (!next.done) {
+            generators.push(gen)
+            yield next.value
+          }
+        }
+      }
+    }
+
+    if (from instanceof Function) {
+      let generators = this.vertices.filter(from).map(rec)
+      return (function* () {
+        while (generators.length) {
+          let prev = generators
+          generators = []
+          for (let gen of prev) {
+            const next = gen.next()
+            if (!next.done) {
+              generators.push(gen)
+              yield next.value
+            }
+          }
+        }
+      })()
+    } else {
+      return rec(from)
+    }
+  }
+  reverseBreadthFirstSearch(
+    to:
+      | Vertex<VMetadata, EMetadata>
+      | ((vertex: Vertex<VMetadata, EMetadata>) => boolean),
+  ): Generator<Vertex<VMetadata, EMetadata>, void> {
+    const visited: Array<Vertex<VMetadata, EMetadata>> = []
+    function* rec(
+      vertex: Vertex<VMetadata, EMetadata>,
+    ): Generator<Vertex<VMetadata, EMetadata>, void> {
+      if (visited.includes(vertex)) {
+        return
+      }
+      visited.push(vertex)
+      yield vertex
+      let generators = vertex.edges
+        .filter((e) => e.to === vertex)
+        .map((e) => rec(e.from))
+      while (generators.length) {
+        let prev = generators
+        generators = []
+        for (let gen of prev) {
+          const next = gen.next()
+          if (!next.done) {
+            generators.push(gen)
+            yield next.value
+          }
+        }
+      }
+    }
+
+    if (to instanceof Function) {
+      let generators = this.vertices.filter(to).map(rec)
+      return (function* () {
+        while (generators.length) {
+          let prev = generators
+          generators = []
+          for (let gen of prev) {
+            const next = gen.next()
+            if (!next.done) {
+              generators.push(gen)
+              yield next.value
+            }
+          }
+        }
+      })()
+    } else {
+      return rec(to)
+    }
+  }
+  shortestPath(
+    from:
+      | Vertex<VMetadata, EMetadata>
+      | ((vertex: Vertex<VMetadata, EMetadata>) => boolean),
+    to:
+      | Vertex<VMetadata, EMetadata>
+      | ((vertex: Vertex<VMetadata, EMetadata>) => boolean),
+  ): Array<Edge<EMetadata, VMetadata>> | void {
+    const isDone =
+      to instanceof Function
+        ? to
+        : (v: Vertex<VMetadata, EMetadata>) => v === to
+    const path: Array<Edge<EMetadata, VMetadata>> = []
+    const visited: Array<Vertex<VMetadata, EMetadata>> = []
+    function* check(
+      vertex: Vertex<VMetadata, EMetadata>,
+      path: Array<Edge<EMetadata, VMetadata>>,
+    ): Generator<undefined, Array<Edge<EMetadata, VMetadata>> | undefined> {
+      if (isDone(vertex)) {
+        return path
+      }
+      if (visited.includes(vertex)) {
+        return
+      }
+      visited.push(vertex)
+      yield
+      let generators = vertex.edges
+        .filter((e) => e.from === vertex)
+        .map((e) => check(e.to, [...path, e]))
+      while (generators.length) {
+        let prev = generators
+        generators = []
+        for (let gen of prev) {
+          const next = gen.next()
+          if (next.done === true) {
+            if (next.value) {
+              return next.value
+            }
+          } else {
+            generators.push(gen)
+            yield
+          }
+        }
+      }
+    }
+
+    if (from instanceof Function) {
+      let generators = this.vertices.filter(from).map((v) => check(v, []))
+      while (generators.length) {
+        let prev = generators
+        generators = []
+        for (let gen of prev) {
+          const next = gen.next()
+          if (next.done === true) {
+            if (next.value) {
+              return next.value
+            }
+          } else {
+            generators.push(gen)
+          }
+        }
+      }
+    } else {
+      const gen = check(from, [])
+      while (true) {
+        const next = gen.next()
+        if (next.done) {
+          return next.value
+        }
+      }
+    }
+  }
+}

sdk/lib/util/index.ts

@@ -3,10 +3,11 @@ import "./fileHelper"
 import "../store/getStore"
 import "./deepEqual"
 import "./deepMerge"
-import "./Overlay"
+import "./SubContainer"
 import "./once"
 
 export { GetServiceInterface, getServiceInterface } from "./getServiceInterface"
+export { asError } from "./asError"
 export { getServiceInterfaces } from "./getServiceInterfaces"
 export { addressHostToUrl } from "./getServiceInterface"
 export { hostnameInfoToAddress } from "./Hostname"

sdk/lib/util/typeHelpers.ts

@@ -21,3 +21,96 @@ export type NoAny<A> = NeverPossible extends A
     ? never
     : A
   : A
+
+type CapitalLetters =
+  | "A"
+  | "B"
+  | "C"
+  | "D"
+  | "E"
+  | "F"
+  | "G"
+  | "H"
+  | "I"
+  | "J"
+  | "K"
+  | "L"
+  | "M"
+  | "N"
+  | "O"
+  | "P"
+  | "Q"
+  | "R"
+  | "S"
+  | "T"
+  | "U"
+  | "V"
+  | "W"
+  | "X"
+  | "Y"
+  | "Z"
+
+type Numbers = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
+
+type CapitalChars = CapitalLetters | Numbers
+
+export type ToKebab<S extends string> = S extends string
+  ? S extends `${infer Head}${CapitalChars}${infer Tail}` // string has a capital char somewhere
+    ? Head extends "" // there is a capital char in the first position
+      ? Tail extends ""
+        ? Lowercase<S> /*  'A' */
+        : S extends `${infer Caps}${Tail}` // tail exists, has capital characters
+          ? Caps extends CapitalChars
+            ? Tail extends CapitalLetters
+              ? `${Lowercase<Caps>}-${Lowercase<Tail>}` /* 'AB' */
+              : Tail extends `${CapitalLetters}${string}`
+                ? `${ToKebab<Caps>}-${ToKebab<Tail>}` /* first tail char is upper? 'ABcd' */
+                : `${ToKebab<Caps>}${ToKebab<Tail>}` /* 'AbCD','AbcD',  */ /* TODO: if tail is only numbers, append without underscore */
+            : never /* never reached, used for inference of caps */
+          : never
+      : Tail extends "" /* 'aB' 'abCD' 'ABCD' 'AB' */
+        ? S extends `${Head}${infer Caps}`
+          ? Caps extends CapitalChars
+            ? Head extends Lowercase<Head> /* 'abcD' */
+              ? Caps extends Numbers
+                ? // Head exists and is lowercase, tail does not, Caps is a number, we may be in a sub-select
+                  // if head ends with number, don't split head an Caps, keep contiguous numbers together
+                  Head extends `${string}${Numbers}`
+                  ? never
+                  : // head does not end in number, safe to split. 'abc2' -> 'abc-2'
+                    `${ToKebab<Head>}-${Caps}`
+                : `${ToKebab<Head>}-${ToKebab<Caps>}` /* 'abcD' 'abc25' */
+              : never /* stop union type forming */
+            : never
+          : never /* never reached, used for inference of caps */
+        : S extends `${Head}${infer Caps}${Tail}` /* 'abCd' 'ABCD' 'AbCd' 'ABcD' */
+          ? Caps extends CapitalChars
+            ? Head extends Lowercase<Head> /* is 'abCd' 'abCD' ? */
+              ? Tail extends CapitalLetters /* is 'abCD' where Caps = 'C' */
+                ? `${ToKebab<Head>}-${ToKebab<Caps>}-${Lowercase<Tail>}` /* aBCD Tail = 'D', Head = 'aB' */
+                : Tail extends `${CapitalLetters}${string}` /* is 'aBCd' where Caps = 'B' */
+                  ? Head extends Numbers
+                    ? never /* stop union type forming */
+                    : Head extends `${string}${Numbers}`
+                      ? never /* stop union type forming */
+                      : `${Head}-${ToKebab<Caps>}-${ToKebab<Tail>}` /* 'aBCd' => `${'a'}-${Lowercase<'B'>}-${ToSnake<'Cd'>}` */
+                  : `${ToKebab<Head>}-${Lowercase<Caps>}${ToKebab<Tail>}` /* 'aBcD' where Caps = 'B' tail starts as lowercase */
+              : never
+            : never
+          : never
+    : S /* 'abc'  */
+  : never
+
+export type StringObject = Record<string, unknown>
+
+function test() {
+  // prettier-ignore
+  const t = <A, B>(a: (
+    A extends B ? (
+      B extends A ? null : never
+    ) : never
+  )) =>{ }
+  t<"foo-bar", ToKebab<"FooBar">>(null)
+  // @ts-expect-error
+  t<"foo-3ar", ToKebab<"FooBar">>(null)
+}

sdk/lib/version/VersionGraph.ts

@@ -0,0 +1,200 @@
+import { ExtendedVersion, VersionRange } from "../exver"
+
+import * as T from "../types"
+import { Graph, Vertex } from "../util/graph"
+import { once } from "../util/once"
+import { IMPOSSIBLE, VersionInfo } from "./VersionInfo"
+
+export class VersionGraph<CurrentVersion extends string> {
+  private readonly graph: () => Graph<
+    ExtendedVersion | VersionRange,
+    ((opts: { effects: T.Effects }) => Promise<void>) | undefined
+  >
+  private constructor(
+    readonly current: VersionInfo<CurrentVersion>,
+    versions: Array<VersionInfo<any>>,
+  ) {
+    this.graph = once(() => {
+      const graph = new Graph<
+        ExtendedVersion | VersionRange,
+        ((opts: { effects: T.Effects }) => Promise<void>) | undefined
+      >()
+      const flavorMap: Record<
+        string,
+        [
+          ExtendedVersion,
+          VersionInfo<any>,
+          Vertex<
+            ExtendedVersion | VersionRange,
+            ((opts: { effects: T.Effects }) => Promise<void>) | undefined
+          >,
+        ][]
+      > = {}
+      for (let version of [current, ...versions]) {
+        const v = ExtendedVersion.parse(version.options.version)
+        const vertex = graph.addVertex(v, [], [])
+        const flavor = v.flavor || ""
+        if (!flavorMap[flavor]) {
+          flavorMap[flavor] = []
+        }
+        flavorMap[flavor].push([v, version, vertex])
+      }
+      for (let flavor in flavorMap) {
+        flavorMap[flavor].sort((a, b) => a[0].compareForSort(b[0]))
+        let prev:
+          | [
+              ExtendedVersion,
+              VersionInfo<any>,
+              Vertex<
+                ExtendedVersion | VersionRange,
+                (opts: { effects: T.Effects }) => Promise<void>
+              >,
+            ]
+          | undefined = undefined
+        for (let [v, version, vertex] of flavorMap[flavor]) {
+          if (version.options.migrations.up !== IMPOSSIBLE) {
+            let range
+            if (prev) {
+              graph.addEdge(version.options.migrations.up, prev[2], vertex)
+              range = VersionRange.anchor(">=", prev[0]).and(
+                VersionRange.anchor("<", v),
+              )
+            } else {
+              range = VersionRange.anchor("<", v)
+            }
+            const vRange = graph.addVertex(range, [], [])
+            graph.addEdge(version.options.migrations.up, vRange, vertex)
+          }
+
+          if (version.options.migrations.down !== IMPOSSIBLE) {
+            let range
+            if (prev) {
+              graph.addEdge(version.options.migrations.down, vertex, prev[2])
+              range = VersionRange.anchor(">=", prev[0]).and(
+                VersionRange.anchor("<", v),
+              )
+            } else {
+              range = VersionRange.anchor("<", v)
+            }
+            const vRange = graph.addVertex(range, [], [])
+            graph.addEdge(version.options.migrations.down, vertex, vRange)
+          }
+
+          if (version.options.migrations.other) {
+            for (let rangeStr in version.options.migrations.other) {
+              const range = VersionRange.parse(rangeStr)
+              const vRange = graph.addVertex(range, [], [])
+              graph.addEdge(
+                version.options.migrations.other[rangeStr],
+                vRange,
+                vertex,
+              )
+              for (let matching of graph.findVertex(
+                (v) =>
+                  v.metadata instanceof ExtendedVersion &&
+                  v.metadata.satisfies(range),
+              )) {
+                graph.addEdge(
+                  version.options.migrations.other[rangeStr],
+                  matching,
+                  vertex,
+                )
+              }
+            }
+          }
+        }
+      }
+      return graph
+    })
+  }
+  currentVersion = once(() =>
+    ExtendedVersion.parse(this.current.options.version),
+  )
+  static of<
+    CurrentVersion extends string,
+    OtherVersions extends Array<VersionInfo<any>>,
+  >(
+    currentVersion: VersionInfo<CurrentVersion>,
+    ...other: EnsureUniqueId<OtherVersions, OtherVersions, CurrentVersion>
+  ) {
+    return new VersionGraph(currentVersion, other as Array<VersionInfo<any>>)
+  }
+  async migrate({
+    effects,
+    from,
+    to,
+  }: {
+    effects: T.Effects
+    from: ExtendedVersion
+    to: ExtendedVersion
+  }) {
+    const graph = this.graph()
+    if (from && to) {
+      const path = graph.shortestPath(
+        (v) =>
+          (v.metadata instanceof VersionRange &&
+            v.metadata.satisfiedBy(from)) ||
+          (v.metadata instanceof ExtendedVersion && v.metadata.equals(from)),
+        (v) =>
+          (v.metadata instanceof VersionRange && v.metadata.satisfiedBy(to)) ||
+          (v.metadata instanceof ExtendedVersion && v.metadata.equals(to)),
+      )
+      if (path) {
+        for (let edge of path) {
+          if (edge.metadata) {
+            await edge.metadata({ effects })
+          }
+          await effects.setDataVersion({ version: edge.to.metadata.toString() })
+        }
+        return
+      }
+    }
+    throw new Error()
+  }
+  canMigrateFrom = once(() =>
+    Array.from(
+      this.graph().reverseBreadthFirstSearch(
+        (v) =>
+          (v.metadata instanceof VersionRange &&
+            v.metadata.satisfiedBy(this.currentVersion())) ||
+          (v.metadata instanceof ExtendedVersion &&
+            v.metadata.equals(this.currentVersion())),
+      ),
+    ).reduce(
+      (acc, x) =>
+        acc.or(
+          x.metadata instanceof VersionRange
+            ? x.metadata
+            : VersionRange.anchor("=", x.metadata),
+        ),
+      VersionRange.none(),
+    ),
+  )
+  canMigrateTo = once(() =>
+    Array.from(
+      this.graph().breadthFirstSearch(
+        (v) =>
+          (v.metadata instanceof VersionRange &&
+            v.metadata.satisfiedBy(this.currentVersion())) ||
+          (v.metadata instanceof ExtendedVersion &&
+            v.metadata.equals(this.currentVersion())),
+      ),
+    ).reduce(
+      (acc, x) =>
+        acc.or(
+          x.metadata instanceof VersionRange
+            ? x.metadata
+            : VersionRange.anchor("=", x.metadata),
+        ),
+      VersionRange.none(),
+    ),
+  )
+}
+
+// prettier-ignore
+export type EnsureUniqueId<A, B = A, OtherVersions = never> =
+  B extends [] ? A : 
+  B extends [VersionInfo<infer Version>, ...infer Rest] ? (
+    Version extends OtherVersions ?  "One or more versions are not unique"[] :
+    EnsureUniqueId<A, Rest, Version | OtherVersions>
+  ) : "There exists a migration that is not a Migration"[]

sdk/lib/version/VersionInfo.ts

@@ -0,0 +1,78 @@
+import { ValidateExVer } from "../exver"
+import * as T from "../types"
+
+export const IMPOSSIBLE = Symbol("IMPOSSIBLE")
+
+export type VersionOptions<Version extends string> = {
+  /** The version being described */
+  version: Version & ValidateExVer<Version>
+  /** The release notes for this version */
+  releaseNotes: string
+  /** Data migrations for this version */
+  migrations: {
+    /**
+     * A migration from the previous version
+     *    Leave blank to indicate no migration is necessary
+     *    Set to `IMPOSSIBLE` to indicate migrating from the previous version is not possible
+     */
+    up?: ((opts: { effects: T.Effects }) => Promise<void>) | typeof IMPOSSIBLE
+    /**
+     * A migration to the previous version
+     *    Leave blank to indicate no migration is necessary
+     *    Set to `IMPOSSIBLE` to indicate downgrades are prohibited
+     */
+    down?: ((opts: { effects: T.Effects }) => Promise<void>) | typeof IMPOSSIBLE
+    /**
+     * Additional migrations, such as fast-forward migrations, or migrations from other flavors
+     */
+    other?: Record<string, (opts: { effects: T.Effects }) => Promise<void>>
+  }
+}
+
+export class VersionInfo<Version extends string> {
+  private _version: null | Version = null
+  private constructor(
+    readonly options: VersionOptions<Version> & { satisfies: string[] },
+  ) {}
+  static of<Version extends string>(options: VersionOptions<Version>) {
+    return new VersionInfo<Version>({ ...options, satisfies: [] })
+  }
+  /** Specify a version that this version is 100% backwards compatible to */
+  satisfies<V extends string>(
+    version: V & ValidateExVer<V>,
+  ): VersionInfo<Version> {
+    return new VersionInfo({
+      ...this.options,
+      satisfies: [...this.options.satisfies, version],
+    })
+  }
+}
+
+function __type_tests() {
+  const version: VersionInfo<"1.0.0:0"> = VersionInfo.of({
+    version: "1.0.0:0",
+    releaseNotes: "",
+    migrations: {},
+  })
+    .satisfies("#other:1.0.0:0")
+    .satisfies("#other:2.0.0:0")
+    // @ts-expect-error
+    .satisfies("#other:2.f.0:0")
+
+  let a: VersionInfo<"1.0.0:0"> = version
+  // @ts-expect-error
+  let b: VersionInfo<"1.0.0:3"> = version
+
+  VersionInfo.of({
+    // @ts-expect-error
+    version: "test",
+    releaseNotes: "",
+    migrations: {},
+  })
+  VersionInfo.of({
+    // @ts-expect-error
+    version: "test" as string,
+    releaseNotes: "",
+    migrations: {},
+  })
+}

sdk/lib/version/index.ts

@@ -0,0 +1,2 @@
+export * from "./VersionGraph"
+export * from "./VersionInfo"

sdk/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.3.6-alpha6",
+  "version": "0.3.6-alpha8",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@start9labs/start-sdk",
-      "version": "0.3.6-alpha6",
+      "version": "0.3.6-alpha8",
       "license": "MIT",
       "dependencies": {
         "@iarna/toml": "^2.2.5",

sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.3.6-alpha6",
+  "version": "0.3.6-alpha8",
   "description": "Software development kit to facilitate packaging services for StartOS",
   "main": "./cjs/lib/index.js",
   "types": "./cjs/lib/index.d.ts",