Feature/sdk040dependencies (#2609)

* update registry upload to take id for new admin permissions (#2605)

* wip

* wip: Get the get dependencies

* wip check_dependencies

* wip: Get the build working to the vm

* wip: Add in the last of the things that where needed for the new sdk

* Add fix

* wip: implement the changes

* wip: Fix the naming

---------

Co-authored-by: Lucy <12953208+elvece@users.noreply.github.com>

sdk/lib/StartSdk.ts

@@ -24,7 +24,7 @@ import {
   ValidIfNoStupidEscape,
 } from "./types"
 import * as patterns from "./util/patterns"
-import { DependencyConfig, Update } from "./dependencyConfig/DependencyConfig"
+import { DependencyConfig, Update } from "./dependencies/DependencyConfig"
 import { BackupSet, Backups } from "./backup/Backups"
 import { smtpConfig } from "./config/configConstants"
 import { Daemons } from "./mainFn/Daemons"
@@ -35,7 +35,7 @@ import { List } from "./config/builder/list"
 import { Migration } from "./inits/migrations/Migration"
 import { Install, InstallFn } from "./inits/setupInstall"
 import { setupActions } from "./actions/setupActions"
-import { setupDependencyConfig } from "./dependencyConfig/setupDependencyConfig"
+import { setupDependencyConfig } from "./dependencies/setupDependencyConfig"
 import { SetupBackupsParams, setupBackups } from "./backup/setupBackups"
 import { setupInit } from "./inits/setupInit"
 import {
@@ -77,6 +77,7 @@ import * as T from "./types"
 import { Checker, EmVer } from "./emverLite/mod"
 import { ExposedStorePaths } from "./store/setupExposeStore"
 import { PathBuilder, extractJsonPath, pathBuilder } from "./store/PathBuilder"
+import { checkAllDependencies } from "./dependencies/dependencies"
 
 // prettier-ignore
 type AnyNeverCond<T extends any[], Then, Else> = 
@@ -124,6 +125,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
     }
 
     return {
+      checkAllDependencies,
       serviceInterface: {
         getOwn: <E extends Effects>(effects: E, id: ServiceInterfaceId) =>
           removeConstType<E>()(
@@ -284,7 +286,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
         Type extends Record<string, any> = ExtractConfigType<ConfigType>,
       >(
         spec: ConfigType,
-        write: Save<Store, Type, Manifest>,
+        write: Save<Type>,
         read: Read<Manifest, Store, Type>,
       ) => setupConfig<Store, ConfigType, Manifest, Type>(spec, write, read),
       setupConfigRead: <
@@ -301,7 +303,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
           | Config<Record<string, never>, never>,
       >(
         _configSpec: ConfigSpec,
-        fn: Save<Store, ConfigSpec, Manifest>,
+        fn: Save<ConfigSpec>,
       ) => fn,
       setupDependencyConfig: <Input extends Record<string, any>>(
         config: Config<Input, Store> | Config<Input, never>,

sdk/lib/config/configDependencies.ts

@@ -1,9 +1,12 @@
 import { SDKManifest } from "../manifest/ManifestTypes"
-import { Dependency } from "../types"
+import { Dependencies } from "../types"
 
 export type ConfigDependencies<T extends SDKManifest> = {
-  exists(id: keyof T["dependencies"]): Dependency
-  running(id: keyof T["dependencies"], healthChecks: string[]): Dependency
+  exists(id: keyof T["dependencies"]): Dependencies[number]
+  running(
+    id: keyof T["dependencies"],
+    healthChecks: string[],
+  ): Dependencies[number]
 }
 
 export const configDependenciesSet = <
@@ -13,7 +16,7 @@ export const configDependenciesSet = <
     return {
       id,
       kind: "exists",
-    } as Dependency
+    } as Dependencies[number]
   },
 
   running(id: keyof T["dependencies"], healthChecks: string[]) {
@@ -21,6 +24,6 @@ export const configDependenciesSet = <
       id,
       kind: "running",
       healthChecks,
-    } as Dependency
+    } as Dependencies[number]
   },
 })

sdk/lib/config/setupConfig.ts

@@ -11,16 +11,13 @@ export type DependenciesReceipt = void & {
 }
 
 export type Save<
-  Store,
   A extends
     | Record<string, any>
     | Config<Record<string, any>, any>
     | Config<Record<string, never>, never>,
-  Manifest extends SDKManifest,
 > = (options: {
   effects: Effects
   input: ExtractConfigType<A> & Record<string, any>
-  dependencies: D.ConfigDependencies<Manifest>
 }) => Promise<{
   dependenciesReceipt: DependenciesReceipt
   interfacesReceipt: InterfacesReceipt
@@ -53,7 +50,7 @@ export function setupConfig<
   Type extends Record<string, any> = ExtractConfigType<ConfigType>,
 >(
   spec: Config<Type, Store> | Config<Type, never>,
-  write: Save<Store, Type, Manifest>,
+  write: Save<Type>,
   read: Read<Manifest, Store, Type>,
 ) {
   const validator = spec.validator
@@ -66,9 +63,8 @@ export function setupConfig<
       await effects.clearBindings()
       await effects.clearServiceInterfaces()
       const { restart } = await write({
-        input: JSON.parse(JSON.stringify(input)),
+        input: JSON.parse(JSON.stringify(input)) as any,
         effects,
-        dependencies: D.configDependenciesSet<Manifest>(),
       })
       if (restart) {
         await effects.restart()

sdk/lib/dependencies/dependencies.ts

@@ -0,0 +1,115 @@
+import {
+  Effects,
+  PackageId,
+  DependencyRequirement,
+  SetHealth,
+  CheckDependencyResult,
+} from "../types"
+
+export type CheckAllDependencies = {
+  notRunning: () => Promise<CheckDependencyResult[]>
+
+  notInstalled: () => Promise<CheckDependencyResult[]>
+
+  healthErrors: () => Promise<{ [id: string]: SetHealth[] }>
+  throwIfNotRunning: () => Promise<void>
+  throwIfNotValid: () => Promise<undefined>
+  throwIfNotInstalled: () => Promise<void>
+  throwIfError: () => Promise<void>
+  isValid: () => Promise<boolean>
+}
+export function checkAllDependencies(effects: Effects): CheckAllDependencies {
+  const dependenciesPromise = effects.getDependencies()
+  const resultsPromise = dependenciesPromise.then((dependencies) =>
+    effects.checkDependencies({
+      packageIds: dependencies.map((dep) => dep.id),
+    }),
+  )
+
+  const dependenciesByIdPromise = dependenciesPromise.then((d) =>
+    d.reduce(
+      (acc, dep) => {
+        acc[dep.id] = dep
+        return acc
+      },
+      {} as { [id: PackageId]: DependencyRequirement },
+    ),
+  )
+
+  const healthErrors = async () => {
+    const results = await resultsPromise
+    const dependenciesById = await dependenciesByIdPromise
+    const answer: { [id: PackageId]: SetHealth[] } = {}
+    for (const result of results) {
+      const dependency = dependenciesById[result.packageId]
+      if (!dependency) continue
+      if (dependency.kind !== "running") continue
+
+      const healthChecks = result.healthChecks
+        .filter((x) => dependency.healthChecks.includes(x.id))
+        .filter((x) => !!x.message)
+      if (healthChecks.length === 0) continue
+      answer[result.packageId] = healthChecks
+    }
+    return answer
+  }
+  const notInstalled = () =>
+    resultsPromise.then((x) => x.filter((x) => !x.isInstalled))
+  const notRunning = async () => {
+    const results = await resultsPromise
+    const dependenciesById = await dependenciesByIdPromise
+    return results.filter((x) => {
+      const dependency = dependenciesById[x.packageId]
+      if (!dependency) return false
+      if (dependency.kind !== "running") return false
+      return !x.isRunning
+    })
+  }
+  const entries = <B>(x: { [k: string]: B }) => Object.entries(x)
+  const first = <A>(x: A[]): A | undefined => x[0]
+  const sinkVoid = <A>(x: A) => void 0
+  const throwIfError = () =>
+    healthErrors()
+      .then(entries)
+      .then(first)
+      .then((x) => {
+        if (!x) return
+        const [id, healthChecks] = x
+        if (healthChecks.length > 0)
+          throw `Package ${id} has the following errors: ${healthChecks.map((x) => x.message).join(", ")}`
+      })
+  const throwIfNotRunning = () =>
+    notRunning().then((results) => {
+      if (results[0])
+        throw new Error(`Package ${results[0].packageId} is not running`)
+    })
+
+  const throwIfNotInstalled = () =>
+    notInstalled().then((results) => {
+      if (results[0])
+        throw new Error(`Package ${results[0].packageId} is not installed`)
+    })
+  const throwIfNotValid = async () =>
+    Promise.all([
+      throwIfNotRunning(),
+      throwIfNotInstalled(),
+      throwIfError(),
+    ]).then(sinkVoid)
+
+  const isValid = () =>
+    throwIfNotValid().then(
+      () => true,
+      () => false,
+    )
+
+  return {
+    notRunning,
+    notInstalled,
+    healthErrors,
+    throwIfNotRunning,
+    throwIfNotValid,
+    throwIfNotInstalled,
+    throwIfError,
+    isValid,
+  }
+}

sdk/lib/health/HealthCheck.ts

@@ -7,6 +7,7 @@ import { TriggerInput } from "../trigger/TriggerInput"
 import { defaultTrigger } from "../trigger/defaultTrigger"
 import { once } from "../util/once"
 import { Overlay } from "../util/Overlay"
+import { object, unknown } from "ts-matches"
 
 export function healthCheck(o: {
   effects: Effects
@@ -66,8 +67,7 @@ export function healthCheck(o: {
   return {} as HealthReceipt
 }
 function asMessage(e: unknown) {
-  if (typeof e === "object" && e != null && "message" in e)
-    return String(e.message)
+  if (object({ message: unknown }).test(e)) return String(e.message)
   const value = String(e)
   if (value.length == null) return null
   return value

sdk/lib/index.browser.ts

@@ -4,7 +4,7 @@ export { setupExposeStore } from "./store/setupExposeStore"
 export * as config from "./config"
 export * as CB from "./config/builder"
 export * as CT from "./config/configTypes"
-export * as dependencyConfig from "./dependencyConfig"
+export * as dependencyConfig from "./dependencies"
 export * as manifest from "./manifest"
 export * as types from "./types"
 export * as T from "./types"

sdk/lib/index.ts

@@ -12,7 +12,7 @@ export * as backup from "./backup"
 export * as config from "./config"
 export * as CB from "./config/builder"
 export * as CT from "./config/configTypes"
-export * as dependencyConfig from "./dependencyConfig"
+export * as dependencyConfig from "./dependencies"
 export * as daemons from "./mainFn/Daemons"
 export * as health from "./health"
 export * as healthFns from "./health/checkFns"

sdk/lib/interfaces/Host.ts

@@ -161,7 +161,7 @@ export class Host {
     options: BindOptionsByKnownProtocol,
     protoInfo: KnownProtocols[keyof KnownProtocols],
   ): AddSslOptions | null {
-    if ("noAddSsl" in options && options.noAddSsl) return null
+    if (inObject("noAddSsl", options) && options.noAddSsl) return null
     if ("withSsl" in protoInfo && protoInfo.withSsl)
       return {
         // addXForwardedHeaders: null,
@@ -174,6 +174,13 @@ export class Host {
   }
 }
 
+function inObject<Key extends string>(
+  key: Key,
+  obj: any,
+): obj is { [K in Key]: unknown } {
+  return key in obj
+}
+
 export class StaticHost extends Host {
   constructor(options: { effects: Effects; id: string }) {
     super({ ...options, kind: "static" })

sdk/lib/osBindings/CheckDependenciesParam.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { PackageId } from "./PackageId"
+
+export type CheckDependenciesParam = { packageIds: Array<PackageId> | null }

sdk/lib/osBindings/CheckDependenciesResult.ts

@@ -0,0 +1,11 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { HealthCheckResult } from "./HealthCheckResult"
+import type { PackageId } from "./PackageId"
+
+export type CheckDependenciesResult = {
+  packageId: PackageId
+  isInstalled: boolean
+  isRunning: boolean
+  healthChecks: Array<HealthCheckResult>
+  version: string | null
+}

sdk/lib/osBindings/index.ts

@@ -12,6 +12,8 @@ export { BindInfo } from "./BindInfo"
 export { BindOptions } from "./BindOptions"
 export { BindParams } from "./BindParams"
 export { Callback } from "./Callback"
+export { CheckDependenciesParam } from "./CheckDependenciesParam"
+export { CheckDependenciesResult } from "./CheckDependenciesResult"
 export { ChrootParams } from "./ChrootParams"
 export { CreateOverlayedImageParams } from "./CreateOverlayedImageParams"
 export { CurrentDependencies } from "./CurrentDependencies"

sdk/lib/test/startosTypeValidation.test.ts

@@ -1,5 +1,5 @@
 import { Effects } from "../types"
-import { ExecuteAction } from ".././osBindings"
+import { CheckDependenciesParam, ExecuteAction } from ".././osBindings"
 import { CreateOverlayedImageParams } from ".././osBindings"
 import { DestroyOverlayedImageParams } from ".././osBindings"
 import { BindParams } from ".././osBindings"
@@ -64,6 +64,8 @@ describe("startosTypeValidation ", () => {
       removeAction: {} as RemoveActionParams,
       reverseProxy: {} as ReverseProxyParams,
       mount: {} as MountParams,
+      checkDependencies: {} as CheckDependenciesParam,
+      getDependencies: undefined,
     })
     typeEquality<Parameters<Effects["executeAction"]>[0]>(
       testInput as ExecuteAction,

sdk/lib/types.ts

@@ -1,6 +1,11 @@
 export * as configTypes from "./config/configTypes"
-import { HealthCheckId } from "./osBindings"
-import { HealthCheckResult } from "./osBindings"
+
+import {
+  DependencyRequirement,
+  SetHealth,
+  HealthCheckResult,
+} from "./osBindings"
+
 import { MainEffects, ServiceInterfaceType, Signals } from "./StartSdk"
 import { InputSpec } from "./config/configTypes"
 import { DependenciesReceipt } from "./config/setupConfig"
@@ -11,6 +16,7 @@ import { ExposedStorePaths } from "./store/setupExposeStore"
 import { UrlString } from "./util/getServiceInterface"
 export * from "./osBindings"
 export { SDKManifest } from "./manifest/ManifestTypes"
+export { HealthReceipt } from "./health/HealthReceipt"
 
 export type ExportedAction = (options: {
   effects: Effects
@@ -471,16 +477,22 @@ export type Effects = {
     algorithm: "ecdsa" | "ed25519" | null
   }) => Promise<string>
 
-  setHealth(
-    o: HealthCheckResult & {
-      id: HealthCheckId
-    },
-  ): Promise<void>
+  setHealth(o: SetHealth): Promise<void>
 
   /** Set the dependencies of what the service needs, usually ran during the set config as a best practice */
   setDependencies(options: {
     dependencies: Dependencies
   }): Promise<DependenciesReceipt>
+
+  /** Get the list of the dependencies, both the dynamic set by the effect of setDependencies and the end result any required in the manifest  */
+  getDependencies(): Promise<DependencyRequirement[]>
+
+  /** When one wants to checks the status of several services during the checking of dependencies. The result will include things like the status
+   * of the service and what the current health checks are.
+   */
+  checkDependencies(options: {
+    packageIds: PackageId[] | null
+  }): Promise<CheckDependencyResult[]>
   /** Exists could be useful during the runtime to know if some service exists, option dep */
   exists(options: { packageId: PackageId }): Promise<boolean>
   /** Exists could be useful during the runtime to know if some service is running, option dep */
@@ -578,13 +590,17 @@ export type KnownError =
       errorCode: [number, string] | readonly [number, string]
     }
 
-export type Dependency = {
-  id: PackageId
-  versionSpec: string
-  registryUrl: string
-} & ({ kind: "exists" } | { kind: "running"; healthChecks: string[] })
-export type Dependencies = Array<Dependency>
+export type Dependencies = Array<DependencyRequirement>
 
 export type DeepPartial<T> = T extends {}
   ? { [P in keyof T]?: DeepPartial<T[P]> }
   : T
+
+export type CheckDependencyResult = {
+  packageId: PackageId
+  isInstalled: boolean
+  isRunning: boolean
+  healthChecks: SetHealth[]
+  version: string | null
+}
+export type CheckResults = CheckDependencyResult[]

sdk/lib/util/Overlay.ts

@@ -77,7 +77,7 @@ export class Overlay {
     stdout: string | Buffer
     stderr: string | Buffer
   }> {
-    const imageMeta = await fs
+    const imageMeta: any = await fs
       .readFile(`/media/startos/images/${this.imageId}.json`, {
         encoding: "utf8",
       })
@@ -147,7 +147,7 @@ export class Overlay {
     command: string[],
     options?: CommandOptions,
   ): Promise<cp.ChildProcessWithoutNullStreams> {
-    const imageMeta = await fs
+    const imageMeta: any = await fs
       .readFile(`/media/startos/images/${this.imageId}.json`, {
         encoding: "utf8",
       })