merge 036, everything broken

sdk/.gitignore

@@ -0,0 +1,5 @@
+.vscode
+dist/
+node_modules/
+lib/coverage
+lib/test/output.ts

sdk/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Start9 Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sdk/Makefile

@@ -0,0 +1,44 @@
+TS_FILES := $(shell find ./**/*.ts )
+version = $(shell git tag --sort=committerdate | tail -1)
+test: $(TS_FILES) lib/test/output.ts
+	npm test
+
+clean:
+	rm -rf dist/* | true
+	
+lib/test/output.ts: lib/test/makeOutput.ts scripts/oldSpecToBuilder.ts
+	npm run buildOutput
+
+buildOutput: lib/test/output.ts fmt
+	echo 'done'
+
+
+bundle: $(TS_FILES) package.json .FORCE node_modules test fmt
+	npx tsc
+	npx tsc --project tsconfig-cjs.json
+	cp package.json dist/package.json
+	cp README.md dist/README.md
+	cp LICENSE dist/LICENSE
+	touch dist
+
+full-bundle:
+	make clean
+	make bundle
+
+check:
+	npm run check
+
+fmt: node_modules
+	npx prettier --write "**/*.ts"
+
+node_modules: package.json
+	npm install
+
+publish: clean bundle	 package.json README.md LICENSE
+	cd dist && npm publish --access=public
+link: bundle	
+	cp package.json dist/package.json
+	cp README.md dist/README.md
+	cp LICENSE dist/LICENSE
+	cd dist && npm link
+.FORCE: 

sdk/README.md

@@ -0,0 +1,18 @@
+# Start SDK
+
+## Config Conversion
+
+- Copy the old config json (from the getConfig.ts)
+- Install the start-sdk with `npm i`
+- paste the config into makeOutput.ts::oldSpecToBuilder (second param)
+- Make the third param
+
+```ts
+  {
+    StartSdk: "start-sdk/lib",
+  }
+```
+
+- run the script `npm run buildOutput` to make the output.ts
+- Copy this whole file into startos/procedures/config/spec.ts
+- Fix all the TODO

sdk/jest.config.js

@@ -0,0 +1,8 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: "ts-jest",
+  automock: false,
+  testEnvironment: "node",
+  rootDir: "./lib/",
+  modulePathIgnorePatterns: ["./dist/"],
+};

sdk/lib/StartSdk.ts

@@ -0,0 +1,656 @@
+import { ManifestVersion, SDKManifest } from "./manifest/ManifestTypes"
+import { RequiredDefault, Value } from "./config/builder/value"
+import { Config, ExtractConfigType, LazyBuild } from "./config/builder/config"
+import {
+  DefaultString,
+  ListValueSpecText,
+  Pattern,
+  RandomString,
+  UniqueBy,
+  ValueSpecDatetime,
+  ValueSpecText,
+} from "./config/configTypes"
+import { Variants } from "./config/builder/variants"
+import { CreatedAction, createAction } from "./actions/createAction"
+import {
+  ActionMetadata,
+  Effects,
+  ActionResult,
+  BackupOptions,
+  DeepPartial,
+  MaybePromise,
+  ServiceInterfaceId,
+  PackageId,
+  EnsureStorePath,
+  ExtractStore,
+  DaemonReturned,
+  ValidIfNoStupidEscape,
+} from "./types"
+import * as patterns from "./util/patterns"
+import { DependencyConfig, Update } from "./dependencyConfig/DependencyConfig"
+import { BackupSet, Backups } from "./backup/Backups"
+import { smtpConfig } from "./config/configConstants"
+import { Daemons } from "./mainFn/Daemons"
+import { healthCheck } from "./health/HealthCheck"
+import { checkPortListening } from "./health/checkFns/checkPortListening"
+import { checkWebUrl, runHealthScript } from "./health/checkFns"
+import { List } from "./config/builder/list"
+import { Migration } from "./inits/migrations/Migration"
+import { Install, InstallFn } from "./inits/setupInstall"
+import { setupActions } from "./actions/setupActions"
+import { setupDependencyConfig } from "./dependencyConfig/setupDependencyConfig"
+import { SetupBackupsParams, setupBackups } from "./backup/setupBackups"
+import { setupInit } from "./inits/setupInit"
+import {
+  EnsureUniqueId,
+  Migrations,
+  setupMigrations,
+} from "./inits/migrations/setupMigrations"
+import { Uninstall, UninstallFn, setupUninstall } from "./inits/setupUninstall"
+import { setupMain } from "./mainFn"
+import { defaultTrigger } from "./trigger/defaultTrigger"
+import { changeOnFirstSuccess, cooldownTrigger } from "./trigger"
+import setupConfig, { Read, Save } from "./config/setupConfig"
+import {
+  InterfacesReceipt,
+  SetInterfaces,
+  setupInterfaces,
+} from "./interfaces/setupInterfaces"
+import { successFailure } from "./trigger/successFailure"
+import { SetupExports } from "./inits/setupExports"
+import { HealthReceipt } from "./health/HealthReceipt"
+import { MultiHost, Scheme, SingleHost, StaticHost } from "./interfaces/Host"
+import { ServiceInterfaceBuilder } from "./interfaces/ServiceInterfaceBuilder"
+import { GetSystemSmtp } from "./util/GetSystemSmtp"
+import nullIfEmpty from "./util/nullIfEmpty"
+import {
+  GetServiceInterface,
+  getServiceInterface,
+} from "./util/getServiceInterface"
+import { getServiceInterfaces } from "./util/getServiceInterfaces"
+import { getStore } from "./store/getStore"
+import { CommandOptions, MountOptions, Overlay } from "./util/Overlay"
+import { splitCommand } from "./util/splitCommand"
+import { Mounts } from "./mainFn/Mounts"
+
+// prettier-ignore
+type AnyNeverCond<T extends any[], Then, Else> = 
+    T extends [] ? Else :
+    T extends [never, ...Array<any>] ? Then :
+    T extends [any, ...infer U] ? AnyNeverCond<U,Then, Else> :
+    never
+
+export type ServiceInterfaceType = "ui" | "p2p" | "api"
+export type MainEffects = Effects & { _type: "main" }
+export type Signals = NodeJS.Signals
+export const SIGTERM: Signals = "SIGTERM"
+export const SIGKILL: Signals = "SIGTERM"
+export const NO_TIMEOUT = -1
+
+function removeConstType<E>() {
+  return <T>(t: T) => t as T & (E extends MainEffects ? {} : { const: never })
+}
+
+export class StartSdk<Manifest extends SDKManifest, Store> {
+  private constructor(readonly manifest: Manifest) {}
+  static of() {
+    return new StartSdk<never, never>(null as never)
+  }
+  withManifest<Manifest extends SDKManifest = never>(manifest: Manifest) {
+    return new StartSdk<Manifest, Store>(manifest)
+  }
+  withStore<Store extends Record<string, any>>() {
+    return new StartSdk<Manifest, Store>(this.manifest)
+  }
+
+  build(isReady: AnyNeverCond<[Manifest, Store], "Build not ready", true>) {
+    return {
+      serviceInterface: {
+        getOwn: <E extends Effects>(effects: E, id: ServiceInterfaceId) =>
+          removeConstType<E>()(
+            getServiceInterface(effects, {
+              id,
+              packageId: null,
+            }),
+          ),
+        get: <E extends Effects>(
+          effects: E,
+          opts: { id: ServiceInterfaceId; packageId: PackageId },
+        ) => removeConstType<E>()(getServiceInterface(effects, opts)),
+        getAllOwn: <E extends Effects>(effects: E) =>
+          removeConstType<E>()(
+            getServiceInterfaces(effects, {
+              packageId: null,
+            }),
+          ),
+        getAll: <E extends Effects>(
+          effects: E,
+          opts: { packageId: PackageId },
+        ) => removeConstType<E>()(getServiceInterfaces(effects, opts)),
+      },
+
+      store: {
+        get: <E extends Effects, Path extends string = never>(
+          effects: E,
+          packageId: string,
+          path: EnsureStorePath<Store, Path>,
+        ) =>
+          removeConstType<E>()(
+            getStore<Store, Path>(effects, path as any, {
+              packageId,
+            }),
+          ),
+        getOwn: <E extends Effects, Path extends string>(
+          effects: E,
+          path: EnsureStorePath<Store, Path>,
+        ) => removeConstType<E>()(getStore<Store, Path>(effects, path as any)),
+        setOwn: <E extends Effects, Path extends string | never>(
+          effects: E,
+          path: EnsureStorePath<Store, Path>,
+          value: ExtractStore<Store, Path>,
+        ) => effects.store.set<Store, Path>({ value, path: path as any }),
+      },
+
+      host: {
+        static: (effects: Effects, id: string) =>
+          new StaticHost({ id, effects }),
+        single: (effects: Effects, id: string) =>
+          new SingleHost({ id, effects }),
+        multi: (effects: Effects, id: string) => new MultiHost({ id, effects }),
+      },
+      nullIfEmpty,
+
+      configConstants: { smtpConfig },
+      createInterface: (
+        effects: Effects,
+        options: {
+          name: string
+          id: string
+          description: string
+          hasPrimary: boolean
+          disabled: boolean
+          type: ServiceInterfaceType
+          username: null | string
+          path: string
+          search: Record<string, string>
+          schemeOverride: { ssl: Scheme; noSsl: Scheme } | null
+          masked: boolean
+        },
+      ) => new ServiceInterfaceBuilder({ ...options, effects }),
+      createAction: <
+        ConfigType extends
+          | Record<string, any>
+          | Config<any, any>
+          | Config<any, never>,
+        Type extends Record<string, any> = ExtractConfigType<ConfigType>,
+      >(
+        metaData: Omit<ActionMetadata, "input"> & {
+          input: Config<Type, Store> | Config<Type, never>
+        },
+        fn: (options: {
+          effects: Effects
+          input: Type
+        }) => Promise<ActionResult>,
+      ) => {
+        const { input, ...rest } = metaData
+        return createAction<Manifest, Store, ConfigType, Type>(rest, fn, input)
+      },
+      getSystemSmtp: <E extends Effects>(effects: E) =>
+        removeConstType<E>()(new GetSystemSmtp(effects)),
+      runCommand: async <A extends string>(
+        effects: Effects,
+        imageId: Manifest["images"][number],
+        command: ValidIfNoStupidEscape<A> | [string, ...string[]],
+        options: CommandOptions & {
+          mounts?: { path: string; options: MountOptions }[]
+        },
+      ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> => {
+        const commands = splitCommand(command)
+        const overlay = await Overlay.of(effects, imageId)
+        try {
+          for (let mount of options.mounts || []) {
+            await overlay.mount(mount.options, mount.path)
+          }
+          return await overlay.exec(commands)
+        } finally {
+          await overlay.destroy()
+        }
+      },
+
+      createDynamicAction: <
+        ConfigType extends
+          | Record<string, any>
+          | Config<any, any>
+          | Config<any, never>,
+        Type extends Record<string, any> = ExtractConfigType<ConfigType>,
+      >(
+        metaData: (options: {
+          effects: Effects
+        }) => MaybePromise<Omit<ActionMetadata, "input">>,
+        fn: (options: {
+          effects: Effects
+          input: Type
+        }) => Promise<ActionResult>,
+        input: Config<Type, Store> | Config<Type, never>,
+      ) => {
+        return createAction<Manifest, Store, ConfigType, Type>(
+          metaData,
+          fn,
+          input,
+        )
+      },
+      HealthCheck: {
+        of: healthCheck,
+      },
+      healthCheck: {
+        checkPortListening,
+        checkWebUrl,
+        runHealthScript,
+      },
+      patterns,
+      setupActions: (...createdActions: CreatedAction<any, any, any>[]) =>
+        setupActions<Manifest, Store>(...createdActions),
+      setupBackups: (...args: SetupBackupsParams<Manifest>) =>
+        setupBackups<Manifest>(...args),
+      setupConfig: <
+        ConfigType extends Config<any, Store> | Config<any, never>,
+        Type extends Record<string, any> = ExtractConfigType<ConfigType>,
+      >(
+        spec: ConfigType,
+        write: Save<Store, Type, Manifest>,
+        read: Read<Manifest, Store, Type>,
+      ) => setupConfig<Store, ConfigType, Manifest, Type>(spec, write, read),
+      setupConfigRead: <
+        ConfigSpec extends
+          | Config<Record<string, any>, any>
+          | Config<Record<string, never>, never>,
+      >(
+        _configSpec: ConfigSpec,
+        fn: Read<Manifest, Store, ConfigSpec>,
+      ) => fn,
+      setupConfigSave: <
+        ConfigSpec extends
+          | Config<Record<string, any>, any>
+          | Config<Record<string, never>, never>,
+      >(
+        _configSpec: ConfigSpec,
+        fn: Save<Store, ConfigSpec, Manifest>,
+      ) => fn,
+      setupDependencyConfig: <Input extends Record<string, any>>(
+        config: Config<Input, Store> | Config<Input, never>,
+        autoConfigs: {
+          [K in keyof Manifest["dependencies"]]: DependencyConfig<
+            Manifest,
+            Store,
+            Input,
+            any
+          >
+        },
+      ) => setupDependencyConfig<Store, Input, Manifest>(config, autoConfigs),
+      setupExports: (fn: SetupExports<Store>) => fn,
+      setupInit: (
+        migrations: Migrations<Manifest, Store>,
+        install: Install<Manifest, Store>,
+        uninstall: Uninstall<Manifest, Store>,
+        setInterfaces: SetInterfaces<Manifest, Store, any, any>,
+        setupExports: SetupExports<Store>,
+      ) =>
+        setupInit<Manifest, Store>(
+          migrations,
+          install,
+          uninstall,
+          setInterfaces,
+          setupExports,
+        ),
+      setupInstall: (fn: InstallFn<Manifest, Store>) => Install.of(fn),
+      setupInterfaces: <
+        ConfigInput extends Record<string, any>,
+        Output extends InterfacesReceipt,
+      >(
+        config: Config<ConfigInput, Store>,
+        fn: SetInterfaces<Manifest, Store, ConfigInput, Output>,
+      ) => setupInterfaces(config, fn),
+      setupMain: (
+        fn: (o: {
+          effects: MainEffects
+          started(onTerm: () => PromiseLike<void>): PromiseLike<void>
+        }) => Promise<Daemons<Manifest, any>>,
+      ) => setupMain<Manifest, Store>(fn),
+      setupMigrations: <
+        Migrations extends Array<Migration<Manifest, Store, any>>,
+      >(
+        ...migrations: EnsureUniqueId<Migrations>
+      ) =>
+        setupMigrations<Manifest, Store, Migrations>(
+          this.manifest,
+          ...migrations,
+        ),
+      setupUninstall: (fn: UninstallFn<Manifest, Store>) =>
+        setupUninstall<Manifest, Store>(fn),
+      trigger: {
+        defaultTrigger,
+        cooldownTrigger,
+        changeOnFirstSuccess,
+        successFailure,
+      },
+      Mounts: {
+        of() {
+          return Mounts.of<Manifest>()
+        },
+      },
+      Backups: {
+        volumes: (
+          ...volumeNames: Array<Manifest["volumes"][number] & string>
+        ) => Backups.volumes<Manifest>(...volumeNames),
+        addSets: (
+          ...options: BackupSet<Manifest["volumes"][number] & string>[]
+        ) => Backups.addSets<Manifest>(...options),
+        withOptions: (options?: Partial<BackupOptions>) =>
+          Backups.with_options<Manifest>(options),
+      },
+      Config: {
+        of: <
+          Spec extends Record<string, Value<any, Store> | Value<any, never>>,
+        >(
+          spec: Spec,
+        ) => Config.of<Spec, Store>(spec),
+      },
+      Daemons: {
+        of(config: {
+          effects: Effects
+          started: (onTerm: () => PromiseLike<void>) => PromiseLike<void>
+          healthReceipts: HealthReceipt[]
+        }) {
+          return Daemons.of<Manifest>(config)
+        },
+      },
+      DependencyConfig: {
+        of<
+          LocalConfig extends Record<string, any>,
+          RemoteConfig extends Record<string, any>,
+        >({
+          localConfig,
+          remoteConfig,
+          dependencyConfig,
+          update,
+        }: {
+          localConfig: Config<LocalConfig, Store> | Config<LocalConfig, never>
+          remoteConfig: Config<RemoteConfig, any> | Config<RemoteConfig, never>
+          dependencyConfig: (options: {
+            effects: Effects
+            localConfig: LocalConfig
+          }) => Promise<void | DeepPartial<RemoteConfig>>
+          update?: Update<void | DeepPartial<RemoteConfig>, RemoteConfig>
+        }) {
+          return new DependencyConfig<
+            Manifest,
+            Store,
+            LocalConfig,
+            RemoteConfig
+          >(dependencyConfig, update)
+        },
+      },
+      List: {
+        text: List.text,
+        number: List.number,
+        obj: <Type extends Record<string, any>>(
+          a: {
+            name: string
+            description?: string | null
+            warning?: string | null
+            /** Default [] */
+            default?: []
+            minLength?: number | null
+            maxLength?: number | null
+          },
+          aSpec: {
+            spec: Config<Type, Store>
+            displayAs?: null | string
+            uniqueBy?: null | UniqueBy
+          },
+        ) => List.obj<Type, Store>(a, aSpec),
+        dynamicText: (
+          getA: LazyBuild<
+            Store,
+            {
+              name: string
+              description?: string | null
+              warning?: string | null
+              /** Default = [] */
+              default?: string[]
+              minLength?: number | null
+              maxLength?: number | null
+              disabled?: false | string
+              generate?: null | RandomString
+              spec: {
+                /** Default = false */
+                masked?: boolean
+                placeholder?: string | null
+                minLength?: number | null
+                maxLength?: number | null
+                patterns: Pattern[]
+                /** Default = "text" */
+                inputmode?: ListValueSpecText["inputmode"]
+              }
+            }
+          >,
+        ) => List.dynamicText<Store>(getA),
+        dynamicNumber: (
+          getA: LazyBuild<
+            Store,
+            {
+              name: string
+              description?: string | null
+              warning?: string | null
+              /** Default = [] */
+              default?: string[]
+              minLength?: number | null
+              maxLength?: number | null
+              disabled?: false | string
+              spec: {
+                integer: boolean
+                min?: number | null
+                max?: number | null
+                step?: number | null
+                units?: string | null
+                placeholder?: string | null
+              }
+            }
+          >,
+        ) => List.dynamicNumber<Store>(getA),
+      },
+      Migration: {
+        of: <Version extends ManifestVersion>(options: {
+          version: Version
+          up: (opts: { effects: Effects }) => Promise<void>
+          down: (opts: { effects: Effects }) => Promise<void>
+        }) => Migration.of<Manifest, Store, Version>(options),
+      },
+      Value: {
+        toggle: Value.toggle,
+        text: Value.text,
+        textarea: Value.textarea,
+        number: Value.number,
+        color: Value.color,
+        datetime: Value.datetime,
+        select: Value.select,
+        multiselect: Value.multiselect,
+        object: Value.object,
+        union: Value.union,
+        list: Value.list,
+        dynamicToggle: (
+          a: LazyBuild<
+            Store,
+            {
+              name: string
+              description?: string | null
+              warning?: string | null
+              default: boolean
+              disabled?: false | string
+            }
+          >,
+        ) => Value.dynamicToggle<Store>(a),
+        dynamicText: (
+          getA: LazyBuild<
+            Store,
+            {
+              name: string
+              description?: string | null
+              warning?: string | null
+              required: RequiredDefault<DefaultString>
+
+              /** Default = false */
+              masked?: boolean
+              placeholder?: string | null
+              minLength?: number | null
+              maxLength?: number | null
+              patterns?: Pattern[]
+              /** Default = 'text' */
+              inputmode?: ValueSpecText["inputmode"]
+              generate?: null | RandomString
+            }
+          >,
+        ) => Value.dynamicText<Store>(getA),
+        dynamicTextarea: (
+          getA: LazyBuild<
+            Store,
+            {
+              name: string
+              description?: string | null
+              warning?: string | null
+              required: boolean
+              minLength?: number | null
+              maxLength?: number | null
+              placeholder?: string | null
+              disabled?: false | string
+              generate?: null | RandomString
+            }
+          >,
+        ) => Value.dynamicTextarea<Store>(getA),
+        dynamicNumber: (
+          getA: LazyBuild<
+            Store,
+            {
+              name: string
+              description?: string | null
+              warning?: string | null
+              required: RequiredDefault<number>
+              min?: number | null
+              max?: number | null
+              /** Default = '1' */
+              step?: number | null
+              integer: boolean
+              units?: string | null
+              placeholder?: string | null
+              disabled?: false | string
+            }
+          >,
+        ) => Value.dynamicNumber<Store>(getA),
+        dynamicColor: (
+          getA: LazyBuild<
+            Store,
+            {
+              name: string
+              description?: string | null
+              warning?: string | null
+              required: RequiredDefault<string>
+
+              disabled?: false | string
+            }
+          >,
+        ) => Value.dynamicColor<Store>(getA),
+        dynamicDatetime: (
+          getA: LazyBuild<
+            Store,
+            {
+              name: string
+              description?: string | null
+              warning?: string | null
+              required: RequiredDefault<string>
+              /** Default = 'datetime-local' */
+              inputmode?: ValueSpecDatetime["inputmode"]
+              min?: string | null
+              max?: string | null
+              disabled?: false | string
+            }
+          >,
+        ) => Value.dynamicDatetime<Store>(getA),
+        dynamicSelect: (
+          getA: LazyBuild<
+            Store,
+            {
+              name: string
+              description?: string | null
+              warning?: string | null
+              required: RequiredDefault<string>
+              values: Record<string, string>
+              disabled?: false | string
+            }
+          >,
+        ) => Value.dynamicSelect<Store>(getA),
+        dynamicMultiselect: (
+          getA: LazyBuild<
+            Store,
+            {
+              name: string
+              description?: string | null
+              warning?: string | null
+              default: string[]
+              values: Record<string, string>
+              minLength?: number | null
+              maxLength?: number | null
+              disabled?: false | string
+            }
+          >,
+        ) => Value.dynamicMultiselect<Store>(getA),
+        filteredUnion: <
+          Required extends RequiredDefault<string>,
+          Type extends Record<string, any>,
+        >(
+          getDisabledFn: LazyBuild<Store, string[]>,
+          a: {
+            name: string
+            description?: string | null
+            warning?: string | null
+            required: Required
+          },
+          aVariants: Variants<Type, Store> | Variants<Type, never>,
+        ) =>
+          Value.filteredUnion<Required, Type, Store>(
+            getDisabledFn,
+            a,
+            aVariants,
+          ),
+
+        dynamicUnion: <
+          Required extends RequiredDefault<string>,
+          Type extends Record<string, any>,
+        >(
+          getA: LazyBuild<
+            Store,
+            {
+              disabled: string[] | false | string
+              name: string
+              description?: string | null
+              warning?: string | null
+              required: Required
+            }
+          >,
+          aVariants: Variants<Type, Store> | Variants<Type, never>,
+        ) => Value.dynamicUnion<Required, Type, Store>(getA, aVariants),
+      },
+      Variants: {
+        of: <
+          VariantValues extends {
+            [K in string]: {
+              name: string
+              spec: Config<any, Store>
+            }
+          },
+        >(
+          a: VariantValues,
+        ) => Variants.of<VariantValues, Store>(a),
+      },
+    }
+  }
+}

sdk/lib/actions/createAction.ts

@@ -0,0 +1,85 @@
+import { Config, ExtractConfigType } from "../config/builder/config"
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { ActionMetadata, ActionResult, Effects, ExportedAction } from "../types"
+
+export type MaybeFn<Manifest extends SDKManifest, Store, Value> =
+  | Value
+  | ((options: { effects: Effects }) => Promise<Value> | Value)
+export class CreatedAction<
+  Manifest extends SDKManifest,
+  Store,
+  ConfigType extends
+    | Record<string, any>
+    | Config<any, Store>
+    | Config<any, never>,
+  Type extends Record<string, any> = ExtractConfigType<ConfigType>,
+> {
+  private constructor(
+    public readonly myMetaData: MaybeFn<
+      Manifest,
+      Store,
+      Omit<ActionMetadata, "input">
+    >,
+    readonly fn: (options: {
+      effects: Effects
+      input: Type
+    }) => Promise<ActionResult>,
+    readonly input: Config<Type, Store>,
+    public validator = input.validator,
+  ) {}
+
+  static of<
+    Manifest extends SDKManifest,
+    Store,
+    ConfigType extends
+      | Record<string, any>
+      | Config<any, any>
+      | Config<any, never>,
+    Type extends Record<string, any> = ExtractConfigType<ConfigType>,
+  >(
+    metaData: MaybeFn<Manifest, Store, Omit<ActionMetadata, "input">>,
+    fn: (options: { effects: Effects; input: Type }) => Promise<ActionResult>,
+    inputConfig: Config<Type, Store> | Config<Type, never>,
+  ) {
+    return new CreatedAction<Manifest, Store, ConfigType, Type>(
+      metaData,
+      fn,
+      inputConfig as Config<Type, Store>,
+    )
+  }
+
+  exportedAction: ExportedAction = ({ effects, input }) => {
+    return this.fn({
+      effects,
+      input: this.validator.unsafeCast(input),
+    })
+  }
+
+  run = async ({ effects, input }: { effects: Effects; input?: Type }) => {
+    return this.fn({
+      effects,
+      input: this.validator.unsafeCast(input),
+    })
+  }
+
+  async metaData(options: { effects: Effects }) {
+    if (this.myMetaData instanceof Function)
+      return await this.myMetaData(options)
+    return this.myMetaData
+  }
+
+  async ActionMetadata(options: { effects: Effects }): Promise<ActionMetadata> {
+    return {
+      ...(await this.metaData(options)),
+      input: await this.input.build(options),
+    }
+  }
+
+  async getConfig({ effects }: { effects: Effects }) {
+    return this.input.build({
+      effects,
+    })
+  }
+}
+
+export const createAction = CreatedAction.of

sdk/lib/actions/index.ts

@@ -0,0 +1,3 @@
+import "./createAction"
+
+import "./setupActions"

sdk/lib/actions/setupActions.ts

@@ -0,0 +1,31 @@
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { Effects, ExpectedExports } from "../types"
+import { once } from "../util/once"
+import { CreatedAction } from "./createAction"
+
+export function setupActions<Manifest extends SDKManifest, Store>(
+  ...createdActions: CreatedAction<Manifest, Store, any>[]
+) {
+  const myActions = async (options: { effects: Effects }) => {
+    const actions: Record<string, CreatedAction<Manifest, Store, any>> = {}
+    for (const action of createdActions) {
+      const actionMetadata = await action.metaData(options)
+      actions[actionMetadata.id] = action
+    }
+    return actions
+  }
+  const answer: {
+    actions: ExpectedExports.actions
+    actionsMetadata: ExpectedExports.actionsMetadata
+  } = {
+    actions(options: { effects: Effects }) {
+      return myActions(options)
+    },
+    async actionsMetadata({ effects }: { effects: Effects }) {
+      return Promise.all(
+        createdActions.map((x) => x.ActionMetadata({ effects })),
+      )
+    },
+  }
+  return answer
+}

sdk/lib/backup/Backups.ts

@@ -0,0 +1,181 @@
+import { SDKManifest } from "../manifest/ManifestTypes"
+import * as T from "../types"
+
+export type BACKUP = "BACKUP"
+export const DEFAULT_OPTIONS: T.BackupOptions = {
+  delete: true,
+  force: true,
+  ignoreExisting: false,
+  exclude: [],
+}
+export type BackupSet<Volumes extends string> = {
+  srcPath: string
+  srcVolume: Volumes | BACKUP
+  dstPath: string
+  dstVolume: Volumes | BACKUP
+  options?: Partial<T.BackupOptions>
+}
+/**
+ * This utility simplifies the volume backup process.
+ * ```ts
+ * export const { createBackup, restoreBackup } = Backups.volumes("main").build();
+ * ```
+ *
+ * Changing the options of the rsync, (ie exludes) use either
+ * ```ts
+ *  Backups.volumes("main").set_options({exclude: ['bigdata/']}).volumes('excludedVolume').build()
+ * // or
+ *  Backups.with_options({exclude: ['bigdata/']}).volumes('excludedVolume').build()
+ * ```
+ *
+ * Using the more fine control, using the addSets for more control
+ * ```ts
+ * Backups.addSets({
+ * srcVolume: 'main', srcPath:'smallData/', dstPath: 'main/smallData/', dstVolume: : Backups.BACKUP
+ * }, {
+ * srcVolume: 'main', srcPath:'bigData/', dstPath: 'main/bigData/', dstVolume: : Backups.BACKUP, options: {exclude:['bigData/excludeThis']}}
+ * ).build()q
+ * ```
+ */
+export class Backups<M extends SDKManifest> {
+  static BACKUP: BACKUP = "BACKUP"
+
+  private constructor(
+    private options = DEFAULT_OPTIONS,
+    private backupSet = [] as BackupSet<M["volumes"][number]>[],
+  ) {}
+  static volumes<M extends SDKManifest = never>(
+    ...volumeNames: Array<M["volumes"][0]>
+  ): Backups<M> {
+    return new Backups<M>().addSets(
+      ...volumeNames.map((srcVolume) => ({
+        srcVolume,
+        srcPath: "./",
+        dstPath: `./${srcVolume}/`,
+        dstVolume: Backups.BACKUP,
+      })),
+    )
+  }
+  static addSets<M extends SDKManifest = never>(
+    ...options: BackupSet<M["volumes"][0]>[]
+  ) {
+    return new Backups().addSets(...options)
+  }
+  static with_options<M extends SDKManifest = never>(
+    options?: Partial<T.BackupOptions>,
+  ) {
+    return new Backups({ ...DEFAULT_OPTIONS, ...options })
+  }
+
+  static withOptions = Backups.with_options
+  setOptions(options?: Partial<T.BackupOptions>) {
+    this.options = {
+      ...this.options,
+      ...options,
+    }
+    return this
+  }
+  volumes(...volumeNames: Array<M["volumes"][0]>) {
+    return this.addSets(
+      ...volumeNames.map((srcVolume) => ({
+        srcVolume,
+        srcPath: "./",
+        dstPath: `./${srcVolume}/`,
+        dstVolume: Backups.BACKUP,
+      })),
+    )
+  }
+  addSets(...options: BackupSet<M["volumes"][0]>[]) {
+    options.forEach((x) =>
+      this.backupSet.push({ ...x, options: { ...this.options, ...x.options } }),
+    )
+    return this
+  }
+  build() {
+    const createBackup: T.ExpectedExports.createBackup = async ({
+      effects,
+    }) => {
+      // const previousItems = (
+      //   await effects
+      //     .readDir({
+      //       volumeId: Backups.BACKUP,
+      //       path: ".",
+      //     })
+      //     .catch(() => [])
+      // ).map((x) => `${x}`)
+      // const backupPaths = this.backupSet
+      //   .filter((x) => x.dstVolume === Backups.BACKUP)
+      //   .map((x) => x.dstPath)
+      //   .map((x) => x.replace(/\.\/([^]*)\//, "$1"))
+      // const filteredItems = previousItems.filter(
+      //   (x) => backupPaths.indexOf(x) === -1,
+      // )
+      // for (const itemToRemove of filteredItems) {
+      //   effects.console.error(`Trying to remove ${itemToRemove}`)
+      //   await effects
+      //     .removeDir({
+      //       volumeId: Backups.BACKUP,
+      //       path: itemToRemove,
+      //     })
+      //     .catch(() =>
+      //       effects.removeFile({
+      //         volumeId: Backups.BACKUP,
+      //         path: itemToRemove,
+      //       }),
+      //     )
+      //     .catch(() => {
+      //       console.warn(`Failed to remove ${itemToRemove} from backup volume`)
+      //     })
+      // }
+      for (const item of this.backupSet) {
+        // if (notEmptyPath(item.dstPath)) {
+        //   await effects.createDir({
+        //     volumeId: item.dstVolume,
+        //     path: item.dstPath,
+        //   })
+        // }
+        // await effects
+        //   .runRsync({
+        //     ...item,
+        //     options: {
+        //       ...this.options,
+        //       ...item.options,
+        //     },
+        //   })
+        //   .wait()
+      }
+      return
+    }
+    const restoreBackup: T.ExpectedExports.restoreBackup = async ({
+      effects,
+    }) => {
+      for (const item of this.backupSet) {
+        // if (notEmptyPath(item.srcPath)) {
+        //   await new Promise((resolve, reject) => fs.mkdir(items.src)).createDir(
+        //     {
+        //       volumeId: item.srcVolume,
+        //       path: item.srcPath,
+        //     },
+        //   )
+        // }
+        // await effects
+        //   .runRsync({
+        //     options: {
+        //       ...this.options,
+        //       ...item.options,
+        //     },
+        //     srcVolume: item.dstVolume,
+        //     dstVolume: item.srcVolume,
+        //     srcPath: item.dstPath,
+        //     dstPath: item.srcPath,
+        //   })
+        //   .wait()
+      }
+      return
+    }
+    return { createBackup, restoreBackup }
+  }
+}
+function notEmptyPath(file: string) {
+  return ["", ".", "./"].indexOf(file) === -1
+}

sdk/lib/backup/index.ts

@@ -0,0 +1,3 @@
+import "./Backups"
+
+import "./setupBackups"

sdk/lib/backup/setupBackups.ts

@@ -0,0 +1,43 @@
+import { Backups } from "./Backups"
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { ExpectedExports } from "../types"
+import { _ } from "../util"
+
+export type SetupBackupsParams<M extends SDKManifest> = Array<
+  M["volumes"][number] | Backups<M>
+>
+
+export function setupBackups<M extends SDKManifest>(
+  ...args: _<SetupBackupsParams<M>>
+) {
+  const backups = Array<Backups<M>>()
+  const volumes = new Set<M["volumes"][0]>()
+  for (const arg of args) {
+    if (arg instanceof Backups) {
+      backups.push(arg)
+    } else {
+      volumes.add(arg)
+    }
+  }
+  backups.push(Backups.volumes(...volumes))
+  const answer: {
+    createBackup: ExpectedExports.createBackup
+    restoreBackup: ExpectedExports.restoreBackup
+  } = {
+    get createBackup() {
+      return (async (options) => {
+        for (const backup of backups) {
+          await backup.build().createBackup(options)
+        }
+      }) as ExpectedExports.createBackup
+    },
+    get restoreBackup() {
+      return (async (options) => {
+        for (const backup of backups) {
+          await backup.build().restoreBackup(options)
+        }
+      }) as ExpectedExports.restoreBackup
+    },
+  }
+  return answer
+}

sdk/lib/config/builder/config.ts

@@ -0,0 +1,137 @@
+import { ValueSpec } from "../configTypes"
+import { Value } from "./value"
+import { _ } from "../../util"
+import { Effects } from "../../types"
+import { Parser, object } from "ts-matches"
+
+export type LazyBuildOptions<Store> = {
+  effects: Effects
+}
+export type LazyBuild<Store, ExpectedOut> = (
+  options: LazyBuildOptions<Store>,
+) => Promise<ExpectedOut> | ExpectedOut
+
+// prettier-ignore
+export type ExtractConfigType<A extends Record<string, any> | Config<Record<string, any>, any> | Config<Record<string, any>, never>> = 
+  A extends Config<infer B, any> | Config<infer B, never> ? B :
+  A
+
+export type ConfigSpecOf<A extends Record<string, any>, Store = never> = {
+  [K in keyof A]: Value<A[K], Store>
+}
+
+export type MaybeLazyValues<A> = LazyBuild<any, A> | A
+/**
+ * Configs are the specs that are used by the os configuration form for this service.
+ * Here is an example of a simple configuration
+  ```ts
+    const smallConfig = Config.of({
+      test: Value.boolean({
+        name: "Test",
+        description: "This is the description for the test",
+        warning: null,
+        default: false,
+      }),
+    });
+  ```
+
+  The idea of a config is that now the form is going to ask for
+  Test: [ ] and the value is going to be checked as a boolean.
+  There are more complex values like selects, lists, and objects. See {@link Value}
+
+  Also, there is the ability to get a validator/parser from this config spec.
+  ```ts
+  const matchSmallConfig = smallConfig.validator();
+  type SmallConfig = typeof matchSmallConfig._TYPE;
+  ```
+
+  Here is an example of a more complex configuration which came from a configuration for a service
+  that works with bitcoin, like c-lightning.
+  ```ts
+
+    export const hostname = Value.string({
+  name: "Hostname",
+  default: null,
+  description: "Domain or IP address of bitcoin peer",
+  warning: null,
+  required: true,
+  masked: false,
+  placeholder: null,
+  pattern:
+    "(^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$)|((^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$)|(^[a-z2-7]{16}\\.onion$)|(^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$))",
+  patternDescription:
+    "Must be either a domain name, or an IPv4 or IPv6 address. Do not include protocol scheme (eg 'http://') or port.",
+});
+export const port = Value.number({
+  name: "Port",
+  default: null,
+  description: "Port that peer is listening on for inbound p2p connections",
+  warning: null,
+  required: false,
+  range: "[0,65535]",
+  integral: true,
+  units: null,
+  placeholder: null,
+});
+export const addNodesSpec = Config.of({ hostname: hostname, port: port });
+
+  ```
+ */
+export class Config<Type extends Record<string, any>, Store = never> {
+  private constructor(
+    private readonly spec: {
+      [K in keyof Type]: Value<Type[K], Store> | Value<Type[K], never>
+    },
+    public validator: Parser<unknown, Type>,
+  ) {}
+  async build(options: LazyBuildOptions<Store>) {
+    const answer = {} as {
+      [K in keyof Type]: ValueSpec
+    }
+    for (const k in this.spec) {
+      answer[k] = await this.spec[k].build(options as any)
+    }
+    return answer
+  }
+
+  static of<
+    Spec extends Record<string, Value<any, Store> | Value<any, never>>,
+    Store = never,
+  >(spec: Spec) {
+    const validatorObj = {} as {
+      [K in keyof Spec]: Parser<unknown, any>
+    }
+    for (const key in spec) {
+      validatorObj[key] = spec[key].validator
+    }
+    const validator = object(validatorObj)
+    return new Config<
+      {
+        [K in keyof Spec]: Spec[K] extends
+          | Value<infer T, Store>
+          | Value<infer T, never>
+          ? T
+          : never
+      },
+      Store
+    >(spec, validator as any)
+  }
+
+  /**
+   * Use this during the times that the input needs a more specific type.
+   * Used in types that the value/ variant/ list/ config is constructed somewhere else.
+  ```ts
+  const a = Config.text({
+    name: "a",
+    required: false,
+  })
+
+  return Config.of<Store>()({
+    myValue: a.withStore(),
+  })
+  ```
+   */
+  withStore<NewStore extends Store extends never ? any : Store>() {
+    return this as any as Config<Type, NewStore>
+  }
+}

sdk/lib/config/builder/index.ts

@@ -0,0 +1,4 @@
+import "./config"
+import "./list"
+import "./value"
+import "./variants"

sdk/lib/config/builder/list.ts

@@ -0,0 +1,279 @@
+import { Config, LazyBuild } from "./config"
+import {
+  ListValueSpecText,
+  Pattern,
+  RandomString,
+  UniqueBy,
+  ValueSpecList,
+  ValueSpecListOf,
+  ValueSpecText,
+} from "../configTypes"
+import { Parser, arrayOf, number, string } from "ts-matches"
+/**
+ * Used as a subtype of Value.list
+```ts
+export const authorizationList = List.string({
+  "name": "Authorization",
+  "range": "[0,*)",
+  "default": [],
+  "description": "Username and hashed password for JSON-RPC connections. RPC clients connect using the usual http basic authentication.",
+  "warning": null
+}, {"masked":false,"placeholder":null,"pattern":"^[a-zA-Z0-9_-]+:([0-9a-fA-F]{2})+\\$([0-9a-fA-F]{2})+$","patternDescription":"Each item must be of the form \"<USERNAME>:<SALT>$<HASH>\"."});
+export const auth = Value.list(authorizationList);
+```
+*/
+export class List<Type, Store> {
+  private constructor(
+    public build: LazyBuild<Store, ValueSpecList>,
+    public validator: Parser<unknown, Type>,
+  ) {}
+  static text(
+    a: {
+      name: string
+      description?: string | null
+      warning?: string | null
+      /** Default = [] */
+      default?: string[]
+      minLength?: number | null
+      maxLength?: number | null
+    },
+    aSpec: {
+      /** Default = false */
+      masked?: boolean
+      placeholder?: string | null
+      minLength?: number | null
+      maxLength?: number | null
+      patterns: Pattern[]
+      /** Default = "text" */
+      inputmode?: ListValueSpecText["inputmode"]
+      generate?: null | RandomString
+    },
+  ) {
+    return new List<string[], never>(() => {
+      const spec = {
+        type: "text" as const,
+        placeholder: null,
+        minLength: null,
+        maxLength: null,
+        masked: false,
+        inputmode: "text" as const,
+        generate: null,
+        ...aSpec,
+      }
+      const built: ValueSpecListOf<"text"> = {
+        description: null,
+        warning: null,
+        default: [],
+        type: "list" as const,
+        minLength: null,
+        maxLength: null,
+        disabled: false,
+        ...a,
+        spec,
+      }
+      return built
+    }, arrayOf(string))
+  }
+  static dynamicText<Store = never>(
+    getA: LazyBuild<
+      Store,
+      {
+        name: string
+        description?: string | null
+        warning?: string | null
+        /** Default = [] */
+        default?: string[]
+        minLength?: number | null
+        maxLength?: number | null
+        disabled?: false | string
+        generate?: null | RandomString
+        spec: {
+          /** Default = false */
+          masked?: boolean
+          placeholder?: string | null
+          minLength?: number | null
+          maxLength?: number | null
+          patterns: Pattern[]
+          /** Default = "text" */
+          inputmode?: ListValueSpecText["inputmode"]
+        }
+      }
+    >,
+  ) {
+    return new List<string[], Store>(async (options) => {
+      const { spec: aSpec, ...a } = await getA(options)
+      const spec = {
+        type: "text" as const,
+        placeholder: null,
+        minLength: null,
+        maxLength: null,
+        masked: false,
+        inputmode: "text" as const,
+        generate: null,
+        ...aSpec,
+      }
+      const built: ValueSpecListOf<"text"> = {
+        description: null,
+        warning: null,
+        default: [],
+        type: "list" as const,
+        minLength: null,
+        maxLength: null,
+        disabled: false,
+        ...a,
+        spec,
+      }
+      return built
+    }, arrayOf(string))
+  }
+  static number(
+    a: {
+      name: string
+      description?: string | null
+      warning?: string | null
+      /** Default = [] */
+      default?: string[]
+      minLength?: number | null
+      maxLength?: number | null
+    },
+    aSpec: {
+      integer: boolean
+      min?: number | null
+      max?: number | null
+      step?: number | null
+      units?: string | null
+      placeholder?: string | null
+    },
+  ) {
+    return new List<number[], never>(() => {
+      const spec = {
+        type: "number" as const,
+        placeholder: null,
+        min: null,
+        max: null,
+        step: null,
+        units: null,
+        ...aSpec,
+      }
+      const built: ValueSpecListOf<"number"> = {
+        description: null,
+        warning: null,
+        minLength: null,
+        maxLength: null,
+        default: [],
+        type: "list" as const,
+        disabled: false,
+        ...a,
+        spec,
+      }
+      return built
+    }, arrayOf(number))
+  }
+  static dynamicNumber<Store = never>(
+    getA: LazyBuild<
+      Store,
+      {
+        name: string
+        description?: string | null
+        warning?: string | null
+        /** Default = [] */
+        default?: string[]
+        minLength?: number | null
+        maxLength?: number | null
+        disabled?: false | string
+        spec: {
+          integer: boolean
+          min?: number | null
+          max?: number | null
+          step?: number | null
+          units?: string | null
+          placeholder?: string | null
+        }
+      }
+    >,
+  ) {
+    return new List<number[], Store>(async (options) => {
+      const { spec: aSpec, ...a } = await getA(options)
+      const spec = {
+        type: "number" as const,
+        placeholder: null,
+        min: null,
+        max: null,
+        step: null,
+        units: null,
+        ...aSpec,
+      }
+      return {
+        description: null,
+        warning: null,
+        minLength: null,
+        maxLength: null,
+        default: [],
+        type: "list" as const,
+        disabled: false,
+        ...a,
+        spec,
+      }
+    }, arrayOf(number))
+  }
+  static obj<Type extends Record<string, any>, Store>(
+    a: {
+      name: string
+      description?: string | null
+      warning?: string | null
+      /** Default [] */
+      default?: []
+      minLength?: number | null
+      maxLength?: number | null
+    },
+    aSpec: {
+      spec: Config<Type, Store>
+      displayAs?: null | string
+      uniqueBy?: null | UniqueBy
+    },
+  ) {
+    return new List<Type[], Store>(async (options) => {
+      const { spec: previousSpecSpec, ...restSpec } = aSpec
+      const specSpec = await previousSpecSpec.build(options)
+      const spec = {
+        type: "object" as const,
+        displayAs: null,
+        uniqueBy: null,
+        ...restSpec,
+        spec: specSpec,
+      }
+      const value = {
+        spec,
+        default: [],
+        ...a,
+      }
+      return {
+        description: null,
+        warning: null,
+        minLength: null,
+        maxLength: null,
+        type: "list" as const,
+        disabled: false,
+        ...value,
+      }
+    }, arrayOf(aSpec.spec.validator))
+  }
+
+  /**
+   * Use this during the times that the input needs a more specific type.
+   * Used in types that the value/ variant/ list/ config is constructed somewhere else.
+  ```ts
+  const a = Config.text({
+    name: "a",
+    required: false,
+  })
+
+  return Config.of<Store>()({
+    myValue: a.withStore(),
+  })
+  ```
+   */
+  withStore<NewStore extends Store extends never ? any : Store>() {
+    return this as any as List<Type, NewStore>
+  }
+}

sdk/lib/config/builder/value.ts

@@ -0,0 +1,783 @@
+import { Config, LazyBuild, LazyBuildOptions } from "./config"
+import { List } from "./list"
+import { Variants } from "./variants"
+import {
+  FilePath,
+  Pattern,
+  RandomString,
+  ValueSpec,
+  ValueSpecDatetime,
+  ValueSpecText,
+  ValueSpecTextarea,
+} from "../configTypes"
+import { DefaultString } from "../configTypes"
+import { _ } from "../../util"
+import {
+  Parser,
+  anyOf,
+  arrayOf,
+  boolean,
+  literal,
+  literals,
+  number,
+  object,
+  string,
+  unknown,
+} from "ts-matches"
+import { once } from "../../util/once"
+
+export type RequiredDefault<A> =
+  | false
+  | {
+      default: A | null
+    }
+
+function requiredLikeToAbove<Input extends RequiredDefault<A>, A>(
+  requiredLike: Input,
+) {
+  // prettier-ignore
+  return {
+    required: (typeof requiredLike === 'object' ? true : requiredLike) as (
+      Input extends { default: unknown} ? true:
+      Input extends true ? true :
+      false
+    ),
+    default:(typeof requiredLike === 'object' ? requiredLike.default : null) as (
+      Input extends { default: infer Default } ? Default :
+      null
+    )
+  };
+}
+type AsRequired<Type, MaybeRequiredType> = MaybeRequiredType extends
+  | { default: unknown }
+  | never
+  ? Type
+  : Type | null | undefined
+
+type InputAsRequired<A, Type> = A extends
+  | { required: { default: any } | never }
+  | never
+  ? Type
+  : Type | null | undefined
+const testForAsRequiredParser = once(
+  () => object({ required: object({ default: unknown }) }).test,
+)
+function asRequiredParser<
+  Type,
+  Input,
+  Return extends
+    | Parser<unknown, Type>
+    | Parser<unknown, Type | null | undefined>,
+>(parser: Parser<unknown, Type>, input: Input): Return {
+  if (testForAsRequiredParser()(input)) return parser as any
+  return parser.optional() as any
+}
+
+/**
+ * A value is going to be part of the form in the FE of the OS.
+ * Something like a boolean, a string, a number, etc.
+ * in the fe it will ask for the name of value, and use the rest of the value to determine how to render it.
+ * While writing with a value, you will start with `Value.` then let the IDE suggest the rest.
+ * for things like string, the options are going to be in {}.
+ * Keep an eye out for another config builder types as params.
+ * Note, usually this is going to be used in a `Config` {@link Config} builder.
+ ```ts
+const username = Value.string({
+  name: "Username",
+  default: "bitcoin",
+  description: "The username for connecting to Bitcoin over RPC.",
+  warning: null,
+  required: true,
+  masked: true,
+  placeholder: null,
+  pattern: "^[a-zA-Z0-9_]+$",
+  patternDescription: "Must be alphanumeric (can contain underscore).",
+});
+ ```
+ */
+export class Value<Type, Store> {
+  protected constructor(
+    public build: LazyBuild<Store, ValueSpec>,
+    public validator: Parser<unknown, Type>,
+  ) {}
+  static toggle(a: {
+    name: string
+    description?: string | null
+    warning?: string | null
+    default: boolean
+    /**  Immutable means it can only be configed at the first config then never again 
+    Default is false */
+    immutable?: boolean
+  }) {
+    return new Value<boolean, never>(
+      async () => ({
+        description: null,
+        warning: null,
+        type: "toggle" as const,
+        disabled: false,
+        immutable: a.immutable ?? false,
+        ...a,
+      }),
+      boolean,
+    )
+  }
+  static dynamicToggle<Store = never>(
+    a: LazyBuild<
+      Store,
+      {
+        name: string
+        description?: string | null
+        warning?: string | null
+        default: boolean
+        disabled?: false | string
+      }
+    >,
+  ) {
+    return new Value<boolean, Store>(
+      async (options) => ({
+        description: null,
+        warning: null,
+        type: "toggle" as const,
+        disabled: false,
+        immutable: false,
+        ...(await a(options)),
+      }),
+      boolean,
+    )
+  }
+  static text<Required extends RequiredDefault<DefaultString>>(a: {
+    name: string
+    description?: string | null
+    warning?: string | null
+    required: Required
+
+    /** Default = false */
+    masked?: boolean
+    placeholder?: string | null
+    minLength?: number | null
+    maxLength?: number | null
+    patterns?: Pattern[]
+    /** Default = 'text' */
+    inputmode?: ValueSpecText["inputmode"]
+    /**  Immutable means it can only be configured at the first config then never again
+     * Default is false
+     */
+    immutable?: boolean
+    generate?: null | RandomString
+  }) {
+    return new Value<AsRequired<string, Required>, never>(
+      async () => ({
+        type: "text" as const,
+        description: null,
+        warning: null,
+        masked: false,
+        placeholder: null,
+        minLength: null,
+        maxLength: null,
+        patterns: [],
+        inputmode: "text",
+        disabled: false,
+        immutable: a.immutable ?? false,
+        generate: a.generate ?? null,
+        ...a,
+        ...requiredLikeToAbove(a.required),
+      }),
+      asRequiredParser(string, a),
+    )
+  }
+  static dynamicText<Store = never>(
+    getA: LazyBuild<
+      Store,
+      {
+        name: string
+        description?: string | null
+        warning?: string | null
+        required: RequiredDefault<DefaultString>
+
+        /** Default = false */
+        masked?: boolean
+        placeholder?: string | null
+        minLength?: number | null
+        maxLength?: number | null
+        patterns?: Pattern[]
+        /** Default = 'text' */
+        inputmode?: ValueSpecText["inputmode"]
+        disabled?: string | false
+        /**  Immutable means it can only be configured at the first config then never again
+         * Default is false
+         */
+        generate?: null | RandomString
+      }
+    >,
+  ) {
+    return new Value<string | null | undefined, Store>(async (options) => {
+      const a = await getA(options)
+      return {
+        type: "text" as const,
+        description: null,
+        warning: null,
+        masked: false,
+        placeholder: null,
+        minLength: null,
+        maxLength: null,
+        patterns: [],
+        inputmode: "text",
+        disabled: false,
+        immutable: false,
+        generate: a.generate ?? null,
+        ...a,
+        ...requiredLikeToAbove(a.required),
+      }
+    }, string.optional())
+  }
+  static textarea(a: {
+    name: string
+    description?: string | null
+    warning?: string | null
+    required: boolean
+    minLength?: number | null
+    maxLength?: number | null
+    placeholder?: string | null
+    /**  Immutable means it can only be configed at the first config then never again 
+    Default is false */
+    immutable?: boolean
+  }) {
+    return new Value<string, never>(async () => {
+      const built: ValueSpecTextarea = {
+        description: null,
+        warning: null,
+        minLength: null,
+        maxLength: null,
+        placeholder: null,
+        type: "textarea" as const,
+        disabled: false,
+        immutable: a.immutable ?? false,
+        ...a,
+      }
+      return built
+    }, string)
+  }
+  static dynamicTextarea<Store = never>(
+    getA: LazyBuild<
+      Store,
+      {
+        name: string
+        description?: string | null
+        warning?: string | null
+        required: boolean
+        minLength?: number | null
+        maxLength?: number | null
+        placeholder?: string | null
+        disabled?: false | string
+      }
+    >,
+  ) {
+    return new Value<string, Store>(async (options) => {
+      const a = await getA(options)
+      return {
+        description: null,
+        warning: null,
+        minLength: null,
+        maxLength: null,
+        placeholder: null,
+        type: "textarea" as const,
+        disabled: false,
+        immutable: false,
+        ...a,
+      }
+    }, string)
+  }
+  static number<Required extends RequiredDefault<number>>(a: {
+    name: string
+    description?: string | null
+    warning?: string | null
+    required: Required
+    min?: number | null
+    max?: number | null
+    /** Default = '1' */
+    step?: number | null
+    integer: boolean
+    units?: string | null
+    placeholder?: string | null
+    /**  Immutable means it can only be configed at the first config then never again 
+    Default is false */
+    immutable?: boolean
+  }) {
+    return new Value<AsRequired<number, Required>, never>(
+      () => ({
+        type: "number" as const,
+        description: null,
+        warning: null,
+        min: null,
+        max: null,
+        step: null,
+        units: null,
+        placeholder: null,
+        disabled: false,
+        immutable: a.immutable ?? false,
+        ...a,
+        ...requiredLikeToAbove(a.required),
+      }),
+      asRequiredParser(number, a),
+    )
+  }
+  static dynamicNumber<Store = never>(
+    getA: LazyBuild<
+      Store,
+      {
+        name: string
+        description?: string | null
+        warning?: string | null
+        required: RequiredDefault<number>
+        min?: number | null
+        max?: number | null
+        /** Default = '1' */
+        step?: number | null
+        integer: boolean
+        units?: string | null
+        placeholder?: string | null
+        disabled?: false | string
+      }
+    >,
+  ) {
+    return new Value<number | null | undefined, Store>(async (options) => {
+      const a = await getA(options)
+      return {
+        type: "number" as const,
+        description: null,
+        warning: null,
+        min: null,
+        max: null,
+        step: null,
+        units: null,
+        placeholder: null,
+        disabled: false,
+        immutable: false,
+        ...a,
+        ...requiredLikeToAbove(a.required),
+      }
+    }, number.optional())
+  }
+  static color<Required extends RequiredDefault<string>>(a: {
+    name: string
+    description?: string | null
+    warning?: string | null
+    required: Required
+    /**  Immutable means it can only be configed at the first config then never again 
+    Default is false */
+    immutable?: boolean
+  }) {
+    return new Value<AsRequired<string, Required>, never>(
+      () => ({
+        type: "color" as const,
+        description: null,
+        warning: null,
+        disabled: false,
+        immutable: a.immutable ?? false,
+        ...a,
+        ...requiredLikeToAbove(a.required),
+      }),
+
+      asRequiredParser(string, a),
+    )
+  }
+
+  static dynamicColor<Store = never>(
+    getA: LazyBuild<
+      Store,
+      {
+        name: string
+        description?: string | null
+        warning?: string | null
+        required: RequiredDefault<string>
+        disabled?: false | string
+      }
+    >,
+  ) {
+    return new Value<string | null | undefined, Store>(async (options) => {
+      const a = await getA(options)
+      return {
+        type: "color" as const,
+        description: null,
+        warning: null,
+        disabled: false,
+        immutable: false,
+        ...a,
+        ...requiredLikeToAbove(a.required),
+      }
+    }, string.optional())
+  }
+  static datetime<Required extends RequiredDefault<string>>(a: {
+    name: string
+    description?: string | null
+    warning?: string | null
+    required: Required
+    /** Default = 'datetime-local' */
+    inputmode?: ValueSpecDatetime["inputmode"]
+    min?: string | null
+    max?: string | null
+    /**  Immutable means it can only be configed at the first config then never again 
+    Default is false */
+    immutable?: boolean
+  }) {
+    return new Value<AsRequired<string, Required>, never>(
+      () => ({
+        type: "datetime" as const,
+        description: null,
+        warning: null,
+        inputmode: "datetime-local",
+        min: null,
+        max: null,
+        step: null,
+        disabled: false,
+        immutable: a.immutable ?? false,
+        ...a,
+        ...requiredLikeToAbove(a.required),
+      }),
+      asRequiredParser(string, a),
+    )
+  }
+  static dynamicDatetime<Store = never>(
+    getA: LazyBuild<
+      Store,
+      {
+        name: string
+        description?: string | null
+        warning?: string | null
+        required: RequiredDefault<string>
+        /** Default = 'datetime-local' */
+        inputmode?: ValueSpecDatetime["inputmode"]
+        min?: string | null
+        max?: string | null
+        disabled?: false | string
+      }
+    >,
+  ) {
+    return new Value<string | null | undefined, Store>(async (options) => {
+      const a = await getA(options)
+      return {
+        type: "datetime" as const,
+        description: null,
+        warning: null,
+        inputmode: "datetime-local",
+        min: null,
+        max: null,
+        disabled: false,
+        immutable: false,
+        ...a,
+        ...requiredLikeToAbove(a.required),
+      }
+    }, string.optional())
+  }
+  static select<
+    Required extends RequiredDefault<string>,
+    B extends Record<string, string>,
+  >(a: {
+    name: string
+    description?: string | null
+    warning?: string | null
+    required: Required
+    values: B
+    /**
+     * Disabled:  false means that there is nothing disabled, good to modify
+     *           string means that this is the message displayed and the whole thing is disabled
+     *           string[] means that the options are disabled
+     */
+    disabled?: false | string | (string & keyof B)[]
+    /**  Immutable means it can only be configed at the first config then never again 
+    Default is false */
+    immutable?: boolean
+  }) {
+    return new Value<AsRequired<keyof B, Required>, never>(
+      () => ({
+        description: null,
+        warning: null,
+        type: "select" as const,
+        disabled: false,
+        immutable: a.immutable ?? false,
+        ...a,
+        ...requiredLikeToAbove(a.required),
+      }),
+      asRequiredParser(
+        anyOf(
+          ...Object.keys(a.values).map((x: keyof B & string) => literal(x)),
+        ),
+        a,
+      ) as any,
+    )
+  }
+  static dynamicSelect<Store = never>(
+    getA: LazyBuild<
+      Store,
+      {
+        name: string
+        description?: string | null
+        warning?: string | null
+        required: RequiredDefault<string>
+        values: Record<string, string>
+        /**
+         * Disabled:  false means that there is nothing disabled, good to modify
+         *           string means that this is the message displayed and the whole thing is disabled
+         *           string[] means that the options are disabled
+         */
+        disabled?: false | string | string[]
+      }
+    >,
+  ) {
+    return new Value<string | null | undefined, Store>(async (options) => {
+      const a = await getA(options)
+      return {
+        description: null,
+        warning: null,
+        type: "select" as const,
+        disabled: false,
+        immutable: false,
+        ...a,
+        ...requiredLikeToAbove(a.required),
+      }
+    }, string.optional())
+  }
+  static multiselect<Values extends Record<string, string>>(a: {
+    name: string
+    description?: string | null
+    warning?: string | null
+    default: string[]
+    values: Values
+    minLength?: number | null
+    maxLength?: number | null
+    /**  Immutable means it can only be configed at the first config then never again 
+    Default is false */
+    immutable?: boolean
+    /**
+     * Disabled:  false means that there is nothing disabled, good to modify
+     *           string means that this is the message displayed and the whole thing is disabled
+     *           string[] means that the options are disabled
+     */
+    disabled?: false | string | (string & keyof Values)[]
+  }) {
+    return new Value<(keyof Values)[], never>(
+      () => ({
+        type: "multiselect" as const,
+        minLength: null,
+        maxLength: null,
+        warning: null,
+        description: null,
+        disabled: false,
+        immutable: a.immutable ?? false,
+        ...a,
+      }),
+      arrayOf(
+        literals(...(Object.keys(a.values) as any as [keyof Values & string])),
+      ),
+    )
+  }
+  static dynamicMultiselect<Store = never>(
+    getA: LazyBuild<
+      Store,
+      {
+        name: string
+        description?: string | null
+        warning?: string | null
+        default: string[]
+        values: Record<string, string>
+        minLength?: number | null
+        maxLength?: number | null
+        /**
+         * Disabled:  false means that there is nothing disabled, good to modify
+         *           string means that this is the message displayed and the whole thing is disabled
+         *           string[] means that the options are disabled
+         */
+        disabled?: false | string | string[]
+      }
+    >,
+  ) {
+    return new Value<string[], Store>(async (options) => {
+      const a = await getA(options)
+      return {
+        type: "multiselect" as const,
+        minLength: null,
+        maxLength: null,
+        warning: null,
+        description: null,
+        disabled: false,
+        immutable: false,
+        ...a,
+      }
+    }, arrayOf(string))
+  }
+  static object<Type extends Record<string, any>, Store>(
+    a: {
+      name: string
+      description?: string | null
+      warning?: string | null
+    },
+    spec: Config<Type, Store>,
+  ) {
+    return new Value<Type, Store>(async (options) => {
+      const built = await spec.build(options as any)
+      return {
+        type: "object" as const,
+        description: null,
+        warning: null,
+        ...a,
+        spec: built,
+      }
+    }, spec.validator)
+  }
+  static file<Required extends RequiredDefault<string>, Store>(a: {
+    name: string
+    description?: string | null
+    warning?: string | null
+    extensions: string[]
+    required: Required
+  }) {
+    const buildValue = {
+      type: "file" as const,
+      description: null,
+      warning: null,
+      ...a,
+    }
+    return new Value<AsRequired<FilePath, Required>, Store>(
+      () => ({
+        ...buildValue,
+
+        ...requiredLikeToAbove(a.required),
+      }),
+      asRequiredParser(object({ filePath: string }), a),
+    )
+  }
+  static dynamicFile<Required extends boolean, Store>(
+    a: LazyBuild<
+      Store,
+      {
+        name: string
+        description?: string | null
+        warning?: string | null
+        extensions: string[]
+        required: Required
+      }
+    >,
+  ) {
+    return new Value<string | null | undefined, Store>(
+      async (options) => ({
+        type: "file" as const,
+        description: null,
+        warning: null,
+        ...(await a(options)),
+      }),
+      string.optional(),
+    )
+  }
+  static union<Required extends RequiredDefault<string>, Type, Store>(
+    a: {
+      name: string
+      description?: string | null
+      warning?: string | null
+      required: Required
+      /**  Immutable means it can only be configed at the first config then never again 
+      Default is false */
+      immutable?: boolean
+      /**
+       * Disabled:  false means that there is nothing disabled, good to modify
+       *           string means that this is the message displayed and the whole thing is disabled
+       *           string[] means that the options are disabled
+       */
+      disabled?: false | string | string[]
+    },
+    aVariants: Variants<Type, Store>,
+  ) {
+    return new Value<AsRequired<Type, Required>, Store>(
+      async (options) => ({
+        type: "union" as const,
+        description: null,
+        warning: null,
+        disabled: false,
+        ...a,
+        variants: await aVariants.build(options as any),
+        ...requiredLikeToAbove(a.required),
+        immutable: a.immutable ?? false,
+      }),
+      asRequiredParser(aVariants.validator, a),
+    )
+  }
+  static filteredUnion<
+    Required extends RequiredDefault<string>,
+    Type extends Record<string, any>,
+    Store = never,
+  >(
+    getDisabledFn: LazyBuild<Store, string[] | false | string>,
+    a: {
+      name: string
+      description?: string | null
+      warning?: string | null
+      required: Required
+    },
+    aVariants: Variants<Type, Store> | Variants<Type, never>,
+  ) {
+    return new Value<AsRequired<Type, Required>, Store>(
+      async (options) => ({
+        type: "union" as const,
+        description: null,
+        warning: null,
+        ...a,
+        variants: await aVariants.build(options as any),
+        ...requiredLikeToAbove(a.required),
+        disabled: (await getDisabledFn(options)) || false,
+        immutable: false,
+      }),
+      asRequiredParser(aVariants.validator, a),
+    )
+  }
+  static dynamicUnion<
+    Required extends RequiredDefault<string>,
+    Type extends Record<string, any>,
+    Store = never,
+  >(
+    getA: LazyBuild<
+      Store,
+      {
+        disabled: string[] | false | string
+        name: string
+        description?: string | null
+        warning?: string | null
+        required: Required
+      }
+    >,
+    aVariants: Variants<Type, Store> | Variants<Type, never>,
+  ) {
+    return new Value<Type | null | undefined, Store>(async (options) => {
+      const newValues = await getA(options)
+      return {
+        type: "union" as const,
+        description: null,
+        warning: null,
+        ...newValues,
+        variants: await aVariants.build(options as any),
+        ...requiredLikeToAbove(newValues.required),
+        immutable: false,
+      }
+    }, aVariants.validator.optional())
+  }
+
+  static list<Type, Store>(a: List<Type, Store>) {
+    return new Value<Type, Store>((options) => a.build(options), a.validator)
+  }
+
+  /**
+   * Use this during the times that the input needs a more specific type.
+   * Used in types that the value/ variant/ list/ config is constructed somewhere else.
+  ```ts
+  const a = Config.text({
+    name: "a",
+    required: false,
+  })
+
+  return Config.of<Store>()({
+    myValue: a.withStore(),
+  })
+  ```
+   */
+  withStore<NewStore extends Store extends never ? any : Store>() {
+    return this as any as Value<Type, NewStore>
+  }
+}

sdk/lib/config/builder/variants.ts

@@ -0,0 +1,120 @@
+import { InputSpec, ValueSpecUnion } from "../configTypes"
+import { LazyBuild, Config } from "./config"
+import { Parser, anyOf, literals, object } from "ts-matches"
+
+/**
+ * Used in the the Value.select { @link './value.ts' }
+ * to indicate the type of select variants that are available. The key for the record passed in will be the
+ * key to the tag.id in the Value.select
+```ts
+ 
+export const disabled = Config.of({});
+export const size = Value.number({
+  name: "Max Chain Size",
+  default: 550,
+  description: "Limit of blockchain size on disk.",
+  warning: "Increasing this value will require re-syncing your node.",
+  required: true,
+  range: "[550,1000000)",
+  integral: true,
+  units: "MiB",
+  placeholder: null,
+});
+export const automatic = Config.of({ size: size });
+export const size1 = Value.number({
+  name: "Failsafe Chain Size",
+  default: 65536,
+  description: "Prune blockchain if size expands beyond this.",
+  warning: null,
+  required: true,
+  range: "[550,1000000)",
+  integral: true,
+  units: "MiB",
+  placeholder: null,
+});
+export const manual = Config.of({ size: size1 });
+export const pruningSettingsVariants = Variants.of({
+  disabled: { name: "Disabled", spec: disabled },
+  automatic: { name: "Automatic", spec: automatic },
+  manual: { name: "Manual", spec: manual },
+});
+export const pruning = Value.union(
+  {
+    name: "Pruning Settings",
+    description:
+      '- Disabled: Disable pruning\n- Automatic: Limit blockchain size on disk to a certain number of megabytes\n- Manual: Prune blockchain with the "pruneblockchain" RPC\n',
+    warning: null,
+    required: true,
+    default: "disabled",
+  },
+  pruningSettingsVariants
+);
+```
+ */
+export class Variants<Type, Store> {
+  static text: any
+  private constructor(
+    public build: LazyBuild<Store, ValueSpecUnion["variants"]>,
+    public validator: Parser<unknown, Type>,
+  ) {}
+  static of<
+    VariantValues extends {
+      [K in string]: {
+        name: string
+        spec: Config<any, Store> | Config<any, never>
+      }
+    },
+    Store = never,
+  >(a: VariantValues) {
+    const validator = anyOf(
+      ...Object.entries(a).map(([name, { spec }]) =>
+        object({
+          unionSelectKey: literals(name),
+          unionValueKey: spec.validator,
+        }),
+      ),
+    ) as Parser<unknown, any>
+
+    return new Variants<
+      {
+        [K in keyof VariantValues]: {
+          unionSelectKey: K
+          // prettier-ignore
+          unionValueKey: 
+            VariantValues[K]["spec"] extends (Config<infer B, Store> | Config<infer B, never>) ? B :
+            never
+        }
+      }[keyof VariantValues],
+      Store
+    >(async (options) => {
+      const variants = {} as {
+        [K in keyof VariantValues]: { name: string; spec: InputSpec }
+      }
+      for (const key in a) {
+        const value = a[key]
+        variants[key] = {
+          name: value.name,
+          spec: await value.spec.build(options as any),
+        }
+      }
+      return variants
+    }, validator)
+  }
+  /**
+   * Use this during the times that the input needs a more specific type.
+   * Used in types that the value/ variant/ list/ config is constructed somewhere else.
+  ```ts
+  const a = Config.text({
+    name: "a",
+    required: false,
+  })
+
+  return Config.of<Store>()({
+    myValue: a.withStore(),
+  })
+  ```
+   */
+  withStore<NewStore extends Store extends never ? any : Store>() {
+    return this as any as Variants<Type, NewStore>
+  }
+}

sdk/lib/config/configConstants.ts

@@ -0,0 +1,81 @@
+import { SmtpValue } from "../types"
+import { GetSystemSmtp } from "../util/GetSystemSmtp"
+import { email } from "../util/patterns"
+import { Config, ConfigSpecOf } from "./builder/config"
+import { Value } from "./builder/value"
+import { Variants } from "./builder/variants"
+
+/**
+ * Base SMTP settings, to be used by StartOS for system wide SMTP
+ */
+export const customSmtp = Config.of<ConfigSpecOf<SmtpValue>, never>({
+  server: Value.text({
+    name: "SMTP Server",
+    required: {
+      default: null,
+    },
+  }),
+  port: Value.number({
+    name: "Port",
+    required: { default: 587 },
+    min: 1,
+    max: 65535,
+    integer: true,
+  }),
+  from: Value.text({
+    name: "From Address",
+    required: {
+      default: null,
+    },
+    placeholder: "<name>test@example.com",
+    inputmode: "email",
+    patterns: [email],
+  }),
+  login: Value.text({
+    name: "Login",
+    required: {
+      default: null,
+    },
+  }),
+  password: Value.text({
+    name: "Password",
+    required: false,
+    masked: true,
+  }),
+})
+
+/**
+ * For service config. Gives users 3 options for SMTP: (1) disabled, (2) use system SMTP settings, (3) use custom SMTP settings
+ */
+export const smtpConfig = Value.filteredUnion(
+  async ({ effects }) => {
+    const smtp = await new GetSystemSmtp(effects).once()
+    return smtp ? [] : ["system"]
+  },
+  {
+    name: "SMTP",
+    description: "Optionally provide an SMTP server for sending emails",
+    required: { default: "disabled" },
+  },
+  Variants.of({
+    disabled: { name: "Disabled", spec: Config.of({}) },
+    system: {
+      name: "System Credentials",
+      spec: Config.of({
+        customFrom: Value.text({
+          name: "Custom From Address",
+          description:
+            "A custom from address for this service. If not provided, the system from address will be used.",
+          required: false,
+          placeholder: "<name>test@example.com",
+          inputmode: "email",
+          patterns: [email],
+        }),
+      }),
+    },
+    custom: {
+      name: "Custom Credentials",
+      spec: customSmtp,
+    },
+  }),
+)

sdk/lib/config/configDependencies.ts

@@ -0,0 +1,26 @@
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { Dependency } from "../types"
+
+export type ConfigDependencies<T extends SDKManifest> = {
+  exists(id: keyof T["dependencies"]): Dependency
+  running(id: keyof T["dependencies"], healthChecks: string[]): Dependency
+}
+
+export const configDependenciesSet = <
+  T extends SDKManifest,
+>(): ConfigDependencies<T> => ({
+  exists(id: keyof T["dependencies"]) {
+    return {
+      id,
+      kind: "exists",
+    } as Dependency
+  },
+
+  running(id: keyof T["dependencies"], healthChecks: string[]) {
+    return {
+      id,
+      kind: "running",
+      healthChecks,
+    } as Dependency
+  },
+})

sdk/lib/config/configTypes.ts

@@ -0,0 +1,249 @@
+export type InputSpec = Record<string, ValueSpec>
+export type ValueType =
+  | "text"
+  | "textarea"
+  | "number"
+  | "color"
+  | "datetime"
+  | "toggle"
+  | "select"
+  | "multiselect"
+  | "list"
+  | "object"
+  | "file"
+  | "union"
+export type ValueSpec = ValueSpecOf<ValueType>
+/** core spec types. These types provide the metadata for performing validations */
+// prettier-ignore
+export type ValueSpecOf<T extends ValueType> = T extends "text"
+  ? ValueSpecText
+  : T extends "textarea"
+    ? ValueSpecTextarea
+    : T extends "number"
+      ? ValueSpecNumber
+      : T extends "color"
+        ? ValueSpecColor
+        : T extends "datetime"
+          ? ValueSpecDatetime
+          : T extends "toggle"
+            ? ValueSpecToggle
+            : T extends "select"
+              ? ValueSpecSelect
+              : T extends "multiselect"
+                ? ValueSpecMultiselect
+                : T extends "list"
+                  ? ValueSpecList
+                  : T extends "object"
+                    ? ValueSpecObject
+                    : T extends "file"
+                      ? ValueSpecFile
+                      : T extends "union"
+                        ? ValueSpecUnion
+                        : never
+
+export interface ValueSpecText extends ListValueSpecText, WithStandalone {
+  required: boolean
+  default: DefaultString | null
+  disabled: false | string
+  generate: null | RandomString
+  /** Immutable means it can only be configed at the first config then never again */
+  immutable: boolean
+}
+export interface ValueSpecTextarea extends WithStandalone {
+  type: "textarea"
+  placeholder: string | null
+  minLength: number | null
+  maxLength: number | null
+  required: boolean
+  disabled: false | string
+  /** Immutable means it can only be configed at the first config then never again */
+  immutable: boolean
+}
+
+export type FilePath = {
+  filePath: string
+}
+export interface ValueSpecNumber extends ListValueSpecNumber, WithStandalone {
+  required: boolean
+  default: number | null
+  disabled: false | string
+  /** Immutable means it can only be configed at the first config then never again */
+  immutable: boolean
+}
+export interface ValueSpecColor extends WithStandalone {
+  type: "color"
+  required: boolean
+  default: string | null
+  disabled: false | string
+  /** Immutable means it can only be configed at the first config then never again */
+  immutable: boolean
+}
+export interface ValueSpecDatetime extends WithStandalone {
+  type: "datetime"
+  required: boolean
+  inputmode: "date" | "time" | "datetime-local"
+  min: string | null
+  max: string | null
+  default: string | null
+  disabled: false | string
+  /** Immutable means it can only be configed at the first config then never again */
+  immutable: boolean
+}
+export interface ValueSpecSelect extends SelectBase, WithStandalone {
+  type: "select"
+  required: boolean
+  default: string | null
+  /**
+   * Disabled:  false means that there is nothing disabled, good to modify
+   *           string means that this is the message displayed and the whole thing is disabled
+   *           string[] means that the options are disabled
+   */
+  disabled: false | string | string[]
+  /** Immutable means it can only be configed at the first config then never again */
+  immutable: boolean
+}
+export interface ValueSpecMultiselect extends SelectBase, WithStandalone {
+  type: "multiselect"
+  minLength: number | null
+  maxLength: number | null
+  /**
+   * Disabled:  false means that there is nothing disabled, good to modify
+   *           string means that this is the message displayed and the whole thing is disabled
+   *           string[] means that the options are disabled
+   */
+  disabled: false | string | string[]
+  default: string[]
+  /** Immutable means it can only be configed at the first config then never again */
+  immutable: boolean
+}
+export interface ValueSpecToggle extends WithStandalone {
+  type: "toggle"
+  default: boolean | null
+  disabled: false | string
+  /** Immutable means it can only be configed at the first config then never again */
+  immutable: boolean
+}
+export interface ValueSpecUnion extends WithStandalone {
+  type: "union"
+  variants: Record<
+    string,
+    {
+      name: string
+      spec: InputSpec
+    }
+  >
+  /**
+   * Disabled:  false means that there is nothing disabled, good to modify
+   *           string means that this is the message displayed and the whole thing is disabled
+   *           string[] means that the options are disabled
+   */
+  disabled: false | string | string[]
+  required: boolean
+  default: string | null
+  /** Immutable means it can only be configed at the first config then never again */
+  immutable: boolean
+}
+export interface ValueSpecFile extends WithStandalone {
+  type: "file"
+  extensions: string[]
+  required: boolean
+}
+export interface ValueSpecObject extends WithStandalone {
+  type: "object"
+  spec: InputSpec
+}
+export interface WithStandalone {
+  name: string
+  description: string | null
+  warning: string | null
+}
+export interface SelectBase {
+  values: Record<string, string>
+}
+export type ListValueSpecType = "text" | "number" | "object"
+/** represents a spec for the values of a list */
+export type ListValueSpecOf<T extends ListValueSpecType> = T extends "text"
+  ? ListValueSpecText
+  : T extends "number"
+    ? ListValueSpecNumber
+    : T extends "object"
+      ? ListValueSpecObject
+      : never
+/** represents a spec for a list */
+export type ValueSpecList = ValueSpecListOf<ListValueSpecType>
+export interface ValueSpecListOf<T extends ListValueSpecType>
+  extends WithStandalone {
+  type: "list"
+  spec: ListValueSpecOf<T>
+  minLength: number | null
+  maxLength: number | null
+  disabled: false | string
+  default:
+    | string[]
+    | number[]
+    | DefaultString[]
+    | Record<string, unknown>[]
+    | readonly string[]
+    | readonly number[]
+    | readonly DefaultString[]
+    | readonly Record<string, unknown>[]
+}
+export interface Pattern {
+  regex: string
+  description: string
+}
+export interface ListValueSpecText {
+  type: "text"
+  patterns: Pattern[]
+  minLength: number | null
+  maxLength: number | null
+  masked: boolean
+
+  generate: null | RandomString
+  inputmode: "text" | "email" | "tel" | "url"
+  placeholder: string | null
+}
+export interface ListValueSpecNumber {
+  type: "number"
+  min: number | null
+  max: number | null
+  integer: boolean
+  step: number | null
+  units: string | null
+  placeholder: string | null
+}
+export interface ListValueSpecObject {
+  type: "object"
+  /** this is a mapped type of the config object at this level, replacing the object's values with specs on those values */
+  spec: InputSpec
+  /** indicates whether duplicates can be permitted in the list */
+  uniqueBy: UniqueBy
+  /** this should be a handlebars template which can make use of the entire config which corresponds to 'spec' */
+  displayAs: string | null
+}
+export type UniqueBy =
+  | null
+  | string
+  | {
+      any: readonly UniqueBy[] | UniqueBy[]
+    }
+  | {
+      all: readonly UniqueBy[] | UniqueBy[]
+    }
+export type DefaultString = string | RandomString
+export type RandomString = {
+  charset: string
+  len: number
+}
+// sometimes the type checker needs just a little bit of help
+export function isValueSpecListOf<S extends ListValueSpecType>(
+  t: ValueSpec,
+  s: S,
+): t is ValueSpecListOf<S> & { spec: ListValueSpecOf<S> } {
+  return "spec" in t && t.spec.type === s
+}
+export const unionSelectKey = "unionSelectKey" as const
+export type UnionSelectKey = typeof unionSelectKey
+
+export const unionValueKey = "unionValueKey" as const
+export type UnionValueKey = typeof unionValueKey

sdk/lib/config/index.ts

@@ -0,0 +1,5 @@
+import "./builder"
+
+import "./setupConfig"
+import "./configDependencies"
+import "./configConstants"

sdk/lib/config/setupConfig.ts

@@ -0,0 +1,90 @@
+import { Effects, ExpectedExports } from "../types"
+import { SDKManifest } from "../manifest/ManifestTypes"
+import * as D from "./configDependencies"
+import { Config, ExtractConfigType } from "./builder/config"
+import nullIfEmpty from "../util/nullIfEmpty"
+import { InterfaceReceipt } from "../interfaces/interfaceReceipt"
+import { InterfacesReceipt as InterfacesReceipt } from "../interfaces/setupInterfaces"
+
+declare const dependencyProof: unique symbol
+export type DependenciesReceipt = void & {
+  [dependencyProof]: never
+}
+
+export type Save<
+  Store,
+  A extends
+    | Record<string, any>
+    | Config<Record<string, any>, any>
+    | Config<Record<string, never>, never>,
+  Manifest extends SDKManifest,
+> = (options: {
+  effects: Effects
+  input: ExtractConfigType<A> & Record<string, any>
+  dependencies: D.ConfigDependencies<Manifest>
+}) => Promise<{
+  dependenciesReceipt: DependenciesReceipt
+  interfacesReceipt: InterfacesReceipt
+  restart: boolean
+}>
+export type Read<
+  Manifest extends SDKManifest,
+  Store,
+  A extends
+    | Record<string, any>
+    | Config<Record<string, any>, any>
+    | Config<Record<string, any>, never>,
+> = (options: {
+  effects: Effects
+}) => Promise<void | (ExtractConfigType<A> & Record<string, any>)>
+/**
+ * We want to setup a config export with a get and set, this
+ * is going to be the default helper to setup config, because it will help
+ * enforce that we have a spec, write, and reading.
+ * @param options
+ * @returns
+ */
+export function setupConfig<
+  Store,
+  ConfigType extends
+    | Record<string, any>
+    | Config<any, any>
+    | Config<any, never>,
+  Manifest extends SDKManifest,
+  Type extends Record<string, any> = ExtractConfigType<ConfigType>,
+>(
+  spec: Config<Type, Store> | Config<Type, never>,
+  write: Save<Store, Type, Manifest>,
+  read: Read<Manifest, Store, Type>,
+) {
+  const validator = spec.validator
+  return {
+    setConfig: (async ({ effects, input }) => {
+      if (!validator.test(input)) {
+        await console.error(String(validator.errorMessage(input)))
+        return { error: "Set config type error for config" }
+      }
+      await effects.clearBindings()
+      await effects.clearServiceInterfaces()
+      const { restart } = await write({
+        input: JSON.parse(JSON.stringify(input)),
+        effects,
+        dependencies: D.configDependenciesSet<Manifest>(),
+      })
+      if (restart) {
+        await effects.restart()
+      }
+    }) as ExpectedExports.setConfig,
+    getConfig: (async ({ effects }) => {
+      const configValue = nullIfEmpty((await read({ effects })) || null)
+      return {
+        spec: await spec.build({
+          effects,
+        }),
+        config: configValue,
+      }
+    }) as ExpectedExports.getConfig,
+  }
+}
+
+export default setupConfig

sdk/lib/dependencyConfig/DependencyConfig.ts

@@ -0,0 +1,44 @@
+import {
+  DependencyConfig as DependencyConfigType,
+  DeepPartial,
+  Effects,
+} from "../types"
+import { deepEqual } from "../util/deepEqual"
+import { deepMerge } from "../util/deepMerge"
+import { SDKManifest } from "../manifest/ManifestTypes"
+
+export type Update<QueryResults, RemoteConfig> = (options: {
+  remoteConfig: RemoteConfig
+  queryResults: QueryResults
+}) => Promise<RemoteConfig>
+
+export class DependencyConfig<
+  Manifest extends SDKManifest,
+  Store,
+  Input extends Record<string, any>,
+  RemoteConfig extends Record<string, any>,
+> {
+  static defaultUpdate = async (options: {
+    queryResults: unknown
+    remoteConfig: unknown
+  }): Promise<unknown> => {
+    return deepMerge({}, options.remoteConfig, options.queryResults || {})
+  }
+  constructor(
+    readonly dependencyConfig: (options: {
+      effects: Effects
+      localConfig: Input
+    }) => Promise<void | DeepPartial<RemoteConfig>>,
+    readonly update: Update<
+      void | DeepPartial<RemoteConfig>,
+      RemoteConfig
+    > = DependencyConfig.defaultUpdate as any,
+  ) {}
+
+  async query(options: { effects: Effects; localConfig: unknown }) {
+    return this.dependencyConfig({
+      localConfig: options.localConfig as Input,
+      effects: options.effects,
+    })
+  }
+}

sdk/lib/dependencyConfig/index.ts

@@ -0,0 +1,9 @@
+// prettier-ignore
+export type ReadonlyDeep<A> = 
+    A extends Function ? A :
+    A extends {} ? { readonly [K in keyof A]: ReadonlyDeep<A[K]> } : A;
+export type MaybePromise<A> = Promise<A> | A
+export type Message = string
+
+import "./DependencyConfig"
+import "./setupDependencyConfig"

sdk/lib/dependencyConfig/setupDependencyConfig.ts

@@ -0,0 +1,22 @@
+import { Config } from "../config/builder/config"
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { ExpectedExports } from "../types"
+import { DependencyConfig } from "./DependencyConfig"
+
+export function setupDependencyConfig<
+  Store,
+  Input extends Record<string, any>,
+  Manifest extends SDKManifest,
+>(
+  _config: Config<Input, Store> | Config<Input, never>,
+  autoConfigs: {
+    [key in keyof Manifest["dependencies"] & string]: DependencyConfig<
+      Manifest,
+      Store,
+      Input,
+      any
+    >
+  },
+): ExpectedExports.dependencyConfig {
+  return autoConfigs
+}

sdk/lib/emverLite/mod.ts

@@ -0,0 +1,307 @@
+import * as matches from "ts-matches"
+
+const starSub = /((\d+\.)*\d+)\.\*/
+// prettier-ignore
+export type ValidEmVer = `${number}${`.${number}` | ""}${`.${number}` | ""}${`-${string}` | ""}`;
+// prettier-ignore
+export type ValidEmVerRange = `${'>=' | '<='| '<' | '>' | ''}${'^' | '~' | ''}${number | '*'}${`.${number | '*'}` | ""}${`.${number | '*'}` | ""}${`-${string}` | ""}`;
+
+function incrementLastNumber(list: number[]) {
+  const newList = [...list]
+  newList[newList.length - 1]++
+  return newList
+}
+/**
+ * Will take in a range, like `>1.2` or `<1.2.3.4` or `=1.2` or `1.*`
+ * and return a checker, that has the check function for checking that a version is in the valid
+ * @param range
+ * @returns
+ */
+export function rangeOf(range: string | Checker): Checker {
+  return Checker.parse(range)
+}
+
+/**
+ * Used to create a checker that will `and` all the ranges passed in
+ * @param ranges
+ * @returns
+ */
+export function rangeAnd(...ranges: (string | Checker)[]): Checker {
+  if (ranges.length === 0) {
+    throw new Error("No ranges given")
+  }
+  const [firstCheck, ...rest] = ranges
+  return Checker.parse(firstCheck).and(...rest)
+}
+
+/**
+ * Used to create a checker that will `or` all the ranges passed in
+ * @param ranges
+ * @returns
+ */
+export function rangeOr(...ranges: (string | Checker)[]): Checker {
+  if (ranges.length === 0) {
+    throw new Error("No ranges given")
+  }
+  const [firstCheck, ...rest] = ranges
+  return Checker.parse(firstCheck).or(...rest)
+}
+
+/**
+ * This will negate the checker, so given a checker that checks for >= 1.0.0, it will check for < 1.0.0
+ * @param range
+ * @returns
+ */
+export function notRange(range: string | Checker): Checker {
+  return rangeOf(range).not()
+}
+
+/**
+ * EmVer is a set of versioning of any pattern like 1 or 1.2 or 1.2.3 or 1.2.3.4 or ..
+ */
+export class EmVer {
+  /**
+   * Convert the range, should be 1.2.* or * into a emver
+   * Or an already made emver
+   * IsUnsafe
+   */
+  static from(range: string | EmVer): EmVer {
+    if (range instanceof EmVer) {
+      return range
+    }
+    return EmVer.parse(range)
+  }
+  /**
+   * Convert the range, should be 1.2.* or * into a emver
+   * IsUnsafe
+   */
+  static parse(rangeExtra: string): EmVer {
+    const [range, extra] = rangeExtra.split("-")
+    const values = range.split(".").map((x) => parseInt(x))
+    for (const value of values) {
+      if (isNaN(value)) {
+        throw new Error(`Couldn't parse range: ${range}`)
+      }
+    }
+    return new EmVer(values, extra)
+  }
+  private constructor(
+    public readonly values: number[],
+    readonly extra: string | null,
+  ) {}
+
+  /**
+   * Used when we need a new emver that has the last number incremented, used in the 1.* like things
+   */
+  public withLastIncremented() {
+    return new EmVer(incrementLastNumber(this.values), null)
+  }
+
+  public greaterThan(other: EmVer): boolean {
+    for (const i in this.values) {
+      if (other.values[i] == null) {
+        return true
+      }
+      if (this.values[i] > other.values[i]) {
+        return true
+      }
+
+      if (this.values[i] < other.values[i]) {
+        return false
+      }
+    }
+    return false
+  }
+
+  public equals(other: EmVer): boolean {
+    if (other.values.length !== this.values.length) {
+      return false
+    }
+    for (const i in this.values) {
+      if (this.values[i] !== other.values[i]) {
+        return false
+      }
+    }
+    return true
+  }
+  public greaterThanOrEqual(other: EmVer): boolean {
+    return this.greaterThan(other) || this.equals(other)
+  }
+  public lessThanOrEqual(other: EmVer): boolean {
+    return !this.greaterThan(other)
+  }
+  public lessThan(other: EmVer): boolean {
+    return !this.greaterThanOrEqual(other)
+  }
+  /**
+   * Return a enum string that describes (used for switching/iffs)
+   * to know comparison
+   * @param other
+   * @returns
+   */
+  public compare(other: EmVer) {
+    if (this.equals(other)) {
+      return "equal" as const
+    } else if (this.greaterThan(other)) {
+      return "greater" as const
+    } else {
+      return "less" as const
+    }
+  }
+  /**
+   * Used when sorting emver's in a list using the sort method
+   * @param other
+   * @returns
+   */
+  public compareForSort(other: EmVer) {
+    return matches
+      .matches(this.compare(other))
+      .when("equal", () => 0 as const)
+      .when("greater", () => 1 as const)
+      .when("less", () => -1 as const)
+      .unwrap()
+  }
+
+  toString() {
+    return `${this.values.join(".")}${this.extra ? `-${this.extra}` : ""}`
+  }
+}
+
+/**
+ * A checker is a function that takes a version and returns true if the version matches the checker.
+ * Used when we are doing range checking, like saying ">=1.0.0".check("1.2.3") will be true
+ */
+export class Checker {
+  /**
+   * Will take in a range, like `>1.2` or `<1.2.3.4` or `=1.2` or `1.*`
+   * and return a checker, that has the check function for checking that a version is in the valid
+   * @param range
+   * @returns
+   */
+  static parse(range: string | Checker): Checker {
+    if (range instanceof Checker) {
+      return range
+    }
+    range = range.trim()
+    if (range.indexOf("||") !== -1) {
+      return rangeOr(...range.split("||").map((x) => Checker.parse(x)))
+    }
+    if (range.indexOf("&&") !== -1) {
+      return rangeAnd(...range.split("&&").map((x) => Checker.parse(x)))
+    }
+    if (range === "*") {
+      return new Checker((version) => {
+        EmVer.from(version)
+        return true
+      })
+    }
+    if (range.startsWith("!")) {
+      return Checker.parse(range.substring(1)).not()
+    }
+    const starSubMatches = starSub.exec(range)
+    if (starSubMatches != null) {
+      const emVarLower = EmVer.parse(starSubMatches[1])
+      const emVarUpper = emVarLower.withLastIncremented()
+
+      return new Checker((version) => {
+        const v = EmVer.from(version)
+        return (
+          (v.greaterThan(emVarLower) || v.equals(emVarLower)) &&
+          !v.greaterThan(emVarUpper) &&
+          !v.equals(emVarUpper)
+        )
+      })
+    }
+
+    switch (range.substring(0, 2)) {
+      case ">=": {
+        const emVar = EmVer.parse(range.substring(2))
+        return new Checker((version) => {
+          const v = EmVer.from(version)
+          return v.greaterThanOrEqual(emVar)
+        })
+      }
+      case "<=": {
+        const emVar = EmVer.parse(range.substring(2))
+        return new Checker((version) => {
+          const v = EmVer.from(version)
+          return v.lessThanOrEqual(emVar)
+        })
+      }
+    }
+
+    switch (range.substring(0, 1)) {
+      case ">": {
+        const emVar = EmVer.parse(range.substring(1))
+        return new Checker((version) => {
+          const v = EmVer.from(version)
+          return v.greaterThan(emVar)
+        })
+      }
+      case "<": {
+        const emVar = EmVer.parse(range.substring(1))
+        return new Checker((version) => {
+          const v = EmVer.from(version)
+          return v.lessThan(emVar)
+        })
+      }
+      case "=": {
+        const emVar = EmVer.parse(range.substring(1))
+        return new Checker((version) => {
+          const v = EmVer.from(version)
+          return v.equals(emVar)
+        })
+      }
+    }
+    throw new Error("Couldn't parse range: " + range)
+  }
+  constructor(
+    /**
+     * Check is the function that will be given a emver or unparsed emver and should give if it follows
+     * a pattern
+     */
+    public readonly check: (value: ValidEmVer | EmVer) => boolean,
+  ) {}
+
+  /**
+   * Used when we want the `and` condition with another checker
+   */
+  public and(...others: (Checker | string)[]): Checker {
+    return new Checker((value) => {
+      if (!this.check(value)) {
+        return false
+      }
+      for (const other of others) {
+        if (!Checker.parse(other).check(value)) {
+          return false
+        }
+      }
+      return true
+    })
+  }
+
+  /**
+   * Used when we want the `or` condition with another checker
+   */
+  public or(...others: (Checker | string)[]): Checker {
+    return new Checker((value) => {
+      if (this.check(value)) {
+        return true
+      }
+      for (const other of others) {
+        if (Checker.parse(other).check(value)) {
+          return true
+        }
+      }
+      return false
+    })
+  }
+
+  /**
+   * A useful example is making sure we don't match an exact version, like !=1.2.3
+   * @returns
+   */
+  public not(): Checker {
+    return new Checker((value) => !this.check(value))
+  }
+}

sdk/lib/health/HealthCheck.ts

@@ -0,0 +1,72 @@
+import { InterfaceReceipt } from "../interfaces/interfaceReceipt"
+import { Daemon, Effects } from "../types"
+import { CheckResult } from "./checkFns/CheckResult"
+import { HealthReceipt } from "./HealthReceipt"
+import { Trigger } from "../trigger"
+import { TriggerInput } from "../trigger/TriggerInput"
+import { defaultTrigger } from "../trigger/defaultTrigger"
+import { once } from "../util/once"
+import { Overlay } from "../util/Overlay"
+
+export function healthCheck(o: {
+  effects: Effects
+  name: string
+  imageId: string
+  trigger?: Trigger
+  fn(overlay: Overlay): Promise<CheckResult> | CheckResult
+  onFirstSuccess?: () => unknown | Promise<unknown>
+}) {
+  new Promise(async () => {
+    const overlay = await Overlay.of(o.effects, o.imageId)
+    try {
+      let currentValue: TriggerInput = {
+        hadSuccess: false,
+      }
+      const getCurrentValue = () => currentValue
+      const trigger = (o.trigger ?? defaultTrigger)(getCurrentValue)
+      const triggerFirstSuccess = once(() =>
+        Promise.resolve(
+          "onFirstSuccess" in o && o.onFirstSuccess
+            ? o.onFirstSuccess()
+            : undefined,
+        ),
+      )
+      for (
+        let res = await trigger.next();
+        !res.done;
+        res = await trigger.next()
+      ) {
+        try {
+          const { status, message } = await o.fn(overlay)
+          await o.effects.setHealth({
+            name: o.name,
+            status,
+            message,
+          })
+          currentValue.hadSuccess = true
+          currentValue.lastResult = "passing"
+          await triggerFirstSuccess().catch((err) => {
+            console.error(err)
+          })
+        } catch (e) {
+          await o.effects.setHealth({
+            name: o.name,
+            status: "failure",
+            message: asMessage(e),
+          })
+          currentValue.lastResult = "failure"
+        }
+      }
+    } finally {
+      await overlay.destroy()
+    }
+  })
+  return {} as HealthReceipt
+}
+function asMessage(e: unknown) {
+  if (typeof e === "object" && e != null && "message" in e)
+    return String(e.message)
+  const value = String(e)
+  if (value.length == null) return null
+  return value
+}

sdk/lib/health/HealthReceipt.ts

@@ -0,0 +1,4 @@
+declare const HealthProof: unique symbol
+export type HealthReceipt = {
+  [HealthProof]: never
+}

sdk/lib/health/checkFns/CheckResult.ts

@@ -0,0 +1,6 @@
+import { HealthStatus } from "../../types"
+
+export type CheckResult = {
+  status: HealthStatus
+  message: string | null
+}

sdk/lib/health/checkFns/checkPortListening.ts

@@ -0,0 +1,67 @@
+import { Effects } from "../../types"
+import { stringFromStdErrOut } from "../../util/stringFromStdErrOut"
+import { CheckResult } from "./CheckResult"
+
+import { promisify } from "node:util"
+import * as CP from "node:child_process"
+
+const cpExec = promisify(CP.exec)
+const cpExecFile = promisify(CP.execFile)
+export function containsAddress(x: string, port: number) {
+  const readPorts = x
+    .split("\n")
+    .filter(Boolean)
+    .splice(1)
+    .map((x) => x.split(" ").filter(Boolean)[1]?.split(":")?.[1])
+    .filter(Boolean)
+    .map((x) => Number.parseInt(x, 16))
+    .filter(Number.isFinite)
+  return readPorts.indexOf(port) >= 0
+}
+
+/**
+ * This is used to check if a port is listening on the system.
+ * Used during the health check fn or the check main fn.
+ */
+export async function checkPortListening(
+  effects: Effects,
+  port: number,
+  options: {
+    errorMessage: string
+    successMessage: string
+    timeoutMessage?: string
+    timeout?: number
+  },
+): Promise<CheckResult> {
+  return Promise.race<CheckResult>([
+    Promise.resolve().then(async () => {
+      const hasAddress =
+        containsAddress(
+          await cpExec(`cat /proc/net/tcp`, {}).then(stringFromStdErrOut),
+          port,
+        ) ||
+        containsAddress(
+          await cpExec("cat /proc/net/udp", {}).then(stringFromStdErrOut),
+          port,
+        )
+      if (hasAddress) {
+        return { status: "passing", message: options.successMessage }
+      }
+      return {
+        status: "failure",
+        message: options.errorMessage,
+      }
+    }),
+    new Promise((resolve) => {
+      setTimeout(
+        () =>
+          resolve({
+            status: "failure",
+            message:
+              options.timeoutMessage || `Timeout trying to check port ${port}`,
+          }),
+        options.timeout ?? 1_000,
+      )
+    }),
+  ])
+}

sdk/lib/health/checkFns/checkWebUrl.ts

@@ -0,0 +1,35 @@
+import { Effects } from "../../types"
+import { CheckResult } from "./CheckResult"
+import { timeoutPromise } from "./index"
+import "isomorphic-fetch"
+
+/**
+ * This is a helper function to check if a web url is reachable.
+ * @param url
+ * @param createSuccess
+ * @returns
+ */
+export const checkWebUrl = async (
+  effects: Effects,
+  url: string,
+  {
+    timeout = 1000,
+    successMessage = `Reached ${url}`,
+    errorMessage = `Error while fetching URL: ${url}`,
+  } = {},
+): Promise<CheckResult> => {
+  return Promise.race([fetch(url), timeoutPromise(timeout)])
+    .then(
+      (x) =>
+        ({
+          status: "passing",
+          message: successMessage,
+        }) as const,
+    )
+    .catch((e) => {
+      console.warn(`Error while fetching URL: ${url}`)
+      console.error(JSON.stringify(e))
+      console.error(e.toString())
+      return { status: "failure" as const, message: errorMessage }
+    })
+}

sdk/lib/health/checkFns/index.ts

@@ -0,0 +1,11 @@
+import { runHealthScript } from "./runHealthScript"
+export { checkPortListening } from "./checkPortListening"
+export { CheckResult } from "./CheckResult"
+export { checkWebUrl } from "./checkWebUrl"
+
+export function timeoutPromise(ms: number, { message = "Timed out" } = {}) {
+  return new Promise<never>((resolve, reject) =>
+    setTimeout(() => reject(new Error(message)), ms),
+  )
+}
+export { runHealthScript }

sdk/lib/health/checkFns/runHealthScript.ts

@@ -0,0 +1,38 @@
+import { Effects } from "../../types"
+import { Overlay } from "../../util/Overlay"
+import { stringFromStdErrOut } from "../../util/stringFromStdErrOut"
+import { CheckResult } from "./CheckResult"
+import { timeoutPromise } from "./index"
+
+/**
+ * Running a health script, is used when we want to have a simple
+ * script in bash or something like that. It should return something that is useful
+ * in {result: string} else it is considered an error
+ * @param param0
+ * @returns
+ */
+export const runHealthScript = async (
+  effects: Effects,
+  runCommand: string[],
+  overlay: Overlay,
+  {
+    timeout = 30000,
+    errorMessage = `Error while running command: ${runCommand}`,
+    message = (res: string) =>
+      `Have ran script ${runCommand} and the result: ${res}`,
+  } = {},
+): Promise<CheckResult> => {
+  const res = await Promise.race([
+    overlay.exec(runCommand),
+    timeoutPromise(timeout),
+  ]).catch((e) => {
+    console.warn(errorMessage)
+    console.warn(JSON.stringify(e))
+    console.warn(e.toString())
+    throw { status: "failure", message: errorMessage } as CheckResult
+  })
+  return {
+    status: "passing",
+    message: message(res.stdout.toString()),
+  } as CheckResult
+}

sdk/lib/health/index.ts

@@ -0,0 +1,3 @@
+import "./checkFns"
+
+import "./HealthReceipt"

sdk/lib/index.ts

@@ -0,0 +1,25 @@
+export { Daemons } from "./mainFn/Daemons"
+export { EmVer } from "./emverLite/mod"
+export { Overlay } from "./util/Overlay"
+export { StartSdk } from "./StartSdk"
+export { setupManifest } from "./manifest/setupManifest"
+export { FileHelper } from "./util/fileHelper"
+export * as actions from "./actions"
+export * as backup from "./backup"
+export * as config from "./config"
+export * as configBuilder from "./config/builder"
+export * as configTypes from "./config/configTypes"
+export * as dependencyConfig from "./dependencyConfig"
+export * as health from "./health"
+export * as healthFns from "./health/checkFns"
+export * as inits from "./inits"
+export * as mainFn from "./mainFn"
+export * as manifest from "./manifest"
+export * as toml from "@iarna/toml"
+export * as types from "./types"
+export * as util from "./util"
+export * as yaml from "yaml"
+
+export * as matches from "ts-matches"
+export * as YAML from "yaml"
+export * as TOML from "@iarna/toml"

sdk/lib/inits/index.ts

@@ -0,0 +1,3 @@
+import "./setupInit"
+import "./setupUninstall"
+import "./setupInstall"

sdk/lib/inits/migrations/Migration.ts

@@ -0,0 +1,35 @@
+import { ManifestVersion, SDKManifest } from "../../manifest/ManifestTypes"
+import { Effects } from "../../types"
+
+export class Migration<
+  Manifest extends SDKManifest,
+  Store,
+  Version extends ManifestVersion,
+> {
+  constructor(
+    readonly options: {
+      version: Version
+      up: (opts: { effects: Effects }) => Promise<void>
+      down: (opts: { effects: Effects }) => Promise<void>
+    },
+  ) {}
+  static of<
+    Manifest extends SDKManifest,
+    Store,
+    Version extends ManifestVersion,
+  >(options: {
+    version: Version
+    up: (opts: { effects: Effects }) => Promise<void>
+    down: (opts: { effects: Effects }) => Promise<void>
+  }) {
+    return new Migration<Manifest, Store, Version>(options)
+  }
+
+  async up(opts: { effects: Effects }) {
+    this.up(opts)
+  }
+
+  async down(opts: { effects: Effects }) {
+    this.down(opts)
+  }
+}

sdk/lib/inits/migrations/setupMigrations.ts

@@ -0,0 +1,73 @@
+import { EmVer } from "../../emverLite/mod"
+import { SDKManifest } from "../../manifest/ManifestTypes"
+import { ExpectedExports } from "../../types"
+import { once } from "../../util/once"
+import { Migration } from "./Migration"
+
+export class Migrations<Manifest extends SDKManifest, Store> {
+  private constructor(
+    readonly manifest: SDKManifest,
+    readonly migrations: Array<Migration<Manifest, Store, any>>,
+  ) {}
+  private sortedMigrations = once(() => {
+    const migrationsAsVersions = (
+      this.migrations as Array<Migration<Manifest, Store, any>>
+    ).map((x) => [EmVer.parse(x.options.version), x] as const)
+    migrationsAsVersions.sort((a, b) => a[0].compareForSort(b[0]))
+    return migrationsAsVersions
+  })
+  private currentVersion = once(() => EmVer.parse(this.manifest.version))
+  static of<
+    Manifest extends SDKManifest,
+    Store,
+    Migrations extends Array<Migration<Manifest, Store, any>>,
+  >(manifest: SDKManifest, ...migrations: EnsureUniqueId<Migrations>) {
+    return new Migrations(
+      manifest,
+      migrations as Array<Migration<Manifest, Store, any>>,
+    )
+  }
+  async init({
+    effects,
+    previousVersion,
+  }: Parameters<ExpectedExports.init>[0]) {
+    if (!!previousVersion) {
+      const previousVersionEmVer = EmVer.parse(previousVersion)
+      for (const [_, migration] of this.sortedMigrations()
+        .filter((x) => x[0].greaterThan(previousVersionEmVer))
+        .filter((x) => x[0].lessThanOrEqual(this.currentVersion()))) {
+        await migration.up({ effects })
+      }
+    }
+  }
+  async uninit({
+    effects,
+    nextVersion,
+  }: Parameters<ExpectedExports.uninit>[0]) {
+    if (!!nextVersion) {
+      const nextVersionEmVer = EmVer.parse(nextVersion)
+      const reversed = [...this.sortedMigrations()].reverse()
+      for (const [_, migration] of reversed
+        .filter((x) => x[0].greaterThan(nextVersionEmVer))
+        .filter((x) => x[0].lessThanOrEqual(this.currentVersion()))) {
+        await migration.down({ effects })
+      }
+    }
+  }
+}
+
+export function setupMigrations<
+  Manifest extends SDKManifest,
+  Store,
+  Migrations extends Array<Migration<Manifest, Store, any>>,
+>(manifest: SDKManifest, ...migrations: EnsureUniqueId<Migrations>) {
+  return Migrations.of<Manifest, Store, Migrations>(manifest, ...migrations)
+}
+
+// prettier-ignore
+export type EnsureUniqueId<A, B = A, ids = never> =
+  B extends [] ? A : 
+  B extends [Migration<any, any, infer id>, ...infer Rest] ? (
+    id extends ids ? "One of the ids are not unique"[] :
+    EnsureUniqueId<A, Rest, id | ids>
+  ) : "There exists a migration that is not a Migration"[]

sdk/lib/inits/setupExports.ts

@@ -0,0 +1,14 @@
+import { Effects, ExposeServicePaths, ExposeUiPaths } from "../types"
+
+export type SetupExports<Store> = (opts: { effects: Effects }) =>
+  | {
+      ui: { [k: string]: ExposeUiPaths<Store> }
+      services: ExposeServicePaths<Store>
+    }
+  | Promise<{
+      ui: { [k: string]: ExposeUiPaths<Store> }
+      services: ExposeServicePaths<Store>
+    }>
+
+export const setupExports = <Store>(fn: (opts: SetupExports<Store>) => void) =>
+  fn

sdk/lib/inits/setupInit.ts

@@ -0,0 +1,63 @@
+import { SetInterfaces } from "../interfaces/setupInterfaces"
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { ExpectedExports, ExposeUiPaths, ExposeUiPathsAll } from "../types"
+import { Migrations } from "./migrations/setupMigrations"
+import { SetupExports } from "./setupExports"
+import { Install } from "./setupInstall"
+import { Uninstall } from "./setupUninstall"
+
+export function setupInit<Manifest extends SDKManifest, Store>(
+  migrations: Migrations<Manifest, Store>,
+  install: Install<Manifest, Store>,
+  uninstall: Uninstall<Manifest, Store>,
+  setInterfaces: SetInterfaces<Manifest, Store, any, any>,
+  setupExports: SetupExports<Store>,
+): {
+  init: ExpectedExports.init
+  uninit: ExpectedExports.uninit
+} {
+  return {
+    init: async (opts) => {
+      await migrations.init(opts)
+      await install.init(opts)
+      await setInterfaces({
+        ...opts,
+        input: null,
+      })
+      const { services, ui } = await setupExports(opts)
+      await opts.effects.exposeForDependents(services)
+      await opts.effects.exposeUi(forExpose(ui))
+    },
+    uninit: async (opts) => {
+      await migrations.uninit(opts)
+      await uninstall.uninit(opts)
+    },
+  }
+}
+function forExpose<Store>(ui: { [key: string]: ExposeUiPaths<Store> }) {
+  return Object.fromEntries(
+    Object.entries(ui).map(([key, value]) => [key, forExpose_(value)]),
+  )
+}
+
+function forExpose_<Store>(ui: ExposeUiPaths<Store>): ExposeUiPathsAll {
+  if (ui.type === ("object" as const)) {
+    return {
+      type: "object" as const,
+      value: Object.fromEntries(
+        Object.entries(ui.value).map(([key, value]) => [
+          key,
+          forExpose_(value),
+        ]),
+      ),
+      description: ui.description ?? null,
+    }
+  }
+  return {
+    description: null,
+
+    copyable: null,
+    qr: null,
+    ...ui,
+  }
+}

sdk/lib/inits/setupInstall.ts

@@ -0,0 +1,30 @@
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { Effects, ExpectedExports } from "../types"
+
+export type InstallFn<Manifest extends SDKManifest, Store> = (opts: {
+  effects: Effects
+}) => Promise<void>
+export class Install<Manifest extends SDKManifest, Store> {
+  private constructor(readonly fn: InstallFn<Manifest, Store>) {}
+  static of<Manifest extends SDKManifest, Store>(
+    fn: InstallFn<Manifest, Store>,
+  ) {
+    return new Install(fn)
+  }
+
+  async init({
+    effects,
+    previousVersion,
+  }: Parameters<ExpectedExports.init>[0]) {
+    if (!previousVersion)
+      await this.fn({
+        effects,
+      })
+  }
+}
+
+export function setupInstall<Manifest extends SDKManifest, Store>(
+  fn: InstallFn<Manifest, Store>,
+) {
+  return Install.of(fn)
+}

sdk/lib/inits/setupUninstall.ts

@@ -0,0 +1,30 @@
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { Effects, ExpectedExports } from "../types"
+
+export type UninstallFn<Manifest extends SDKManifest, Store> = (opts: {
+  effects: Effects
+}) => Promise<void>
+export class Uninstall<Manifest extends SDKManifest, Store> {
+  private constructor(readonly fn: UninstallFn<Manifest, Store>) {}
+  static of<Manifest extends SDKManifest, Store>(
+    fn: UninstallFn<Manifest, Store>,
+  ) {
+    return new Uninstall(fn)
+  }
+
+  async uninit({
+    effects,
+    nextVersion,
+  }: Parameters<ExpectedExports.uninit>[0]) {
+    if (!nextVersion)
+      await this.fn({
+        effects,
+      })
+  }
+}
+
+export function setupUninstall<Manifest extends SDKManifest, Store>(
+  fn: UninstallFn<Manifest, Store>,
+) {
+  return Uninstall.of(fn)
+}

sdk/lib/interfaces/AddressReceipt.ts

@@ -0,0 +1,4 @@
+declare const AddressProof: unique symbol
+export type AddressReceipt = {
+  [AddressProof]: never
+}

sdk/lib/interfaces/Host.ts

@@ -0,0 +1,205 @@
+import { object, string } from "ts-matches"
+import { Effects } from "../types"
+import { Origin } from "./Origin"
+
+const knownProtocols = {
+  http: {
+    secure: false,
+    ssl: false,
+    defaultPort: 80,
+    withSsl: "https",
+  },
+  https: {
+    secure: true,
+    ssl: true,
+    defaultPort: 443,
+  },
+  ws: {
+    secure: false,
+    ssl: false,
+    defaultPort: 80,
+    withSsl: "wss",
+  },
+  wss: {
+    secure: true,
+    ssl: true,
+    defaultPort: 443,
+  },
+  ssh: {
+    secure: true,
+    ssl: false,
+    defaultPort: 22,
+  },
+  bitcoin: {
+    secure: true,
+    ssl: false,
+    defaultPort: 8333,
+  },
+  lightning: {
+    secure: true,
+    ssl: true,
+    defaultPort: 9735,
+  },
+  grpc: {
+    secure: true,
+    ssl: true,
+    defaultPort: 50051,
+  },
+  dns: {
+    secure: true,
+    ssl: false,
+    defaultPort: 53,
+  },
+} as const
+
+export type Scheme = string | null
+
+type AddSslOptions = {
+  scheme: Scheme
+  preferredExternalPort: number
+  addXForwardedHeaders: boolean | null /** default: false */
+}
+type Security = { ssl: boolean }
+export type BindOptions = {
+  scheme: Scheme
+  preferredExternalPort: number
+  addSsl: AddSslOptions | null
+  secure: Security | null
+}
+type KnownProtocols = typeof knownProtocols
+type ProtocolsWithSslVariants = {
+  [K in keyof KnownProtocols]: KnownProtocols[K] extends {
+    withSsl: string
+  }
+    ? K
+    : never
+}[keyof KnownProtocols]
+type NotProtocolsWithSslVariants = Exclude<
+  keyof KnownProtocols,
+  ProtocolsWithSslVariants
+>
+
+type BindOptionsByKnownProtocol =
+  | {
+      protocol: ProtocolsWithSslVariants
+      preferredExternalPort?: number
+      scheme?: Scheme
+      addSsl?: Partial<AddSslOptions>
+    }
+  | {
+      protocol: NotProtocolsWithSslVariants
+      preferredExternalPort?: number
+      scheme?: Scheme
+      addSsl?: AddSslOptions
+    }
+type BindOptionsByProtocol = BindOptionsByKnownProtocol | BindOptions
+
+export type HostKind = "static" | "single" | "multi"
+
+const hasStringProtocol = object({
+  protocol: string,
+}).test
+
+export class Host {
+  constructor(
+    readonly options: {
+      effects: Effects
+      kind: HostKind
+      id: string
+    },
+  ) {}
+
+  async bindPort(
+    internalPort: number,
+    options: BindOptionsByProtocol,
+  ): Promise<Origin<this>> {
+    if (hasStringProtocol(options)) {
+      return await this.bindPortForKnown(options, internalPort)
+    } else {
+      return await this.bindPortForUnknown(internalPort, options)
+    }
+  }
+
+  private async bindPortForUnknown(
+    internalPort: number,
+    options: {
+      scheme: Scheme
+      preferredExternalPort: number
+      addSsl: AddSslOptions | null
+      secure: { ssl: boolean } | null
+    },
+  ) {
+    await this.options.effects.bind({
+      kind: this.options.kind,
+      id: this.options.id,
+      internalPort: internalPort,
+      ...options,
+    })
+
+    return new Origin(this, options)
+  }
+
+  private async bindPortForKnown(
+    options: BindOptionsByKnownProtocol,
+    internalPort: number,
+  ) {
+    const scheme =
+      options.scheme === undefined ? options.protocol : options.scheme
+    const protoInfo = knownProtocols[options.protocol]
+    const preferredExternalPort =
+      options.preferredExternalPort ||
+      knownProtocols[options.protocol].defaultPort
+    const addSsl = this.getAddSsl(options, protoInfo)
+
+    const secure: Security | null = !protoInfo.secure ? null : { ssl: false }
+
+    const newOptions = {
+      scheme,
+      preferredExternalPort,
+      addSsl,
+      secure,
+    }
+
+    await this.options.effects.bind({
+      kind: this.options.kind,
+      id: this.options.id,
+      internalPort,
+      ...newOptions,
+    })
+
+    return new Origin(this, newOptions)
+  }
+
+  private getAddSsl(
+    options: BindOptionsByKnownProtocol,
+    protoInfo: KnownProtocols[keyof KnownProtocols],
+  ): AddSslOptions | null {
+    if ("noAddSsl" in options && options.noAddSsl) return null
+    if ("withSsl" in protoInfo && protoInfo.withSsl)
+      return {
+        addXForwardedHeaders: null,
+        preferredExternalPort: knownProtocols[protoInfo.withSsl].defaultPort,
+        scheme: protoInfo.withSsl,
+        ...("addSsl" in options ? options.addSsl : null),
+      }
+    return null
+  }
+}
+
+export class StaticHost extends Host {
+  constructor(options: { effects: Effects; id: string }) {
+    super({ ...options, kind: "static" })
+  }
+}
+
+export class SingleHost extends Host {
+  constructor(options: { effects: Effects; id: string }) {
+    super({ ...options, kind: "single" })
+  }
+}
+
+export class MultiHost extends Host {
+  constructor(options: { effects: Effects; id: string }) {
+    super({ ...options, kind: "multi" })
+  }
+}

sdk/lib/interfaces/Origin.ts

@@ -0,0 +1,97 @@
+import { AddressInfo } from "../types"
+import { AddressReceipt } from "./AddressReceipt"
+import { Host, BindOptions, Scheme } from "./Host"
+import { ServiceInterfaceBuilder } from "./ServiceInterfaceBuilder"
+
+export class Origin<T extends Host> {
+  constructor(
+    readonly host: T,
+    readonly options: BindOptions,
+  ) {}
+
+  build({ username, path, search, schemeOverride }: BuildOptions): AddressInfo {
+    const qpEntries = Object.entries(search)
+      .map(
+        ([key, val]) => `${encodeURIComponent(key)}=${encodeURIComponent(val)}`,
+      )
+      .join("&")
+
+    const qp = qpEntries.length ? `?${qpEntries}` : ""
+
+    return {
+      hostId: this.host.options.id,
+      bindOptions: {
+        ...this.options,
+        scheme: schemeOverride ? schemeOverride.noSsl : this.options.scheme,
+        addSsl: this.options.addSsl
+          ? {
+              ...this.options.addSsl,
+              scheme: schemeOverride
+                ? schemeOverride.ssl
+                : this.options.addSsl.scheme,
+            }
+          : null,
+      },
+      suffix: `${path}${qp}`,
+      username,
+    }
+  }
+
+  /**
+   * A function to register a group of origins (<PROTOCOL> :// <HOSTNAME> : <PORT>) with StartOS
+   *
+   * The returned addressReceipt serves as proof that the addresses were registered
+   *
+   * @param addressInfo
+   * @returns
+   */
+  async export(
+    serviceInterfaces: ServiceInterfaceBuilder[],
+  ): Promise<AddressInfo[] & AddressReceipt> {
+    const addressesInfo = []
+    for (let serviceInterface of serviceInterfaces) {
+      const {
+        name,
+        description,
+        hasPrimary,
+        disabled,
+        id,
+        type,
+        username,
+        path,
+        search,
+        schemeOverride,
+        masked,
+      } = serviceInterface.options
+
+      const addressInfo = this.build({
+        username,
+        path,
+        search,
+        schemeOverride,
+      })
+
+      await serviceInterface.options.effects.exportServiceInterface({
+        id,
+        name,
+        description,
+        hasPrimary,
+        disabled,
+        addressInfo,
+        type,
+        masked,
+      })
+
+      addressesInfo.push(addressInfo)
+    }
+
+    return addressesInfo as AddressInfo[] & AddressReceipt
+  }
+}
+
+type BuildOptions = {
+  schemeOverride: { ssl: Scheme; noSsl: Scheme } | null
+  username: string | null
+  path: string
+  search: Record<string, string>
+}

sdk/lib/interfaces/ServiceInterfaceBuilder.ts

@@ -0,0 +1,33 @@
+import { ServiceInterfaceType } from "../StartSdk"
+import { Effects } from "../types"
+import { Scheme } from "./Host"
+
+/**
+ * A helper class for creating a Network Interface
+ *
+ * Network Interfaces are collections of web addresses that expose the same API or other resource,
+ * display to the user with under a common name and description.
+ *
+ * All URIs on an interface inherit the same ui: bool, basic auth credentials, path, and search (query) params
+ *
+ * @param options
+ * @returns
+ */
+export class ServiceInterfaceBuilder {
+  constructor(
+    readonly options: {
+      effects: Effects
+      name: string
+      id: string
+      description: string
+      hasPrimary: boolean
+      disabled: boolean
+      type: ServiceInterfaceType
+      username: string | null
+      path: string
+      search: Record<string, string>
+      schemeOverride: { ssl: Scheme; noSsl: Scheme } | null
+      masked: boolean
+    },
+  ) {}
+}

sdk/lib/interfaces/interfaceReceipt.ts

@@ -0,0 +1,4 @@
+declare const InterfaceProof: unique symbol
+export type InterfaceReceipt = {
+  [InterfaceProof]: never
+}

sdk/lib/interfaces/setupInterfaces.ts

@@ -0,0 +1,23 @@
+import { Config } from "../config/builder/config"
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { AddressInfo, Effects } from "../types"
+import { AddressReceipt } from "./AddressReceipt"
+
+export type InterfacesReceipt = Array<AddressInfo[] & AddressReceipt>
+export type SetInterfaces<
+  Manifest extends SDKManifest,
+  Store,
+  ConfigInput extends Record<string, any>,
+  Output extends InterfacesReceipt,
+> = (opts: { effects: Effects; input: null | ConfigInput }) => Promise<Output>
+export type SetupInterfaces = <
+  Manifest extends SDKManifest,
+  Store,
+  ConfigInput extends Record<string, any>,
+  Output extends InterfacesReceipt,
+>(
+  config: Config<ConfigInput, Store>,
+  fn: SetInterfaces<Manifest, Store, ConfigInput, Output>,
+) => SetInterfaces<Manifest, Store, ConfigInput, Output>
+export const NO_INTERFACE_CHANGES = [] as InterfacesReceipt
+export const setupInterfaces: SetupInterfaces = (_config, fn) => fn

sdk/lib/mainFn/Daemons.ts

@@ -0,0 +1,256 @@
+import { NO_TIMEOUT, SIGKILL, SIGTERM, Signals } from "../StartSdk"
+import { HealthReceipt } from "../health/HealthReceipt"
+import { CheckResult } from "../health/checkFns"
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { Trigger } from "../trigger"
+import { TriggerInput } from "../trigger/TriggerInput"
+import { defaultTrigger } from "../trigger/defaultTrigger"
+import { DaemonReturned, Effects, ValidIfNoStupidEscape } from "../types"
+import { Mounts } from "./Mounts"
+import { CommandOptions, MountOptions, Overlay } from "../util/Overlay"
+import { splitCommand } from "../util/splitCommand"
+
+import { promisify } from "node:util"
+import * as CP from "node:child_process"
+
+const cpExec = promisify(CP.exec)
+const cpExecFile = promisify(CP.execFile)
+async function psTree(pid: number, overlay: Overlay): Promise<number[]> {
+  const { stdout } = await cpExec(`pstree -p ${pid}`)
+  const regex: RegExp = /\((\d+)\)/g
+  return [...stdout.toString().matchAll(regex)].map(([_all, pid]) =>
+    parseInt(pid),
+  )
+}
+type Daemon<
+  Manifest extends SDKManifest,
+  Ids extends string,
+  Command extends string,
+  Id extends string,
+> = {
+  id: "" extends Id ? never : Id
+  command: ValidIfNoStupidEscape<Command> | [string, ...string[]]
+  imageId: Manifest["images"][number]
+  mounts: Mounts<Manifest>
+  env?: Record<string, string>
+  ready: {
+    display: string | null
+    fn: () => Promise<CheckResult> | CheckResult
+    trigger?: Trigger
+  }
+  requires: Exclude<Ids, Id>[]
+}
+
+type ErrorDuplicateId<Id extends string> = `The id '${Id}' is already used`
+
+const runDaemon =
+  <Manifest extends SDKManifest>() =>
+  async <A extends string>(
+    effects: Effects,
+    imageId: Manifest["images"][number],
+    command: ValidIfNoStupidEscape<A> | [string, ...string[]],
+    options: CommandOptions & {
+      mounts?: { path: string; options: MountOptions }[]
+      overlay?: Overlay
+    },
+  ): Promise<DaemonReturned> => {
+    const commands = splitCommand(command)
+    const overlay = options.overlay || (await Overlay.of(effects, imageId))
+    for (let mount of options.mounts || []) {
+      await overlay.mount(mount.options, mount.path)
+    }
+    const childProcess = await overlay.spawn(commands, {
+      env: options.env,
+    })
+    const answer = new Promise<null>((resolve, reject) => {
+      childProcess.stdout.on("data", (data: any) => {
+        console.log(data.toString())
+      })
+      childProcess.stderr.on("data", (data: any) => {
+        console.error(data.toString())
+      })
+
+      childProcess.on("exit", (code: any) => {
+        if (code === 0) {
+          return resolve(null)
+        }
+        return reject(new Error(`${commands[0]} exited with code ${code}`))
+      })
+    })
+
+    const pid = childProcess.pid
+    return {
+      async wait() {
+        const pids = pid ? await psTree(pid, overlay) : []
+        try {
+          return await answer
+        } finally {
+          for (const process of pids) {
+            cpExecFile("kill", [`-9`, String(process)]).catch((_) => {})
+          }
+        }
+      },
+      async term({ signal = SIGTERM, timeout = NO_TIMEOUT } = {}) {
+        const pids = pid ? await psTree(pid, overlay) : []
+        try {
+          childProcess.kill(signal)
+
+          if (timeout > NO_TIMEOUT) {
+            const didTimeout = await Promise.race([
+              new Promise((resolve) => setTimeout(resolve, timeout)).then(
+                () => true,
+              ),
+              answer.then(() => false),
+            ])
+            if (didTimeout) {
+              childProcess.kill(SIGKILL)
+            }
+          } else {
+            await answer
+          }
+        } finally {
+          await overlay.destroy()
+        }
+
+        try {
+          for (const process of pids) {
+            await cpExecFile("kill", [`-${signal}`, String(process)])
+          }
+        } finally {
+          for (const process of pids) {
+            cpExecFile("kill", [`-9`, String(process)]).catch((_) => {})
+          }
+        }
+      },
+    }
+  }
+
+/**
+ * A class for defining and controlling the service daemons
+```ts
+Daemons.of({
+  effects,
+  started,
+  interfaceReceipt, // Provide the interfaceReceipt to prove it was completed
+  healthReceipts, // Provide the healthReceipts or [] to prove they were at least considered
+}).addDaemon('webui', {
+  command: 'hello-world', // The command to start the daemon
+  ready: {
+    display: 'Web Interface',
+    // The function to run to determine the health status of the daemon
+    fn: () =>
+      checkPortListening(effects, 80, {
+        successMessage: 'The web interface is ready',
+        errorMessage: 'The web interface is not ready',
+      }),
+  },
+  requires: [],
+})
+```
+ */
+export class Daemons<Manifest extends SDKManifest, Ids extends string> {
+  private constructor(
+    readonly effects: Effects,
+    readonly started: (onTerm: () => PromiseLike<void>) => PromiseLike<void>,
+    readonly daemons?: Daemon<Manifest, Ids, "command", Ids>[],
+  ) {}
+  /**
+   * Returns an empty new Daemons class with the provided config.
+   *
+   * Call .addDaemon() on the returned class to add a daemon.
+   *
+   * Daemons run in the order they are defined, with latter daemons being capable of
+   * depending on prior daemons
+   * @param config
+   * @returns
+   */
+  static of<Manifest extends SDKManifest>(config: {
+    effects: Effects
+    started: (onTerm: () => PromiseLike<void>) => PromiseLike<void>
+    healthReceipts: HealthReceipt[]
+  }) {
+    return new Daemons<Manifest, never>(config.effects, config.started)
+  }
+  /**
+   * Returns the complete list of daemons, including the one defined here
+   * @param id
+   * @param newDaemon
+   * @returns
+   */
+  addDaemon<Id extends string, Command extends string>(
+    // prettier-ignore
+    id: 
+      "" extends Id ? never :
+      ErrorDuplicateId<Id> extends Id ? never :
+      Id extends Ids ? ErrorDuplicateId<Id> :
+      Id,
+    newDaemon: Omit<Daemon<Manifest, Ids, Command, Id>, "id">,
+  ) {
+    const daemons = ((this?.daemons ?? []) as any[]).concat({
+      ...newDaemon,
+      id,
+    })
+    return new Daemons<Manifest, Ids | Id>(this.effects, this.started, daemons)
+  }
+
+  async build() {
+    const daemonsStarted = {} as Record<Ids, Promise<DaemonReturned>>
+    const { effects } = this
+    const daemons = this.daemons ?? []
+    for (const daemon of daemons) {
+      const requiredPromise = Promise.all(
+        daemon.requires?.map((id) => daemonsStarted[id]) ?? [],
+      )
+      daemonsStarted[daemon.id] = requiredPromise.then(async () => {
+        const { command, imageId } = daemon
+
+        const child = runDaemon<Manifest>()(effects, imageId, command, {
+          env: daemon.env,
+          mounts: daemon.mounts.build(),
+        })
+        let currentInput: TriggerInput = {}
+        const getCurrentInput = () => currentInput
+        const trigger = (daemon.ready.trigger ?? defaultTrigger)(
+          getCurrentInput,
+        )
+        return new Promise(async (resolve) => {
+          for (
+            let res = await trigger.next();
+            !res.done;
+            res = await trigger.next()
+          ) {
+            const response = await Promise.resolve(daemon.ready.fn()).catch(
+              (err) =>
+                ({
+                  status: "failure",
+                  message: "message" in err ? err.message : String(err),
+                }) as CheckResult,
+            )
+            currentInput.lastResult = response.status || null
+            if (!currentInput.hadSuccess && response.status === "passing") {
+              currentInput.hadSuccess = true
+              resolve(child)
+            }
+          }
+          resolve(child)
+        })
+      })
+    }
+    return {
+      async term(options?: { signal?: Signals; timeout?: number }) {
+        await Promise.all(
+          Object.values<Promise<DaemonReturned>>(daemonsStarted).map((x) =>
+            x.then((x) => x.term(options)),
+          ),
+        )
+      },
+      async wait() {
+        await Promise.all(
+          Object.values<Promise<DaemonReturned>>(daemonsStarted).map((x) =>
+            x.then((x) => x.wait()),
+          ),
+        )
+      },
+    }
+  }
+}

sdk/lib/mainFn/Mounts.ts

@@ -0,0 +1,126 @@
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { Effects } from "../types"
+import { MountOptions } from "../util/Overlay"
+
+type MountArray = { path: string; options: MountOptions }[]
+
+export class Mounts<Manifest extends SDKManifest> {
+  private constructor(
+    readonly volumes: {
+      id: Manifest["volumes"][number]
+      subpath: string | null
+      mountpoint: string
+      readonly: boolean
+    }[],
+    readonly assets: {
+      id: Manifest["assets"][number]
+      subpath: string | null
+      mountpoint: string
+    }[],
+    readonly dependencies: {
+      dependencyId: string
+      volumeId: string
+      subpath: string | null
+      mountpoint: string
+      readonly: boolean
+    }[],
+  ) {}
+
+  static of<Manifest extends SDKManifest>() {
+    return new Mounts<Manifest>([], [], [])
+  }
+
+  addVolume(
+    id: Manifest["volumes"][number],
+    subpath: string | null,
+    mountpoint: string,
+    readonly: boolean,
+  ) {
+    this.volumes.push({
+      id,
+      subpath,
+      mountpoint,
+      readonly,
+    })
+    return this
+  }
+
+  addAssets(
+    id: Manifest["assets"][number],
+    subpath: string | null,
+    mountpoint: string,
+  ) {
+    this.assets.push({
+      id,
+      subpath,
+      mountpoint,
+    })
+    return this
+  }
+
+  addDependency<DependencyManifest extends SDKManifest>(
+    dependencyId: keyof Manifest["dependencies"] & string,
+    volumeId: DependencyManifest["volumes"][number],
+    subpath: string | null,
+    mountpoint: string,
+    readonly: boolean,
+  ) {
+    this.dependencies.push({
+      dependencyId,
+      volumeId,
+      subpath,
+      mountpoint,
+      readonly,
+    })
+    return this
+  }
+
+  build(): MountArray {
+    const mountpoints = new Set()
+    for (let mountpoint of this.volumes
+      .map((v) => v.mountpoint)
+      .concat(this.assets.map((a) => a.mountpoint))
+      .concat(this.dependencies.map((d) => d.mountpoint))) {
+      if (mountpoints.has(mountpoint)) {
+        throw new Error(
+          `cannot mount more than once to mountpoint ${mountpoint}`,
+        )
+      }
+      mountpoints.add(mountpoint)
+    }
+    return ([] as MountArray)
+      .concat(
+        this.volumes.map((v) => ({
+          path: v.mountpoint,
+          options: {
+            type: "volume",
+            id: v.id,
+            subpath: v.subpath,
+            readonly: v.readonly,
+          },
+        })),
+      )
+      .concat(
+        this.assets.map((a) => ({
+          path: a.mountpoint,
+          options: {
+            type: "assets",
+            id: a.id,
+            subpath: a.subpath,
+          },
+        })),
+      )
+      .concat(
+        this.dependencies.map((d) => ({
+          path: d.mountpoint,
+          options: {
+            type: "pointer",
+            packageId: d.dependencyId,
+            volumeId: d.volumeId,
+            subpath: d.subpath,
+            readonly: d.readonly,
+          },
+        })),
+      )
+  }
+}

sdk/lib/mainFn/index.ts

@@ -0,0 +1,30 @@
+import { ExpectedExports } from "../types"
+import { Daemons } from "./Daemons"
+import "../interfaces/ServiceInterfaceBuilder"
+import "../interfaces/Origin"
+
+import "./Daemons"
+import { SDKManifest } from "../manifest/ManifestTypes"
+import { MainEffects } from "../StartSdk"
+
+/**
+ * Used to ensure that the main function is running with the valid proofs.
+ * We first do the folowing order of things
+ * 1. We get the interfaces
+ * 2. We setup all the commands to setup the system
+ * 3. We create the health checks
+ * 4. We setup the daemons init system
+ * @param fn
+ * @returns
+ */
+export const setupMain = <Manifest extends SDKManifest, Store>(
+  fn: (o: {
+    effects: MainEffects
+    started(onTerm: () => PromiseLike<void>): PromiseLike<void>
+  }) => Promise<Daemons<Manifest, any>>,
+): ExpectedExports.main => {
+  return async (options) => {
+    const result = await fn(options)
+    return result
+  }
+}

sdk/lib/manifest/ManifestTypes.ts

@@ -0,0 +1,105 @@
+import { ValidEmVer } from "../emverLite/mod"
+import { ActionMetadata } from "../types"
+
+export interface Container {
+  /** This should be pointing to a docker container name */
+  image: string
+  /** These should match the manifest data volumes */
+  mounts: Record<string, string>
+  /** Default is 64mb */
+  shmSizeMb?: `${number}${"mb" | "gb" | "b" | "kb"}`
+  /** if more than 30s to shutdown */
+  sigtermTimeout?: `${number}${"s" | "m" | "h"}`
+}
+
+export type ManifestVersion = ValidEmVer
+
+export type SDKManifest = {
+  /**  The package identifier used by the OS. This must be unique amongst all other known packages */
+  readonly id: string
+  /** A human readable service title */
+  readonly title: string
+  /** Service version - accepts up to four digits, where the last confirms to revisions necessary for StartOs
+   * - see documentation: https://github.com/Start9Labs/emver-rs. This value will change with each release of
+   * the service
+   */
+  readonly version: ManifestVersion
+  /** Release notes for the update - can be a string, paragraph or URL */
+  readonly releaseNotes: string
+  /** The type of license for the project. Include the LICENSE in the root of the project directory. A license is required for a Start9 package.*/
+  readonly license: string // name of license
+  /** A list of normie (hosted, SaaS, custodial, etc) services this services intends to replace */
+  readonly replaces: Readonly<string[]>
+  /** The Start9 wrapper repository URL for the package. This repo contains the manifest file (this),
+   * any scripts necessary for configuration, backups, actions, or health checks (more below). This key
+   * must exist. But could be embedded into the source repository
+   */
+  readonly wrapperRepo: string
+  /** The original project repository URL. There is no upstream repo in this example */
+  readonly upstreamRepo: string
+  /**  URL to the support site / channel for the project. This key can be omitted if none exists, or it can link to the original project repository issues */
+  readonly supportSite: string
+  /** URL to the marketing site for the project. If there is no marketing site, it can link to the original project repository */
+  readonly marketingSite: string
+  /** URL where users can donate to the upstream project */
+  readonly donationUrl: string | null
+  /**Human readable descriptions for the service. These are used throughout the StartOS user interface, primarily in the marketplace. */
+  readonly description: {
+    /**This is the first description visible to the user in the marketplace */
+    readonly short: string
+    /** This description will display with additional details in the service's individual marketplace page */
+    readonly long: string
+  }
+
+  /** Defines the os images needed to run the container processes */
+  readonly images: string[]
+  /** This denotes readonly asset directories that should be available to mount to the container.
+   * Assuming that there will be three files with names along the lines:
+   * icon.* : the icon that will be this packages icon on the ui
+   * LICENSE : What the license is for this service
+   * Instructions : to be seen in the ui section of the package
+   * */
+  readonly assets: string[]
+  /** This denotes any data volumes that should be available to mount to the container */
+  readonly volumes: string[]
+
+  readonly alerts: {
+    readonly install: string | null
+    readonly update: string | null
+    readonly uninstall: string | null
+    readonly restore: string | null
+    readonly start: string | null
+    readonly stop: string | null
+  }
+  readonly dependencies: Readonly<Record<string, ManifestDependency>>
+}
+
+export interface ManifestDependency {
+  /** The range of versions that would satisfy the dependency
+   *
+   * ie: >=3.4.5 <4.0.0
+   */
+  version: string
+  /**
+   * A human readable explanation on what the dependency is used for
+   */
+  description: string | null
+  requirement:
+    | {
+        type: "opt-in"
+        /**
+         * The human readable explanation on how to opt-in to the dependency
+         */
+        how: string
+      }
+    | {
+        type: "opt-out"
+        /**
+         * The human readable explanation on how to opt-out to the dependency
+         */
+        how: string
+      }
+    | {
+        type: "required"
+      }
+}

sdk/lib/manifest/index.ts

@@ -0,0 +1,2 @@
+import "./setupManifest"
+import "./ManifestTypes"

sdk/lib/manifest/setupManifest.ts

@@ -0,0 +1,20 @@
+import { SDKManifest, ManifestVersion } from "./ManifestTypes"
+
+export function setupManifest<
+  Id extends string,
+  Version extends ManifestVersion,
+  Dependencies extends Record<string, unknown>,
+  VolumesTypes extends string,
+  AssetTypes extends string,
+  ImagesTypes extends string,
+  Manifest extends SDKManifest & {
+    dependencies: Dependencies
+    id: Id
+    version: Version
+    assets: AssetTypes[]
+    images: ImagesTypes[]
+    volumes: VolumesTypes[]
+  },
+>(manifest: Manifest): Manifest {
+  return manifest
+}

sdk/lib/store/getStore.ts

@@ -0,0 +1,61 @@
+import { Effects, EnsureStorePath } from "../types"
+
+export class GetStore<Store, Path extends string> {
+  constructor(
+    readonly effects: Effects,
+    readonly path: Path & EnsureStorePath<Store, Path>,
+    readonly options: {
+      /** Defaults to what ever the package currently in */
+      packageId?: string | undefined
+    } = {},
+  ) {}
+
+  /**
+   * Returns the value of Store at the provided path. Restart the service if the value changes
+   */
+  const() {
+    return this.effects.store.get<Store, Path>({
+      ...this.options,
+      path: this.path as any,
+      callback: this.effects.restart,
+    })
+  }
+  /**
+   * Returns the value of Store at the provided path. Does nothing if the value changes
+   */
+  once() {
+    return this.effects.store.get<Store, Path>({
+      ...this.options,
+      path: this.path as any,
+      callback: () => {},
+    })
+  }
+
+  /**
+   * Watches the value of Store at the provided path. Takes a custom callback function to run whenever the value changes
+   */
+  async *watch() {
+    while (true) {
+      let callback: () => void
+      const waitForNext = new Promise<void>((resolve) => {
+        callback = resolve
+      })
+      yield await this.effects.store.get<Store, Path>({
+        ...this.options,
+        path: this.path as any,
+        callback: () => callback(),
+      })
+      await waitForNext
+    }
+  }
+}
+export function getStore<Store, Path extends string>(
+  effects: Effects,
+  path: Path & EnsureStorePath<Store, Path>,
+  options: {
+    /** Defaults to what ever the package currently in */
+    packageId?: string | undefined
+  } = {},
+) {
+  return new GetStore<Store, Path>(effects, path as any, options)
+}

sdk/lib/test/configBuilder.test.ts

@@ -0,0 +1,864 @@
+import { testOutput } from "./output.test"
+import { Config } from "../config/builder/config"
+import { List } from "../config/builder/list"
+import { Value } from "../config/builder/value"
+import { Variants } from "../config/builder/variants"
+import { ValueSpec } from "../config/configTypes"
+import { setupManifest } from "../manifest/setupManifest"
+import { StartSdk } from "../StartSdk"
+
+describe("builder tests", () => {
+  test("text", async () => {
+    const bitcoinPropertiesBuilt: {
+      "peer-tor-address": ValueSpec
+    } = await Config.of({
+      "peer-tor-address": Value.text({
+        name: "Peer tor address",
+        description: "The Tor address of the peer interface",
+        required: { default: null },
+      }),
+    }).build({} as any)
+    expect(bitcoinPropertiesBuilt).toMatchObject({
+      "peer-tor-address": {
+        type: "text",
+        description: "The Tor address of the peer interface",
+        warning: null,
+        masked: false,
+        placeholder: null,
+        minLength: null,
+        maxLength: null,
+        patterns: [],
+        disabled: false,
+        inputmode: "text",
+        name: "Peer tor address",
+        required: true,
+        default: null,
+      },
+    })
+  })
+})
+
+describe("values", () => {
+  test("toggle", async () => {
+    const value = Value.toggle({
+      name: "Testing",
+      description: null,
+      warning: null,
+      default: false,
+    })
+    const validator = value.validator
+    validator.unsafeCast(false)
+    testOutput<typeof validator._TYPE, boolean>()(null)
+  })
+  test("text", async () => {
+    const value = Value.text({
+      name: "Testing",
+      required: { default: null },
+    })
+    const validator = value.validator
+    const rawIs = await value.build({} as any)
+    validator.unsafeCast("test text")
+    expect(() => validator.unsafeCast(null)).toThrowError()
+    testOutput<typeof validator._TYPE, string>()(null)
+  })
+  test("text with default", async () => {
+    const value = Value.text({
+      name: "Testing",
+      required: { default: "this is a default value" },
+    })
+    const validator = value.validator
+    const rawIs = await value.build({} as any)
+    validator.unsafeCast("test text")
+    expect(() => validator.unsafeCast(null)).toThrowError()
+    testOutput<typeof validator._TYPE, string>()(null)
+  })
+  test("optional text", async () => {
+    const value = Value.text({
+      name: "Testing",
+      required: false,
+    })
+    const validator = value.validator
+    const rawIs = await value.build({} as any)
+    validator.unsafeCast("test text")
+    validator.unsafeCast(null)
+    testOutput<typeof validator._TYPE, string | null | undefined>()(null)
+  })
+  test("color", async () => {
+    const value = Value.color({
+      name: "Testing",
+      required: false,
+      description: null,
+      warning: null,
+    })
+    const validator = value.validator
+    validator.unsafeCast("#000000")
+    testOutput<typeof validator._TYPE, string | null | undefined>()(null)
+  })
+  test("datetime", async () => {
+    const value = Value.datetime({
+      name: "Testing",
+      required: { default: null },
+      description: null,
+      warning: null,
+      inputmode: "date",
+      min: null,
+      max: null,
+    })
+    const validator = value.validator
+    validator.unsafeCast("2021-01-01")
+    testOutput<typeof validator._TYPE, string>()(null)
+  })
+  test("optional datetime", async () => {
+    const value = Value.datetime({
+      name: "Testing",
+      required: false,
+      description: null,
+      warning: null,
+      inputmode: "date",
+      min: null,
+      max: null,
+    })
+    const validator = value.validator
+    validator.unsafeCast("2021-01-01")
+    testOutput<typeof validator._TYPE, string | null | undefined>()(null)
+  })
+  test("textarea", async () => {
+    const value = Value.textarea({
+      name: "Testing",
+      required: false,
+      description: null,
+      warning: null,
+      minLength: null,
+      maxLength: null,
+      placeholder: null,
+    })
+    const validator = value.validator
+    validator.unsafeCast("test text")
+    testOutput<typeof validator._TYPE, string>()(null)
+  })
+  test("number", async () => {
+    const value = Value.number({
+      name: "Testing",
+      required: { default: null },
+      integer: false,
+      description: null,
+      warning: null,
+      min: null,
+      max: null,
+      step: null,
+      units: null,
+      placeholder: null,
+    })
+    const validator = value.validator
+    validator.unsafeCast(2)
+    testOutput<typeof validator._TYPE, number>()(null)
+  })
+  test("optional number", async () => {
+    const value = Value.number({
+      name: "Testing",
+      required: false,
+      integer: false,
+      description: null,
+      warning: null,
+      min: null,
+      max: null,
+      step: null,
+      units: null,
+      placeholder: null,
+    })
+    const validator = value.validator
+    validator.unsafeCast(2)
+    testOutput<typeof validator._TYPE, number | null | undefined>()(null)
+  })
+  test("select", async () => {
+    const value = Value.select({
+      name: "Testing",
+      required: { default: null },
+      values: {
+        a: "A",
+        b: "B",
+      },
+      description: null,
+      warning: null,
+    })
+    const validator = value.validator
+    validator.unsafeCast("a")
+    validator.unsafeCast("b")
+    expect(() => validator.unsafeCast("c")).toThrowError()
+    testOutput<typeof validator._TYPE, "a" | "b">()(null)
+  })
+  test("nullable select", async () => {
+    const value = Value.select({
+      name: "Testing",
+      required: false,
+      values: {
+        a: "A",
+        b: "B",
+      },
+      description: null,
+      warning: null,
+    })
+    const validator = value.validator
+    validator.unsafeCast("a")
+    validator.unsafeCast("b")
+    validator.unsafeCast(null)
+    testOutput<typeof validator._TYPE, "a" | "b" | null | undefined>()(null)
+  })
+  test("multiselect", async () => {
+    const value = Value.multiselect({
+      name: "Testing",
+      values: {
+        a: "A",
+        b: "B",
+      },
+      default: [],
+      description: null,
+      warning: null,
+      minLength: null,
+      maxLength: null,
+    })
+    const validator = value.validator
+    validator.unsafeCast([])
+    validator.unsafeCast(["a", "b"])
+
+    expect(() => validator.unsafeCast(["e"])).toThrowError()
+    expect(() => validator.unsafeCast([4])).toThrowError()
+    testOutput<typeof validator._TYPE, Array<"a" | "b">>()(null)
+  })
+  test("object", async () => {
+    const value = Value.object(
+      {
+        name: "Testing",
+        description: null,
+        warning: null,
+      },
+      Config.of({
+        a: Value.toggle({
+          name: "test",
+          description: null,
+          warning: null,
+          default: false,
+        }),
+      }),
+    )
+    const validator = value.validator
+    validator.unsafeCast({ a: true })
+    testOutput<typeof validator._TYPE, { a: boolean }>()(null)
+  })
+  test("union", async () => {
+    const value = Value.union(
+      {
+        name: "Testing",
+        required: { default: null },
+        description: null,
+        warning: null,
+      },
+      Variants.of({
+        a: {
+          name: "a",
+          spec: Config.of({
+            b: Value.toggle({
+              name: "b",
+              description: null,
+              warning: null,
+              default: false,
+            }),
+          }),
+        },
+      }),
+    )
+    const validator = value.validator
+    validator.unsafeCast({ unionSelectKey: "a", unionValueKey: { b: false } })
+    type Test = typeof validator._TYPE
+    testOutput<Test, { unionSelectKey: "a"; unionValueKey: { b: boolean } }>()(
+      null,
+    )
+  })
+  test("list", async () => {
+    const value = Value.list(
+      List.number(
+        {
+          name: "test",
+        },
+        {
+          integer: false,
+        },
+      ),
+    )
+    const validator = value.validator
+    validator.unsafeCast([1, 2, 3])
+    testOutput<typeof validator._TYPE, number[]>()(null)
+  })
+
+  describe("dynamic", () => {
+    const fakeOptions = {
+      config: "config",
+      effects: "effects",
+      utils: "utils",
+    } as any
+    test("toggle", async () => {
+      const value = Value.dynamicToggle(async () => ({
+        name: "Testing",
+        description: null,
+        warning: null,
+        default: false,
+      }))
+      const validator = value.validator
+      validator.unsafeCast(false)
+      expect(() => validator.unsafeCast(null)).toThrowError()
+      testOutput<typeof validator._TYPE, boolean>()(null)
+      expect(await value.build(fakeOptions)).toMatchObject({
+        name: "Testing",
+        description: null,
+        warning: null,
+        default: false,
+      })
+    })
+    test("text", async () => {
+      const value = Value.dynamicText(async () => ({
+        name: "Testing",
+        required: { default: null },
+      }))
+      const validator = value.validator
+      const rawIs = await value.build({} as any)
+      validator.unsafeCast("test text")
+      validator.unsafeCast(null)
+      testOutput<typeof validator._TYPE, string | null | undefined>()(null)
+      expect(await value.build(fakeOptions)).toMatchObject({
+        name: "Testing",
+        required: true,
+        default: null,
+      })
+    })
+    test("text with default", async () => {
+      const value = Value.dynamicText(async () => ({
+        name: "Testing",
+        required: { default: "this is a default value" },
+      }))
+      const validator = value.validator
+      validator.unsafeCast("test text")
+      validator.unsafeCast(null)
+      testOutput<typeof validator._TYPE, string | null | undefined>()(null)
+      expect(await value.build(fakeOptions)).toMatchObject({
+        name: "Testing",
+        required: true,
+        default: "this is a default value",
+      })
+    })
+    test("optional text", async () => {
+      const value = Value.dynamicText(async () => ({
+        name: "Testing",
+        required: false,
+      }))
+      const validator = value.validator
+      const rawIs = await value.build({} as any)
+      validator.unsafeCast("test text")
+      validator.unsafeCast(null)
+      testOutput<typeof validator._TYPE, string | null | undefined>()(null)
+      expect(await value.build(fakeOptions)).toMatchObject({
+        name: "Testing",
+        required: false,
+        default: null,
+      })
+    })
+    test("color", async () => {
+      const value = Value.dynamicColor(async () => ({
+        name: "Testing",
+        required: false,
+        description: null,
+        warning: null,
+      }))
+      const validator = value.validator
+      validator.unsafeCast("#000000")
+      validator.unsafeCast(null)
+      testOutput<typeof validator._TYPE, string | null | undefined>()(null)
+      expect(await value.build(fakeOptions)).toMatchObject({
+        name: "Testing",
+        required: false,
+        default: null,
+        description: null,
+        warning: null,
+      })
+    })
+    test("datetime", async () => {
+      const sdk = StartSdk.of()
+        .withManifest(
+          setupManifest({
+            id: "testOutput",
+            title: "",
+            version: "1.0",
+            releaseNotes: "",
+            license: "",
+            replaces: [],
+            wrapperRepo: "",
+            upstreamRepo: "",
+            supportSite: "",
+            marketingSite: "",
+            donationUrl: null,
+            description: {
+              short: "",
+              long: "",
+            },
+            containers: {},
+            images: [],
+            volumes: [],
+            assets: [],
+            alerts: {
+              install: null,
+              update: null,
+              uninstall: null,
+              restore: null,
+              start: null,
+              stop: null,
+            },
+            dependencies: {
+              remoteTest: {
+                description: "",
+                requirement: { how: "", type: "opt-in" },
+                version: "1.0",
+              },
+            },
+          }),
+        )
+        .withStore<{ test: "a" }>()
+        .build(true)
+
+      const value = Value.dynamicDatetime<{ test: "a" }>(
+        async ({ effects }) => {
+          ;async () => {
+            ;(await sdk.store.getOwn(effects, "/test").once()) satisfies "a"
+          }
+
+          return {
+            name: "Testing",
+            required: { default: null },
+            inputmode: "date",
+          }
+        },
+      )
+      const validator = value.validator
+      validator.unsafeCast("2021-01-01")
+      validator.unsafeCast(null)
+      testOutput<typeof validator._TYPE, string | null | undefined>()(null)
+      expect(await value.build(fakeOptions)).toMatchObject({
+        name: "Testing",
+        required: true,
+        default: null,
+        description: null,
+        warning: null,
+        inputmode: "date",
+      })
+    })
+    test("textarea", async () => {
+      const value = Value.dynamicTextarea(async () => ({
+        name: "Testing",
+        required: false,
+        description: null,
+        warning: null,
+        minLength: null,
+        maxLength: null,
+        placeholder: null,
+      }))
+      const validator = value.validator
+      validator.unsafeCast("test text")
+      expect(() => validator.unsafeCast(null)).toThrowError()
+      testOutput<typeof validator._TYPE, string>()(null)
+      expect(await value.build(fakeOptions)).toMatchObject({
+        name: "Testing",
+        required: false,
+      })
+    })
+    test("number", async () => {
+      const value = Value.dynamicNumber(() => ({
+        name: "Testing",
+        required: { default: null },
+        integer: false,
+        description: null,
+        warning: null,
+        min: null,
+        max: null,
+        step: null,
+        units: null,
+        placeholder: null,
+      }))
+      const validator = value.validator
+      validator.unsafeCast(2)
+      validator.unsafeCast(null)
+      expect(() => validator.unsafeCast("null")).toThrowError()
+      testOutput<typeof validator._TYPE, number | null | undefined>()(null)
+      expect(await value.build(fakeOptions)).toMatchObject({
+        name: "Testing",
+        required: true,
+      })
+    })
+    test("select", async () => {
+      const value = Value.dynamicSelect(() => ({
+        name: "Testing",
+        required: { default: null },
+        values: {
+          a: "A",
+          b: "B",
+        },
+        description: null,
+        warning: null,
+      }))
+      const validator = value.validator
+      validator.unsafeCast("a")
+      validator.unsafeCast("b")
+      validator.unsafeCast("c")
+      validator.unsafeCast(null)
+      testOutput<typeof validator._TYPE, string | null | undefined>()(null)
+      expect(await value.build(fakeOptions)).toMatchObject({
+        name: "Testing",
+        required: true,
+      })
+    })
+    test("multiselect", async () => {
+      const value = Value.dynamicMultiselect(() => ({
+        name: "Testing",
+        values: {
+          a: "A",
+          b: "B",
+        },
+        default: [],
+        description: null,
+        warning: null,
+        minLength: null,
+        maxLength: null,
+      }))
+      const validator = value.validator
+      validator.unsafeCast([])
+      validator.unsafeCast(["a", "b"])
+      validator.unsafeCast(["c"])
+
+      expect(() => validator.unsafeCast([4])).toThrowError()
+      expect(() => validator.unsafeCast(null)).toThrowError()
+      testOutput<typeof validator._TYPE, Array<string>>()(null)
+      expect(await value.build(fakeOptions)).toMatchObject({
+        name: "Testing",
+        default: [],
+      })
+    })
+  })
+  describe("filtering", () => {
+    test("union", async () => {
+      const value = Value.filteredUnion(
+        () => ["a", "c"],
+        {
+          name: "Testing",
+          required: { default: null },
+          description: null,
+          warning: null,
+        },
+        Variants.of({
+          a: {
+            name: "a",
+            spec: Config.of({
+              b: Value.toggle({
+                name: "b",
+                description: null,
+                warning: null,
+                default: false,
+              }),
+            }),
+          },
+          b: {
+            name: "b",
+            spec: Config.of({
+              b: Value.toggle({
+                name: "b",
+                description: null,
+                warning: null,
+                default: false,
+              }),
+            }),
+          },
+        }),
+      )
+      const validator = value.validator
+      validator.unsafeCast({ unionSelectKey: "a", unionValueKey: { b: false } })
+      type Test = typeof validator._TYPE
+      testOutput<
+        Test,
+        | { unionSelectKey: "a"; unionValueKey: { b: boolean } }
+        | { unionSelectKey: "b"; unionValueKey: { b: boolean } }
+      >()(null)
+
+      const built = await value.build({} as any)
+      expect(built).toMatchObject({
+        name: "Testing",
+        variants: {
+          b: {},
+        },
+      })
+      expect(built).toMatchObject({
+        name: "Testing",
+        variants: {
+          a: {},
+          b: {},
+        },
+      })
+      expect(built).toMatchObject({
+        name: "Testing",
+        variants: {
+          a: {},
+          b: {},
+        },
+        disabled: ["a", "c"],
+      })
+    })
+  })
+  test("dynamic union", async () => {
+    const value = Value.dynamicUnion(
+      () => ({
+        disabled: ["a", "c"],
+        name: "Testing",
+        required: { default: null },
+        description: null,
+        warning: null,
+      }),
+      Variants.of({
+        a: {
+          name: "a",
+          spec: Config.of({
+            b: Value.toggle({
+              name: "b",
+              description: null,
+              warning: null,
+              default: false,
+            }),
+          }),
+        },
+        b: {
+          name: "b",
+          spec: Config.of({
+            b: Value.toggle({
+              name: "b",
+              description: null,
+              warning: null,
+              default: false,
+            }),
+          }),
+        },
+      }),
+    )
+    const validator = value.validator
+    validator.unsafeCast({ unionSelectKey: "a", unionValueKey: { b: false } })
+    type Test = typeof validator._TYPE
+    testOutput<
+      Test,
+      | { unionSelectKey: "a"; unionValueKey: { b: boolean } }
+      | { unionSelectKey: "b"; unionValueKey: { b: boolean } }
+      | null
+      | undefined
+    >()(null)
+
+    const built = await value.build({} as any)
+    expect(built).toMatchObject({
+      name: "Testing",
+      variants: {
+        b: {},
+      },
+    })
+    expect(built).toMatchObject({
+      name: "Testing",
+      variants: {
+        a: {},
+        b: {},
+      },
+    })
+    expect(built).toMatchObject({
+      name: "Testing",
+      variants: {
+        a: {},
+        b: {},
+      },
+      disabled: ["a", "c"],
+    })
+  })
+})
+
+describe("Builder List", () => {
+  test("obj", async () => {
+    const value = Value.list(
+      List.obj(
+        {
+          name: "test",
+        },
+        {
+          spec: Config.of({
+            test: Value.toggle({
+              name: "test",
+              description: null,
+              warning: null,
+              default: false,
+            }),
+          }),
+        },
+      ),
+    )
+    const validator = value.validator
+    validator.unsafeCast([{ test: true }])
+    testOutput<typeof validator._TYPE, { test: boolean }[]>()(null)
+  })
+  test("text", async () => {
+    const value = Value.list(
+      List.text(
+        {
+          name: "test",
+        },
+        {
+          patterns: [],
+        },
+      ),
+    )
+    const validator = value.validator
+    validator.unsafeCast(["test", "text"])
+    testOutput<typeof validator._TYPE, string[]>()(null)
+  })
+  describe("dynamic", () => {
+    test("text", async () => {
+      const value = Value.list(
+        List.dynamicText(() => ({
+          name: "test",
+          spec: { patterns: [] },
+        })),
+      )
+      const validator = value.validator
+      validator.unsafeCast(["test", "text"])
+      expect(() => validator.unsafeCast([3, 4])).toThrowError()
+      expect(() => validator.unsafeCast(null)).toThrowError()
+      testOutput<typeof validator._TYPE, string[]>()(null)
+      expect(await value.build({} as any)).toMatchObject({
+        name: "test",
+        spec: { patterns: [] },
+      })
+    })
+  })
+  test("number", async () => {
+    const value = Value.list(
+      List.dynamicNumber(() => ({
+        name: "test",
+        spec: { integer: true },
+      })),
+    )
+    const validator = value.validator
+    expect(() => validator.unsafeCast(["test", "text"])).toThrowError()
+    validator.unsafeCast([4, 2])
+    expect(() => validator.unsafeCast(null)).toThrowError()
+    validator.unsafeCast([])
+    testOutput<typeof validator._TYPE, number[]>()(null)
+    expect(await value.build({} as any)).toMatchObject({
+      name: "test",
+      spec: { integer: true },
+    })
+  })
+})
+
+describe("Nested nullable values", () => {
+  test("Testing text", async () => {
+    const value = Config.of({
+      a: Value.text({
+        name: "Temp Name",
+        description:
+          "If no name is provided, the name from config will be used",
+        required: false,
+      }),
+    })
+    const validator = value.validator
+    validator.unsafeCast({ a: null })
+    validator.unsafeCast({ a: "test" })
+    expect(() => validator.unsafeCast({ a: 4 })).toThrowError()
+    testOutput<typeof validator._TYPE, { a: string | null | undefined }>()(null)
+  })
+  test("Testing number", async () => {
+    const value = Config.of({
+      a: Value.number({
+        name: "Temp Name",
+        description:
+          "If no name is provided, the name from config will be used",
+        required: false,
+        warning: null,
+        placeholder: null,
+        integer: false,
+        min: null,
+        max: null,
+        step: null,
+        units: null,
+      }),
+    })
+    const validator = value.validator
+    validator.unsafeCast({ a: null })
+    validator.unsafeCast({ a: 5 })
+    expect(() => validator.unsafeCast({ a: "4" })).toThrowError()
+    testOutput<typeof validator._TYPE, { a: number | null | undefined }>()(null)
+  })
+  test("Testing color", async () => {
+    const value = Config.of({
+      a: Value.color({
+        name: "Temp Name",
+        description:
+          "If no name is provided, the name from config will be used",
+        required: false,
+        warning: null,
+      }),
+    })
+    const validator = value.validator
+    validator.unsafeCast({ a: null })
+    validator.unsafeCast({ a: "5" })
+    expect(() => validator.unsafeCast({ a: 4 })).toThrowError()
+    testOutput<typeof validator._TYPE, { a: string | null | undefined }>()(null)
+  })
+  test("Testing select", async () => {
+    const value = Config.of({
+      a: Value.select({
+        name: "Temp Name",
+        description:
+          "If no name is provided, the name from config will be used",
+        required: false,
+        warning: null,
+        values: {
+          a: "A",
+        },
+      }),
+    })
+    const higher = await Value.select({
+      name: "Temp Name",
+      description: "If no name is provided, the name from config will be used",
+      required: false,
+      warning: null,
+      values: {
+        a: "A",
+      },
+    }).build({} as any)
+
+    const validator = value.validator
+    validator.unsafeCast({ a: null })
+    validator.unsafeCast({ a: "a" })
+    expect(() => validator.unsafeCast({ a: "4" })).toThrowError()
+    testOutput<typeof validator._TYPE, { a: "a" | null | undefined }>()(null)
+  })
+  test("Testing multiselect", async () => {
+    const value = Config.of({
+      a: Value.multiselect({
+        name: "Temp Name",
+        description:
+          "If no name is provided, the name from config will be used",
+
+        warning: null,
+        default: [],
+        values: {
+          a: "A",
+        },
+        minLength: null,
+        maxLength: null,
+      }),
+    })
+    const validator = value.validator
+    validator.unsafeCast({ a: [] })
+    validator.unsafeCast({ a: ["a"] })
+    expect(() => validator.unsafeCast({ a: ["4"] })).toThrowError()
+    expect(() => validator.unsafeCast({ a: "4" })).toThrowError()
+    testOutput<typeof validator._TYPE, { a: "a"[] }>()(null)
+  })
+})

sdk/lib/test/configTypes.test.ts

@@ -0,0 +1,32 @@
+import {
+  ListValueSpecOf,
+  ValueSpec,
+  isValueSpecListOf,
+} from "../config/configTypes"
+import { Config } from "../config/builder/config"
+import { List } from "../config/builder/list"
+import { Value } from "../config/builder/value"
+
+describe("Config Types", () => {
+  test("isValueSpecListOf", async () => {
+    const options = [List.obj, List.text, List.number]
+    for (const option of options) {
+      const test = (option as any)(
+        {} as any,
+        { spec: Config.of({}) } as any,
+      ) as any
+      const someList = await Value.list(test).build({} as any)
+      if (isValueSpecListOf(someList, "text")) {
+        someList.spec satisfies ListValueSpecOf<"text">
+      } else if (isValueSpecListOf(someList, "number")) {
+        someList.spec satisfies ListValueSpecOf<"number">
+      } else if (isValueSpecListOf(someList, "object")) {
+        someList.spec satisfies ListValueSpecOf<"object">
+      } else {
+        throw new Error(
+          "Failed to figure out the type: " + JSON.stringify(someList),
+        )
+      }
+    }
+  })
+})

sdk/lib/test/emverList.test.ts

@@ -0,0 +1,262 @@
+import { EmVer, notRange, rangeAnd, rangeOf, rangeOr } from "../emverLite/mod"
+describe("EmVer", () => {
+  {
+    {
+      const checker = rangeOf("*")
+      test("rangeOf('*')", () => {
+        checker.check("1")
+        checker.check("1.2")
+        checker.check("1.2.3")
+        checker.check("1.2.3.4")
+        // @ts-expect-error
+        checker.check("1.2.3.4.5")
+        // @ts-expect-error
+        checker.check("1.2.3.4.5.6")
+        expect(checker.check("1")).toEqual(true)
+        expect(checker.check("1.2")).toEqual(true)
+        expect(checker.check("1.2.3.4")).toEqual(true)
+      })
+      test("rangeOf('*') invalid", () => {
+        // @ts-expect-error
+        expect(() => checker.check("a")).toThrow()
+        // @ts-expect-error
+        expect(() => checker.check("")).toThrow()
+        expect(() => checker.check("1..3")).toThrow()
+      })
+    }
+
+    {
+      const checker = rangeOf(">1.2.3.4")
+      test(`rangeOf(">1.2.3.4") valid`, () => {
+        expect(checker.check("2-beta123")).toEqual(true)
+        expect(checker.check("2")).toEqual(true)
+        expect(checker.check("1.2.3.5")).toEqual(true)
+        // @ts-expect-error
+        expect(checker.check("1.2.3.4.1")).toEqual(true)
+      })
+
+      test(`rangeOf(">1.2.3.4") invalid`, () => {
+        expect(checker.check("1.2.3.4")).toEqual(false)
+        expect(checker.check("1.2.3")).toEqual(false)
+        expect(checker.check("1")).toEqual(false)
+      })
+    }
+    {
+      const checker = rangeOf("=1.2.3")
+      test(`rangeOf("=1.2.3") valid`, () => {
+        expect(checker.check("1.2.3")).toEqual(true)
+      })
+
+      test(`rangeOf("=1.2.3") invalid`, () => {
+        expect(checker.check("2")).toEqual(false)
+        expect(checker.check("1.2.3.1")).toEqual(false)
+        expect(checker.check("1.2")).toEqual(false)
+      })
+    }
+    {
+      const checker = rangeOf(">=1.2.3.4")
+      test(`rangeOf(">=1.2.3.4") valid`, () => {
+        expect(checker.check("2")).toEqual(true)
+        expect(checker.check("1.2.3.5")).toEqual(true)
+        // @ts-expect-error
+        expect(checker.check("1.2.3.4.1")).toEqual(true)
+        expect(checker.check("1.2.3.4")).toEqual(true)
+      })
+
+      test(`rangeOf(">=1.2.3.4") invalid`, () => {
+        expect(checker.check("1.2.3")).toEqual(false)
+        expect(checker.check("1")).toEqual(false)
+      })
+    }
+    {
+      const checker = rangeOf("<1.2.3.4")
+      test(`rangeOf("<1.2.3.4") invalid`, () => {
+        expect(checker.check("2")).toEqual(false)
+        expect(checker.check("1.2.3.5")).toEqual(false)
+        // @ts-expect-error
+        expect(checker.check("1.2.3.4.1")).toEqual(false)
+        expect(checker.check("1.2.3.4")).toEqual(false)
+      })
+
+      test(`rangeOf("<1.2.3.4") valid`, () => {
+        expect(checker.check("1.2.3")).toEqual(true)
+        expect(checker.check("1")).toEqual(true)
+      })
+    }
+    {
+      const checker = rangeOf("<=1.2.3.4")
+      test(`rangeOf("<=1.2.3.4") invalid`, () => {
+        expect(checker.check("2")).toEqual(false)
+        expect(checker.check("1.2.3.5")).toEqual(false)
+        // @ts-expect-error
+        expect(checker.check("1.2.3.4.1")).toEqual(false)
+      })
+
+      test(`rangeOf("<=1.2.3.4") valid`, () => {
+        expect(checker.check("1.2.3")).toEqual(true)
+        expect(checker.check("1")).toEqual(true)
+        expect(checker.check("1.2.3.4")).toEqual(true)
+      })
+    }
+
+    {
+      const checkA = rangeOf(">1")
+      const checkB = rangeOf("<=2")
+
+      const checker = rangeAnd(checkA, checkB)
+      test(`simple and(checkers) valid`, () => {
+        expect(checker.check("2")).toEqual(true)
+
+        expect(checker.check("1.1")).toEqual(true)
+      })
+      test(`simple and(checkers) invalid`, () => {
+        expect(checker.check("2.1")).toEqual(false)
+        expect(checker.check("1")).toEqual(false)
+        expect(checker.check("0")).toEqual(false)
+      })
+    }
+    {
+      const checkA = rangeOf("<1")
+      const checkB = rangeOf("=2")
+
+      const checker = rangeOr(checkA, checkB)
+      test(`simple or(checkers) valid`, () => {
+        expect(checker.check("2")).toEqual(true)
+        expect(checker.check("0.1")).toEqual(true)
+      })
+      test(`simple or(checkers) invalid`, () => {
+        expect(checker.check("2.1")).toEqual(false)
+        expect(checker.check("1")).toEqual(false)
+        expect(checker.check("1.1")).toEqual(false)
+      })
+    }
+
+    {
+      const checker = rangeOf("1.2.*")
+      test(`rangeOf(1.2.*) valid`, () => {
+        expect(checker.check("1.2")).toEqual(true)
+        expect(checker.check("1.2.1")).toEqual(true)
+      })
+      test(`rangeOf(1.2.*) invalid`, () => {
+        expect(checker.check("1.3")).toEqual(false)
+        expect(checker.check("1.3.1")).toEqual(false)
+
+        expect(checker.check("1.1.1")).toEqual(false)
+        expect(checker.check("1.1")).toEqual(false)
+        expect(checker.check("1")).toEqual(false)
+
+        expect(checker.check("2")).toEqual(false)
+      })
+    }
+
+    {
+      const checker = notRange(rangeOf("1.2.*"))
+      test(`notRange(rangeOf(1.2.*)) valid`, () => {
+        expect(checker.check("1.3")).toEqual(true)
+        expect(checker.check("1.3.1")).toEqual(true)
+
+        expect(checker.check("1.1.1")).toEqual(true)
+        expect(checker.check("1.1")).toEqual(true)
+        expect(checker.check("1")).toEqual(true)
+
+        expect(checker.check("2")).toEqual(true)
+      })
+      test(`notRange(rangeOf(1.2.*)) invalid `, () => {
+        expect(checker.check("1.2")).toEqual(false)
+        expect(checker.check("1.2.1")).toEqual(false)
+      })
+    }
+    {
+      const checker = rangeOf("!1.2.*")
+      test(`!(rangeOf(1.2.*)) valid`, () => {
+        expect(checker.check("1.3")).toEqual(true)
+        expect(checker.check("1.3.1")).toEqual(true)
+
+        expect(checker.check("1.1.1")).toEqual(true)
+        expect(checker.check("1.1")).toEqual(true)
+        expect(checker.check("1")).toEqual(true)
+
+        expect(checker.check("2")).toEqual(true)
+      })
+      test(`!(rangeOf(1.2.*)) invalid `, () => {
+        expect(checker.check("1.2")).toEqual(false)
+        expect(checker.check("1.2.1")).toEqual(false)
+      })
+    }
+    {
+      test(`no and ranges`, () => {
+        expect(() => rangeAnd()).toThrow()
+      })
+      test(`no or ranges`, () => {
+        expect(() => rangeOr()).toThrow()
+      })
+    }
+    {
+      const checker = rangeOf("!>1.2.3.4")
+      test(`rangeOf("!>1.2.3.4") invalid`, () => {
+        expect(checker.check("2")).toEqual(false)
+        expect(checker.check("1.2.3.5")).toEqual(false)
+        // @ts-expect-error
+        expect(checker.check("1.2.3.4.1")).toEqual(false)
+      })
+
+      test(`rangeOf("!>1.2.3.4") valid`, () => {
+        expect(checker.check("1.2.3.4")).toEqual(true)
+        expect(checker.check("1.2.3")).toEqual(true)
+        expect(checker.check("1")).toEqual(true)
+      })
+    }
+
+    {
+      test(">1 && =1.2", () => {
+        const checker = rangeOf(">1 && =1.2")
+
+        expect(checker.check("1.2")).toEqual(true)
+        expect(checker.check("1.2.1")).toEqual(false)
+      })
+      test("=1 || =2", () => {
+        const checker = rangeOf("=1 || =2")
+
+        expect(checker.check("1")).toEqual(true)
+        expect(checker.check("2")).toEqual(true)
+        expect(checker.check("3")).toEqual(false)
+      })
+
+      test(">1 && =1.2 || =2", () => {
+        const checker = rangeOf(">1 && =1.2 || =2")
+
+        expect(checker.check("1.2")).toEqual(true)
+        expect(checker.check("1")).toEqual(false)
+        expect(checker.check("2")).toEqual(true)
+        expect(checker.check("3")).toEqual(false)
+      })
+
+      test("&& before || order of operationns:  <1.5 && >1 || >1.5 && <3", () => {
+        const checker = rangeOf("<1.5 && >1 || >1.5 && <3")
+        expect(checker.check("1.1")).toEqual(true)
+        expect(checker.check("2")).toEqual(true)
+
+        expect(checker.check("1.5")).toEqual(false)
+        expect(checker.check("1")).toEqual(false)
+        expect(checker.check("3")).toEqual(false)
+      })
+
+      test("Compare function on the emver", () => {
+        const a = EmVer.from("1.2.3")
+        const b = EmVer.from("1.2.4")
+
+        expect(a.compare(b)).toEqual("less")
+        expect(b.compare(a)).toEqual("greater")
+        expect(a.compare(a)).toEqual("equal")
+      })
+      test("Compare for sort function on the emver", () => {
+        const a = EmVer.from("1.2.3")
+        const b = EmVer.from("1.2.4")
+
+        expect(a.compareForSort(b)).toEqual(-1)
+        expect(b.compareForSort(a)).toEqual(1)
+        expect(a.compareForSort(a)).toEqual(0)
+      })
+    }
+  }
+})

sdk/lib/test/health.readyCheck.test.ts

@@ -0,0 +1,17 @@
+import { containsAddress } from "../health/checkFns/checkPortListening"
+
+describe("Health ready check", () => {
+  it("Should be able to parse an example information", () => {
+    let input = `
+  
+  sl  local_address rem_address   st tx_queue rx_queue tr tm->when retrnsmt   uid  timeout inode                                                     
+   0: 00000000:1F90 00000000:0000 0A 00000000:00000000 00:00000000 00000000     0        0 21634478 1 0000000000000000 100 0 0 10 0                  
+   1: 00000000:0050 00000000:0000 0A 00000000:00000000 00:00000000 00000000     0        0 21634477 1 0000000000000000 100 0 0 10 0                  
+   2: 0B00007F:9671 00000000:0000 0A 00000000:00000000 00:00000000 00000000     0        0 21635458 1 0000000000000000 100 0 0 10 0                  
+   3: 00000000:0D73 00000000:0000 0A 00000000:00000000 00:00000000 00000000     0        0 21634479 1 0000000000000000 100 0 0 10 0   
+  `
+
+    expect(containsAddress(input, 80)).toBe(true)
+    expect(containsAddress(input, 1234)).toBe(false)
+  })
+})

sdk/lib/test/host.test.ts

@@ -0,0 +1,30 @@
+import { ServiceInterfaceBuilder } from "../interfaces/ServiceInterfaceBuilder"
+import { Effects } from "../types"
+import { sdk } from "./output.sdk"
+
+describe("host", () => {
+  test("Testing that the types work", () => {
+    async function test(effects: Effects) {
+      const foo = sdk.host.multi(effects, "foo")
+      const fooOrigin = await foo.bindPort(80, {
+        protocol: "http" as const,
+      })
+      const fooInterface = new ServiceInterfaceBuilder({
+        effects,
+        name: "Foo",
+        id: "foo",
+        description: "A Foo",
+        hasPrimary: false,
+        disabled: false,
+        type: "ui",
+        username: "bar",
+        path: "/baz",
+        search: { qux: "yes" },
+        schemeOverride: null,
+        masked: false,
+      })
+
+      await fooOrigin.export([fooInterface])
+    }
+  })
+})

sdk/lib/test/makeOutput.ts

@@ -0,0 +1,428 @@
+import { oldSpecToBuilder } from "../../scripts/oldSpecToBuilder"
+
+oldSpecToBuilder(
+  // Make the location
+  "./lib/test/output.ts",
+  // Put the config here
+  {
+    mediasources: {
+      type: "list",
+      subtype: "enum",
+      name: "Media Sources",
+      description: "List of Media Sources to use with Jellyfin",
+      range: "[1,*)",
+      default: ["nextcloud"],
+      spec: {
+        values: ["nextcloud", "filebrowser"],
+        "value-names": {
+          nextcloud: "NextCloud",
+          filebrowser: "File Browser",
+        },
+      },
+    },
+    testListUnion: {
+      type: "list",
+      subtype: "union",
+      name: "Lightning Nodes",
+      description: "List of Lightning Network node instances to manage",
+      range: "[1,*)",
+      default: ["lnd"],
+      spec: {
+        type: "string",
+        "display-as": "{{name}}",
+        "unique-by": "name",
+        name: "Node Implementation",
+        tag: {
+          id: "type",
+          name: "Type",
+          description:
+            "- LND: Lightning Network Daemon from Lightning Labs\n- CLN: Core Lightning from Blockstream\n",
+          "variant-names": {
+            lnd: "Lightning Network Daemon (LND)",
+            "c-lightning": "Core Lightning (CLN)",
+          },
+        },
+        default: "lnd",
+        variants: {
+          lnd: {
+            name: {
+              type: "string",
+              name: "Node Name",
+              description: "Name of this node in the list",
+              default: "LND Wrapper",
+              nullable: false,
+            },
+          },
+        },
+      },
+    },
+    rpc: {
+      type: "object",
+      name: "RPC Settings",
+      description: "RPC configuration options.",
+      spec: {
+        enable: {
+          type: "boolean",
+          name: "Enable",
+          description: "Allow remote RPC requests.",
+          default: true,
+        },
+        username: {
+          type: "string",
+          nullable: false,
+          name: "Username",
+          description: "The username for connecting to Bitcoin over RPC.",
+          default: "bitcoin",
+          masked: true,
+          pattern: "^[a-zA-Z0-9_]+$",
+          "pattern-description":
+            "Must be alphanumeric (can contain underscore).",
+        },
+        password: {
+          type: "string",
+          nullable: false,
+          name: "RPC Password",
+          description: "The password for connecting to Bitcoin over RPC.",
+          default: {
+            charset: "a-z,2-7",
+            len: 20,
+          },
+          pattern: '^[^\\n"]*$',
+          "pattern-description":
+            "Must not contain newline or quote characters.",
+          copyable: true,
+          masked: true,
+        },
+        bio: {
+          type: "string",
+          nullable: false,
+          name: "Username",
+          description: "The username for connecting to Bitcoin over RPC.",
+          default: "bitcoin",
+          masked: true,
+          pattern: "^[a-zA-Z0-9_]+$",
+          "pattern-description":
+            "Must be alphanumeric (can contain underscore).",
+          textarea: true,
+        },
+        advanced: {
+          type: "object",
+          name: "Advanced",
+          description: "Advanced RPC Settings",
+          spec: {
+            auth: {
+              name: "Authorization",
+              description:
+                "Username and hashed password for JSON-RPC connections. RPC clients connect using the usual http basic authentication.",
+              type: "list",
+              subtype: "string",
+              default: [],
+              spec: {
+                pattern:
+                  "^[a-zA-Z0-9_-]+:([0-9a-fA-F]{2})+\\$([0-9a-fA-F]{2})+$",
+                "pattern-description":
+                  'Each item must be of the form "<USERNAME>:<SALT>$<HASH>".',
+                masked: false,
+              },
+              range: "[0,*)",
+            },
+            serialversion: {
+              name: "Serialization Version",
+              description:
+                "Return raw transaction or block hex with Segwit or non-SegWit serialization.",
+              type: "enum",
+              values: ["non-segwit", "segwit"],
+              "value-names": {},
+              default: "segwit",
+            },
+            servertimeout: {
+              name: "Rpc Server Timeout",
+              description:
+                "Number of seconds after which an uncompleted RPC call will time out.",
+              type: "number",
+              nullable: false,
+              range: "[5,300]",
+              integral: true,
+              units: "seconds",
+              default: 30,
+            },
+            threads: {
+              name: "Threads",
+              description:
+                "Set the number of threads for handling RPC calls. You may wish to increase this if you are making lots of calls via an integration.",
+              type: "number",
+              nullable: false,
+              default: 16,
+              range: "[1,64]",
+              integral: true,
+            },
+            workqueue: {
+              name: "Work Queue",
+              description:
+                "Set the depth of the work queue to service RPC calls. Determines how long the backlog of RPC requests can get before it just rejects new ones.",
+              type: "number",
+              nullable: false,
+              default: 128,
+              range: "[8,256]",
+              integral: true,
+              units: "requests",
+            },
+          },
+        },
+      },
+    },
+    "zmq-enabled": {
+      type: "boolean",
+      name: "ZeroMQ Enabled",
+      description: "Enable the ZeroMQ interface",
+      default: true,
+    },
+    txindex: {
+      type: "boolean",
+      name: "Transaction Index",
+      description: "Enable the Transaction Index (txindex)",
+      default: true,
+    },
+    wallet: {
+      type: "object",
+      name: "Wallet",
+      description: "Wallet Settings",
+      spec: {
+        enable: {
+          name: "Enable Wallet",
+          description: "Load the wallet and enable wallet RPC calls.",
+          type: "boolean",
+          default: true,
+        },
+        avoidpartialspends: {
+          name: "Avoid Partial Spends",
+          description:
+            "Group outputs by address, selecting all or none, instead of selecting on a per-output basis. This improves privacy at the expense of higher transaction fees.",
+          type: "boolean",
+          default: true,
+        },
+        discardfee: {
+          name: "Discard Change Tolerance",
+          description:
+            "The fee rate (in BTC/kB) that indicates your tolerance for discarding change by adding it to the fee.",
+          type: "number",
+          nullable: false,
+          default: 0.0001,
+          range: "[0,.01]",
+          integral: false,
+          units: "BTC/kB",
+        },
+      },
+    },
+    advanced: {
+      type: "object",
+      name: "Advanced",
+      description: "Advanced Settings",
+      spec: {
+        mempool: {
+          type: "object",
+          name: "Mempool",
+          description: "Mempool Settings",
+          spec: {
+            mempoolfullrbf: {
+              name: "Enable Full RBF",
+              description:
+                "Policy for your node to use for relaying and mining unconfirmed transactions.  For details, see https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-24.0.md#notice-of-new-option-for-transaction-replacement-policies",
+              type: "boolean",
+              default: false,
+            },
+            persistmempool: {
+              type: "boolean",
+              name: "Persist Mempool",
+              description: "Save the mempool on shutdown and load on restart.",
+              default: true,
+            },
+            maxmempool: {
+              type: "number",
+              nullable: false,
+              name: "Max Mempool Size",
+              description:
+                "Keep the transaction memory pool below <n> megabytes.",
+              range: "[1,*)",
+              integral: true,
+              units: "MiB",
+              default: 300,
+            },
+            mempoolexpiry: {
+              type: "number",
+              nullable: false,
+              name: "Mempool Expiration",
+              description:
+                "Do not keep transactions in the mempool longer than <n> hours.",
+              range: "[1,*)",
+              integral: true,
+              units: "Hr",
+              default: 336,
+            },
+          },
+        },
+        peers: {
+          type: "object",
+          name: "Peers",
+          description: "Peer Connection Settings",
+          spec: {
+            listen: {
+              type: "boolean",
+              name: "Make Public",
+              description:
+                "Allow other nodes to find your server on the network.",
+              default: true,
+            },
+            onlyconnect: {
+              type: "boolean",
+              name: "Disable Peer Discovery",
+              description: "Only connect to specified peers.",
+              default: false,
+            },
+            onlyonion: {
+              type: "boolean",
+              name: "Disable Clearnet",
+              description: "Only connect to peers over Tor.",
+              default: false,
+            },
+            addnode: {
+              name: "Add Nodes",
+              description: "Add addresses of nodes to connect to.",
+              type: "list",
+              subtype: "object",
+              range: "[0,*)",
+              default: [],
+              spec: {
+                "unique-by": null,
+                spec: {
+                  hostname: {
+                    type: "string",
+                    nullable: true,
+                    name: "Hostname",
+                    description: "Domain or IP address of bitcoin peer",
+                    pattern:
+                      "(^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$)|((^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$)|(^[a-z2-7]{16}\\.onion$)|(^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$))",
+                    "pattern-description":
+                      "Must be either a domain name, or an IPv4 or IPv6 address. Do not include protocol scheme (eg 'http://') or port.",
+                    masked: false,
+                  },
+                  port: {
+                    type: "number",
+                    nullable: true,
+                    name: "Port",
+                    description:
+                      "Port that peer is listening on for inbound p2p connections",
+                    range: "[0,65535]",
+                    integral: true,
+                  },
+                },
+              },
+            },
+          },
+        },
+        dbcache: {
+          type: "number",
+          nullable: true,
+          name: "Database Cache",
+          description:
+            "How much RAM to allocate for caching the TXO set. Higher values improve syncing performance, but increase your chance of using up all your system's memory or corrupting your database in the event of an ungraceful shutdown. Set this high but comfortably below your system's total RAM during IBD, then turn down to 450 (or leave blank) once the sync completes.",
+          warning:
+            "WARNING: Increasing this value results in a higher chance of ungraceful shutdowns, which can leave your node unusable if it happens during the initial block download. Use this setting with caution. Be sure to set this back to the default (450 or leave blank) once your node is synced. DO NOT press the STOP button if your dbcache is large. Instead, set this number back to the default, hit save, and wait for bitcoind to restart on its own.",
+          range: "(0,*)",
+          integral: true,
+          units: "MiB",
+        },
+        pruning: {
+          type: "union",
+          name: "Pruning Settings",
+          description:
+            "Blockchain Pruning Options\nReduce the blockchain size on disk\n",
+          warning:
+            "If you set pruning to Manual and your disk is smaller than the total size of the blockchain, you MUST have something running that prunes these blocks or you may overfill your disk!\nDisabling pruning will convert your node into a full archival node. This requires a resync of the entire blockchain, a process that may take several days. Make sure you have enough free disk space or you may fill up your disk.\n",
+          tag: {
+            id: "mode",
+            name: "Pruning Mode",
+            description:
+              '- Disabled: Disable pruning\n- Automatic: Limit blockchain size on disk to a certain number of megabytes\n- Manual: Prune blockchain with the "pruneblockchain" RPC\n',
+            "variant-names": {
+              disabled: "Disabled",
+              automatic: "Automatic",
+              manual: "Manual",
+            },
+          },
+          variants: {
+            disabled: {},
+            automatic: {
+              size: {
+                type: "number",
+                nullable: false,
+                name: "Max Chain Size",
+                description: "Limit of blockchain size on disk.",
+                warning:
+                  "Increasing this value will require re-syncing your node.",
+                default: 550,
+                range: "[550,1000000)",
+                integral: true,
+                units: "MiB",
+              },
+            },
+            manual: {
+              size: {
+                type: "number",
+                nullable: false,
+                name: "Failsafe Chain Size",
+                description: "Prune blockchain if size expands beyond this.",
+                default: 65536,
+                range: "[550,1000000)",
+                integral: true,
+                units: "MiB",
+              },
+            },
+          },
+          default: "disabled",
+        },
+        blockfilters: {
+          type: "object",
+          name: "Block Filters",
+          description: "Settings for storing and serving compact block filters",
+          spec: {
+            blockfilterindex: {
+              type: "boolean",
+              name: "Compute Compact Block Filters (BIP158)",
+              description:
+                "Generate Compact Block Filters during initial sync (IBD) to enable 'getblockfilter' RPC. This is useful if dependent services need block filters to efficiently scan for addresses/transactions etc.",
+              default: true,
+            },
+            peerblockfilters: {
+              type: "boolean",
+              name: "Serve Compact Block Filters to Peers (BIP157)",
+              description:
+                "Serve Compact Block Filters as a peer service to other nodes on the network. This is useful if you wish to connect an SPV client to your node to make it efficient to scan transactions without having to download all block data.  'Compute Compact Block Filters (BIP158)' is required.",
+              default: false,
+            },
+          },
+        },
+        bloomfilters: {
+          type: "object",
+          name: "Bloom Filters (BIP37)",
+          description: "Setting for serving Bloom Filters",
+          spec: {
+            peerbloomfilters: {
+              type: "boolean",
+              name: "Serve Bloom Filters to Peers",
+              description:
+                "Peers have the option of setting filters on each connection they make after the version handshake has completed. Bloom filters are for clients implementing SPV (Simplified Payment Verification) that want to check that block headers  connect together correctly, without needing to verify the full blockchain.  The client must trust that the transactions in the chain are in fact valid.  It is highly recommended AGAINST using for anything except Bisq integration.",
+              warning:
+                "This is ONLY for use with Bisq integration, please use Block Filters for all other applications.",
+              default: false,
+            },
+          },
+        },
+      },
+    },
+  },
+  {
+    // convert this to `start-sdk/lib` for conversions
+    StartSdk: "./output.sdk",
+  },
+)

sdk/lib/test/output.sdk.ts

@@ -0,0 +1,45 @@
+import { StartSdk } from "../StartSdk"
+import { setupManifest } from "../manifest/setupManifest"
+
+export type Manifest = any
+export const sdk = StartSdk.of()
+  .withManifest(
+    setupManifest({
+      id: "testOutput",
+      title: "",
+      version: "1.0",
+      releaseNotes: "",
+      license: "",
+      replaces: [],
+      wrapperRepo: "",
+      upstreamRepo: "",
+      supportSite: "",
+      marketingSite: "",
+      donationUrl: null,
+      description: {
+        short: "",
+        long: "",
+      },
+      containers: {},
+      images: [],
+      volumes: [],
+      assets: [],
+      alerts: {
+        install: null,
+        update: null,
+        uninstall: null,
+        restore: null,
+        start: null,
+        stop: null,
+      },
+      dependencies: {
+        remoteTest: {
+          description: "",
+          requirement: { how: "", type: "opt-in" },
+          version: "1.0",
+        },
+      },
+    }),
+  )
+  .withStore<{ storeRoot: { storeLeaf: "value" } }>()
+  .build(true)

sdk/lib/test/output.test.ts

@@ -0,0 +1,152 @@
+import {
+  UnionSelectKey,
+  unionSelectKey,
+  UnionValueKey,
+  unionValueKey,
+} from "../config/configTypes"
+import { ConfigSpec, matchConfigSpec } from "./output"
+import * as _I from "../index"
+import { camelCase } from "../../scripts/oldSpecToBuilder"
+import { deepMerge } from "../util/deepMerge"
+
+export type IfEquals<T, U, Y = unknown, N = never> =
+  (<G>() => G extends T ? 1 : 2) extends <G>() => G extends U ? 1 : 2 ? Y : N
+export function testOutput<A, B>(): (c: IfEquals<A, B>) => null {
+  return () => null
+}
+
+/// Testing the types of the input spec
+testOutput<ConfigSpec["rpc"]["enable"], boolean>()(null)
+testOutput<ConfigSpec["rpc"]["username"], string>()(null)
+testOutput<ConfigSpec["rpc"]["username"], string>()(null)
+
+testOutput<ConfigSpec["rpc"]["advanced"]["auth"], string[]>()(null)
+testOutput<
+  ConfigSpec["rpc"]["advanced"]["serialversion"],
+  "segwit" | "non-segwit"
+>()(null)
+testOutput<ConfigSpec["rpc"]["advanced"]["servertimeout"], number>()(null)
+testOutput<
+  ConfigSpec["advanced"]["peers"]["addnode"][0]["hostname"],
+  string | null | undefined
+>()(null)
+testOutput<
+  ConfigSpec["testListUnion"][0]["union"][UnionValueKey]["name"],
+  string
+>()(null)
+testOutput<ConfigSpec["testListUnion"][0]["union"][UnionSelectKey], "lnd">()(
+  null,
+)
+testOutput<ConfigSpec["mediasources"], Array<"filebrowser" | "nextcloud">>()(
+  null,
+)
+
+// @ts-expect-error Because enable should be a boolean
+testOutput<ConfigSpec["rpc"]["enable"], string>()(null)
+// prettier-ignore
+// @ts-expect-error Expect that the string is the one above
+testOutput<ConfigSpec["testListUnion"][0][UnionSelectKey][UnionSelectKey], "unionSelectKey">()(null);
+
+/// Here we test the output of the matchConfigSpec function
+describe("Inputs", () => {
+  const validInput: ConfigSpec = {
+    mediasources: ["filebrowser"],
+    testListUnion: [
+      {
+        union: { [unionSelectKey]: "lnd", [unionValueKey]: { name: "string" } },
+      },
+    ],
+    rpc: {
+      enable: true,
+      bio: "This is a bio",
+      username: "test",
+      password: "test",
+      advanced: {
+        auth: ["test"],
+        serialversion: "segwit",
+        servertimeout: 6,
+        threads: 3,
+        workqueue: 9,
+      },
+    },
+    "zmq-enabled": false,
+    txindex: false,
+    wallet: { enable: false, avoidpartialspends: false, discardfee: 0.0001 },
+    advanced: {
+      mempool: {
+        maxmempool: 1,
+        persistmempool: true,
+        mempoolexpiry: 23,
+        mempoolfullrbf: true,
+      },
+      peers: {
+        listen: true,
+        onlyconnect: true,
+        onlyonion: true,
+        addnode: [
+          {
+            hostname: "test",
+            port: 1,
+          },
+        ],
+      },
+      dbcache: 5,
+      pruning: {
+        unionSelectKey: "disabled",
+        unionValueKey: {},
+      },
+      blockfilters: {
+        blockfilterindex: false,
+        peerblockfilters: false,
+      },
+      bloomfilters: { peerbloomfilters: false },
+    },
+  }
+
+  test("test valid input", () => {
+    const output = matchConfigSpec.unsafeCast(validInput)
+    expect(output).toEqual(validInput)
+  })
+  test("test no longer care about the conversion of min/max and validating", () => {
+    matchConfigSpec.unsafeCast(
+      deepMerge({}, validInput, { rpc: { advanced: { threads: 0 } } }),
+    )
+  })
+  test("test errors should throw for number in string", () => {
+    expect(() =>
+      matchConfigSpec.unsafeCast(
+        deepMerge({}, validInput, { rpc: { enable: 2 } }),
+      ),
+    ).toThrowError()
+  })
+  test("Test that we set serialversion to something not segwit or non-segwit", () => {
+    expect(() =>
+      matchConfigSpec.unsafeCast(
+        deepMerge({}, validInput, {
+          rpc: { advanced: { serialversion: "testing" } },
+        }),
+      ),
+    ).toThrowError()
+  })
+})
+
+describe("camelCase", () => {
+  test("'EquipmentClass name'", () => {
+    expect(camelCase("EquipmentClass name")).toEqual("equipmentClassName")
+  })
+  test("'Equipment className'", () => {
+    expect(camelCase("Equipment className")).toEqual("equipmentClassName")
+  })
+  test("'equipment class name'", () => {
+    expect(camelCase("equipment class name")).toEqual("equipmentClassName")
+  })
+  test("'Equipment Class Name'", () => {
+    expect(camelCase("Equipment Class Name")).toEqual("equipmentClassName")
+  })
+  test("'hyphen-name-format'", () => {
+    expect(camelCase("hyphen-name-format")).toEqual("hyphenNameFormat")
+  })
+  test("'underscore_name_format'", () => {
+    expect(camelCase("underscore_name_format")).toEqual("underscoreNameFormat")
+  })
+})

sdk/lib/test/setupDependencyConfig.test.ts

@@ -0,0 +1,27 @@
+import { sdk } from "./output.sdk"
+
+describe("setupDependencyConfig", () => {
+  test("test", () => {
+    const testConfig = sdk.Config.of({
+      test: sdk.Value.text({
+        name: "testValue",
+        required: false,
+      }),
+    })
+
+    const testConfig2 = sdk.Config.of({
+      test2: sdk.Value.text({
+        name: "testValue2",
+        required: false,
+      }),
+    })
+    const remoteTest = sdk.DependencyConfig.of({
+      localConfig: testConfig,
+      remoteConfig: testConfig2,
+      dependencyConfig: async ({}) => {},
+    })
+    sdk.setupDependencyConfig(testConfig, {
+      remoteTest,
+    })
+  })
+})

sdk/lib/test/startosTypeValidation.test.ts

@@ -0,0 +1,75 @@
+import { Effects } from "../types"
+import { ExecuteAction } from "../../../core/startos/bindings/ExecuteAction"
+import { CreateOverlayedImageParams } from "../../../core/startos/bindings/CreateOverlayedImageParams"
+import { DestroyOverlayedImageParams } from "../../../core/startos/bindings/DestroyOverlayedImageParams"
+import { BindParams } from "../../../core/startos/bindings/BindParams"
+import { GetHostInfoParams } from "../../../core/startos/bindings/GetHostInfoParams"
+import { ParamsPackageId } from "../../../core/startos/bindings/ParamsPackageId"
+import { ParamsMaybePackageId } from "../../../core/startos/bindings/ParamsMaybePackageId"
+import { SetConfigured } from "../../../core/startos/bindings/SetConfigured"
+import { SetHealth } from "../../../core/startos/bindings/SetHealth"
+import { ExposeForDependentsParams } from "../../../core/startos/bindings/ExposeForDependentsParams"
+import { ExposeUiParams } from "../../../core/startos/bindings/ExposeUiParams"
+import { GetSslCertificateParams } from "../../../core/startos/bindings/GetSslCertificateParams"
+import { GetSslKeyParams } from "../../../core/startos/bindings/GetSslKeyParams"
+import { GetServiceInterfaceParams } from "../../../core/startos/bindings/GetServiceInterfaceParams"
+import { SetDependenciesParams } from "../../../core/startos/bindings/SetDependenciesParams"
+import { GetSystemSmtpParams } from "../../../core/startos/bindings/GetSystemSmtpParams"
+import { GetServicePortForwardParams } from "../../../core/startos/bindings/GetServicePortForwardParams"
+import { ExportServiceInterfaceParams } from "../../../core/startos/bindings/ExportServiceInterfaceParams"
+import { GetPrimaryUrlParams } from "../../../core/startos/bindings/GetPrimaryUrlParams"
+import { ListServiceInterfacesParams } from "../../../core/startos/bindings/ListServiceInterfacesParams"
+import { RemoveAddressParams } from "../../../core/startos/bindings/RemoveAddressParams"
+import { ExportActionParams } from "../../../core/startos/bindings/ExportActionParams"
+import { RemoveActionParams } from "../../../core/startos/bindings/RemoveActionParams"
+import { ReverseProxyParams } from "../../../core/startos/bindings/ReverseProxyParams"
+import { MountParams } from "../../../core/startos/bindings/MountParams"
+import { ExposedUI } from "../../../core/startos/bindings/ExposedUI"
+function typeEquality<ExpectedType>(_a: ExpectedType) {}
+describe("startosTypeValidation ", () => {
+  test(`checking the params match`, () => {
+    const testInput: any = {}
+    typeEquality<{
+      [K in keyof Effects]: Effects[K] extends (args: infer A) => any
+        ? A
+        : never
+    }>({
+      executeAction: {} as ExecuteAction,
+      createOverlayedImage: {} as CreateOverlayedImageParams,
+      destroyOverlayedImage: {} as DestroyOverlayedImageParams,
+      clearBindings: undefined,
+      bind: {} as BindParams,
+      getHostInfo: {} as GetHostInfoParams,
+      exists: {} as ParamsPackageId,
+      getConfigured: undefined,
+      stopped: {} as ParamsMaybePackageId,
+      running: {} as ParamsPackageId,
+      restart: undefined,
+      shutdown: undefined,
+      setConfigured: {} as SetConfigured,
+      setHealth: {} as SetHealth,
+      exposeForDependents: {} as ExposeForDependentsParams,
+      exposeUi: {} as { [key: string]: ExposedUI },
+      getSslCertificate: {} as GetSslCertificateParams,
+      getSslKey: {} as GetSslKeyParams,
+      getServiceInterface: {} as GetServiceInterfaceParams,
+      setDependencies: {} as SetDependenciesParams,
+      store: {} as never,
+      getSystemSmtp: {} as GetSystemSmtpParams,
+      getContainerIp: undefined,
+      getServicePortForward: {} as GetServicePortForwardParams,
+      clearServiceInterfaces: undefined,
+      exportServiceInterface: {} as ExportServiceInterfaceParams,
+      getPrimaryUrl: {} as GetPrimaryUrlParams,
+      listServiceInterfaces: {} as ListServiceInterfacesParams,
+      removeAddress: {} as RemoveAddressParams,
+      exportAction: {} as ExportActionParams,
+      removeAction: {} as RemoveActionParams,
+      reverseProxy: {} as ReverseProxyParams,
+      mount: {} as MountParams,
+    })
+    typeEquality<Parameters<Effects["executeAction"]>[0]>(
+      testInput as ExecuteAction,
+    )
+  })
+})

sdk/lib/test/store.test.ts

@@ -0,0 +1,117 @@
+import { MainEffects, StartSdk } from "../StartSdk"
+import { Effects } from "../types"
+
+type Store = {
+  config: {
+    someValue: "a" | "b"
+  }
+}
+type Manifest = any
+const todo = <A>(): A => {
+  throw new Error("not implemented")
+}
+const noop = () => {}
+
+const sdk = StartSdk.of()
+  .withManifest({} as Manifest)
+  .withStore<Store>()
+  .build(true)
+
+describe("Store", () => {
+  test("types", async () => {
+    ;async () => {
+      sdk.store.setOwn(todo<Effects>(), "/config", {
+        someValue: "a",
+      })
+      sdk.store.setOwn(todo<Effects>(), "/config/someValue", "b")
+      sdk.store.setOwn(todo<Effects>(), "", {
+        config: { someValue: "b" },
+      })
+      sdk.store.setOwn(
+        todo<Effects>(),
+        "/config/someValue",
+
+        // @ts-expect-error Type is wrong for the setting value
+        5,
+      )
+      sdk.store.setOwn(
+        todo<Effects>(),
+        // @ts-expect-error Path is wrong
+        "/config/someVae3lue",
+        "someValue",
+      )
+
+      todo<Effects>().store.set<Store, "/config/someValue">({
+        path: "/config/someValue",
+        value: "b",
+      })
+      todo<Effects>().store.set<Store, "/config/some2Value">({
+        //@ts-expect-error Path is wrong
+        path: "/config/someValue",
+        //@ts-expect-error Path is wrong
+        value: "someValueIn",
+      })
+      todo<Effects>().store.set<Store, "/config/someValue">({
+        //@ts-expect-error Path is wrong
+        path: "/config/some2Value",
+        value: "a",
+      })
+      ;(await sdk.store
+        .getOwn(todo<MainEffects>(), "/config/someValue")
+        .const()) satisfies string
+      ;(await sdk.store
+        .getOwn(todo<MainEffects>(), "/config")
+        .const()) satisfies Store["config"]
+      await sdk.store // @ts-expect-error Path is wrong
+        .getOwn(todo<MainEffects>(), "/config/somdsfeValue")
+        .const()
+      ///  ----------------- ERRORS -----------------
+
+      sdk.store.setOwn(todo<MainEffects>(), "", {
+        // @ts-expect-error Type is wrong for the setting value
+        config: { someValue: "notInAOrB" },
+      })
+      sdk.store.setOwn(
+        todo<MainEffects>(),
+        "/config/someValue",
+        // @ts-expect-error Type is wrong for the setting value
+        "notInAOrB",
+      )
+      ;(await sdk.store
+        .getOwn(todo<Effects>(), "/config/someValue")
+        // @ts-expect-error Const should normally not be callable
+        .const()) satisfies string
+      ;(await sdk.store
+        .getOwn(todo<Effects>(), "/config")
+        // @ts-expect-error Const should normally not be callable
+        .const()) satisfies Store["config"]
+      await sdk.store // @ts-expect-error Path is wrong
+        .getOwn("/config/somdsfeValue")
+        // @ts-expect-error Const should normally not be callable
+        .const()
+
+      ///
+      ;(await sdk.store
+        .getOwn(todo<MainEffects>(), "/config/someValue")
+        // @ts-expect-error satisfies type is wrong
+        .const()) satisfies number
+      ;(await sdk.store // @ts-expect-error Path is wrong
+        .getOwn(todo<MainEffects>(), "/config/")
+        .const()) satisfies Store["config"]
+      ;(await todo<Effects>().store.get<Store, "/config/someValue">({
+        path: "/config/someValue",
+        callback: noop,
+      })) satisfies string
+      await todo<Effects>().store.get<Store, "/config/someValue">({
+        // @ts-expect-error Path is wrong as in it doesn't match above
+        path: "/config/someV2alue",
+        callback: noop,
+      })
+      await todo<Effects>().store.get<Store, "/config/someV2alue">({
+        // @ts-expect-error Path is wrong as in it doesn't exists in wrapper type
+        path: "/config/someV2alue",
+        callback: noop,
+      })
+    }
+  })
+})

sdk/lib/test/util.deepMerge.test.ts

@@ -0,0 +1,26 @@
+import { deepEqual } from "../util/deepEqual"
+import { deepMerge } from "../util/deepMerge"
+
+describe("deepMerge", () => {
+  test("deepMerge({}, {a: 1}, {b: 2}) should return {a: 1, b: 2}", () => {
+    expect(deepMerge({}, { a: 1 }, { b: 2 })).toEqual({ a: 1, b: 2 })
+  })
+  test("deepMerge(null, [1,2,3]) should equal [1,2,3]", () => {
+    expect(deepMerge(null, [1, 2, 3])).toEqual([1, 2, 3])
+  })
+  test("deepMerge({a: {b: 1, c:2}}, {a: {b: 3}}) should equal {a: {b: 3, c: 2}}", () => {
+    expect(deepMerge({ a: { b: 1, c: 2 } }, { a: { b: 3 } })).toEqual({
+      a: { b: 3, c: 2 },
+    })
+  })
+  test("deepMerge({a: {b: 1, c:2}}, {a: {b: 3}}) should equal {a: {b: 3, c: 2}} with deep equal", () => {
+    expect(
+      deepEqual(deepMerge({ a: { b: 1, c: 2 } }, { a: { b: 3 } }), {
+        a: { b: 3, c: 2 },
+      }),
+    ).toBeTruthy()
+  })
+  test("deepMerge([1,2,3], [2,3,4]) should equal [2,3,4]", () => {
+    expect(deepMerge([1, 2, 3], [2, 3, 4])).toEqual([2, 3, 4])
+  })
+})

sdk/lib/test/util.getNetworkInterface.test.ts

@@ -0,0 +1,20 @@
+import { getHostname } from "../util/getServiceInterface"
+
+describe("getHostname ", () => {
+  const inputToExpected = [
+    ["http://localhost:3000", "localhost"],
+    ["http://localhost", "localhost"],
+    ["localhost", "localhost"],
+    ["http://127.0.0.1/", "127.0.0.1"],
+    ["http://127.0.0.1/testing/1234?314345", "127.0.0.1"],
+    ["127.0.0.1/", "127.0.0.1"],
+    ["http://mail.google.com/", "mail.google.com"],
+    ["mail.google.com/", "mail.google.com"],
+  ]
+
+  for (const [input, expectValue] of inputToExpected) {
+    test(`should return ${expectValue} for ${input}`, () => {
+      expect(getHostname(input)).toEqual(expectValue)
+    })
+  }
+})

sdk/lib/test/utils.splitCommand.test.ts

@@ -0,0 +1,42 @@
+import { getHostname } from "../util/getServiceInterface"
+import { splitCommand } from "../util/splitCommand"
+
+describe("splitCommand ", () => {
+  const inputToExpected = [
+    ["cat", ["cat"]],
+    [["cat"], ["cat"]],
+    [
+      ["cat", "hello all my homies"],
+      ["cat", "hello all my homies"],
+    ],
+    ["cat hello world", ["cat", "hello", "world"]],
+    ["cat hello 'big world'", ["cat", "hello", "big world"]],
+    [`cat hello "big world"`, ["cat", "hello", "big world"]],
+    [
+      `cat hello "big world's are the greatest"`,
+      ["cat", "hello", "big world's are the greatest"],
+    ],
+    // Too many spaces
+    ["cat      ", ["cat"]],
+    [["cat   "], ["cat   "]],
+    [
+      ["cat   ", "hello all my homies    "],
+      ["cat   ", "hello all my homies    "],
+    ],
+    ["cat        hello   world ", ["cat", "hello", "world"]],
+    [
+      "        cat      hello     'big  world'       ",
+      ["cat", "hello", "big  world"],
+    ],
+    [
+      `     cat     hello    "big   world"       `,
+      ["cat", "hello", "big   world"],
+    ],
+  ]
+
+  for (const [input, expectValue] of inputToExpected) {
+    test(`should return ${expectValue} for ${input}`, () => {
+      expect(splitCommand(input as any)).toEqual(expectValue)
+    })
+  }
+})

sdk/lib/trigger/TriggerInput.ts

@@ -0,0 +1,6 @@
+import { HealthStatus } from "../types"
+
+export type TriggerInput = {
+  lastResult?: HealthStatus
+  hadSuccess?: boolean
+}

sdk/lib/trigger/changeOnFirstSuccess.ts

@@ -0,0 +1,30 @@
+import { Trigger } from "./index"
+
+export function changeOnFirstSuccess(o: {
+  beforeFirstSuccess: Trigger
+  afterFirstSuccess: Trigger
+}): Trigger {
+  return async function* (getInput) {
+    const beforeFirstSuccess = o.beforeFirstSuccess(getInput)
+    yield
+    let currentValue = getInput()
+    beforeFirstSuccess.next()
+    for (
+      let res = await beforeFirstSuccess.next();
+      currentValue?.lastResult !== "passing" && !res.done;
+      res = await beforeFirstSuccess.next()
+    ) {
+      yield
+      currentValue = getInput()
+    }
+    const afterFirstSuccess = o.afterFirstSuccess(getInput)
+    for (
+      let res = await afterFirstSuccess.next();
+      !res.done;
+      res = await afterFirstSuccess.next()
+    ) {
+      yield
+      currentValue = getInput()
+    }
+  }
+}

sdk/lib/trigger/cooldownTrigger.ts

@@ -0,0 +1,8 @@
+export function cooldownTrigger(timeMs: number) {
+  return async function* () {
+    while (true) {
+      await new Promise((resolve) => setTimeout(resolve, timeMs))
+      yield
+    }
+  }
+}

sdk/lib/trigger/defaultTrigger.ts

@@ -0,0 +1,8 @@
+import { cooldownTrigger } from "./cooldownTrigger"
+import { changeOnFirstSuccess } from "./changeOnFirstSuccess"
+import { successFailure } from "./successFailure"
+
+export const defaultTrigger = successFailure({
+  duringSuccess: cooldownTrigger(0),
+  duringError: cooldownTrigger(30000),
+})

sdk/lib/trigger/index.ts

@@ -0,0 +1,7 @@
+import { TriggerInput } from "./TriggerInput"
+export { changeOnFirstSuccess } from "./changeOnFirstSuccess"
+export { cooldownTrigger } from "./cooldownTrigger"
+
+export type Trigger = (
+  getInput: () => TriggerInput,
+) => AsyncIterator<unknown, unknown, never>

sdk/lib/trigger/successFailure.ts

@@ -0,0 +1,32 @@
+import { Trigger } from "."
+
+export function successFailure(o: {
+  duringSuccess: Trigger
+  duringError: Trigger
+}): Trigger {
+  return async function* (getInput) {
+    while (true) {
+      const beforeSuccess = o.duringSuccess(getInput)
+      yield
+      let currentValue = getInput()
+      beforeSuccess.next()
+      for (
+        let res = await beforeSuccess.next();
+        currentValue?.lastResult !== "passing" && !res.done;
+        res = await beforeSuccess.next()
+      ) {
+        yield
+        currentValue = getInput()
+      }
+      const duringError = o.duringError(getInput)
+      for (
+        let res = await duringError.next();
+        currentValue?.lastResult === "passing" && !res.done;
+        res = await duringError.next()
+      ) {
+        yield
+        currentValue = getInput()
+      }
+    }
+  }
+}

sdk/lib/types.ts

@@ -0,0 +1,607 @@
+export * as configTypes from "./config/configTypes"
+import { AddSslOptions } from "../../core/startos/bindings/AddSslOptions"
+import { MainEffects, ServiceInterfaceType, Signals } from "./StartSdk"
+import { InputSpec } from "./config/configTypes"
+import { DependenciesReceipt } from "./config/setupConfig"
+import { BindOptions, Scheme } from "./interfaces/Host"
+import { Daemons } from "./mainFn/Daemons"
+import { UrlString } from "./util/getServiceInterface"
+
+export type ExportedAction = (options: {
+  effects: Effects
+  input?: Record<string, unknown>
+}) => Promise<ActionResult>
+export type MaybePromise<A> = Promise<A> | A
+export namespace ExpectedExports {
+  version: 1
+  /** Set configuration is called after we have modified and saved the configuration in the start9 ui. Use this to make a file for the docker to read from for configuration.  */
+  export type setConfig = (options: {
+    effects: Effects
+    input: Record<string, unknown>
+  }) => Promise<void>
+  /** Get configuration returns a shape that describes the format that the start9 ui will generate, and later send to the set config  */
+  export type getConfig = (options: { effects: Effects }) => Promise<ConfigRes>
+  // /** These are how we make sure the our dependency configurations are valid and if not how to fix them. */
+  // export type dependencies = Dependencies;
+  /** For backing up service data though the startOS UI */
+  export type createBackup = (options: { effects: Effects }) => Promise<unknown>
+  /** For restoring service data that was previously backed up using the startOS UI create backup flow. Backup restores are also triggered via the startOS UI, or doing a system restore flow during setup. */
+  export type restoreBackup = (options: {
+    effects: Effects
+  }) => Promise<unknown>
+
+  // /** Health checks are used to determine if the service is working properly after starting
+  //  * A good use case is if we are using a web server, seeing if we can get to the web server.
+  //  */
+  // export type health = {
+  //   /** Should be the health check id */
+  //   [id: string]: (options: { effects: Effects; input: TimeMs }) => Promise<unknown>;
+  // };
+
+  /**
+   * Actions are used so we can effect the service, like deleting a directory.
+   * One old use case is to add a action where we add a file, that will then be run during the
+   * service starting, and that file would indicate that it would rescan all the data.
+   */
+  export type actions = (options: { effects: Effects }) => MaybePromise<{
+    [id: string]: {
+      run: ExportedAction
+      getConfig: (options: { effects: Effects }) => Promise<InputSpec>
+    }
+  }>
+
+  export type actionsMetadata = (options: {
+    effects: Effects
+  }) => Promise<Array<ActionMetadata>>
+
+  /**
+   * This is the entrypoint for the main container. Used to start up something like the service that the
+   * package represents, like running a bitcoind in a bitcoind-wrapper.
+   */
+  export type main = (options: {
+    effects: MainEffects
+    started(onTerm: () => PromiseLike<void>): PromiseLike<void>
+  }) => Promise<Daemons<any, any>>
+
+  /**
+   * After a shutdown, if we wanted to do any operations to clean up things, like
+   * set the action as unavailable or something.
+   */
+  export type afterShutdown = (options: {
+    effects: Effects
+  }) => Promise<unknown>
+
+  /**
+   * Every time a package completes an install, this function is called before the main.
+   * Can be used to do migration like things.
+   */
+  export type init = (options: {
+    effects: Effects
+    previousVersion: null | string
+  }) => Promise<unknown>
+  /** This will be ran during any time a package is uninstalled, for example during a update
+   * this will be called.
+   */
+  export type uninit = (options: {
+    effects: Effects
+    nextVersion: null | string
+  }) => Promise<unknown>
+
+  /** Auto configure is used to make sure that other dependencies have the values t
+   * that this service could use.
+   */
+  export type dependencyConfig = Record<PackageId, DependencyConfig>
+}
+export type TimeMs = number
+export type VersionString = string
+
+/**
+ * AutoConfigure is used as the value to the key of package id,
+ * this is used to make sure that other dependencies have the values that this service could use.
+ */
+export type DependencyConfig = {
+  /** During autoconfigure, we have access to effects and local data. We are going to figure out all the data that we need and send it to update. For the sdk it is the desired delta */
+  query(options: { effects: Effects; localConfig: unknown }): Promise<unknown>
+  /** This is the second part. Given the query results off the previous function, we will determine what to change the remote config to. In our sdk normall we are going to use the previous as a deep merge. */
+  update(options: {
+    queryResults: unknown
+    remoteConfig: unknown
+  }): Promise<unknown>
+}
+
+export type ValidIfNoStupidEscape<A> = A extends
+  | `${string}'"'"'${string}`
+  | `${string}\\"${string}`
+  ? never
+  : "" extends A & ""
+    ? never
+    : A
+
+export type ConfigRes = {
+  /** This should be the previous config, that way during set config we start with the previous */
+  config?: null | Record<string, unknown>
+  /** Shape that is describing the form in the ui */
+  spec: InputSpec
+}
+
+declare const DaemonProof: unique symbol
+export type DaemonReceipt = {
+  [DaemonProof]: never
+}
+export type Daemon = {
+  wait(): Promise<string>
+  term(): Promise<void>
+  [DaemonProof]: never
+}
+
+export type HealthStatus =
+  | `passing`
+  | `disabled`
+  | `starting`
+  | `warning`
+  | `failure`
+
+export type SmtpValue = {
+  server: string
+  port: number
+  from: string
+  login: string
+  password: string | null | undefined
+}
+
+export type CommandType<A extends string> =
+  | ValidIfNoStupidEscape<A>
+  | [string, ...string[]]
+
+export type DaemonReturned = {
+  wait(): Promise<null>
+  term(options?: { signal?: Signals; timeout?: number }): Promise<void>
+}
+
+export type ActionMetadata = {
+  name: string
+  description: string
+  id: string
+  input: InputSpec
+  allowedStatuses: "only-running" | "only-stopped" | "any" | "disabled"
+  /**
+   * So the ordering of the actions is by alphabetical order of the group, then followed by the alphabetical of the actions
+   */
+  group: string | null
+}
+export declare const hostName: unique symbol
+// asdflkjadsf.onion | 1.2.3.4
+export type Hostname = string & { [hostName]: never }
+
+/** ${scheme}://${username}@${host}:${externalPort}${suffix} */
+export type AddressInfo = {
+  username: string | null
+  hostId: string
+  bindOptions: BindOptions
+  suffix: string
+}
+
+export type HostnameInfoIp = {
+  kind: "ip"
+  networkInterfaceId: string
+  public: boolean
+  hostname:
+    | {
+        kind: "ipv4" | "ipv6" | "local"
+        value: string
+        port: number | null
+        sslPort: number | null
+      }
+    | {
+        kind: "domain"
+        domain: string
+        subdomain: string | null
+        port: number | null
+        sslPort: number | null
+      }
+}
+
+export type HostnameInfoOnion = {
+  kind: "onion"
+  hostname: { value: string; port: number | null; sslPort: number | null }
+}
+
+export type HostnameInfo = HostnameInfoIp | HostnameInfoOnion
+
+export type SingleHost = {
+  id: string
+  kind: "single" | "static"
+  hostname: HostnameInfo | null
+}
+
+export type MultiHost = {
+  id: string
+  kind: "multi"
+  hostnames: HostnameInfo[]
+}
+
+export type HostInfo = SingleHost | MultiHost
+
+export type ServiceInterfaceId = string
+
+export type ServiceInterface = {
+  id: ServiceInterfaceId
+  /** The title of this field to be displayed */
+  name: string
+  /** Human readable description, used as tooltip usually */
+  description: string
+  /** Whether or not one address must be the primary address */
+  hasPrimary: boolean
+  /** Disabled interfaces do not serve, but they retain their metadata and addresses */
+  disabled: boolean
+  /** Whether or not to mask the URIs for this interface. Useful if the URIs contain sensitive information, such as a password, macaroon, or API key */
+  masked: boolean
+  /** URI Information */
+  addressInfo: AddressInfo
+  /** The network interface could be several types, something like ui, p2p, or network */
+  type: ServiceInterfaceType
+}
+
+export type ServiceInterfaceWithHostInfo = ServiceInterface & {
+  hostInfo: HostInfo
+}
+
+// prettier-ignore
+export type ExposeAllServicePaths<Store, PreviousPath extends string = ""> = 
+  Store extends never ? string :
+  Store extends Record<string, unknown> ? {[K in keyof Store & string]: ExposeAllServicePaths<Store[K], `${PreviousPath}/${K & string}`>}[keyof Store & string] :
+  PreviousPath
+// prettier-ignore
+export type ExposeAllUiPaths<Store, PreviousPath extends string = ""> = 
+  Store extends Record<string, unknown> ? {[K in keyof Store & string]: ExposeAllUiPaths<Store[K], `${PreviousPath}/${K & string}`>}[keyof Store & string] :
+  Store extends string ? PreviousPath : 
+  never
+export type ExposeServicePaths<Store = never> = {
+  /** The path to the value in the Store. [JsonPath](https://jsonpath.com/)  */
+  paths: Store extends never ? string[] : ExposeAllServicePaths<Store>[]
+}
+
+export type ExposeUiPaths<Store> =
+  | {
+      type: "object"
+      value: { [k: string]: ExposeUiPaths<Store> }
+      description?: string
+    }
+  | {
+      type: "string"
+      /** The path to the value in the Store. [JsonPath](https://jsonpath.com/)  */
+      path: ExposeAllUiPaths<Store>
+      /** A human readable description or explanation of the value */
+      description?: string
+      /** (string/number only) Whether or not to mask the value, for example, when displaying a password */
+      masked: boolean
+      /** (string/number only) Whether or not to include a button for copying the value to clipboard */
+      copyable?: boolean
+      /** (string/number only) Whether or not to include a button for displaying the value as a QR code */
+      qr?: boolean
+    }
+export type ExposeUiPathsAll =
+  | {
+      type: "object"
+      value: { [k: string]: ExposeUiPathsAll }
+      description: string | null
+    }
+  | {
+      type: "string"
+      /** The path to the value in the Store. [JsonPath](https://jsonpath.com/)  */
+      path: string
+      /** A human readable description or explanation of the value */
+      description: string | null
+      /** (string/number only) Whether or not to mask the value, for example, when displaying a password */
+      masked: boolean
+      /** (string/number only) Whether or not to include a button for copying the value to clipboard */
+      copyable: boolean | null
+      /** (string/number only) Whether or not to include a button for displaying the value as a QR code */
+      qr: boolean | null
+    }
+
+/** Used to reach out from the pure js runtime */
+export type Effects = {
+  executeAction<Input>(opts: {
+    serviceId: string | null
+    input: Input
+  }): Promise<unknown>
+
+  /** A low level api used by makeOverlay */
+  createOverlayedImage(options: { imageId: string }): Promise<[string, string]>
+
+  /** A low level api used by destroyOverlay + makeOverlay:destroy */
+  destroyOverlayedImage(options: { guid: string }): Promise<void>
+
+  /** Removes all network bindings */
+  clearBindings(): Promise<void>
+  /** Creates a host connected to the specified port with the provided options */
+  bind(options: {
+    kind: "static" | "single" | "multi"
+    id: string
+    internalPort: number
+
+    scheme: Scheme
+    preferredExternalPort: number
+    addSsl: AddSslOptions | null
+    secure: { ssl: boolean } | null
+  }): Promise<void>
+  /** Retrieves the current hostname(s) associated with a host id */
+  // getHostInfo(options: {
+  //   kind: "static" | "single"
+  //   serviceInterfaceId: string
+  //   packageId: string | null
+  //   callback: () => void
+  // }): Promise<SingleHost>
+  getHostInfo(options: {
+    kind: "multi" | null
+    serviceInterfaceId: string
+    packageId: string | null
+    callback: () => void
+  }): Promise<MultiHost>
+
+  // /**
+  //  * Run rsync between two volumes. This is used to backup data between volumes.
+  //  * This is a long running process, and a structure that we can either wait for, or get the progress of.
+  //  */
+  // runRsync(options: {
+  //   srcVolume: string
+  //   dstVolume: string
+  //   srcPath: string
+  //   dstPath: string
+  //   // rsync options: https://linux.die.net/man/1/rsync
+  //   options: BackupOptions
+  // }): {
+  //   id: () => Promise<string>
+  //   wait: () => Promise<null>
+  //   progress: () => Promise<number>
+  // }
+
+  store: {
+    /** Get a value in a json like data, can be observed and subscribed */
+    get<Store = never, Path extends string = never>(options: {
+      /** If there is no packageId it is assumed the current package */
+      packageId?: string
+      /** The path defaults to root level, using the [JsonPath](https://jsonpath.com/) */
+      path: Path & EnsureStorePath<Store, Path>
+      callback: (config: unknown, previousConfig: unknown) => void
+    }): Promise<ExtractStore<Store, Path>>
+    /** Used to store values that can be accessed and subscribed to */
+    set<Store = never, Path extends string = never>(options: {
+      /** Sets the value for the wrapper at the path, it will override, using the [JsonPath](https://jsonpath.com/)  */
+      path: Path & EnsureStorePath<Store, Path>
+      value: ExtractStore<Store, Path>
+    }): Promise<void>
+  }
+
+  getSystemSmtp(input: {
+    callback: (config: unknown, previousConfig: unknown) => void
+  }): Promise<SmtpValue>
+
+  /** Get the IP address of the container */
+  getContainerIp(): Promise<string>
+  /**
+   * Get the port address for another service
+   */
+  getServicePortForward(options: {
+    internalPort: number
+    packageId: string | null
+  }): Promise<number>
+
+  /** Removes all network interfaces */
+  clearServiceInterfaces(): Promise<void>
+  /** When we want to create a link in the front end interfaces, and example is
+   * exposing a url to view a web service
+   */
+  exportServiceInterface(options: ServiceInterface): Promise<string>
+
+  exposeForDependents(options: { paths: string[] }): Promise<void>
+
+  exposeUi(options: { [key: string]: ExposeUiPathsAll }): Promise<void>
+  /**
+   * There are times that we want to see the addresses that where exported
+   * @param options.addressId If we want to filter the address id
+   *
+   * Note: any auth should be filtered out already
+   */
+  getServiceInterface(options: {
+    packageId: PackageId | null
+    serviceInterfaceId: ServiceInterfaceId
+    callback: () => void
+  }): Promise<ServiceInterface>
+
+  /**
+   * The user sets the primary url for a interface
+   * @param options
+   */
+  getPrimaryUrl(options: {
+    packageId: PackageId | null
+    serviceInterfaceId: ServiceInterfaceId
+    callback: () => void
+  }): Promise<UrlString | null>
+
+  /**
+   * There are times that we want to see the addresses that where exported
+   * @param options.addressId If we want to filter the address id
+   *
+   * Note: any auth should be filtered out already
+   */
+  listServiceInterfaces(options: {
+    packageId: PackageId | null
+    callback: () => void
+  }): Promise<ServiceInterface[]>
+
+  /**
+   *Remove an address that was exported. Used problably during main or during setConfig.
+   * @param options
+   */
+  removeAddress(options: { id: string }): Promise<void>
+
+  /**
+   *
+   * @param options
+   */
+  exportAction(options: ActionMetadata): Promise<void>
+  /**
+   * Remove an action that was exported. Used problably during main or during setConfig.
+   */
+  removeAction(options: { id: string }): Promise<void>
+
+  getConfigured(): Promise<boolean>
+  /**
+   * This called after a valid set config as well as during init.
+   * @param configured
+   */
+  setConfigured(options: { configured: boolean }): Promise<void>
+
+  /**
+   *
+   * @returns  PEM encoded fullchain (ecdsa)
+   */
+  getSslCertificate: (options: {
+    packageId: string | null
+    hostId: string
+    algorithm: "ecdsa" | "ed25519" | null
+  }) => Promise<[string, string, string]>
+  /**
+   * @returns PEM encoded ssl key (ecdsa)
+   */
+  getSslKey: (options: {
+    packageId: string | null
+    hostId: string
+    algorithm: "ecdsa" | "ed25519" | null
+  }) => Promise<string>
+
+  setHealth(o: {
+    name: string
+    status: HealthStatus
+    message: string | null
+  }): Promise<void>
+
+  /** Set the dependencies of what the service needs, usually ran during the set config as a best practice */
+  setDependencies(options: {
+    dependencies: Dependencies
+  }): Promise<DependenciesReceipt>
+  /** Exists could be useful during the runtime to know if some service exists, option dep */
+  exists(options: { packageId: PackageId }): Promise<boolean>
+  /** Exists could be useful during the runtime to know if some service is running, option dep */
+  running(options: { packageId: PackageId }): Promise<boolean>
+
+  /** Instead of creating proxies with nginx, we have a utility to create and maintain a proxy in the lifetime of this running. */
+  reverseProxy(options: {
+    bind: {
+      /** Optional, default is 0.0.0.0 */
+      ip: string | null
+      port: number
+      ssl: boolean
+    }
+    dst: {
+      /** Optional: default is 127.0.0.1 */
+      ip: string | null // optional, default 127.0.0.1
+      port: number
+      ssl: boolean
+    }
+    http: {
+      // optional, will do TCP layer proxy only if not present
+      headers: Record<string, string> | null
+    } | null
+  }): Promise<{ stop(): Promise<void> }>
+  restart(): void
+  shutdown(): void
+
+  mount(options: {
+    location: string
+    target: {
+      packageId: string
+      volumeId: string
+      subpath: string | null
+      readonly: boolean
+    }
+  }): Promise<string>
+
+  stopped(options: { packageId: string | null }): Promise<boolean>
+}
+
+// prettier-ignore
+export type ExtractStore<Store, Path extends string> = 
+  Path extends `/${infer A }/${infer Rest }` ? (A extends keyof Store ? ExtractStore<Store[A], `/${Rest}`> : never) :
+  Path extends `/${infer A }` ? (A extends keyof Store ? Store[A] : never) :
+  Path extends '' ? Store :
+  never
+
+// prettier-ignore
+type _EnsureStorePath<Store, Path extends string, Origin extends string> = 
+  Path extends`/${infer A }/${infer Rest}` ? (Store extends {[K in A & string]: infer NextStore} ? _EnsureStorePath<NextStore, `/${Rest}`, Origin> : never) :
+  Path extends `/${infer A }`  ? (Store extends {[K in A]: infer B} ? Origin : never) :
+  Path extends '' ? Origin :
+  never
+// prettier-ignore
+export type EnsureStorePath<Store, Path extends string> = _EnsureStorePath<Store, Path, Path>
+
+/** rsync options: https://linux.die.net/man/1/rsync
+ */
+export type BackupOptions = {
+  delete: boolean
+  force: boolean
+  ignoreExisting: boolean
+  exclude: string[]
+}
+/**
+ * This is the metadata that is returned from the metadata call.
+ */
+export type Metadata = {
+  fileType: string
+  isDir: boolean
+  isFile: boolean
+  isSymlink: boolean
+  len: number
+  modified?: Date
+  accessed?: Date
+  created?: Date
+  readonly: boolean
+  uid: number
+  gid: number
+  mode: number
+}
+
+export type MigrationRes = {
+  configured: boolean
+}
+
+export type ActionResult = {
+  message: string
+  value: null | {
+    value: string
+    copyable: boolean
+    qr: boolean
+  }
+}
+export type SetResult = {
+  /** These are the unix process signals */
+  signal: Signals
+  "depends-on": DependsOn
+}
+
+export type PackageId = string
+export type Message = string
+export type DependencyKind = "running" | "exists"
+
+export type DependsOn = {
+  [packageId: string]: string[]
+}
+
+export type KnownError =
+  | { error: string }
+  | {
+      "error-code": [number, string] | readonly [number, string]
+    }
+
+export type Dependency = {
+  id: PackageId
+  kind: DependencyKind
+} & ({ kind: "exists" } | { kind: "running"; healthChecks: string[] })
+export type Dependencies = Array<Dependency>
+
+export type DeepPartial<T> = T extends {}
+  ? { [P in keyof T]?: DeepPartial<T[P]> }
+  : T

sdk/lib/util/GetSystemSmtp.ts

@@ -0,0 +1,37 @@
+import { Effects } from "../types"
+
+export class GetSystemSmtp {
+  constructor(readonly effects: Effects) {}
+
+  /**
+   * Returns the system SMTP credentials. Restarts the service if the credentials change
+   */
+  const() {
+    return this.effects.getSystemSmtp({
+      callback: this.effects.restart,
+    })
+  }
+  /**
+   * Returns the system SMTP credentials. Does nothing if the credentials change
+   */
+  once() {
+    return this.effects.getSystemSmtp({
+      callback: () => {},
+    })
+  }
+  /**
+   * Watches the system SMTP credentials. Takes a custom callback function to run whenever the credentials change
+   */
+  async *watch() {
+    while (true) {
+      let callback: () => void
+      const waitForNext = new Promise<void>((resolve) => {
+        callback = resolve
+      })
+      yield await this.effects.getSystemSmtp({
+        callback: () => callback(),
+      })
+      await waitForNext
+    }
+  }
+}

sdk/lib/util/Overlay.ts

@@ -0,0 +1,169 @@
+import * as fs from "fs/promises"
+import * as T from "../types"
+import * as cp from "child_process"
+import { promisify } from "util"
+import { Buffer } from "node:buffer"
+export const execFile = promisify(cp.execFile)
+const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/`
+export class Overlay {
+  private constructor(
+    readonly effects: T.Effects,
+    readonly imageId: string,
+    readonly rootfs: string,
+    readonly guid: string,
+  ) {}
+  static async of(effects: T.Effects, imageId: string) {
+    const [rootfs, guid] = await effects.createOverlayedImage({ imageId })
+
+    for (const dirPart of ["dev", "sys", "proc", "run"] as const) {
+      await fs.mkdir(`${rootfs}/${dirPart}`, { recursive: true })
+      await execFile("mount", [
+        "--rbind",
+        `/${dirPart}`,
+        `${rootfs}/${dirPart}`,
+      ])
+    }
+
+    return new Overlay(effects, imageId, rootfs, guid)
+  }
+
+  async mount(options: MountOptions, path: string): Promise<Overlay> {
+    path = path.startsWith("/")
+      ? `${this.rootfs}${path}`
+      : `${this.rootfs}/${path}`
+    if (options.type === "volume") {
+      const subpath = options.subpath
+        ? options.subpath.startsWith("/")
+          ? options.subpath
+          : `/${options.subpath}`
+        : "/"
+      await execFile("mount", [
+        "--bind",
+        `/media/startos/volumes/${options.id}${subpath}`,
+        path,
+      ])
+    } else if (options.type === "assets") {
+      const subpath = options.subpath
+        ? options.subpath.startsWith("/")
+          ? options.subpath
+          : `/${options.subpath}`
+        : "/"
+      await execFile("mount", [
+        "--bind",
+        `/media/startos/assets/${options.id}${subpath}`,
+        path,
+      ])
+    } else if (options.type === "pointer") {
+      await this.effects.mount({ location: path, target: options })
+    } else {
+      throw new Error(`unknown type ${(options as any).type}`)
+    }
+    return this
+  }
+
+  async destroy() {
+    const imageId = this.imageId
+    const guid = this.guid
+    await this.effects.destroyOverlayedImage({ guid })
+  }
+
+  async exec(
+    command: string[],
+    options?: CommandOptions,
+  ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> {
+    const imageMeta = await fs
+      .readFile(`/media/startos/images/${this.imageId}.json`, {
+        encoding: "utf8",
+      })
+      .catch(() => "{}")
+      .then(JSON.parse)
+    let extra: string[] = []
+    if (options?.user) {
+      extra.push(`--user=${options.user}`)
+      delete options.user
+    }
+    let workdir = imageMeta.workdir || "/"
+    if (options?.cwd) {
+      workdir = options.cwd
+      delete options.cwd
+    }
+    return await execFile(
+      "start-cli",
+      [
+        "chroot",
+        `--env=/media/startos/images/${this.imageId}.env`,
+        `--workdir=${workdir}`,
+        ...extra,
+        this.rootfs,
+        ...command,
+      ],
+      options,
+    )
+  }
+
+  async spawn(
+    command: string[],
+    options?: CommandOptions,
+  ): Promise<cp.ChildProcessWithoutNullStreams> {
+    const imageMeta = await fs
+      .readFile(`/media/startos/images/${this.imageId}.json`, {
+        encoding: "utf8",
+      })
+      .catch(() => "{}")
+      .then(JSON.parse)
+    let extra: string[] = []
+    if (options?.user) {
+      extra.push(`--user=${options.user}`)
+      delete options.user
+    }
+    let workdir = imageMeta.workdir || "/"
+    if (options?.cwd) {
+      workdir = options.cwd
+      delete options.cwd
+    }
+    return cp.spawn(
+      "start-cli",
+      [
+        "chroot",
+        `--env=/media/startos/images/${this.imageId}.env`,
+        `--workdir=${workdir}`,
+        ...extra,
+        this.rootfs,
+        ...command,
+      ],
+      options,
+    )
+  }
+}
+
+export type CommandOptions = {
+  env?: { [variable: string]: string }
+  cwd?: string
+  user?: string
+}
+
+export type MountOptions =
+  | MountOptionsVolume
+  | MountOptionsAssets
+  | MountOptionsPointer
+
+export type MountOptionsVolume = {
+  type: "volume"
+  id: string
+  subpath: string | null
+  readonly: boolean
+}
+
+export type MountOptionsAssets = {
+  type: "assets"
+  id: string
+  subpath: string | null
+}
+
+export type MountOptionsPointer = {
+  type: "pointer"
+  packageId: string
+  volumeId: string
+  subpath: string | null
+  readonly: boolean
+}

sdk/lib/util/deepEqual.ts

@@ -0,0 +1,19 @@
+import { object } from "ts-matches"
+
+export function deepEqual(...args: unknown[]) {
+  if (!object.test(args[args.length - 1])) return args[args.length - 1]
+  const objects = args.filter(object.test)
+  if (objects.length === 0) {
+    for (const x of args) if (x !== args[0]) return false
+    return true
+  }
+  if (objects.length !== args.length) return false
+  const allKeys = new Set(objects.flatMap((x) => Object.keys(x)))
+  for (const key of allKeys) {
+    for (const x of objects) {
+      if (!(key in x)) return false
+      if (!deepEqual((objects[0] as any)[key], (x as any)[key])) return false
+    }
+  }
+  return true
+}

sdk/lib/util/deepMerge.ts

@@ -0,0 +1,17 @@
+import { object } from "ts-matches"
+
+export function deepMerge(...args: unknown[]): unknown {
+  const lastItem = (args as any)[args.length - 1]
+  if (!object.test(lastItem)) return lastItem
+  const objects = args.filter(object.test).filter((x) => !Array.isArray(x))
+  if (objects.length === 0) return lastItem as any
+  if (objects.length === 1) objects.unshift({})
+  const allKeys = new Set(objects.flatMap((x) => Object.keys(x)))
+  for (const key of allKeys) {
+    const filteredValues = objects.flatMap((x) =>
+      key in x ? [(x as any)[key]] : [],
+    )
+    ;(objects as any)[0][key] = deepMerge(...filteredValues)
+  }
+  return objects[0] as any
+}

sdk/lib/util/fileHelper.ts

@@ -0,0 +1,147 @@
+import * as matches from "ts-matches"
+import * as YAML from "yaml"
+import * as TOML from "@iarna/toml"
+import * as T from "../types"
+import * as fs from "fs"
+
+const previousPath = /(.+?)\/([^/]*)$/
+
+/**
+ * Used in the get config and the set config exported functions.
+ * The idea is that we are going to be reading/ writing to a file, or multiple files. And then we use this tool
+ * to keep the same path on the read and write, and have methods for helping with structured data.
+ * And if we are not using a structured data, we can use the raw method which forces the construction of a BiMap
+ * ```ts
+        import {InputSpec} from './InputSpec.ts'
+        import {matches, T} from '../deps.ts';
+        const { object, string, number, boolean, arrayOf, array, anyOf, allOf } = matches
+        const someValidator = object({
+        data: string
+        })
+        const jsonFile = FileHelper.json({
+        path: 'data.json',
+        validator: someValidator,
+        volume: 'main'
+        })
+        const  tomlFile = FileHelper.toml({
+        path: 'data.toml',
+        validator: someValidator,
+        volume: 'main'
+        })
+        const rawFile = FileHelper.raw({
+        path: 'data.amazingSettings',
+        volume: 'main'
+        fromData(dataIn: Data): string {
+            return `myDatais ///- ${dataIn.data}`
+        },
+        toData(rawData: string): Data {
+        const [,data] = /myDatais \/\/\/- (.*)/.match(rawData)
+        return {data}
+        }
+        })
+
+        export const setConfig : T.ExpectedExports.setConfig= async (effects, config) => {
+        await  jsonFile.write({ data: 'here lies data'}, effects)
+        }
+
+        export const getConfig: T.ExpectedExports.getConfig = async (effects, config) => ({
+        spec: InputSpec,
+        config: nullIfEmpty({
+            ...jsonFile.get(effects)
+        })
+    ```
+ */
+export class FileHelper<A> {
+  protected constructor(
+    readonly path: string,
+    readonly writeData: (dataIn: A) => string,
+    readonly readData: (stringValue: string) => A,
+  ) {}
+  async write(data: A, effects: T.Effects) {
+    if (previousPath.exec(this.path)) {
+      await new Promise((resolve, reject) =>
+        fs.mkdir(this.path, (err: any) => (!err ? resolve(null) : reject(err))),
+      )
+    }
+
+    await new Promise((resolve, reject) =>
+      fs.writeFile(this.path, this.writeData(data), (err: any) =>
+        !err ? resolve(null) : reject(err),
+      ),
+    )
+  }
+  async read(effects: T.Effects) {
+    if (!fs.existsSync(this.path)) {
+      return null
+    }
+    return this.readData(
+      await new Promise((resolve, reject) =>
+        fs.readFile(this.path, (err: any, data: any) =>
+          !err ? resolve(data.toString("utf-8")) : reject(err),
+        ),
+      ),
+    )
+  }
+  /**
+   * Create a File Helper for an arbitrary file type.
+   *
+   * Provide custom functions for translating data to the file format and visa versa.
+   */
+  static raw<A>(
+    path: string,
+    toFile: (dataIn: A) => string,
+    fromFile: (rawData: string) => A,
+  ) {
+    return new FileHelper<A>(path, toFile, fromFile)
+  }
+  /**
+   * Create a File Helper for a .json file
+   */
+  static json<A>(path: string, shape: matches.Validator<unknown, A>) {
+    return new FileHelper<A>(
+      path,
+      (inData) => {
+        return JSON.stringify(inData, null, 2)
+      },
+      (inString) => {
+        return shape.unsafeCast(JSON.parse(inString))
+      },
+    )
+  }
+  /**
+   * Create a File Helper for a .toml file
+   */
+  static toml<A extends Record<string, unknown>>(
+    path: string,
+    shape: matches.Validator<unknown, A>,
+  ) {
+    return new FileHelper<A>(
+      path,
+      (inData) => {
+        return TOML.stringify(inData as any)
+      },
+      (inString) => {
+        return shape.unsafeCast(TOML.parse(inString))
+      },
+    )
+  }
+  /**
+   * Create a File Helper for a .yaml file
+   */
+  static yaml<A extends Record<string, unknown>>(
+    path: string,
+    shape: matches.Validator<unknown, A>,
+  ) {
+    return new FileHelper<A>(
+      path,
+      (inData) => {
+        return JSON.stringify(inData, null, 2)
+      },
+      (inString) => {
+        return shape.unsafeCast(YAML.parse(inString))
+      },
+    )
+  }
+}
+
+export default FileHelper

sdk/lib/util/getDefaultString.ts

@@ -0,0 +1,10 @@
+import { DefaultString } from "../config/configTypes"
+import { getRandomString } from "./getRandomString"
+
+export function getDefaultString(defaultSpec: DefaultString): string {
+  if (typeof defaultSpec === "string") {
+    return defaultSpec
+  } else {
+    return getRandomString(defaultSpec)
+  }
+}

sdk/lib/util/getRandomCharInSet.ts

@@ -0,0 +1,98 @@
+// a,g,h,A-Z,,,,-
+
+import * as crypto from "crypto"
+export function getRandomCharInSet(charset: string): string {
+  const set = stringToCharSet(charset)
+  let charIdx = crypto.randomInt(0, set.len)
+  for (let range of set.ranges) {
+    if (range.len > charIdx) {
+      return String.fromCharCode(range.start.charCodeAt(0) + charIdx)
+    }
+    charIdx -= range.len
+  }
+  throw new Error("unreachable")
+}
+function stringToCharSet(charset: string): CharSet {
+  let set: CharSet = { ranges: [], len: 0 }
+  let start: string | null = null
+  let end: string | null = null
+  let in_range = false
+  for (let char of charset) {
+    switch (char) {
+      case ",":
+        if (start !== null && end !== null) {
+          if (start!.charCodeAt(0) > end!.charCodeAt(0)) {
+            throw new Error("start > end of charset")
+          }
+          const len = end.charCodeAt(0) - start.charCodeAt(0) + 1
+          set.ranges.push({
+            start,
+            end,
+            len,
+          })
+          set.len += len
+          start = null
+          end = null
+          in_range = false
+        } else if (start !== null && !in_range) {
+          set.len += 1
+          set.ranges.push({ start, end: start, len: 1 })
+          start = null
+        } else if (start !== null && in_range) {
+          end = ","
+        } else if (start === null && end === null && !in_range) {
+          start = ","
+        } else {
+          throw new Error('unexpected ","')
+        }
+        break
+      case "-":
+        if (start === null) {
+          start = "-"
+        } else if (!in_range) {
+          in_range = true
+        } else if (in_range && end === null) {
+          end = "-"
+        } else {
+          throw new Error('unexpected "-"')
+        }
+        break
+      default:
+        if (start === null) {
+          start = char
+        } else if (in_range && end === null) {
+          end = char
+        } else {
+          throw new Error(`unexpected "${char}"`)
+        }
+    }
+  }
+  if (start !== null && end !== null) {
+    if (start!.charCodeAt(0) > end!.charCodeAt(0)) {
+      throw new Error("start > end of charset")
+    }
+    const len = end.charCodeAt(0) - start.charCodeAt(0) + 1
+    set.ranges.push({
+      start,
+      end,
+      len,
+    })
+    set.len += len
+  } else if (start !== null) {
+    set.len += 1
+    set.ranges.push({
+      start,
+      end: start,
+      len: 1,
+    })
+  }
+  return set
+}
+type CharSet = {
+  ranges: {
+    start: string
+    end: string
+    len: number
+  }[]
+  len: number
+}

sdk/lib/util/getRandomString.ts

@@ -0,0 +1,11 @@
+import { RandomString } from "../config/configTypes"
+import { getRandomCharInSet } from "./getRandomCharInSet"
+
+export function getRandomString(generator: RandomString): string {
+  let s = ""
+  for (let i = 0; i < generator.len; i++) {
+    s = s + getRandomCharInSet(generator.charset)
+  }
+
+  return s
+}

sdk/lib/util/getServiceInterface.ts

@@ -0,0 +1,283 @@
+import { ServiceInterfaceType } from "../StartSdk"
+import {
+  AddressInfo,
+  Effects,
+  HostInfo,
+  Hostname,
+  HostnameInfo,
+} from "../types"
+
+export type UrlString = string
+export type HostId = string
+
+const getHostnameRegex = /^(\w+:\/\/)?([^\/\:]+)(:\d{1,3})?(\/)?/
+export const getHostname = (url: string): Hostname | null => {
+  const founds = url.match(getHostnameRegex)?.[2]
+  if (!founds) return null
+  const parts = founds.split("@")
+  const last = parts[parts.length - 1] as Hostname | null
+  return last
+}
+
+export type Filled = {
+  hostnames: Hostname[]
+  onionHostnames: Hostname[]
+  localHostnames: Hostname[]
+  ipHostnames: Hostname[]
+  ipv4Hostnames: Hostname[]
+  ipv6Hostnames: Hostname[]
+  nonIpHostnames: Hostname[]
+
+  urls: UrlString[]
+  onionUrls: UrlString[]
+  localUrls: UrlString[]
+  ipUrls: UrlString[]
+  ipv4Urls: UrlString[]
+  ipv6Urls: UrlString[]
+  nonIpUrls: UrlString[]
+}
+export type FilledAddressInfo = AddressInfo & Filled
+export type ServiceInterfaceFilled = {
+  id: string
+  /** The title of this field to be displayed */
+  name: string
+  /** Human readable description, used as tooltip usually */
+  description: string
+  /** Whether or not the interface has a primary URL */
+  hasPrimary: boolean
+  /** Whether or not the interface disabled */
+  disabled: boolean
+  /** Whether or not to mask the URIs for this interface. Useful if the URIs contain sensitive information, such as a password, macaroon, or API key */
+  masked: boolean
+  /** Information about the host for this binding */
+  hostInfo: HostInfo
+  /** URI information */
+  addressInfo: FilledAddressInfo
+  /** Indicates if we are a ui/p2p/api for the kind of interface that this is representing */
+  type: ServiceInterfaceType
+  /** The primary hostname for the service, as chosen by the user */
+  primaryHostname: Hostname | null
+  /** The primary URL for the service, as chosen by the user */
+  primaryUrl: UrlString | null
+}
+const either =
+  <A>(...args: ((a: A) => boolean)[]) =>
+  (a: A) =>
+    args.some((x) => x(a))
+const negate =
+  <A>(fn: (a: A) => boolean) =>
+  (a: A) =>
+    !fn(a)
+const unique = <A>(values: A[]) => Array.from(new Set(values))
+function stringifyHostname(info: HostnameInfo): Hostname {
+  let base: string
+  if ("kind" in info.hostname && info.hostname.kind === "domain") {
+    base = info.hostname.subdomain
+      ? `${info.hostname.subdomain}.${info.hostname.domain}`
+      : info.hostname.domain
+  } else {
+    base = info.hostname.value
+  }
+  if (info.hostname.port && info.hostname.sslPort) {
+    return `${base}:${info.hostname.port}` as Hostname
+  } else if (info.hostname.sslPort) {
+    return `${base}:${info.hostname.sslPort}` as Hostname
+  } else if (info.hostname.port) {
+    return `${base}:${info.hostname.port}` as Hostname
+  }
+  return base as Hostname
+}
+const addressHostToUrl = (
+  { bindOptions, username, suffix }: AddressInfo,
+  host: Hostname,
+): UrlString => {
+  const scheme = host.endsWith(".onion")
+    ? bindOptions.scheme
+    : bindOptions.addSsl
+      ? bindOptions.addSsl.scheme
+      : bindOptions.scheme // TODO: encode whether hostname transport is "secure"?
+  return `${scheme ? `${scheme}//` : ""}${
+    username ? `${username}@` : ""
+  }${host}${suffix}`
+}
+export const filledAddress = (
+  hostInfo: HostInfo,
+  addressInfo: AddressInfo,
+): FilledAddressInfo => {
+  const toUrl = addressHostToUrl.bind(null, addressInfo)
+  const hostnameInfo =
+    hostInfo.kind == "multi"
+      ? hostInfo.hostnames
+      : hostInfo.hostname
+        ? [hostInfo.hostname]
+        : []
+  return {
+    ...addressInfo,
+    hostnames: hostnameInfo.flatMap((h) => stringifyHostname(h)),
+    get onionHostnames() {
+      return hostnameInfo
+        .filter((h) => h.kind === "onion")
+        .map((h) => stringifyHostname(h))
+    },
+    get localHostnames() {
+      return hostnameInfo
+        .filter((h) => h.kind === "ip" && h.hostname.kind === "local")
+        .map((h) => stringifyHostname(h))
+    },
+    get ipHostnames() {
+      return hostnameInfo
+        .filter(
+          (h) =>
+            h.kind === "ip" &&
+            (h.hostname.kind === "ipv4" || h.hostname.kind === "ipv6"),
+        )
+        .map((h) => stringifyHostname(h))
+    },
+    get ipv4Hostnames() {
+      return hostnameInfo
+        .filter((h) => h.kind === "ip" && h.hostname.kind === "ipv4")
+        .map((h) => stringifyHostname(h))
+    },
+    get ipv6Hostnames() {
+      return hostnameInfo
+        .filter((h) => h.kind === "ip" && h.hostname.kind === "ipv6")
+        .map((h) => stringifyHostname(h))
+    },
+    get nonIpHostnames() {
+      return hostnameInfo
+        .filter(
+          (h) =>
+            h.kind === "ip" &&
+            h.hostname.kind !== "ipv4" &&
+            h.hostname.kind !== "ipv6",
+        )
+        .map((h) => stringifyHostname(h))
+    },
+    get urls() {
+      return this.hostnames.map(toUrl)
+    },
+    get onionUrls() {
+      return this.onionHostnames.map(toUrl)
+    },
+    get localUrls() {
+      return this.localHostnames.map(toUrl)
+    },
+    get ipUrls() {
+      return this.ipHostnames.map(toUrl)
+    },
+    get ipv4Urls() {
+      return this.ipv4Hostnames.map(toUrl)
+    },
+    get ipv6Urls() {
+      return this.ipv6Hostnames.map(toUrl)
+    },
+    get nonIpUrls() {
+      return this.nonIpHostnames.map(toUrl)
+    },
+  }
+}
+
+const makeInterfaceFilled = async ({
+  effects,
+  id,
+  packageId,
+  callback,
+}: {
+  effects: Effects
+  id: string
+  packageId: string | null
+  callback: () => void
+}) => {
+  const serviceInterfaceValue = await effects.getServiceInterface({
+    serviceInterfaceId: id,
+    packageId,
+    callback,
+  })
+  const hostInfo = await effects.getHostInfo({
+    packageId,
+    kind: null,
+    serviceInterfaceId: serviceInterfaceValue.id,
+    callback,
+  })
+  const primaryUrl = await effects.getPrimaryUrl({
+    serviceInterfaceId: id,
+    packageId,
+    callback,
+  })
+
+  const interfaceFilled: ServiceInterfaceFilled = {
+    ...serviceInterfaceValue,
+    primaryUrl: primaryUrl,
+    hostInfo,
+    addressInfo: filledAddress(hostInfo, serviceInterfaceValue.addressInfo),
+    get primaryHostname() {
+      if (primaryUrl == null) return null
+      return getHostname(primaryUrl)
+    },
+  }
+  return interfaceFilled
+}
+
+export class GetServiceInterface {
+  constructor(
+    readonly effects: Effects,
+    readonly opts: { id: string; packageId: string | null },
+  ) {}
+
+  /**
+   * Returns the value of Store at the provided path. Restart the service if the value changes
+   */
+  async const() {
+    const { id, packageId } = this.opts
+    const callback = this.effects.restart
+    const interfaceFilled: ServiceInterfaceFilled = await makeInterfaceFilled({
+      effects: this.effects,
+      id,
+      packageId,
+      callback,
+    })
+
+    return interfaceFilled
+  }
+  /**
+   * Returns the value of ServiceInterfacesFilled at the provided path. Does nothing if the value changes
+   */
+  async once() {
+    const { id, packageId } = this.opts
+    const callback = () => {}
+    const interfaceFilled: ServiceInterfaceFilled = await makeInterfaceFilled({
+      effects: this.effects,
+      id,
+      packageId,
+      callback,
+    })
+
+    return interfaceFilled
+  }
+
+  /**
+   * Watches the value of ServiceInterfacesFilled at the provided path. Takes a custom callback function to run whenever the value changes
+   */
+  async *watch() {
+    const { id, packageId } = this.opts
+    while (true) {
+      let callback: () => void = () => {}
+      const waitForNext = new Promise<void>((resolve) => {
+        callback = resolve
+      })
+      yield await makeInterfaceFilled({
+        effects: this.effects,
+        id,
+        packageId,
+        callback,
+      })
+      await waitForNext
+    }
+  }
+}
+export function getServiceInterface(
+  effects: Effects,
+  opts: { id: string; packageId: string | null },
+) {
+  return new GetServiceInterface(effects, opts)
+}

sdk/lib/util/getServiceInterfaces.ts

@@ -0,0 +1,127 @@
+import { Effects } from "../types"
+import {
+  ServiceInterfaceFilled,
+  filledAddress,
+  getHostname,
+} from "./getServiceInterface"
+
+const makeManyInterfaceFilled = async ({
+  effects,
+  packageId,
+  callback,
+}: {
+  effects: Effects
+  packageId: string | null
+  callback: () => void
+}) => {
+  const serviceInterfaceValues = await effects.listServiceInterfaces({
+    packageId,
+    callback,
+  })
+  const hostIdsRecord = Object.fromEntries(
+    await Promise.all(
+      Array.from(new Set(serviceInterfaceValues.map((x) => x.id))).map(
+        async (id) =>
+          [
+            id,
+            await effects.getHostInfo({
+              kind: null,
+              packageId,
+              serviceInterfaceId: id,
+              callback,
+            }),
+          ] as const,
+      ),
+    ),
+  )
+
+  const serviceInterfacesFilled: ServiceInterfaceFilled[] = await Promise.all(
+    serviceInterfaceValues.map(async (serviceInterfaceValue) => {
+      const hostInfo = await effects.getHostInfo({
+        kind: null,
+        packageId,
+        serviceInterfaceId: serviceInterfaceValue.id,
+        callback,
+      })
+      const primaryUrl = await effects.getPrimaryUrl({
+        serviceInterfaceId: serviceInterfaceValue.id,
+        packageId,
+        callback,
+      })
+      return {
+        ...serviceInterfaceValue,
+        primaryUrl: primaryUrl,
+        hostInfo,
+        addressInfo: filledAddress(hostInfo, serviceInterfaceValue.addressInfo),
+        get primaryHostname() {
+          if (primaryUrl == null) return null
+          return getHostname(primaryUrl)
+        },
+      }
+    }),
+  )
+  return serviceInterfacesFilled
+}
+
+export class GetServiceInterfaces {
+  constructor(
+    readonly effects: Effects,
+    readonly opts: { packageId: string | null },
+  ) {}
+
+  /**
+   * Returns the value of Store at the provided path. Restart the service if the value changes
+   */
+  async const() {
+    const { packageId } = this.opts
+    const callback = this.effects.restart
+    const interfaceFilled: ServiceInterfaceFilled[] =
+      await makeManyInterfaceFilled({
+        effects: this.effects,
+        packageId,
+        callback,
+      })
+
+    return interfaceFilled
+  }
+  /**
+   * Returns the value of ServiceInterfacesFilled at the provided path. Does nothing if the value changes
+   */
+  async once() {
+    const { packageId } = this.opts
+    const callback = () => {}
+    const interfaceFilled: ServiceInterfaceFilled[] =
+      await makeManyInterfaceFilled({
+        effects: this.effects,
+        packageId,
+        callback,
+      })
+
+    return interfaceFilled
+  }
+
+  /**
+   * Watches the value of ServiceInterfacesFilled at the provided path. Takes a custom callback function to run whenever the value changes
+   */
+  async *watch() {
+    const { packageId } = this.opts
+    while (true) {
+      let callback: () => void = () => {}
+      const waitForNext = new Promise<void>((resolve) => {
+        callback = resolve
+      })
+      yield await makeManyInterfaceFilled({
+        effects: this.effects,
+        packageId,
+        callback,
+      })
+      await waitForNext
+    }
+  }
+}
+export function getServiceInterfaces(
+  effects: Effects,
+  opts: { packageId: string | null },
+) {
+  return new GetServiceInterfaces(effects, opts)
+}

sdk/lib/util/index.ts

@@ -0,0 +1,30 @@
+import * as T from "../types"
+
+import "./nullIfEmpty"
+import "./fileHelper"
+import "../store/getStore"
+import "./deepEqual"
+import "./deepMerge"
+import "./Overlay"
+import "./once"
+import { SDKManifest } from "../manifest/ManifestTypes"
+
+// prettier-ignore
+export type FlattenIntersection<T> = 
+T extends ArrayLike<any> ? T :
+T extends object ? {} & {[P in keyof T]: T[P]} :
+ T;
+
+export type _<T> = FlattenIntersection<T>
+
+export const isKnownError = (e: unknown): e is T.KnownError =>
+  e instanceof Object && ("error" in e || "error-code" in e)
+
+declare const affine: unique symbol
+
+type NeverPossible = { [affine]: string }
+export type NoAny<A> = NeverPossible extends A
+  ? keyof NeverPossible extends keyof A
+    ? never
+    : A
+  : A

sdk/lib/util/nullIfEmpty.ts

@@ -0,0 +1,12 @@
+/**
+ * A useful tool when doing a getConfig.
+ * Look into the config {@link FileHelper} for an example of the use.
+ * @param s
+ * @returns
+ */
+export default function nullIfEmpty<A extends Record<string, any>>(
+  s: null | A,
+) {
+  if (s === null) return null
+  return Object.keys(s).length === 0 ? null : s
+}

sdk/lib/util/once.ts

@@ -0,0 +1,9 @@
+export function once<B>(fn: () => B): () => B {
+  let result: [B] | [] = []
+  return () => {
+    if (!result.length) {
+      result = [fn()]
+    }
+    return result[0]
+  }
+}

sdk/lib/util/patterns.ts

@@ -0,0 +1,59 @@
+import { Pattern } from "../config/configTypes"
+import * as regexes from "./regexes"
+
+export const ipv6: Pattern = {
+  regex: regexes.ipv6.toString(),
+  description: "Must be a valid IPv6 address",
+}
+
+export const ipv4: Pattern = {
+  regex: regexes.ipv4.toString(),
+  description: "Must be a valid IPv4 address",
+}
+
+export const hostname: Pattern = {
+  regex: regexes.hostname.toString(),
+  description: "Must be a valid hostname",
+}
+
+export const localHostname: Pattern = {
+  regex: regexes.localHostname.toString(),
+  description: 'Must be a valid ".local" hostname',
+}
+
+export const torHostname: Pattern = {
+  regex: regexes.torHostname.toString(),
+  description: 'Must be a valid Tor (".onion") hostname',
+}
+
+export const url: Pattern = {
+  regex: regexes.url.toString(),
+  description: "Must be a valid URL",
+}
+
+export const localUrl: Pattern = {
+  regex: regexes.localUrl.toString(),
+  description: 'Must be a valid ".local" URL',
+}
+
+export const torUrl: Pattern = {
+  regex: regexes.torUrl.toString(),
+  description: 'Must be a valid Tor (".onion") URL',
+}
+
+export const ascii: Pattern = {
+  regex: regexes.ascii.toString(),
+  description:
+    "May only contain ASCII characters. See https://www.w3schools.com/charsets/ref_html_ascii.asp",
+}
+
+export const email: Pattern = {
+  regex: regexes.email.toString(),
+  description: "Must be a valid email address",
+}
+
+export const base64: Pattern = {
+  regex: regexes.base64.toString(),
+  description:
+    "May only contain base64 characters. See https://base64.guru/learn/base64-characters",
+}

sdk/lib/util/regexes.ts

@@ -0,0 +1,34 @@
+// https://ihateregex.io/expr/ipv6/
+export const ipv6 =
+  /(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))/
+
+// https://ihateregex.io/expr/ipv4/
+export const ipv4 =
+  /(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}/
+
+export const hostname =
+  /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$/
+
+export const localHostname = /[-a-zA-Z0-9@:%._\+~#=]{1,256}\.local/
+
+export const torHostname = /[-a-zA-Z0-9@:%._\+~#=]{1,256}\.onion/
+
+// https://ihateregex.io/expr/url/
+export const url =
+  /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()!@:%_\+.~#?&\/\/=]*)/
+
+export const localUrl =
+  /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.local\b([-a-zA-Z0-9()!@:%_\+.~#?&\/\/=]*)/
+
+export const torUrl =
+  /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.onion\b([-a-zA-Z0-9()!@:%_\+.~#?&\/\/=]*)/
+
+// https://ihateregex.io/expr/ascii/
+export const ascii = /^[ -~]*$/
+
+//https://ihateregex.io/expr/email/
+export const email = /[^@ \t\r\n]+@[^@ \t\r\n]+\.[^@ \t\r\n]+/
+
+//https://rgxdb.com/r/1NUN74O6
+export const base64 =
+  /^(?:[a-zA-Z0-9+\/]{4})*(?:|(?:[a-zA-Z0-9+\/]{3}=)|(?:[a-zA-Z0-9+\/]{2}==)|(?:[a-zA-Z0-9+\/]{1}===))$/

sdk/lib/util/splitCommand.ts

@@ -0,0 +1,17 @@
+import { arrayOf, string } from "ts-matches"
+import { ValidIfNoStupidEscape } from "../types"
+
+export const splitCommand = <A>(
+  command: ValidIfNoStupidEscape<A> | [string, ...string[]],
+): string[] => {
+  if (arrayOf(string).test(command)) return command
+  return String(command)
+    .split('"')
+    .flatMap((x, i) =>
+      i % 2 !== 0
+        ? [x]
+        : x.split("'").flatMap((x, i) => (i % 2 !== 0 ? [x] : x.split(" "))),
+    )
+    .map((x) => x.trim())
+    .filter(Boolean)
+}

sdk/lib/util/stringFromStdErrOut.ts

@@ -0,0 +1,6 @@
+export async function stringFromStdErrOut(x: {
+  stdout: string
+  stderr: string
+}) {
+  return x?.stderr ? Promise.reject(x.stderr) : x.stdout
+}

sdk/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@start9labs/start-sdk",
+  "version": "0.4.0-rev0.lib0.rc8.beta10",
+  "description": "Software development kit to facilitate packaging services for StartOS",
+  "main": "./cjs/sdk/lib/index.js",
+  "types": "./cjs/sdk/lib/index.d.ts",
+  "module": "./mjs/sdk/lib/index.js",
+  "sideEffects": true,
+  "exports": {
+    ".": {
+      "import": "./mjs/sdk/lib/index.js",
+      "require": "./cjs/sdk/lib/index.js",
+      "types": "./cjs/sdk/lib/index.d.ts"
+    }
+  },
+  "typesVersion": {
+    ">=3.1": {
+      "*": [
+        "cjs/lib/*"
+      ]
+    }
+  },
+  "scripts": {
+    "test": "jest -c ./jest.config.js --coverage",
+    "buildOutput": "ts-node --project ./tsconfig-cjs.json  ./lib/test/makeOutput.ts && npx prettier --write '**/*.ts'",
+    "check": "tsc --noEmit"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Start9Labs/start-sdk.git"
+  },
+  "author": "Start9 Labs",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/Start9Labs/start-sdk/issues"
+  },
+  "homepage": "https://github.com/Start9Labs/start-sdk#readme",
+  "dependencies": {
+    "@iarna/toml": "^2.2.5",
+    "isomorphic-fetch": "^3.0.0",
+    "ts-matches": "^5.4.1",
+    "yaml": "^2.2.2"
+  },
+  "prettier": {
+    "trailingComma": "all",
+    "tabWidth": 2,
+    "semi": false,
+    "singleQuote": false
+  },
+  "devDependencies": {
+    "@types/jest": "^29.4.0",
+    "jest": "^29.4.3",
+    "prettier": "^3.2.5",
+    "ts-jest": "^29.0.5",
+    "ts-node": "^10.9.1",
+    "tsx": "^4.7.1",
+    "typescript": "^5.0.4"
+  }
+}

sdk/scripts/oldSpecToBuilder.ts

@@ -0,0 +1,413 @@
+import * as fs from "fs"
+
+// https://stackoverflow.com/questions/2970525/converting-any-string-into-camel-case
+export function camelCase(value: string) {
+  return value
+    .replace(/([\(\)\[\]])/g, "")
+    .replace(/^([A-Z])|[\s-_](\w)/g, function (match, p1, p2, offset) {
+      if (p2) return p2.toUpperCase()
+      return p1.toLowerCase()
+    })
+}
+
+export async function oldSpecToBuilder(
+  file: string,
+  inputData: Promise<any> | any,
+  options?: Parameters<typeof makeFileContentFromOld>[1],
+) {
+  await fs.writeFile(
+    file,
+    await makeFileContentFromOld(inputData, options),
+    (err) => console.error(err),
+  )
+}
+
+function isString(x: unknown): x is string {
+  return typeof x === "string"
+}
+
+export default async function makeFileContentFromOld(
+  inputData: Promise<any> | any,
+  { StartSdk = "start-sdk", nested = true } = {},
+) {
+  const outputLines: string[] = []
+  outputLines.push(`
+import { sdk } from "${StartSdk}"
+const {Config, List, Value, Variants} = sdk
+`)
+  const data = await inputData
+
+  const namedConsts = new Set(["Config", "Value", "List"])
+  const configName = newConst("configSpec", convertInputSpec(data))
+  const configMatcherName = newConst(
+    "matchConfigSpec",
+    `${configName}.validator`,
+  )
+  outputLines.push(
+    `export type ConfigSpec = typeof ${configMatcherName}._TYPE;`,
+  )
+
+  return outputLines.join("\n")
+
+  function newConst(key: string, data: string, type?: string) {
+    const variableName = getNextConstName(camelCase(key))
+    outputLines.push(
+      `export const ${variableName}${!type ? "" : `: ${type}`} = ${data};`,
+    )
+    return variableName
+  }
+  function maybeNewConst(key: string, data: string) {
+    if (nested) return data
+    return newConst(key, data)
+  }
+  function convertInputSpecInner(data: any) {
+    let answer = "{"
+    for (const [key, value] of Object.entries(data)) {
+      const variableName = maybeNewConst(key, convertValueSpec(value))
+
+      answer += `${JSON.stringify(key)}: ${variableName},`
+    }
+    return `${answer}}`
+  }
+
+  function convertInputSpec(data: any) {
+    return `Config.of(${convertInputSpecInner(data)})`
+  }
+  function convertValueSpec(value: any): string {
+    switch (value.type) {
+      case "string": {
+        if (value.textarea) {
+          return `${rangeToTodoComment(
+            value?.range,
+          )}Value.textarea(${JSON.stringify(
+            {
+              name: value.name || null,
+              description: value.description || null,
+              warning: value.warning || null,
+              required: !(value.nullable || false),
+              placeholder: value.placeholder || null,
+              maxLength: null,
+              minLength: null,
+            },
+            null,
+            2,
+          )})`
+        }
+        return `${rangeToTodoComment(value?.range)}Value.text(${JSON.stringify(
+          {
+            name: value.name || null,
+            // prettier-ignore
+            required: (
+              value.default != null  ? {default: value.default} :
+              value.nullable === false ? {default: null} :
+              !value.nullable
+            ),
+            description: value.description || null,
+            warning: value.warning || null,
+            masked: value.masked || false,
+            placeholder: value.placeholder || null,
+            inputmode: "text",
+            patterns: value.pattern
+              ? [
+                  {
+                    regex: value.pattern,
+                    description: value["pattern-description"],
+                  },
+                ]
+              : [],
+            minLength: null,
+            maxLength: null,
+          },
+          null,
+          2,
+        )})`
+      }
+      case "number": {
+        return `${rangeToTodoComment(
+          value?.range,
+        )}Value.number(${JSON.stringify(
+          {
+            name: value.name || null,
+            description: value.description || null,
+            warning: value.warning || null,
+            // prettier-ignore
+            required: (
+              value.default != null  ? {default: value.default} :
+              value.nullable === false ? {default: null} :
+              !value.nullable
+            ),
+            min: null,
+            max: null,
+            step: null,
+            integer: value.integral || false,
+            units: value.units || null,
+            placeholder: value.placeholder || null,
+          },
+          null,
+          2,
+        )})`
+      }
+      case "boolean": {
+        return `Value.toggle(${JSON.stringify(
+          {
+            name: value.name || null,
+            default: value.default || false,
+            description: value.description || null,
+            warning: value.warning || null,
+          },
+          null,
+          2,
+        )})`
+      }
+      case "enum": {
+        const allValueNames = new Set([
+          ...(value?.["values"] || []),
+          ...Object.keys(value?.["value-names"] || {}),
+        ])
+        const values = Object.fromEntries(
+          Array.from(allValueNames)
+            .filter(isString)
+            .map((key) => [key, value?.spec?.["value-names"]?.[key] || key]),
+        )
+        return `Value.select(${JSON.stringify(
+          {
+            name: value.name || null,
+            description: value.description || null,
+            warning: value.warning || null,
+
+            // prettier-ignore
+            required:(
+              value.default != null  ? {default: value.default} :
+              value.nullable === false ? {default: null} :
+              !value.nullable
+            ),
+            values,
+          },
+          null,
+          2,
+        )} as const)`
+      }
+      case "object": {
+        const specName = maybeNewConst(
+          value.name + "_spec",
+          convertInputSpec(value.spec),
+        )
+        return `Value.object({
+        name: ${JSON.stringify(value.name || null)},
+        description: ${JSON.stringify(value.description || null)},
+        warning: ${JSON.stringify(value.warning || null)},
+      }, ${specName})`
+      }
+      case "union": {
+        const variants = maybeNewConst(
+          value.name + "_variants",
+          convertVariants(value.variants, value.tag["variant-names"] || {}),
+        )
+
+        return `Value.union({
+        name: ${JSON.stringify(value.name || null)},
+        description: ${JSON.stringify(value.tag.description || null)},
+        warning: ${JSON.stringify(value.tag.warning || null)},
+        
+        // prettier-ignore
+        required: ${JSON.stringify(
+          // prettier-ignore
+          value.default != null  ? {default: value.default} :
+          value.nullable === false ? {default: null} :
+          !value.nullable,
+        )},
+      }, ${variants})`
+      }
+      case "list": {
+        if (value.subtype === "enum") {
+          const allValueNames = new Set([
+            ...(value?.spec?.["values"] || []),
+            ...Object.keys(value?.spec?.["value-names"] || {}),
+          ])
+          const values = Object.fromEntries(
+            Array.from(allValueNames)
+              .filter(isString)
+              .map((key: string) => [
+                key,
+                value?.spec?.["value-names"]?.[key] ?? key,
+              ]),
+          )
+          return `Value.multiselect(${JSON.stringify(
+            {
+              name: value.name || null,
+              minLength: null,
+              maxLength: null,
+              default: value.default ?? null,
+              description: value.description || null,
+              warning: value.warning || null,
+              values,
+            },
+            null,
+            2,
+          )})`
+        }
+        const list = maybeNewConst(value.name + "_list", convertList(value))
+        return `Value.list(${list})`
+      }
+      case "pointer": {
+        return `/* TODO deal with point removed point "${value.name}" */null as any`
+      }
+    }
+    throw Error(`Unknown type "${value.type}"`)
+  }
+
+  function convertList(value: any) {
+    switch (value.subtype) {
+      case "string": {
+        return `${rangeToTodoComment(value?.range)}List.text(${JSON.stringify(
+          {
+            name: value.name || null,
+            minLength: null,
+            maxLength: null,
+            default: value.default || null,
+            description: value.description || null,
+            warning: value.warning || null,
+          },
+          null,
+          2,
+        )}, ${JSON.stringify({
+          masked: value?.spec?.masked || false,
+          placeholder: value?.spec?.placeholder || null,
+          patterns: value?.spec?.pattern
+            ? [
+                {
+                  regex: value.spec.pattern,
+                  description: value?.spec?.["pattern-description"],
+                },
+              ]
+            : [],
+          minLength: null,
+          maxLength: null,
+        })})`
+      }
+      case "number": {
+        return `${rangeToTodoComment(value?.range)}List.number(${JSON.stringify(
+          {
+            name: value.name || null,
+            minLength: null,
+            maxLength: null,
+            default: value.default || null,
+            description: value.description || null,
+            warning: value.warning || null,
+          },
+          null,
+          2,
+        )}, ${JSON.stringify({
+          integer: value?.spec?.integral || false,
+          min: null,
+          max: null,
+          units: value?.spec?.units || null,
+          placeholder: value?.spec?.placeholder || null,
+        })})`
+      }
+      case "enum": {
+        return "/* error!! list.enum */"
+      }
+      case "object": {
+        const specName = maybeNewConst(
+          value.name + "_spec",
+          convertInputSpec(value.spec.spec),
+        )
+        return `${rangeToTodoComment(value?.range)}List.obj({
+          name: ${JSON.stringify(value.name || null)},
+          minLength: ${JSON.stringify(null)},
+          maxLength: ${JSON.stringify(null)},
+          default: ${JSON.stringify(value.default || null)},
+          description: ${JSON.stringify(value.description || null)},
+          warning: ${JSON.stringify(value.warning || null)},
+        }, {
+          spec: ${specName},
+          displayAs: ${JSON.stringify(value?.spec?.["display-as"] || null)},
+          uniqueBy: ${JSON.stringify(value?.spec?.["unique-by"] || null)},
+        })`
+      }
+      case "union": {
+        const variants = maybeNewConst(
+          value.name + "_variants",
+          convertVariants(
+            value.spec.variants,
+            value.spec["variant-names"] || {},
+          ),
+        )
+        const unionValueName = maybeNewConst(
+          value.name + "_union",
+          `${rangeToTodoComment(value?.range)}
+          Value.union({
+            name: ${JSON.stringify(value?.spec?.tag?.name || null)},
+            description: ${JSON.stringify(
+              value?.spec?.tag?.description || null,
+            )},
+            warning: ${JSON.stringify(value?.spec?.tag?.warning || null)},
+            required: ${JSON.stringify(
+              // prettier-ignore
+              'default' in value?.spec ? {default: value?.spec?.default} :
+              !!value?.spec?.tag?.nullable || false ? {default: null} :
+              false,
+            )},
+          }, ${variants})
+        `,
+        )
+        const listConfig = maybeNewConst(
+          value.name + "_list_config",
+          `
+          Config.of({
+            "union": ${unionValueName}
+          })
+        `,
+        )
+        return `${rangeToTodoComment(value?.range)}List.obj({
+          name:${JSON.stringify(value.name || null)},
+          minLength:${JSON.stringify(null)},
+          maxLength:${JSON.stringify(null)},
+          default: [],
+          description: ${JSON.stringify(value.description || null)},
+          warning: ${JSON.stringify(value.warning || null)},
+        }, {
+          spec: ${listConfig},
+          displayAs: ${JSON.stringify(value?.spec?.["display-as"] || null)},
+          uniqueBy: ${JSON.stringify(value?.spec?.["unique-by"] || null)},
+        })`
+      }
+    }
+    throw new Error(`Unknown subtype "${value.subtype}"`)
+  }
+
+  function convertVariants(
+    variants: Record<string, unknown>,
+    variantNames: Record<string, string>,
+  ): string {
+    let answer = "Variants.of({"
+    for (const [key, value] of Object.entries(variants)) {
+      const variantSpec = maybeNewConst(key, convertInputSpec(value))
+      answer += `"${key}": {name: "${
+        variantNames[key] || key
+      }", spec: ${variantSpec}},`
+    }
+    return `${answer}})`
+  }
+
+  function getNextConstName(name: string, i = 0): string {
+    const newName = !i ? name : name + i
+    if (namedConsts.has(newName)) {
+      return getNextConstName(name, i + 1)
+    }
+    namedConsts.add(newName)
+    return newName
+  }
+}
+
+function rangeToTodoComment(range: string | undefined) {
+  if (!range) return ""
+  return `/* TODO: Convert range for this value (${range})*/`
+}
+
+// oldSpecToBuilder(
+//   "./config.ts",
+//   // Put config here
+//   {},
+// )

sdk/tsconfig-base.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "module": "esnext",
+    "strict": true,
+    "outDir": "dist",
+    "preserveConstEnums": true,
+    "sourceMap": true,
+    "target": "es2017",
+    "pretty": true,
+    "declaration": true,
+    "noImplicitAny": true,
+    "esModuleInterop": true,
+    "types": ["node", "jest"],
+    "moduleResolution": "node",
+    "skipLibCheck": true
+  },
+  "include": ["lib/**/*"],
+  "exclude": ["lib/**/*.spec.ts", "lib/**/*.gen.ts", "list", "node_modules"]
+}

sdk/tsconfig-cjs.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig-base.json",
+  "compilerOptions": {
+    "module": "commonjs",
+    "outDir": "dist/cjs",
+    "target": "es2018"
+  }
+}

sdk/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig-base.json",
+  "compilerOptions": {
+    "module": "esnext",
+    "outDir": "dist/mjs",
+    "target": "esnext"
+  }
+}