Refactor/actions (#2733)

* store, properties, manifest

* interfaces

* init and backups

* fix init and backups

* file models

* more versions

* dependencies

* config except dynamic types

* clean up config

* remove disabled from non-dynamic vaues

* actions

* standardize example code block formats

* wip: actions refactor

Co-authored-by: Jade <Blu-J@users.noreply.github.com>

* commit types

* fix types

* update types

* update action request type

* update apis

* add description to actionrequest

* clean up imports

* revert package json

* chore: Remove the recursive to the index

* chore: Remove the other thing I was testing

* flatten action requests

* update container runtime with new config paradigm

* new actions strategy

* seems to be working

* misc backend fixes

* fix fe bugs

* only show breakages if breakages

* only show success modal if result

* don't panic on failed removal

* hide config from actions page

* polyfill autoconfig

* use metadata strategy for actions instead of prev

* misc fixes

* chore: split the sdk into 2 libs (#2736)

* follow sideload progress (#2718)

* follow sideload progress

* small bugfix

* shareReplay with no refcount false

* don't wrap sideload progress in RPCResult

* dont present toast

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>

* chore: Add the initial of the creation of the two sdk

* chore: Add in the baseDist

* chore: Add in the baseDist

* chore: Get the web and the runtime-container running

* chore: Remove the empty file

* chore: Fix it so the container-runtime works

---------

Co-authored-by: Matt Hill <MattDHill@users.noreply.github.com>
Co-authored-by: Aiden McClelland <me@drbonez.dev>

* misc fixes

* update todos

* minor clean up

* fix link script

* update node version in CI test

* fix node version syntax in ci build

* wip: fixing callbacks

* fix sdk makefile dependencies

* add support for const outside of main

* update apis

* don't panic!

* Chore: Capture weird case on rpc, and log that

* fix procedure id issue

* pass input value for dep auto config

* handle disabled and warning for actions

* chore: Fix for link not having node_modules

* sdk fixes

* fix build

* fix build

* fix build

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>
Co-authored-by: Jade <Blu-J@users.noreply.github.com>
Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>
Co-authored-by: Matt Hill <MattDHill@users.noreply.github.com>

sdk/base/lib/util/GetSystemSmtp.ts

@@ -0,0 +1,35 @@
+import { Effects } from "../Effects"
+
+export class GetSystemSmtp {
+  constructor(readonly effects: Effects) {}
+
+  /**
+   * Returns the system SMTP credentials. Restarts the service if the credentials change
+   */
+  const() {
+    return this.effects.getSystemSmtp({
+      callback: () => this.effects.constRetry(),
+    })
+  }
+  /**
+   * Returns the system SMTP credentials. Does nothing if the credentials change
+   */
+  once() {
+    return this.effects.getSystemSmtp({})
+  }
+  /**
+   * Watches the system SMTP credentials. Takes a custom callback function to run whenever the credentials change
+   */
+  async *watch() {
+    while (true) {
+      let callback: () => void
+      const waitForNext = new Promise<void>((resolve) => {
+        callback = resolve
+      })
+      yield await this.effects.getSystemSmtp({
+        callback: () => callback(),
+      })
+      await waitForNext
+    }
+  }
+}

sdk/base/lib/util/Hostname.ts

@@ -0,0 +1,25 @@
+import { HostnameInfo } from "../types"
+
+export function hostnameInfoToAddress(hostInfo: HostnameInfo): string {
+  if (hostInfo.kind === "onion") {
+    return `${hostInfo.hostname.value}`
+  }
+  if (hostInfo.kind !== "ip") {
+    throw Error("Expecting that the kind is ip.")
+  }
+  const hostname = hostInfo.hostname
+  if (hostname.kind === "domain") {
+    return `${hostname.subdomain ? `${hostname.subdomain}.` : ""}${hostname.domain}`
+  }
+  const port = hostname.sslPort || hostname.port
+  const portString = port ? `:${port}` : ""
+  if ("ipv4" === hostname.kind || "ipv6" === hostname.kind) {
+    return `${hostname.value}${portString}`
+  }
+  if ("local" === hostname.kind) {
+    return `${hostname.value}${portString}`
+  }
+  throw Error(
+    "Expecting to have a valid hostname kind." + JSON.stringify(hostname),
+  )
+}

sdk/base/lib/util/PathBuilder.ts

@@ -0,0 +1,38 @@
+import { Affine } from "../util"
+
+const pathValue = Symbol("pathValue")
+export type PathValue = typeof pathValue
+
+export type PathBuilderStored<AllStore, Store> = {
+  [K in PathValue]: [AllStore, Store]
+}
+
+export type PathBuilder<AllStore, Store = AllStore> = (Store extends Record<
+  string,
+  unknown
+>
+  ? {
+      [K in keyof Store]: PathBuilder<AllStore, Store[K]>
+    }
+  : {}) &
+  PathBuilderStored<AllStore, Store>
+
+export type StorePath = string & Affine<"StorePath">
+const privateSymbol = Symbol("jsonPath")
+export const extractJsonPath = (builder: PathBuilder<unknown>) => {
+  return (builder as any)[privateSymbol] as StorePath
+}
+
+export const pathBuilder = <Store, StorePath = Store>(
+  paths: string[] = [],
+): PathBuilder<Store, StorePath> => {
+  return new Proxy({} as PathBuilder<Store, StorePath>, {
+    get(target, prop) {
+      if (prop === privateSymbol) {
+        if (paths.length === 0) return ""
+        return `/${paths.join("/")}`
+      }
+      return pathBuilder<any>([...paths, prop as string])
+    },
+  }) as PathBuilder<Store, StorePath>
+}

sdk/base/lib/util/asError.ts

@@ -0,0 +1,9 @@
+export const asError = (e: unknown) => {
+  if (e instanceof Error) {
+    return new Error(e as any)
+  }
+  if (typeof e === "string") {
+    return new Error(`${e}`)
+  }
+  return new Error(`${JSON.stringify(e)}`)
+}

sdk/base/lib/util/deepEqual.ts

@@ -0,0 +1,19 @@
+import { object } from "ts-matches"
+
+export function deepEqual(...args: unknown[]) {
+  if (!object.test(args[args.length - 1])) return args[args.length - 1]
+  const objects = args.filter(object.test)
+  if (objects.length === 0) {
+    for (const x of args) if (x !== args[0]) return false
+    return true
+  }
+  if (objects.length !== args.length) return false
+  const allKeys = new Set(objects.flatMap((x) => Object.keys(x)))
+  for (const key of allKeys) {
+    for (const x of objects) {
+      if (!(key in x)) return false
+      if (!deepEqual((objects[0] as any)[key], (x as any)[key])) return false
+    }
+  }
+  return true
+}

sdk/base/lib/util/deepMerge.ts

@@ -0,0 +1,86 @@
+export function partialDiff<T>(
+  prev: T,
+  next: T,
+): { diff: Partial<T> } | undefined {
+  if (prev === next) {
+    return
+  } else if (Array.isArray(prev) && Array.isArray(next)) {
+    const res = { diff: [] as any[] }
+    for (let newItem of next) {
+      let anyEq = false
+      for (let oldItem of prev) {
+        if (!partialDiff(oldItem, newItem)) {
+          anyEq = true
+          break
+        }
+      }
+      if (!anyEq) {
+        res.diff.push(newItem)
+      }
+    }
+    if (res.diff.length) {
+      return res as any
+    } else {
+      return
+    }
+  } else if (typeof prev === "object" && typeof next === "object") {
+    if (prev === null) {
+      return { diff: next }
+    }
+    if (next === null) return
+    const res = { diff: {} as Record<keyof T, any> }
+    for (let key in next) {
+      const diff = partialDiff(prev[key], next[key])
+      if (diff) {
+        res.diff[key] = diff.diff
+      }
+    }
+    if (Object.keys(res.diff).length) {
+      return res
+    } else {
+      return
+    }
+  } else {
+    return { diff: next }
+  }
+}
+
+export function deepMerge(...args: unknown[]): unknown {
+  const lastItem = (args as any)[args.length - 1]
+  if (typeof lastItem !== "object" || !lastItem) return lastItem
+  if (Array.isArray(lastItem))
+    return deepMergeList(
+      ...(args.filter((x) => Array.isArray(x)) as unknown[][]),
+    )
+  return deepMergeObject(
+    ...(args.filter(
+      (x) => typeof x === "object" && x && !Array.isArray(x),
+    ) as object[]),
+  )
+}
+
+function deepMergeList(...args: unknown[][]): unknown[] {
+  const res: unknown[] = []
+  for (let arg of args) {
+    for (let item of arg) {
+      if (!res.some((x) => !partialDiff(x, item))) {
+        res.push(item)
+      }
+    }
+  }
+  return res
+}
+
+function deepMergeObject(...args: object[]): object {
+  const lastItem = (args as any)[args.length - 1]
+  if (args.length === 0) return lastItem as any
+  if (args.length === 1) args.unshift({})
+  const allKeys = new Set(args.flatMap((x) => Object.keys(x)))
+  for (const key of allKeys) {
+    const filteredValues = args.flatMap((x) =>
+      key in x ? [(x as any)[key]] : [],
+    )
+    ;(args as any)[0][key] = deepMerge(...filteredValues)
+  }
+  return args[0] as any
+}

sdk/base/lib/util/getDefaultString.ts

@@ -0,0 +1,10 @@
+import { DefaultString } from "../actions/input/inputSpecTypes"
+import { getRandomString } from "./getRandomString"
+
+export function getDefaultString(defaultSpec: DefaultString): string {
+  if (typeof defaultSpec === "string") {
+    return defaultSpec
+  } else {
+    return getRandomString(defaultSpec)
+  }
+}

sdk/base/lib/util/getRandomCharInSet.ts

@@ -0,0 +1,99 @@
+// a,g,h,A-Z,,,,-
+
+export function getRandomCharInSet(charset: string): string {
+  const set = stringToCharSet(charset)
+  let charIdx = Math.floor(
+    (crypto.getRandomValues(new Uint32Array(1))[0] / 2 ** 32) * set.len,
+  )
+  for (let range of set.ranges) {
+    if (range.len > charIdx) {
+      return String.fromCharCode(range.start.charCodeAt(0) + charIdx)
+    }
+    charIdx -= range.len
+  }
+  throw new Error("unreachable")
+}
+function stringToCharSet(charset: string): CharSet {
+  let set: CharSet = { ranges: [], len: 0 }
+  let start: string | null = null
+  let end: string | null = null
+  let in_range = false
+  for (let char of charset) {
+    switch (char) {
+      case ",":
+        if (start !== null && end !== null) {
+          if (start!.charCodeAt(0) > end!.charCodeAt(0)) {
+            throw new Error("start > end of charset")
+          }
+          const len = end.charCodeAt(0) - start.charCodeAt(0) + 1
+          set.ranges.push({
+            start,
+            end,
+            len,
+          })
+          set.len += len
+          start = null
+          end = null
+          in_range = false
+        } else if (start !== null && !in_range) {
+          set.len += 1
+          set.ranges.push({ start, end: start, len: 1 })
+          start = null
+        } else if (start !== null && in_range) {
+          end = ","
+        } else if (start === null && end === null && !in_range) {
+          start = ","
+        } else {
+          throw new Error('unexpected ","')
+        }
+        break
+      case "-":
+        if (start === null) {
+          start = "-"
+        } else if (!in_range) {
+          in_range = true
+        } else if (in_range && end === null) {
+          end = "-"
+        } else {
+          throw new Error('unexpected "-"')
+        }
+        break
+      default:
+        if (start === null) {
+          start = char
+        } else if (in_range && end === null) {
+          end = char
+        } else {
+          throw new Error(`unexpected "${char}"`)
+        }
+    }
+  }
+  if (start !== null && end !== null) {
+    if (start!.charCodeAt(0) > end!.charCodeAt(0)) {
+      throw new Error("start > end of charset")
+    }
+    const len = end.charCodeAt(0) - start.charCodeAt(0) + 1
+    set.ranges.push({
+      start,
+      end,
+      len,
+    })
+    set.len += len
+  } else if (start !== null) {
+    set.len += 1
+    set.ranges.push({
+      start,
+      end: start,
+      len: 1,
+    })
+  }
+  return set
+}
+type CharSet = {
+  ranges: {
+    start: string
+    end: string
+    len: number
+  }[]
+  len: number
+}

sdk/base/lib/util/getRandomString.ts

@@ -0,0 +1,11 @@
+import { RandomString } from "../actions/input/inputSpecTypes"
+import { getRandomCharInSet } from "./getRandomCharInSet"
+
+export function getRandomString(generator: RandomString): string {
+  let s = ""
+  for (let i = 0; i < generator.len; i++) {
+    s = s + getRandomCharInSet(generator.charset)
+  }
+
+  return s
+}

sdk/base/lib/util/getServiceInterface.ts

@@ -0,0 +1,284 @@
+import { ServiceInterfaceType } from "../types"
+import { knownProtocols } from "../interfaces/Host"
+import {
+  AddressInfo,
+  Host,
+  HostAddress,
+  Hostname,
+  HostnameInfo,
+  HostnameInfoIp,
+  HostnameInfoOnion,
+  IpInfo,
+} from "../types"
+import { Effects } from "../Effects"
+
+export type UrlString = string
+export type HostId = string
+
+const getHostnameRegex = /^(\w+:\/\/)?([^\/\:]+)(:\d{1,3})?(\/)?/
+export const getHostname = (url: string): Hostname | null => {
+  const founds = url.match(getHostnameRegex)?.[2]
+  if (!founds) return null
+  const parts = founds.split("@")
+  const last = parts[parts.length - 1] as Hostname | null
+  return last
+}
+
+export type Filled = {
+  hostnames: HostnameInfo[]
+  onionHostnames: HostnameInfo[]
+  localHostnames: HostnameInfo[]
+  ipHostnames: HostnameInfo[]
+  ipv4Hostnames: HostnameInfo[]
+  ipv6Hostnames: HostnameInfo[]
+  nonIpHostnames: HostnameInfo[]
+
+  urls: UrlString[]
+  onionUrls: UrlString[]
+  localUrls: UrlString[]
+  ipUrls: UrlString[]
+  ipv4Urls: UrlString[]
+  ipv6Urls: UrlString[]
+  nonIpUrls: UrlString[]
+}
+export type FilledAddressInfo = AddressInfo & Filled
+export type ServiceInterfaceFilled = {
+  id: string
+  /** The title of this field to be displayed */
+  name: string
+  /** Human readable description, used as tooltip usually */
+  description: string
+  /** Whether or not the interface has a primary URL */
+  hasPrimary: boolean
+  /** Whether or not to mask the URIs for this interface. Useful if the URIs contain sensitive information, such as a password, macaroon, or API key */
+  masked: boolean
+  /** Information about the host for this binding */
+  host: Host | null
+  /** URI information */
+  addressInfo: FilledAddressInfo | null
+  /** Indicates if we are a ui/p2p/api for the kind of interface that this is representing */
+  type: ServiceInterfaceType
+  /** The primary hostname for the service, as chosen by the user */
+  primaryHostname: Hostname | null
+  /** The primary URL for the service, as chosen by the user */
+  primaryUrl: UrlString | null
+}
+const either =
+  <A>(...args: ((a: A) => boolean)[]) =>
+  (a: A) =>
+    args.some((x) => x(a))
+const negate =
+  <A>(fn: (a: A) => boolean) =>
+  (a: A) =>
+    !fn(a)
+const unique = <A>(values: A[]) => Array.from(new Set(values))
+export const addressHostToUrl = (
+  { scheme, sslScheme, username, suffix }: AddressInfo,
+  host: HostnameInfo,
+): UrlString[] => {
+  const res = []
+  const fmt = (scheme: string | null, host: HostnameInfo, port: number) => {
+    const excludePort =
+      scheme &&
+      scheme in knownProtocols &&
+      port === knownProtocols[scheme as keyof typeof knownProtocols].defaultPort
+    let hostname
+    if (host.kind === "onion") {
+      hostname = host.hostname.value
+    } else if (host.kind === "ip") {
+      if (host.hostname.kind === "domain") {
+        hostname = `${host.hostname.subdomain ? `${host.hostname.subdomain}.` : ""}${host.hostname.domain}`
+      } else if (host.hostname.kind === "ipv6") {
+        hostname = `[${host.hostname.value}]`
+      } else {
+        hostname = host.hostname.value
+      }
+    }
+    return `${scheme ? `${scheme}://` : ""}${
+      username ? `${username}@` : ""
+    }${hostname}${excludePort ? "" : `:${port}`}${suffix}`
+  }
+  if (host.hostname.sslPort !== null) {
+    res.push(fmt(sslScheme, host, host.hostname.sslPort))
+  }
+  if (host.hostname.port !== null) {
+    res.push(fmt(scheme, host, host.hostname.port))
+  }
+
+  return res
+}
+
+export const filledAddress = (
+  host: Host,
+  addressInfo: AddressInfo,
+): FilledAddressInfo => {
+  const toUrl = addressHostToUrl.bind(null, addressInfo)
+  const hostnames = host.hostnameInfo[addressInfo.internalPort]
+
+  return {
+    ...addressInfo,
+    hostnames,
+    get onionHostnames() {
+      return hostnames.filter((h) => h.kind === "onion")
+    },
+    get localHostnames() {
+      return hostnames.filter(
+        (h) => h.kind === "ip" && h.hostname.kind === "local",
+      )
+    },
+    get ipHostnames() {
+      return hostnames.filter(
+        (h) =>
+          h.kind === "ip" &&
+          (h.hostname.kind === "ipv4" || h.hostname.kind === "ipv6"),
+      )
+    },
+    get ipv4Hostnames() {
+      return hostnames.filter(
+        (h) => h.kind === "ip" && h.hostname.kind === "ipv4",
+      )
+    },
+    get ipv6Hostnames() {
+      return hostnames.filter(
+        (h) => h.kind === "ip" && h.hostname.kind === "ipv6",
+      )
+    },
+    get nonIpHostnames() {
+      return hostnames.filter(
+        (h) =>
+          h.kind === "ip" &&
+          h.hostname.kind !== "ipv4" &&
+          h.hostname.kind !== "ipv6",
+      )
+    },
+    get urls() {
+      return this.hostnames.flatMap(toUrl)
+    },
+    get onionUrls() {
+      return this.onionHostnames.flatMap(toUrl)
+    },
+    get localUrls() {
+      return this.localHostnames.flatMap(toUrl)
+    },
+    get ipUrls() {
+      return this.ipHostnames.flatMap(toUrl)
+    },
+    get ipv4Urls() {
+      return this.ipv4Hostnames.flatMap(toUrl)
+    },
+    get ipv6Urls() {
+      return this.ipv6Hostnames.flatMap(toUrl)
+    },
+    get nonIpUrls() {
+      return this.nonIpHostnames.flatMap(toUrl)
+    },
+  }
+}
+
+const makeInterfaceFilled = async ({
+  effects,
+  id,
+  packageId,
+  callback,
+}: {
+  effects: Effects
+  id: string
+  packageId?: string
+  callback?: () => void
+}) => {
+  const serviceInterfaceValue = await effects.getServiceInterface({
+    serviceInterfaceId: id,
+    packageId,
+    callback,
+  })
+  if (!serviceInterfaceValue) {
+    return null
+  }
+  const hostId = serviceInterfaceValue.addressInfo.hostId
+  const host = await effects.getHostInfo({
+    packageId,
+    hostId,
+    callback,
+  })
+  const primaryUrl = await effects.getPrimaryUrl({
+    hostId,
+    packageId,
+    callback,
+  })
+
+  const interfaceFilled: ServiceInterfaceFilled = {
+    ...serviceInterfaceValue,
+    primaryUrl: primaryUrl,
+    host,
+    addressInfo: host
+      ? filledAddress(host, serviceInterfaceValue.addressInfo)
+      : null,
+    get primaryHostname() {
+      if (primaryUrl == null) return null
+      return getHostname(primaryUrl)
+    },
+  }
+  return interfaceFilled
+}
+
+export class GetServiceInterface {
+  constructor(
+    readonly effects: Effects,
+    readonly opts: { id: string; packageId?: string },
+  ) {}
+
+  /**
+   * Returns the value of Store at the provided path. Restart the service if the value changes
+   */
+  async const() {
+    const { id, packageId } = this.opts
+    const callback = () => this.effects.constRetry()
+    const interfaceFilled = await makeInterfaceFilled({
+      effects: this.effects,
+      id,
+      packageId,
+      callback,
+    })
+
+    return interfaceFilled
+  }
+  /**
+   * Returns the value of ServiceInterfacesFilled at the provided path. Does nothing if the value changes
+   */
+  async once() {
+    const { id, packageId } = this.opts
+    const interfaceFilled = await makeInterfaceFilled({
+      effects: this.effects,
+      id,
+      packageId,
+    })
+
+    return interfaceFilled
+  }
+
+  /**
+   * Watches the value of ServiceInterfacesFilled at the provided path. Takes a custom callback function to run whenever the value changes
+   */
+  async *watch() {
+    const { id, packageId } = this.opts
+    while (true) {
+      let callback: () => void = () => {}
+      const waitForNext = new Promise<void>((resolve) => {
+        callback = resolve
+      })
+      yield await makeInterfaceFilled({
+        effects: this.effects,
+        id,
+        packageId,
+        callback,
+      })
+      await waitForNext
+    }
+  }
+}
+export function getServiceInterface(
+  effects: Effects,
+  opts: { id: string; packageId?: string },
+) {
+  return new GetServiceInterface(effects, opts)
+}

sdk/base/lib/util/getServiceInterfaces.ts

@@ -0,0 +1,114 @@
+import { Effects } from "../Effects"
+import {
+  ServiceInterfaceFilled,
+  filledAddress,
+  getHostname,
+} from "./getServiceInterface"
+
+const makeManyInterfaceFilled = async ({
+  effects,
+  packageId,
+  callback,
+}: {
+  effects: Effects
+  packageId?: string
+  callback?: () => void
+}) => {
+  const serviceInterfaceValues = await effects.listServiceInterfaces({
+    packageId,
+    callback,
+  })
+
+  const serviceInterfacesFilled: ServiceInterfaceFilled[] = await Promise.all(
+    Object.values(serviceInterfaceValues).map(async (serviceInterfaceValue) => {
+      const hostId = serviceInterfaceValue.addressInfo.hostId
+      const host = await effects.getHostInfo({
+        packageId,
+        hostId,
+        callback,
+      })
+      if (!host) {
+        throw new Error(`host ${hostId} not found!`)
+      }
+      const primaryUrl = await effects
+        .getPrimaryUrl({
+          hostId,
+          packageId,
+          callback,
+        })
+        .catch(() => null)
+      return {
+        ...serviceInterfaceValue,
+        primaryUrl: primaryUrl,
+        host,
+        addressInfo: filledAddress(host, serviceInterfaceValue.addressInfo),
+        get primaryHostname() {
+          if (primaryUrl == null) return null
+          return getHostname(primaryUrl)
+        },
+      }
+    }),
+  )
+  return serviceInterfacesFilled
+}
+
+export class GetServiceInterfaces {
+  constructor(
+    readonly effects: Effects,
+    readonly opts: { packageId?: string },
+  ) {}
+
+  /**
+   * Returns the value of Store at the provided path. Restart the service if the value changes
+   */
+  async const() {
+    const { packageId } = this.opts
+    const callback = () => this.effects.constRetry()
+    const interfaceFilled: ServiceInterfaceFilled[] =
+      await makeManyInterfaceFilled({
+        effects: this.effects,
+        packageId,
+        callback,
+      })
+
+    return interfaceFilled
+  }
+  /**
+   * Returns the value of ServiceInterfacesFilled at the provided path. Does nothing if the value changes
+   */
+  async once() {
+    const { packageId } = this.opts
+    const interfaceFilled: ServiceInterfaceFilled[] =
+      await makeManyInterfaceFilled({
+        effects: this.effects,
+        packageId,
+      })
+
+    return interfaceFilled
+  }
+
+  /**
+   * Watches the value of ServiceInterfacesFilled at the provided path. Takes a custom callback function to run whenever the value changes
+   */
+  async *watch() {
+    const { packageId } = this.opts
+    while (true) {
+      let callback: () => void = () => {}
+      const waitForNext = new Promise<void>((resolve) => {
+        callback = resolve
+      })
+      yield await makeManyInterfaceFilled({
+        effects: this.effects,
+        packageId,
+        callback,
+      })
+      await waitForNext
+    }
+  }
+}
+export function getServiceInterfaces(
+  effects: Effects,
+  opts: { packageId?: string },
+) {
+  return new GetServiceInterfaces(effects, opts)
+}

sdk/base/lib/util/graph.ts

@@ -0,0 +1,244 @@
+import { boolean } from "ts-matches"
+
+export type Vertex<VMetadata = void, EMetadata = void> = {
+  metadata: VMetadata
+  edges: Array<Edge<EMetadata, VMetadata>>
+}
+
+export type Edge<EMetadata = void, VMetadata = void> = {
+  metadata: EMetadata
+  from: Vertex<VMetadata, EMetadata>
+  to: Vertex<VMetadata, EMetadata>
+}
+
+export class Graph<VMetadata = void, EMetadata = void> {
+  private readonly vertices: Array<Vertex<VMetadata, EMetadata>> = []
+  constructor() {}
+  addVertex(
+    metadata: VMetadata,
+    fromEdges: Array<Omit<Edge<EMetadata, VMetadata>, "to">>,
+    toEdges: Array<Omit<Edge<EMetadata, VMetadata>, "from">>,
+  ): Vertex<VMetadata, EMetadata> {
+    const vertex: Vertex<VMetadata, EMetadata> = {
+      metadata,
+      edges: [],
+    }
+    for (let edge of fromEdges) {
+      const vEdge = {
+        metadata: edge.metadata,
+        from: edge.from,
+        to: vertex,
+      }
+      edge.from.edges.push(vEdge)
+      vertex.edges.push(vEdge)
+    }
+    for (let edge of toEdges) {
+      const vEdge = {
+        metadata: edge.metadata,
+        from: vertex,
+        to: edge.to,
+      }
+      edge.to.edges.push(vEdge)
+      vertex.edges.push(vEdge)
+    }
+    this.vertices.push(vertex)
+    return vertex
+  }
+  findVertex(
+    predicate: (vertex: Vertex<VMetadata, EMetadata>) => boolean,
+  ): Generator<Vertex<VMetadata, EMetadata>, void> {
+    const veritces = this.vertices
+    function* gen() {
+      for (let vertex of veritces) {
+        if (predicate(vertex)) {
+          yield vertex
+        }
+      }
+    }
+    return gen()
+  }
+  addEdge(
+    metadata: EMetadata,
+    from: Vertex<VMetadata, EMetadata>,
+    to: Vertex<VMetadata, EMetadata>,
+  ): Edge<EMetadata, VMetadata> {
+    const edge = {
+      metadata,
+      from,
+      to,
+    }
+    edge.from.edges.push(edge)
+    edge.to.edges.push(edge)
+    return edge
+  }
+  breadthFirstSearch(
+    from:
+      | Vertex<VMetadata, EMetadata>
+      | ((vertex: Vertex<VMetadata, EMetadata>) => boolean),
+  ): Generator<Vertex<VMetadata, EMetadata>, void> {
+    const visited: Array<Vertex<VMetadata, EMetadata>> = []
+    function* rec(
+      vertex: Vertex<VMetadata, EMetadata>,
+    ): Generator<Vertex<VMetadata, EMetadata>, void> {
+      if (visited.includes(vertex)) {
+        return
+      }
+      visited.push(vertex)
+      yield vertex
+      let generators = vertex.edges
+        .filter((e) => e.from === vertex)
+        .map((e) => rec(e.to))
+      while (generators.length) {
+        let prev = generators
+        generators = []
+        for (let gen of prev) {
+          const next = gen.next()
+          if (!next.done) {
+            generators.push(gen)
+            yield next.value
+          }
+        }
+      }
+    }
+
+    if (from instanceof Function) {
+      let generators = this.vertices.filter(from).map(rec)
+      return (function* () {
+        while (generators.length) {
+          let prev = generators
+          generators = []
+          for (let gen of prev) {
+            const next = gen.next()
+            if (!next.done) {
+              generators.push(gen)
+              yield next.value
+            }
+          }
+        }
+      })()
+    } else {
+      return rec(from)
+    }
+  }
+  reverseBreadthFirstSearch(
+    to:
+      | Vertex<VMetadata, EMetadata>
+      | ((vertex: Vertex<VMetadata, EMetadata>) => boolean),
+  ): Generator<Vertex<VMetadata, EMetadata>, void> {
+    const visited: Array<Vertex<VMetadata, EMetadata>> = []
+    function* rec(
+      vertex: Vertex<VMetadata, EMetadata>,
+    ): Generator<Vertex<VMetadata, EMetadata>, void> {
+      if (visited.includes(vertex)) {
+        return
+      }
+      visited.push(vertex)
+      yield vertex
+      let generators = vertex.edges
+        .filter((e) => e.to === vertex)
+        .map((e) => rec(e.from))
+      while (generators.length) {
+        let prev = generators
+        generators = []
+        for (let gen of prev) {
+          const next = gen.next()
+          if (!next.done) {
+            generators.push(gen)
+            yield next.value
+          }
+        }
+      }
+    }
+
+    if (to instanceof Function) {
+      let generators = this.vertices.filter(to).map(rec)
+      return (function* () {
+        while (generators.length) {
+          let prev = generators
+          generators = []
+          for (let gen of prev) {
+            const next = gen.next()
+            if (!next.done) {
+              generators.push(gen)
+              yield next.value
+            }
+          }
+        }
+      })()
+    } else {
+      return rec(to)
+    }
+  }
+  shortestPath(
+    from:
+      | Vertex<VMetadata, EMetadata>
+      | ((vertex: Vertex<VMetadata, EMetadata>) => boolean),
+    to:
+      | Vertex<VMetadata, EMetadata>
+      | ((vertex: Vertex<VMetadata, EMetadata>) => boolean),
+  ): Array<Edge<EMetadata, VMetadata>> | void {
+    const isDone =
+      to instanceof Function
+        ? to
+        : (v: Vertex<VMetadata, EMetadata>) => v === to
+    const path: Array<Edge<EMetadata, VMetadata>> = []
+    const visited: Array<Vertex<VMetadata, EMetadata>> = []
+    function* check(
+      vertex: Vertex<VMetadata, EMetadata>,
+      path: Array<Edge<EMetadata, VMetadata>>,
+    ): Generator<undefined, Array<Edge<EMetadata, VMetadata>> | undefined> {
+      if (isDone(vertex)) {
+        return path
+      }
+      if (visited.includes(vertex)) {
+        return
+      }
+      visited.push(vertex)
+      yield
+      let generators = vertex.edges
+        .filter((e) => e.from === vertex)
+        .map((e) => check(e.to, [...path, e]))
+      while (generators.length) {
+        let prev = generators
+        generators = []
+        for (let gen of prev) {
+          const next = gen.next()
+          if (next.done === true) {
+            if (next.value) {
+              return next.value
+            }
+          } else {
+            generators.push(gen)
+            yield
+          }
+        }
+      }
+    }
+
+    if (from instanceof Function) {
+      let generators = this.vertices.filter(from).map((v) => check(v, []))
+      while (generators.length) {
+        let prev = generators
+        generators = []
+        for (let gen of prev) {
+          const next = gen.next()
+          if (next.done === true) {
+            if (next.value) {
+              return next.value
+            }
+          } else {
+            generators.push(gen)
+          }
+        }
+      }
+    } else {
+      const gen = check(from, [])
+      while (true) {
+        const next = gen.next()
+        if (next.done) {
+          return next.value
+        }
+      }
+    }
+  }
+}

sdk/base/lib/util/inMs.test.ts

@@ -0,0 +1,34 @@
+import { inMs } from "./inMs"
+
+describe("inMs", () => {
+  test("28.001s", () => {
+    expect(inMs("28.001s")).toBe(28001)
+  })
+  test("28.123s", () => {
+    expect(inMs("28.123s")).toBe(28123)
+  })
+  test(".123s", () => {
+    expect(inMs(".123s")).toBe(123)
+  })
+  test("123ms", () => {
+    expect(inMs("123ms")).toBe(123)
+  })
+  test("1h", () => {
+    expect(inMs("1h")).toBe(3600000)
+  })
+  test("1m", () => {
+    expect(inMs("1m")).toBe(60000)
+  })
+  test("1m", () => {
+    expect(inMs("1d")).toBe(1000 * 60 * 60 * 24)
+  })
+  test("123", () => {
+    expect(() => inMs("123")).toThrowError("Invalid time format: 123")
+  })
+  test("123 as number", () => {
+    expect(inMs(123)).toBe(123)
+  })
+  test.only("undefined", () => {
+    expect(inMs(undefined)).toBe(undefined)
+  })
+})

sdk/base/lib/util/inMs.ts

@@ -0,0 +1,29 @@
+const matchTimeRegex = /^\s*(\d+)?(\.\d+)?\s*(ms|s|m|h|d)/
+
+const unitMultiplier = (unit?: string) => {
+  if (!unit) return 1
+  if (unit === "ms") return 1
+  if (unit === "s") return 1000
+  if (unit === "m") return 1000 * 60
+  if (unit === "h") return 1000 * 60 * 60
+  if (unit === "d") return 1000 * 60 * 60 * 24
+  throw new Error(`Invalid unit: ${unit}`)
+}
+const digitsMs = (digits: string | null, multiplier: number) => {
+  if (!digits) return 0
+  const value = parseInt(digits.slice(1))
+  const divideBy = multiplier / Math.pow(10, digits.length - 1)
+  return Math.round(value * divideBy)
+}
+export const inMs = (time?: string | number) => {
+  if (typeof time === "number") return time
+  if (!time) return undefined
+  const matches = time.match(matchTimeRegex)
+  if (!matches) throw new Error(`Invalid time format: ${time}`)
+  const [_, leftHandSide, digits, unit] = matches
+  const multiplier = unitMultiplier(unit)
+  const firstValue = parseInt(leftHandSide || "0") * multiplier
+  const secondValue = digitsMs(digits, multiplier)
+
+  return firstValue + secondValue
+}

sdk/base/lib/util/index.ts

@@ -0,0 +1,22 @@
+/// Currently being used
+export { addressHostToUrl } from "./getServiceInterface"
+export { getDefaultString } from "./getDefaultString"
+
+/// Not being used, but known to be browser compatible
+export { GetServiceInterface, getServiceInterface } from "./getServiceInterface"
+export { getServiceInterfaces } from "./getServiceInterfaces"
+export { once } from "./once"
+export { asError } from "./asError"
+export * as Patterns from "./patterns"
+export * from "./typeHelpers"
+export { GetSystemSmtp } from "./GetSystemSmtp"
+export { Graph, Vertex } from "./graph"
+export { inMs } from "./inMs"
+export { splitCommand } from "./splitCommand"
+export { nullIfEmpty } from "./nullIfEmpty"
+export { deepMerge, partialDiff } from "./deepMerge"
+export { deepEqual } from "./deepEqual"
+export { hostnameInfoToAddress } from "./Hostname"
+export { PathBuilder, extractJsonPath, StorePath } from "./PathBuilder"
+export * as regexes from "./regexes"
+export { stringFromStdErrOut } from "./stringFromStdErrOut"

sdk/base/lib/util/nullIfEmpty.ts

@@ -0,0 +1,10 @@
+/**
+ * A useful tool when doing a getInputSpec.
+ * Look into the inputSpec {@link FileHelper} for an example of the use.
+ * @param s
+ * @returns
+ */
+export function nullIfEmpty<A extends Record<string, any>>(s: null | A) {
+  if (s === null) return null
+  return Object.keys(s).length === 0 ? null : s
+}

sdk/base/lib/util/once.ts

@@ -0,0 +1,9 @@
+export function once<B>(fn: () => B): () => B {
+  let result: [B] | [] = []
+  return () => {
+    if (!result.length) {
+      result = [fn()]
+    }
+    return result[0]
+  }
+}

sdk/base/lib/util/patterns.ts

@@ -0,0 +1,59 @@
+import { Pattern } from "../actions/input/inputSpecTypes"
+import * as regexes from "./regexes"
+
+export const ipv6: Pattern = {
+  regex: regexes.ipv6.toString(),
+  description: "Must be a valid IPv6 address",
+}
+
+export const ipv4: Pattern = {
+  regex: regexes.ipv4.toString(),
+  description: "Must be a valid IPv4 address",
+}
+
+export const hostname: Pattern = {
+  regex: regexes.hostname.toString(),
+  description: "Must be a valid hostname",
+}
+
+export const localHostname: Pattern = {
+  regex: regexes.localHostname.toString(),
+  description: 'Must be a valid ".local" hostname',
+}
+
+export const torHostname: Pattern = {
+  regex: regexes.torHostname.toString(),
+  description: 'Must be a valid Tor (".onion") hostname',
+}
+
+export const url: Pattern = {
+  regex: regexes.url.toString(),
+  description: "Must be a valid URL",
+}
+
+export const localUrl: Pattern = {
+  regex: regexes.localUrl.toString(),
+  description: 'Must be a valid ".local" URL',
+}
+
+export const torUrl: Pattern = {
+  regex: regexes.torUrl.toString(),
+  description: 'Must be a valid Tor (".onion") URL',
+}
+
+export const ascii: Pattern = {
+  regex: regexes.ascii.toString(),
+  description:
+    "May only contain ASCII characters. See https://www.w3schools.com/charsets/ref_html_ascii.asp",
+}
+
+export const email: Pattern = {
+  regex: regexes.email.toString(),
+  description: "Must be a valid email address",
+}
+
+export const base64: Pattern = {
+  regex: regexes.base64.toString(),
+  description:
+    "May only contain base64 characters. See https://base64.guru/learn/base64-characters",
+}

sdk/base/lib/util/regexes.ts

@@ -0,0 +1,34 @@
+// https://ihateregex.io/expr/ipv6/
+export const ipv6 =
+  /(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))/
+
+// https://ihateregex.io/expr/ipv4/
+export const ipv4 =
+  /(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}/
+
+export const hostname =
+  /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$/
+
+export const localHostname = /[-a-zA-Z0-9@:%._\+~#=]{1,256}\.local/
+
+export const torHostname = /[-a-zA-Z0-9@:%._\+~#=]{1,256}\.onion/
+
+// https://ihateregex.io/expr/url/
+export const url =
+  /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()!@:%_\+.~#?&\/\/=]*)/
+
+export const localUrl =
+  /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.local\b([-a-zA-Z0-9()!@:%_\+.~#?&\/\/=]*)/
+
+export const torUrl =
+  /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.onion\b([-a-zA-Z0-9()!@:%_\+.~#?&\/\/=]*)/
+
+// https://ihateregex.io/expr/ascii/
+export const ascii = /^[ -~]*$/
+
+//https://ihateregex.io/expr/email/
+export const email = /[^@ \t\r\n]+@[^@ \t\r\n]+\.[^@ \t\r\n]+/
+
+//https://rgxdb.com/r/1NUN74O6
+export const base64 =
+  /^(?:[a-zA-Z0-9+\/]{4})*(?:|(?:[a-zA-Z0-9+\/]{3}=)|(?:[a-zA-Z0-9+\/]{2}==)|(?:[a-zA-Z0-9+\/]{1}===))$/

sdk/base/lib/util/splitCommand.ts

@@ -0,0 +1,8 @@
+import { arrayOf, string } from "ts-matches"
+
+export const splitCommand = (
+  command: string | [string, ...string[]],
+): string[] => {
+  if (arrayOf(string).test(command)) return command
+  return ["sh", "-c", command]
+}

sdk/base/lib/util/stringFromStdErrOut.ts

@@ -0,0 +1,6 @@
+export async function stringFromStdErrOut(x: {
+  stdout: string
+  stderr: string
+}) {
+  return x?.stderr ? Promise.reject(x.stderr) : x.stdout
+}

sdk/base/lib/util/typeHelpers.ts

@@ -0,0 +1,116 @@
+import * as T from "../types"
+
+// prettier-ignore
+export type FlattenIntersection<T> = 
+T extends ArrayLike<any> ? T :
+T extends object ? {} & {[P in keyof T]: T[P]} :
+ T;
+
+export type _<T> = FlattenIntersection<T>
+
+export const isKnownError = (e: unknown): e is T.KnownError =>
+  e instanceof Object && ("error" in e || "error-code" in e)
+
+declare const affine: unique symbol
+
+export type Affine<A> = { [affine]: A }
+
+type NeverPossible = { [affine]: string }
+export type NoAny<A> = NeverPossible extends A
+  ? keyof NeverPossible extends keyof A
+    ? never
+    : A
+  : A
+
+type CapitalLetters =
+  | "A"
+  | "B"
+  | "C"
+  | "D"
+  | "E"
+  | "F"
+  | "G"
+  | "H"
+  | "I"
+  | "J"
+  | "K"
+  | "L"
+  | "M"
+  | "N"
+  | "O"
+  | "P"
+  | "Q"
+  | "R"
+  | "S"
+  | "T"
+  | "U"
+  | "V"
+  | "W"
+  | "X"
+  | "Y"
+  | "Z"
+
+type Numbers = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
+
+type CapitalChars = CapitalLetters | Numbers
+
+export type ToKebab<S extends string> = S extends string
+  ? S extends `${infer Head}${CapitalChars}${infer Tail}` // string has a capital char somewhere
+    ? Head extends "" // there is a capital char in the first position
+      ? Tail extends ""
+        ? Lowercase<S> /*  'A' */
+        : S extends `${infer Caps}${Tail}` // tail exists, has capital characters
+          ? Caps extends CapitalChars
+            ? Tail extends CapitalLetters
+              ? `${Lowercase<Caps>}-${Lowercase<Tail>}` /* 'AB' */
+              : Tail extends `${CapitalLetters}${string}`
+                ? `${ToKebab<Caps>}-${ToKebab<Tail>}` /* first tail char is upper? 'ABcd' */
+                : `${ToKebab<Caps>}${ToKebab<Tail>}` /* 'AbCD','AbcD',  */ /* TODO: if tail is only numbers, append without underscore */
+            : never /* never reached, used for inference of caps */
+          : never
+      : Tail extends "" /* 'aB' 'abCD' 'ABCD' 'AB' */
+        ? S extends `${Head}${infer Caps}`
+          ? Caps extends CapitalChars
+            ? Head extends Lowercase<Head> /* 'abcD' */
+              ? Caps extends Numbers
+                ? // Head exists and is lowercase, tail does not, Caps is a number, we may be in a sub-select
+                  // if head ends with number, don't split head an Caps, keep contiguous numbers together
+                  Head extends `${string}${Numbers}`
+                  ? never
+                  : // head does not end in number, safe to split. 'abc2' -> 'abc-2'
+                    `${ToKebab<Head>}-${Caps}`
+                : `${ToKebab<Head>}-${ToKebab<Caps>}` /* 'abcD' 'abc25' */
+              : never /* stop union type forming */
+            : never
+          : never /* never reached, used for inference of caps */
+        : S extends `${Head}${infer Caps}${Tail}` /* 'abCd' 'ABCD' 'AbCd' 'ABcD' */
+          ? Caps extends CapitalChars
+            ? Head extends Lowercase<Head> /* is 'abCd' 'abCD' ? */
+              ? Tail extends CapitalLetters /* is 'abCD' where Caps = 'C' */
+                ? `${ToKebab<Head>}-${ToKebab<Caps>}-${Lowercase<Tail>}` /* aBCD Tail = 'D', Head = 'aB' */
+                : Tail extends `${CapitalLetters}${string}` /* is 'aBCd' where Caps = 'B' */
+                  ? Head extends Numbers
+                    ? never /* stop union type forming */
+                    : Head extends `${string}${Numbers}`
+                      ? never /* stop union type forming */
+                      : `${Head}-${ToKebab<Caps>}-${ToKebab<Tail>}` /* 'aBCd' => `${'a'}-${Lowercase<'B'>}-${ToSnake<'Cd'>}` */
+                  : `${ToKebab<Head>}-${Lowercase<Caps>}${ToKebab<Tail>}` /* 'aBcD' where Caps = 'B' tail starts as lowercase */
+              : never
+            : never
+          : never
+    : S /* 'abc'  */
+  : never
+
+export type StringObject = Record<string, unknown>
+
+function test() {
+  // prettier-ignore
+  const t = <A, B>(a: (
+    A extends B ? (
+      B extends A ? null : never
+    ) : never
+  )) =>{ }
+  t<"foo-bar", ToKebab<"FooBar">>(null)
+  // @ts-expect-error
+  t<"foo-3ar", ToKebab<"FooBar">>(null)
+}