Merge branch 'next/minor' of github.com:Start9Labs/start-os into feature/registry-metrics

sdk/Makefile

@@ -25,7 +25,7 @@ dist: $(TS_FILES) package.json node_modules README.md LICENSE
 	cp LICENSE dist/LICENSE
 	touch dist
 
-full-bundle: clean bundle
+full-bundle: bundle
 
 check:
 	npm run check
@@ -36,7 +36,7 @@ fmt: node_modules
 node_modules: package.json
 	npm ci
 
-publish: clean bundle package.json README.md LICENSE
+publish: bundle package.json README.md LICENSE
 	cd dist && npm publish --access=public
 
 link: bundle

sdk/lib/StartSdk.ts

@@ -59,7 +59,7 @@ import {
 } from "./interfaces/setupInterfaces"
 import { successFailure } from "./trigger/successFailure"
 import { HealthReceipt } from "./health/HealthReceipt"
-import { MultiHost, Scheme, SingleHost, StaticHost } from "./interfaces/Host"
+import { MultiHost, Scheme } from "./interfaces/Host"
 import { ServiceInterfaceBuilder } from "./interfaces/ServiceInterfaceBuilder"
 import { GetSystemSmtp } from "./util/GetSystemSmtp"
 import nullIfEmpty from "./util/nullIfEmpty"
@@ -178,16 +178,19 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
       },
 
       host: {
-        static: (effects: Effects, id: string) =>
-          new StaticHost({ id, effects }),
-        single: (effects: Effects, id: string) =>
-          new SingleHost({ id, effects }),
+        // static: (effects: Effects, id: string) =>
+        //   new StaticHost({ id, effects }),
+        // single: (effects: Effects, id: string) =>
+        //   new SingleHost({ id, effects }),
         multi: (effects: Effects, id: string) => new MultiHost({ id, effects }),
       },
       nullIfEmpty,
       runCommand: async <A extends string>(
         effects: Effects,
-        image: { id: Manifest["images"][number]; sharedRun?: boolean },
+        image: {
+          id: keyof Manifest["images"] & T.ImageId
+          sharedRun?: boolean
+        },
         command: ValidIfNoStupidEscape<A> | [string, ...string[]],
         options: CommandOptions & {
           mounts?: { path: string; options: MountOptions }[]
@@ -396,7 +399,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
       setupProperties:
         (
           fn: (options: { effects: Effects }) => Promise<T.SdkPropertiesReturn>,
-        ): T.ExpectedExports.Properties =>
+        ): T.ExpectedExports.properties =>
         (options) =>
           fn(options).then(nullifyProperties),
       setupUninstall: (fn: UninstallFn<Manifest, Store>) =>
@@ -743,7 +746,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
 
 export async function runCommand<Manifest extends SDKManifest>(
   effects: Effects,
-  image: { id: Manifest["images"][number]; sharedRun?: boolean },
+  image: { id: keyof Manifest["images"] & T.ImageId; sharedRun?: boolean },
   command: string | [string, ...string[]],
   options: CommandOptions & {
     mounts?: { path: string; options: MountOptions }[]

sdk/lib/health/HealthCheck.ts

@@ -8,12 +8,13 @@ import { defaultTrigger } from "../trigger/defaultTrigger"
 import { once } from "../util/once"
 import { Overlay } from "../util/Overlay"
 import { object, unknown } from "ts-matches"
+import { T } from ".."
 
 export type HealthCheckParams<Manifest extends SDKManifest> = {
   effects: Effects
   name: string
   image: {
-    id: Manifest["images"][number]
+    id: keyof Manifest["images"] & T.ImageId
     sharedRun?: boolean
   }
   trigger?: Trigger

sdk/lib/interfaces/Host.ts

@@ -1,14 +1,14 @@
-import { object, string } from "ts-matches"
+import { number, object, string } from "ts-matches"
 import { Effects } from "../types"
 import { Origin } from "./Origin"
-import { AddSslOptions } from ".././osBindings"
+import { AddSslOptions, BindParams } from ".././osBindings"
 import { Security } from ".././osBindings"
 import { BindOptions } from ".././osBindings"
 import { AlpnInfo } from ".././osBindings"
 
 export { AddSslOptions, Security, BindOptions }
 
-const knownProtocols = {
+export const knownProtocols = {
   http: {
     secure: null,
     defaultPort: 80,
@@ -70,18 +70,16 @@ type BindOptionsByKnownProtocol =
   | {
       protocol: ProtocolsWithSslVariants
       preferredExternalPort?: number
-      scheme?: Scheme
       addSsl?: Partial<AddSslOptions>
     }
   | {
       protocol: NotProtocolsWithSslVariants
       preferredExternalPort?: number
-      scheme?: Scheme
       addSsl?: AddSslOptions
     }
-type BindOptionsByProtocol = BindOptionsByKnownProtocol | BindOptions
+export type BindOptionsByProtocol = BindOptionsByKnownProtocol | BindOptions
 
-export type HostKind = "static" | "single" | "multi"
+export type HostKind = BindParams["kind"]
 
 const hasStringProtocol = object({
   protocol: string,
@@ -110,66 +108,62 @@ export class Host {
   private async bindPortForUnknown(
     internalPort: number,
     options: {
-      scheme: Scheme
       preferredExternalPort: number
       addSsl: AddSslOptions | null
       secure: { ssl: boolean } | null
     },
   ) {
-    await this.options.effects.bind({
+    const binderOptions = {
       kind: this.options.kind,
       id: this.options.id,
-      internalPort: internalPort,
+      internalPort,
       ...options,
-    })
+    }
+    await this.options.effects.bind(binderOptions)
 
-    return new Origin(this, options)
+    return new Origin(this, internalPort, null, null)
   }
 
   private async bindPortForKnown(
     options: BindOptionsByKnownProtocol,
     internalPort: number,
   ) {
-    const scheme =
-      options.scheme === undefined ? options.protocol : options.scheme
     const protoInfo = knownProtocols[options.protocol]
     const preferredExternalPort =
       options.preferredExternalPort ||
       knownProtocols[options.protocol].defaultPort
-    const addSsl = this.getAddSsl(options, protoInfo)
+    const sslProto = this.getSslProto(options, protoInfo)
+    const addSsl =
+      sslProto && "alpn" in protoInfo
+        ? {
+            // addXForwardedHeaders: null,
+            preferredExternalPort: knownProtocols[sslProto].defaultPort,
+            scheme: sslProto,
+            alpn: protoInfo.alpn,
+            ...("addSsl" in options ? options.addSsl : null),
+          }
+        : null
 
     const secure: Security | null = !protoInfo.secure ? null : { ssl: false }
 
-    const newOptions = {
-      scheme,
-      preferredExternalPort,
-      addSsl,
-      secure,
-    }
-
     await this.options.effects.bind({
       kind: this.options.kind,
       id: this.options.id,
       internalPort,
-      ...newOptions,
+      preferredExternalPort,
+      addSsl,
+      secure,
     })
 
-    return new Origin(this, newOptions)
+    return new Origin(this, internalPort, options.protocol, sslProto)
   }
 
-  private getAddSsl(
+  private getSslProto(
     options: BindOptionsByKnownProtocol,
     protoInfo: KnownProtocols[keyof KnownProtocols],
-  ): AddSslOptions | null {
+  ) {
     if (inObject("noAddSsl", options) && options.noAddSsl) return null
-    if ("withSsl" in protoInfo && protoInfo.withSsl)
-      return {
-        // addXForwardedHeaders: null,
-        preferredExternalPort: knownProtocols[protoInfo.withSsl].defaultPort,
-        scheme: protoInfo.withSsl,
-        alpn: protoInfo.alpn,
-        ...("addSsl" in options ? options.addSsl : null),
-      }
+    if ("withSsl" in protoInfo && protoInfo.withSsl) return protoInfo.withSsl
     return null
   }
 }
@@ -181,17 +175,17 @@ function inObject<Key extends string>(
   return key in obj
 }
 
-export class StaticHost extends Host {
-  constructor(options: { effects: Effects; id: string }) {
-    super({ ...options, kind: "static" })
-  }
-}
+// export class StaticHost extends Host {
+//   constructor(options: { effects: Effects; id: string }) {
+//     super({ ...options, kind: "static" })
+//   }
+// }
 
-export class SingleHost extends Host {
-  constructor(options: { effects: Effects; id: string }) {
-    super({ ...options, kind: "single" })
-  }
-}
+// export class SingleHost extends Host {
+//   constructor(options: { effects: Effects; id: string }) {
+//     super({ ...options, kind: "single" })
+//   }
+// }
 
 export class MultiHost extends Host {
   constructor(options: { effects: Effects; id: string }) {

sdk/lib/interfaces/Origin.ts

@@ -6,7 +6,9 @@ import { ServiceInterfaceBuilder } from "./ServiceInterfaceBuilder"
 export class Origin<T extends Host> {
   constructor(
     readonly host: T,
-    readonly options: BindOptions,
+    readonly internalPort: number,
+    readonly scheme: string | null,
+    readonly sslScheme: string | null,
   ) {}
 
   build({ username, path, search, schemeOverride }: BuildOptions): AddressInfo {
@@ -20,18 +22,9 @@ export class Origin<T extends Host> {
 
     return {
       hostId: this.host.options.id,
-      bindOptions: {
-        ...this.options,
-        scheme: schemeOverride ? schemeOverride.noSsl : this.options.scheme,
-        addSsl: this.options.addSsl
-          ? {
-              ...this.options.addSsl,
-              scheme: schemeOverride
-                ? schemeOverride.ssl
-                : this.options.addSsl.scheme,
-            }
-          : null,
-      },
+      internalPort: this.internalPort,
+      scheme: schemeOverride ? schemeOverride.noSsl : this.scheme,
+      sslScheme: schemeOverride ? schemeOverride.ssl : this.sslScheme,
       suffix: `${path}${qp}`,
       username,
     }

sdk/lib/mainFn/CommandController.ts

@@ -1,6 +1,6 @@
 import { NO_TIMEOUT, SIGKILL, SIGTERM } from "../StartSdk"
 import { SDKManifest } from "../manifest/ManifestTypes"
-import { Effects, ValidIfNoStupidEscape } from "../types"
+import { Effects, ImageId, ValidIfNoStupidEscape } from "../types"
 import { MountOptions, Overlay } from "../util/Overlay"
 import { splitCommand } from "../util/splitCommand"
 import { cpExecFile, cpExec } from "./Daemons"
@@ -15,7 +15,7 @@ export class CommandController {
     return async <A extends string>(
       effects: Effects,
       imageId: {
-        id: Manifest["images"][number]
+        id: keyof Manifest["images"] & ImageId
         sharedRun?: boolean
       },
       command: ValidIfNoStupidEscape<A> | [string, ...string[]],

sdk/lib/mainFn/Daemon.ts

@@ -1,5 +1,5 @@
 import { SDKManifest } from "../manifest/ManifestTypes"
-import { Effects, ValidIfNoStupidEscape } from "../types"
+import { Effects, ImageId, ValidIfNoStupidEscape } from "../types"
 import { MountOptions, Overlay } from "../util/Overlay"
 import { CommandController } from "./CommandController"
 
@@ -18,7 +18,7 @@ export class Daemon {
     return async <A extends string>(
       effects: Effects,
       imageId: {
-        id: Manifest["images"][number]
+        id: keyof Manifest["images"] & ImageId
         sharedRun?: boolean
       },
       command: ValidIfNoStupidEscape<A> | [string, ...string[]],

sdk/lib/mainFn/Daemons.ts

@@ -5,7 +5,12 @@ import { SDKManifest } from "../manifest/ManifestTypes"
 import { Trigger } from "../trigger"
 import { TriggerInput } from "../trigger/TriggerInput"
 import { defaultTrigger } from "../trigger/defaultTrigger"
-import { DaemonReturned, Effects, ValidIfNoStupidEscape } from "../types"
+import {
+  DaemonReturned,
+  Effects,
+  ImageId,
+  ValidIfNoStupidEscape,
+} from "../types"
 import { Mounts } from "./Mounts"
 import { CommandOptions, MountOptions, Overlay } from "../util/Overlay"
 import { splitCommand } from "../util/splitCommand"
@@ -34,8 +39,8 @@ type DaemonsParams<
   Id extends string,
 > = {
   command: ValidIfNoStupidEscape<Command> | [string, ...string[]]
-  image: { id: Manifest["images"][number]; sharedRun?: boolean }
-  mounts: { path: string; options: MountOptions }[]
+  image: { id: keyof Manifest["images"] & ImageId; sharedRun?: boolean }
+  mounts: Mounts<Manifest>
   env?: Record<string, string>
   ready: Ready
   requires: Exclude<Ids, Id>[]
@@ -116,12 +121,10 @@ export class Daemons<Manifest extends SDKManifest, Ids extends string> {
     options: DaemonsParams<Manifest, Ids, Command, Id>,
   ) {
     const daemonIndex = this.daemons.length
-    const daemon = Daemon.of()(
-      this.effects,
-      options.image,
-      options.command,
-      options,
-    )
+    const daemon = Daemon.of()(this.effects, options.image, options.command, {
+      ...options,
+      mounts: options.mounts.build(),
+    })
     const healthDaemon = new HealthDaemon(
       daemon,
       daemonIndex,
@@ -151,7 +154,7 @@ export class Daemons<Manifest extends SDKManifest, Ids extends string> {
     this.healthDaemons.forEach((x) =>
       x.addWatcher(() => this.updateMainHealth()),
     )
-    return {
+    const built = {
       term: async (options?: { signal?: Signals; timeout?: number }) => {
         try {
           await Promise.all(this.healthDaemons.map((x) => x.term(options)))
@@ -160,13 +163,11 @@ export class Daemons<Manifest extends SDKManifest, Ids extends string> {
         }
       },
     }
+    this.started(() => built.term())
+    return built
   }
 
   private updateMainHealth() {
-    if (this.healthDaemons.every((x) => x.health.status === "success")) {
-      this.effects.setMainStatus({ status: "running" })
-    } else {
-      this.effects.setMainStatus({ status: "starting" })
-    }
+    this.effects.setMainStatus({ status: "running" })
   }
 }

sdk/lib/mainFn/HealthDaemon.ts

@@ -132,14 +132,14 @@ export class HealthDaemon {
       this.effects.setHealth({
         result: status,
         message: health.message,
-        id: display,
+        id: this.id,
         name: display,
       })
     } else {
       this.effects.setHealth({
         result: health.status,
         message: health.message || "",
-        id: display,
+        id: this.id,
         name: display,
       })
     }

sdk/lib/manifest/ManifestTypes.ts

@@ -1,5 +1,5 @@
 import { ValidEmVer } from "../emverLite/mod"
-import { ActionMetadata } from "../types"
+import { ActionMetadata, ImageConfig, ImageId } from "../types"
 
 export interface Container {
   /** This should be pointing to a docker container name */
@@ -28,8 +28,6 @@ export type SDKManifest = {
   readonly releaseNotes: string
   /** The type of license for the project. Include the LICENSE in the root of the project directory. A license is required for a Start9 package.*/
   readonly license: string // name of license
-  /** A list of normie (hosted, SaaS, custodial, etc) services this services intends to replace */
-  readonly replaces: Readonly<string[]>
   /** The Start9 wrapper repository URL for the package. This repo contains the manifest file (this),
    * any scripts necessary for configuration, backups, actions, or health checks (more below). This key
    * must exist. But could be embedded into the source repository
@@ -52,7 +50,7 @@ export type SDKManifest = {
   }
 
   /** Defines the os images needed to run the container processes */
-  readonly images: string[]
+  readonly images: Record<ImageId, ImageConfig>
   /** This denotes readonly asset directories that should be available to mount to the container.
    * Assuming that there will be three files with names along the lines:
    * icon.* : the icon that will be this packages icon on the ui

sdk/lib/manifest/setupManifest.ts

@@ -1,18 +1,19 @@
+import { ImageConfig, ImageId, VolumeId } from "../osBindings"
 import { SDKManifest, ManifestVersion } from "./ManifestTypes"
 
 export function setupManifest<
   Id extends string,
   Version extends ManifestVersion,
   Dependencies extends Record<string, unknown>,
-  VolumesTypes extends string,
-  AssetTypes extends string,
-  ImagesTypes extends string,
+  VolumesTypes extends VolumeId,
+  AssetTypes extends VolumeId,
+  ImagesTypes extends ImageId,
   Manifest extends SDKManifest & {
     dependencies: Dependencies
     id: Id
     version: Version
     assets: AssetTypes[]
-    images: ImagesTypes[]
+    images: Record<ImagesTypes, ImageConfig>
     volumes: VolumesTypes[]
   },
 >(manifest: Manifest): Manifest {

sdk/lib/osBindings/AddSslOptions.ts

@@ -2,7 +2,6 @@
 import type { AlpnInfo } from "./AlpnInfo"
 
 export type AddSslOptions = {
-  scheme: string | null
   preferredExternalPort: number
-  alpn: AlpnInfo
+  alpn: AlpnInfo | null
 }

sdk/lib/osBindings/AddressInfo.ts

@@ -1,10 +1,11 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
-import type { BindOptions } from "./BindOptions"
 import type { HostId } from "./HostId"
 
 export type AddressInfo = {
   username: string | null
   hostId: HostId
-  bindOptions: BindOptions
+  internalPort: number
+  scheme: string | null
+  sslScheme: string | null
   suffix: string
 }

sdk/lib/osBindings/AttachParams.ts

@@ -0,0 +1,7 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { EncryptedWire } from "./EncryptedWire"
+
+export type AttachParams = {
+  startOsPassword: EncryptedWire | null
+  guid: string
+}

sdk/lib/osBindings/BackupTargetFS.ts

@@ -0,0 +1,7 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { BlockDev } from "./BlockDev"
+import type { Cifs } from "./Cifs"
+
+export type BackupTargetFS =
+  | ({ type: "disk" } & BlockDev)
+  | ({ type: "cifs" } & Cifs)

sdk/lib/osBindings/BindInfo.ts

@@ -1,4 +1,5 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { BindOptions } from "./BindOptions"
+import type { LanInfo } from "./LanInfo"
 
-export type BindInfo = { options: BindOptions; assignedLanPort: number | null }
+export type BindInfo = { options: BindOptions; lan: LanInfo }

sdk/lib/osBindings/BindOptions.ts

@@ -3,7 +3,6 @@ import type { AddSslOptions } from "./AddSslOptions"
 import type { Security } from "./Security"
 
 export type BindOptions = {
-  scheme: string | null
   preferredExternalPort: number
   addSsl: AddSslOptions | null
   secure: Security | null

sdk/lib/osBindings/BindParams.ts

@@ -8,7 +8,6 @@ export type BindParams = {
   kind: HostKind
   id: HostId
   internalPort: number
-  scheme: string | null
   preferredExternalPort: number
   addSsl: AddSslOptions | null
   secure: Security | null

sdk/lib/osBindings/BlockDev.ts

@@ -1,3 +1,3 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export type GetHostInfoParamsKind = "multi"
+export type BlockDev = { logicalname: string }

sdk/lib/osBindings/Cifs.ts

@@ -0,0 +1,8 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type Cifs = {
+  hostname: string
+  path: string
+  username: string
+  password: string | null
+}

sdk/lib/osBindings/CreateOverlayedImageParams.ts

@@ -1,3 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { ImageId } from "./ImageId"
 
-export type CreateOverlayedImageParams = { imageId: string }
+export type CreateOverlayedImageParams = { imageId: ImageId }

sdk/lib/osBindings/DestroyOverlayedImageParams.ts

@@ -1,3 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Guid } from "./Guid"
 
-export type DestroyOverlayedImageParams = { guid: string }
+export type DestroyOverlayedImageParams = { guid: Guid }

sdk/lib/osBindings/ExecuteAction.ts

@@ -1,6 +1,8 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Guid } from "./Guid"
 
 export type ExecuteAction = {
+  procedureId: Guid
   serviceId: string | null
   actionId: string
   input: any

sdk/lib/osBindings/ExportServiceInterfaceParams.ts

@@ -1,7 +1,5 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { AddressInfo } from "./AddressInfo"
-import type { ExportedHostnameInfo } from "./ExportedHostnameInfo"
-import type { HostKind } from "./HostKind"
 import type { ServiceInterfaceId } from "./ServiceInterfaceId"
 import type { ServiceInterfaceType } from "./ServiceInterfaceType"
 
@@ -14,6 +12,4 @@ export type ExportServiceInterfaceParams = {
   masked: boolean
   addressInfo: AddressInfo
   type: ServiceInterfaceType
-  hostKind: HostKind
-  hostnames: Array<ExportedHostnameInfo>
 }

sdk/lib/osBindings/ExportedHostInfo.ts

@@ -1,10 +0,0 @@
-// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
-import type { ExportedHostnameInfo } from "./ExportedHostnameInfo"
-import type { HostId } from "./HostId"
-import type { HostKind } from "./HostKind"
-
-export type ExportedHostInfo = {
-  id: HostId
-  kind: HostKind
-  hostnames: Array<ExportedHostnameInfo>
-}

sdk/lib/osBindings/ExportedHostnameInfo.ts

@@ -1,12 +0,0 @@
-// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
-import type { ExportedIpHostname } from "./ExportedIpHostname"
-import type { ExportedOnionHostname } from "./ExportedOnionHostname"
-
-export type ExportedHostnameInfo =
-  | {
-      kind: "ip"
-      networkInterfaceId: string
-      public: boolean
-      hostname: ExportedIpHostname
-    }
-  | { kind: "onion"; hostname: ExportedOnionHostname }

sdk/lib/osBindings/GetHostInfoParams.ts

@@ -1,10 +1,9 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { Callback } from "./Callback"
-import type { GetHostInfoParamsKind } from "./GetHostInfoParamsKind"
+import type { HostId } from "./HostId"
 
 export type GetHostInfoParams = {
-  kind: GetHostInfoParamsKind | null
-  serviceInterfaceId: string
+  hostId: HostId
   packageId: string | null
   callback: Callback
 }

sdk/lib/osBindings/GetPrimaryUrlParams.ts

@@ -1,10 +1,9 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { Callback } from "./Callback"
-import type { HostId } from "./HostId"
+import type { ServiceInterfaceId } from "./ServiceInterfaceId"
 
 export type GetPrimaryUrlParams = {
   packageId: string | null
-  serviceInterfaceId: string
+  serviceInterfaceId: ServiceInterfaceId
   callback: Callback
-  hostId: HostId
 }

sdk/lib/osBindings/GetServiceInterfaceParams.ts

@@ -1,8 +1,9 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { Callback } from "./Callback"
+import type { ServiceInterfaceId } from "./ServiceInterfaceId"
 
 export type GetServiceInterfaceParams = {
   packageId: string | null
-  serviceInterfaceId: string
+  serviceInterfaceId: ServiceInterfaceId
   callback: Callback
 }

sdk/lib/osBindings/Host.ts

@@ -2,10 +2,14 @@
 import type { BindInfo } from "./BindInfo"
 import type { HostAddress } from "./HostAddress"
 import type { HostKind } from "./HostKind"
+import type { HostnameInfo } from "./HostnameInfo"
 
 export type Host = {
   kind: HostKind
   bindings: { [key: number]: BindInfo }
   addresses: Array<HostAddress>
-  primary: HostAddress | null
+  /**
+   * COMPUTED: NetService::update
+   */
+  hostnameInfo: { [key: number]: Array<HostnameInfo> }
 }

sdk/lib/osBindings/HostnameInfo.ts

@@ -0,0 +1,12 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { IpHostname } from "./IpHostname"
+import type { OnionHostname } from "./OnionHostname"
+
+export type HostnameInfo =
+  | {
+      kind: "ip"
+      networkInterfaceId: string
+      public: boolean
+      hostname: IpHostname
+    }
+  | { kind: "onion"; hostname: OnionHostname }

sdk/lib/osBindings/Hosts.ts

@@ -2,4 +2,4 @@
 import type { Host } from "./Host"
 import type { HostId } from "./HostId"
 
-export type HostInfo = { [key: HostId]: Host }
+export type Hosts = { [key: HostId]: Host }

sdk/lib/osBindings/ImageConfig.ts

@@ -0,0 +1,8 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { ImageSource } from "./ImageSource"
+
+export type ImageConfig = {
+  source: ImageSource
+  arch: string[]
+  emulateMissingAs: string | null
+}

sdk/lib/osBindings/ImageMetadata.ts

@@ -1,3 +1,3 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export type ReverseProxyHttp = { headers: null | { [key: string]: string } }
+export type ImageMetadata = { workdir: string; user: string }

sdk/lib/osBindings/ImageSource.ts

@@ -0,0 +1,6 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type ImageSource =
+  | "packed"
+  | { dockerBuild: { workdir: string | null; dockerfile: string | null } }
+  | { dockerTag: string }

sdk/lib/osBindings/InitProgressRes.ts

@@ -0,0 +1,5 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { FullProgress } from "./FullProgress"
+import type { Guid } from "./Guid"
+
+export type InitProgressRes = { progress: FullProgress; guid: Guid }

sdk/lib/osBindings/IpHostname.ts

@@ -1,6 +1,6 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export type ExportedIpHostname =
+export type IpHostname =
   | { kind: "ipv4"; value: string; port: number | null; sslPort: number | null }
   | { kind: "ipv6"; value: string; port: number | null; sslPort: number | null }
   | {

sdk/lib/osBindings/LanInfo.ts

@@ -1,7 +1,6 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export type ReverseProxyDestination = {
-  ip: string | null
-  port: number
-  ssl: boolean
+export type LanInfo = {
+  assignedPort: number | null
+  assignedSslPort: number | null
 }

sdk/lib/osBindings/LoginParams.ts

@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { PasswordType } from "./PasswordType"
+
+export type LoginParams = { password: PasswordType | null; metadata: any }

sdk/lib/osBindings/Manifest.ts

@@ -3,6 +3,7 @@ import type { Alerts } from "./Alerts"
 import type { Dependencies } from "./Dependencies"
 import type { Description } from "./Description"
 import type { HardwareRequirements } from "./HardwareRequirements"
+import type { ImageConfig } from "./ImageConfig"
 import type { ImageId } from "./ImageId"
 import type { PackageId } from "./PackageId"
 import type { Version } from "./Version"
@@ -20,7 +21,7 @@ export type Manifest = {
   marketingSite: string
   donationUrl: string | null
   description: Description
-  images: Array<ImageId>
+  images: { [key: ImageId]: ImageConfig }
   assets: Array<VolumeId>
   volumes: Array<VolumeId>
   alerts: Alerts

sdk/lib/osBindings/OnionHostname.ts

@@ -1,6 +1,6 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export type ExportedOnionHostname = {
+export type OnionHostname = {
   value: string
   port: number | null
   sslPort: number | null

sdk/lib/osBindings/PackageDataEntry.ts

@@ -3,10 +3,10 @@ import type { ActionId } from "./ActionId"
 import type { ActionMetadata } from "./ActionMetadata"
 import type { CurrentDependencies } from "./CurrentDependencies"
 import type { DataUrl } from "./DataUrl"
-import type { HostInfo } from "./HostInfo"
+import type { Hosts } from "./Hosts"
 import type { PackageState } from "./PackageState"
+import type { ServiceInterface } from "./ServiceInterface"
 import type { ServiceInterfaceId } from "./ServiceInterfaceId"
-import type { ServiceInterfaceWithHostInfo } from "./ServiceInterfaceWithHostInfo"
 import type { Status } from "./Status"
 
 export type PackageDataEntry = {
@@ -18,7 +18,7 @@ export type PackageDataEntry = {
   lastBackup: string | null
   currentDependencies: CurrentDependencies
   actions: { [key: ActionId]: ActionMetadata }
-  serviceInterfaces: { [key: ServiceInterfaceId]: ServiceInterfaceWithHostInfo }
-  hosts: HostInfo
+  serviceInterfaces: { [key: ServiceInterfaceId]: ServiceInterface }
+  hosts: Hosts
   storeExposedDependents: string[]
 }

sdk/lib/osBindings/ProcedureId.ts

@@ -1,3 +1,4 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Guid } from "./Guid"
 
-export type ReverseProxyBind = { ip: string | null; port: number; ssl: boolean }
+export type ProcedureId = { procedureId: Guid }

sdk/lib/osBindings/RecoverySource.ts

@@ -0,0 +1,6 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { BackupTargetFS } from "./BackupTargetFS"
+
+export type RecoverySource =
+  | { type: "migrate"; guid: string }
+  | { type: "backup"; target: BackupTargetFS }

sdk/lib/osBindings/ReverseProxyParams.ts

@@ -1,10 +0,0 @@
-// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
-import type { ReverseProxyBind } from "./ReverseProxyBind"
-import type { ReverseProxyDestination } from "./ReverseProxyDestination"
-import type { ReverseProxyHttp } from "./ReverseProxyHttp"
-
-export type ReverseProxyParams = {
-  bind: ReverseProxyBind
-  dst: ReverseProxyDestination
-  http: ReverseProxyHttp
-}

sdk/lib/osBindings/ServiceInterfaceWithHostInfo.ts

@@ -1,17 +0,0 @@
-// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
-import type { AddressInfo } from "./AddressInfo"
-import type { ExportedHostInfo } from "./ExportedHostInfo"
-import type { ServiceInterfaceId } from "./ServiceInterfaceId"
-import type { ServiceInterfaceType } from "./ServiceInterfaceType"
-
-export type ServiceInterfaceWithHostInfo = {
-  hostInfo: ExportedHostInfo
-  id: ServiceInterfaceId
-  name: string
-  description: string
-  hasPrimary: boolean
-  disabled: boolean
-  masked: boolean
-  addressInfo: AddressInfo
-  type: ServiceInterfaceType
-}

sdk/lib/osBindings/SetDependenciesParams.ts

@@ -1,6 +1,8 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { DependencyRequirement } from "./DependencyRequirement"
+import type { Guid } from "./Guid"
 
 export type SetDependenciesParams = {
+  procedureId: Guid
   dependencies: Array<DependencyRequirement>
 }

sdk/lib/osBindings/SetMainStatusStatus.ts

@@ -1,3 +1,3 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export type SetMainStatusStatus = "running" | "stopped" | "starting"
+export type SetMainStatusStatus = "running" | "stopped"

sdk/lib/osBindings/SetupExecuteParams.ts

@@ -0,0 +1,10 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { EncryptedWire } from "./EncryptedWire"
+import type { RecoverySource } from "./RecoverySource"
+
+export type SetupExecuteParams = {
+  startOsLogicalname: string
+  startOsPassword: EncryptedWire
+  recoverySource: RecoverySource | null
+  recoveryPassword: EncryptedWire | null
+}

sdk/lib/osBindings/SetupProgress.ts

@@ -0,0 +1,5 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { FullProgress } from "./FullProgress"
+import type { Guid } from "./Guid"
+
+export type SetupProgress = { progress: FullProgress; guid: Guid }

sdk/lib/osBindings/SetupResult.ts

@@ -0,0 +1,7 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type SetupResult = {
+  torAddress: string
+  lanAddress: string
+  rootCa: string
+}

sdk/lib/osBindings/SetupStatusRes.ts

@@ -0,0 +1,7 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { SetupProgress } from "./SetupProgress"
+import type { SetupResult } from "./SetupResult"
+
+export type SetupStatusRes =
+  | ({ status: "complete" } & SetupResult)
+  | ({ status: "running" } & SetupProgress)

sdk/lib/osBindings/VerifyCifsParams.ts

@@ -0,0 +1,9 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { EncryptedWire } from "./EncryptedWire"
+
+export type VerifyCifsParams = {
+  hostname: string
+  path: string
+  username: string
+  password: EncryptedWire | null
+}

sdk/lib/osBindings/index.ts

@@ -15,17 +15,21 @@ export { AlpnInfo } from "./AlpnInfo"
 export { AnySignature } from "./AnySignature"
 export { AnySigningKey } from "./AnySigningKey"
 export { AnyVerifyingKey } from "./AnyVerifyingKey"
+export { AttachParams } from "./AttachParams"
 export { BackupProgress } from "./BackupProgress"
+export { BackupTargetFS } from "./BackupTargetFS"
 export { Base64 } from "./Base64"
 export { BindInfo } from "./BindInfo"
 export { BindOptions } from "./BindOptions"
 export { BindParams } from "./BindParams"
 export { Blake3Commitment } from "./Blake3Commitment"
+export { BlockDev } from "./BlockDev"
 export { Callback } from "./Callback"
 export { Category } from "./Category"
 export { CheckDependenciesParam } from "./CheckDependenciesParam"
 export { CheckDependenciesResult } from "./CheckDependenciesResult"
 export { ChrootParams } from "./ChrootParams"
+export { Cifs } from "./Cifs"
 export { ContactInfo } from "./ContactInfo"
 export { CreateOverlayedImageParams } from "./CreateOverlayedImageParams"
 export { CurrentDependencies } from "./CurrentDependencies"
@@ -41,15 +45,10 @@ export { Duration } from "./Duration"
 export { EncryptedWire } from "./EncryptedWire"
 export { ExecuteAction } from "./ExecuteAction"
 export { ExportActionParams } from "./ExportActionParams"
-export { ExportedHostInfo } from "./ExportedHostInfo"
-export { ExportedHostnameInfo } from "./ExportedHostnameInfo"
-export { ExportedIpHostname } from "./ExportedIpHostname"
-export { ExportedOnionHostname } from "./ExportedOnionHostname"
 export { ExportServiceInterfaceParams } from "./ExportServiceInterfaceParams"
 export { ExposeForDependentsParams } from "./ExposeForDependentsParams"
 export { FullIndex } from "./FullIndex"
 export { FullProgress } from "./FullProgress"
-export { GetHostInfoParamsKind } from "./GetHostInfoParamsKind"
 export { GetHostInfoParams } from "./GetHostInfoParams"
 export { GetOsAssetParams } from "./GetOsAssetParams"
 export { GetPackageParams } from "./GetPackageParams"
@@ -70,16 +69,24 @@ export { HealthCheckId } from "./HealthCheckId"
 export { HealthCheckResult } from "./HealthCheckResult"
 export { HostAddress } from "./HostAddress"
 export { HostId } from "./HostId"
-export { HostInfo } from "./HostInfo"
 export { HostKind } from "./HostKind"
+export { HostnameInfo } from "./HostnameInfo"
+export { Hosts } from "./Hosts"
 export { Host } from "./Host"
+export { ImageConfig } from "./ImageConfig"
 export { ImageId } from "./ImageId"
+export { ImageMetadata } from "./ImageMetadata"
+export { ImageSource } from "./ImageSource"
+export { InitProgressRes } from "./InitProgressRes"
 export { InstalledState } from "./InstalledState"
 export { InstallingInfo } from "./InstallingInfo"
 export { InstallingState } from "./InstallingState"
+export { IpHostname } from "./IpHostname"
 export { IpInfo } from "./IpInfo"
+export { LanInfo } from "./LanInfo"
 export { ListServiceInterfacesParams } from "./ListServiceInterfacesParams"
 export { ListVersionSignersParams } from "./ListVersionSignersParams"
+export { LoginParams } from "./LoginParams"
 export { MainStatus } from "./MainStatus"
 export { Manifest } from "./Manifest"
 export { MaybeUtf8String } from "./MaybeUtf8String"
@@ -87,6 +94,7 @@ export { MerkleArchiveCommitment } from "./MerkleArchiveCommitment"
 export { MountParams } from "./MountParams"
 export { MountTarget } from "./MountTarget"
 export { NamedProgress } from "./NamedProgress"
+export { OnionHostname } from "./OnionHostname"
 export { OsIndex } from "./OsIndex"
 export { OsVersionInfo } from "./OsVersionInfo"
 export { PackageDataEntry } from "./PackageDataEntry"
@@ -100,17 +108,15 @@ export { PackageVersionInfo } from "./PackageVersionInfo"
 export { ParamsMaybePackageId } from "./ParamsMaybePackageId"
 export { ParamsPackageId } from "./ParamsPackageId"
 export { PasswordType } from "./PasswordType"
+export { ProcedureId } from "./ProcedureId"
 export { Progress } from "./Progress"
 export { Public } from "./Public"
+export { RecoverySource } from "./RecoverySource"
 export { RegistryAsset } from "./RegistryAsset"
 export { RemoveActionParams } from "./RemoveActionParams"
 export { RemoveAddressParams } from "./RemoveAddressParams"
 export { RemoveVersionParams } from "./RemoveVersionParams"
 export { RequestCommitment } from "./RequestCommitment"
-export { ReverseProxyBind } from "./ReverseProxyBind"
-export { ReverseProxyDestination } from "./ReverseProxyDestination"
-export { ReverseProxyHttp } from "./ReverseProxyHttp"
-export { ReverseProxyParams } from "./ReverseProxyParams"
 export { Security } from "./Security"
 export { ServerInfo } from "./ServerInfo"
 export { ServerSpecs } from "./ServerSpecs"
@@ -118,7 +124,6 @@ export { ServerStatus } from "./ServerStatus"
 export { ServiceInterfaceId } from "./ServiceInterfaceId"
 export { ServiceInterface } from "./ServiceInterface"
 export { ServiceInterfaceType } from "./ServiceInterfaceType"
-export { ServiceInterfaceWithHostInfo } from "./ServiceInterfaceWithHostInfo"
 export { SessionList } from "./SessionList"
 export { Sessions } from "./Sessions"
 export { Session } from "./Session"
@@ -129,10 +134,15 @@ export { SetMainStatusStatus } from "./SetMainStatusStatus"
 export { SetMainStatus } from "./SetMainStatus"
 export { SetStoreParams } from "./SetStoreParams"
 export { SetSystemSmtpParams } from "./SetSystemSmtpParams"
+export { SetupExecuteParams } from "./SetupExecuteParams"
+export { SetupProgress } from "./SetupProgress"
+export { SetupResult } from "./SetupResult"
+export { SetupStatusRes } from "./SetupStatusRes"
 export { SignAssetParams } from "./SignAssetParams"
 export { SignerInfo } from "./SignerInfo"
 export { Status } from "./Status"
 export { UpdatingState } from "./UpdatingState"
+export { VerifyCifsParams } from "./VerifyCifsParams"
 export { VersionSignerParams } from "./VersionSignerParams"
 export { Version } from "./Version"
 export { VolumeId } from "./VolumeId"

sdk/lib/test/configBuilder.test.ts

@@ -400,7 +400,7 @@ describe("values", () => {
               long: "",
             },
             containers: {},
-            images: [],
+            images: {},
             volumes: [],
             assets: [],
             alerts: {

sdk/lib/test/host.test.ts

@@ -8,6 +8,7 @@ describe("host", () => {
       const foo = sdk.host.multi(effects, "foo")
       const fooOrigin = await foo.bindPort(80, {
         protocol: "http" as const,
+        preferredExternalPort: 80,
       })
       const fooInterface = new ServiceInterfaceBuilder({
         effects,

sdk/lib/test/output.sdk.ts

@@ -21,7 +21,7 @@ export const sdk = StartSdk.of()
         long: "",
       },
       containers: {},
-      images: [],
+      images: {},
       volumes: [],
       assets: [],
       alerts: {

sdk/lib/test/startosTypeValidation.test.ts

@@ -25,7 +25,6 @@ import { ListServiceInterfacesParams } from ".././osBindings"
 import { RemoveAddressParams } from ".././osBindings"
 import { ExportActionParams } from ".././osBindings"
 import { RemoveActionParams } from ".././osBindings"
-import { ReverseProxyParams } from ".././osBindings"
 import { MountParams } from ".././osBindings"
 function typeEquality<ExpectedType>(_a: ExpectedType) {}
 describe("startosTypeValidation ", () => {
@@ -66,7 +65,6 @@ describe("startosTypeValidation ", () => {
       removeAddress: {} as RemoveAddressParams,
       exportAction: {} as ExportActionParams,
       removeAction: {} as RemoveActionParams,
-      reverseProxy: {} as ReverseProxyParams,
       mount: {} as MountParams,
       checkDependencies: {} as CheckDependenciesParam,
       getDependencies: undefined,

sdk/lib/types.ts

@@ -5,6 +5,13 @@ import {
   SetHealth,
   HealthCheckResult,
   SetMainStatus,
+  ServiceInterface,
+  Host,
+  ExportServiceInterfaceParams,
+  GetPrimaryUrlParams,
+  LanInfo,
+  BindParams,
+  Manifest,
 } from "./osBindings"
 
 import { MainEffects, ServiceInterfaceType, Signals } from "./StartSdk"
@@ -12,7 +19,7 @@ import { InputSpec } from "./config/configTypes"
 import { DependenciesReceipt } from "./config/setupConfig"
 import { BindOptions, Scheme } from "./interfaces/Host"
 import { Daemons } from "./mainFn/Daemons"
-import { PathBuilder, StorePath } from "./store/PathBuilder"
+import { StorePath } from "./store/PathBuilder"
 import { ExposedStorePaths } from "./store/setupExposeStore"
 import { UrlString } from "./util/getServiceInterface"
 export * from "./osBindings"
@@ -104,9 +111,26 @@ export namespace ExpectedExports {
    */
   export type dependencyConfig = Record<PackageId, DependencyConfig | null>
 
-  export type Properties = (options: {
+  export type properties = (options: {
     effects: Effects
   }) => Promise<PropertiesReturn>
+
+  export type manifest = Manifest
+}
+export type ABI = {
+  setConfig: ExpectedExports.setConfig
+  getConfig: ExpectedExports.getConfig
+  createBackup: ExpectedExports.createBackup
+  restoreBackup: ExpectedExports.restoreBackup
+  actions: ExpectedExports.actions
+  actionsMetadata: ExpectedExports.actionsMetadata
+  main: ExpectedExports.main
+  afterShutdown: ExpectedExports.afterShutdown
+  init: ExpectedExports.init
+  uninit: ExpectedExports.uninit
+  dependencyConfig: ExpectedExports.dependencyConfig
+  properties: ExpectedExports.properties
+  manifest: ExpectedExports.manifest
 }
 export type TimeMs = number
 export type VersionString = string
@@ -184,14 +208,6 @@ export declare const hostName: unique symbol
 // asdflkjadsf.onion | 1.2.3.4
 export type Hostname = string & { [hostName]: never }
 
-/** ${scheme}://${username}@${host}:${externalPort}${suffix} */
-export type AddressInfo = {
-  username: string | null
-  hostId: string
-  bindOptions: BindOptions
-  suffix: string
-}
-
 export type HostnameInfoIp = {
   kind: "ip"
   networkInterfaceId: string
@@ -219,44 +235,9 @@ export type HostnameInfoOnion = {
 
 export type HostnameInfo = HostnameInfoIp | HostnameInfoOnion
 
-export type SingleHost = {
-  id: string
-  kind: "single" | "static"
-  hostname: HostnameInfo | null
-}
-
-export type MultiHost = {
-  id: string
-  kind: "multi"
-  hostnames: HostnameInfo[]
-}
-
-export type HostInfo = SingleHost | MultiHost
-
 export type ServiceInterfaceId = string
 
-export type ServiceInterface = {
-  id: ServiceInterfaceId
-  /** The title of this field to be displayed */
-  name: string
-  /** Human readable description, used as tooltip usually */
-  description: string
-  /** Whether or not one address must be the primary address */
-  hasPrimary: boolean
-  /** Disabled interfaces do not serve, but they retain their metadata and addresses */
-  disabled: boolean
-  /** Whether or not to mask the URIs for this interface. Useful if the URIs contain sensitive information, such as a password, macaroon, or API key */
-  masked: boolean
-  /** URI Information */
-  addressInfo: AddressInfo
-  /** The network interface could be several types, something like ui, p2p, or network */
-  type: ServiceInterfaceType
-}
-
-export type ServiceInterfaceWithHostInfo = ServiceInterface & {
-  hostInfo: HostInfo
-}
-
+export { ServiceInterface }
 export type ExposeServicePaths<Store = never> = {
   /** The path to the value in the Store. [JsonPath](https://jsonpath.com/)  */
   paths: ExposedStorePaths
@@ -326,13 +307,7 @@ export type Effects = {
   /** Removes all network bindings */
   clearBindings(): Promise<void>
   /** Creates a host connected to the specified port with the provided options */
-  bind(
-    options: {
-      kind: "static" | "single" | "multi"
-      id: string
-      internalPort: number
-    } & BindOptions,
-  ): Promise<void>
+  bind(options: BindParams): Promise<void>
   /** Retrieves the current hostname(s) associated with a host id */
   // getHostInfo(options: {
   //   kind: "static" | "single"
@@ -341,11 +316,10 @@ export type Effects = {
   //   callback: () => void
   // }): Promise<SingleHost>
   getHostInfo(options: {
-    kind: "multi" | null
-    serviceInterfaceId: string
+    hostId: string
     packageId: string | null
     callback: () => void
-  }): Promise<MultiHost>
+  }): Promise<Host>
 
   // /**
   //  * Run rsync between two volumes. This is used to backup data between volumes.
@@ -395,14 +369,14 @@ export type Effects = {
   getServicePortForward(options: {
     internalPort: number
     packageId: string | null
-  }): Promise<number>
+  }): Promise<LanInfo>
 
   /** Removes all network interfaces */
   clearServiceInterfaces(): Promise<void>
   /** When we want to create a link in the front end interfaces, and example is
    * exposing a url to view a web service
    */
-  exportServiceInterface(options: ServiceInterface): Promise<string>
+  exportServiceInterface(options: ExportServiceInterfaceParams): Promise<string>
 
   exposeForDependents(options: { paths: string[] }): Promise<void>
 
@@ -422,11 +396,7 @@ export type Effects = {
    * The user sets the primary url for a interface
    * @param options
    */
-  getPrimaryUrl(options: {
-    packageId: PackageId | null
-    serviceInterfaceId: ServiceInterfaceId
-    callback: () => void
-  }): Promise<UrlString | null>
+  getPrimaryUrl(options: GetPrimaryUrlParams): Promise<UrlString | null>
 
   /**
    * There are times that we want to see the addresses that where exported
@@ -437,7 +407,7 @@ export type Effects = {
   listServiceInterfaces(options: {
     packageId: PackageId | null
     callback: () => void
-  }): Promise<ServiceInterface[]>
+  }): Promise<Record<ServiceInterfaceId, ServiceInterface>>
 
   /**
    *Remove an address that was exported. Used problably during main or during setConfig.
@@ -501,27 +471,8 @@ export type Effects = {
   /** Exists could be useful during the runtime to know if some service is running, option dep */
   running(options: { packageId: PackageId }): Promise<boolean>
 
-  /** Instead of creating proxies with nginx, we have a utility to create and maintain a proxy in the lifetime of this running. */
-  reverseProxy(options: {
-    bind: {
-      /** Optional, default is 0.0.0.0 */
-      ip: string | null
-      port: number
-      ssl: boolean
-    }
-    dst: {
-      /** Optional: default is 127.0.0.1 */
-      ip: string | null // optional, default 127.0.0.1
-      port: number
-      ssl: boolean
-    }
-    http: {
-      // optional, will do TCP layer proxy only if not present
-      headers: Record<string, string> | null
-    } | null
-  }): Promise<{ stop(): Promise<void> }>
-  restart(): void
-  shutdown(): void
+  restart(): Promise<void>
+  shutdown(): Promise<void>
 
   mount(options: {
     location: string

sdk/lib/util/Hostname.ts

@@ -0,0 +1,25 @@
+import { HostnameInfo } from "../types"
+
+export function hostnameInfoToAddress(hostInfo: HostnameInfo): string {
+  if (hostInfo.kind === "onion") {
+    return `${hostInfo.hostname.value}`
+  }
+  if (hostInfo.kind !== "ip") {
+    throw Error("Expecting that the kind is ip.")
+  }
+  const hostname = hostInfo.hostname
+  if (hostname.kind === "domain") {
+    return `${hostname.subdomain ? `${hostname.subdomain}.` : ""}${hostname.domain}`
+  }
+  const port = hostname.sslPort || hostname.port
+  const portString = port ? `:${port}` : ""
+  if ("ipv4" === hostname.kind || "ipv6" === hostname.kind) {
+    return `${hostname.value}${portString}`
+  }
+  if ("local" === hostname.kind) {
+    return `${hostname.value}${portString}`
+  }
+  throw Error(
+    "Expecting to have a valid hostname kind." + JSON.stringify(hostname),
+  )
+}

sdk/lib/util/Overlay.ts

@@ -8,16 +8,18 @@ const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/`
 export class Overlay {
   private constructor(
     readonly effects: T.Effects,
-    readonly imageId: string,
+    readonly imageId: T.ImageId,
     readonly rootfs: string,
-    readonly guid: string,
+    readonly guid: T.Guid,
   ) {}
   static async of(
     effects: T.Effects,
-    image: { id: string; sharedRun?: boolean },
+    image: { id: T.ImageId; sharedRun?: boolean },
   ) {
-    const { id: imageId, sharedRun } = image
-    const [rootfs, guid] = await effects.createOverlayedImage({ imageId })
+    const { id, sharedRun } = image
+    const [rootfs, guid] = await effects.createOverlayedImage({
+      imageId: id as string,
+    })
 
     const shared = ["dev", "sys", "proc"]
     if (!!sharedRun) {
@@ -25,15 +27,14 @@ export class Overlay {
     }
 
     for (const dirPart of shared) {
-      await fs.mkdir(`${rootfs}/${dirPart}`, { recursive: true })
-      await execFile("mount", [
-        "--rbind",
-        `/${dirPart}`,
-        `${rootfs}/${dirPart}`,
-      ])
+      const from = `/${dirPart}`
+      const to = `${rootfs}/${dirPart}`
+      await fs.mkdir(from, { recursive: true })
+      await fs.mkdir(to, { recursive: true })
+      await execFile("mount", ["--rbind", from, to])
     }
 
-    return new Overlay(effects, imageId, rootfs, guid)
+    return new Overlay(effects, id, rootfs, guid)
   }
 
   async mount(options: MountOptions, path: string): Promise<Overlay> {
@@ -46,22 +47,22 @@ export class Overlay {
           ? options.subpath
           : `/${options.subpath}`
         : "/"
-      await execFile("mount", [
-        "--bind",
-        `/media/startos/volumes/${options.id}${subpath}`,
-        path,
-      ])
+      const from = `/media/startos/volumes/${options.id}${subpath}`
+
+      await fs.mkdir(from, { recursive: true })
+      await fs.mkdir(path, { recursive: true })
+      await await execFile("mount", ["--bind", from, path])
     } else if (options.type === "assets") {
       const subpath = options.subpath
         ? options.subpath.startsWith("/")
           ? options.subpath
           : `/${options.subpath}`
         : "/"
-      await execFile("mount", [
-        "--bind",
-        `/media/startos/assets/${options.id}${subpath}`,
-        path,
-      ])
+      const from = `/media/startos/assets/${options.id}${subpath}`
+
+      await fs.mkdir(from, { recursive: true })
+      await fs.mkdir(path, { recursive: true })
+      await execFile("mount", ["--bind", from, path])
     } else if (options.type === "pointer") {
       await this.effects.mount({ location: path, target: options })
     } else if (options.type === "backup") {
@@ -70,11 +71,11 @@ export class Overlay {
           ? options.subpath
           : `/${options.subpath}`
         : "/"
-      await execFile("mount", [
-        "--bind",
-        `/media/startos/backup${subpath}`,
-        path,
-      ])
+      const from = `/media/startos/backup${subpath}`
+
+      await fs.mkdir(from, { recursive: true })
+      await fs.mkdir(path, { recursive: true })
+      await execFile("mount", ["--bind", from, path])
     } else {
       throw new Error(`unknown type ${(options as any).type}`)
     }
@@ -97,7 +98,7 @@ export class Overlay {
     stdout: string | Buffer
     stderr: string | Buffer
   }> {
-    const imageMeta: any = await fs
+    const imageMeta: T.ImageMetadata = await fs
       .readFile(`/media/startos/images/${this.imageId}.json`, {
         encoding: "utf8",
       })

sdk/lib/util/fileHelper.ts

@@ -1,8 +1,9 @@
 import * as matches from "ts-matches"
 import * as YAML from "yaml"
 import * as TOML from "@iarna/toml"
+import _ from "lodash"
 import * as T from "../types"
-import * as fs from "fs"
+import * as fs from "node:fs/promises"
 
 const previousPath = /(.+?)\/([^/]*)$/
 
@@ -58,30 +59,32 @@ export class FileHelper<A> {
     readonly readData: (stringValue: string) => A,
   ) {}
   async write(data: A, effects: T.Effects) {
-    if (previousPath.exec(this.path)) {
-      await new Promise((resolve, reject) =>
-        fs.mkdir(this.path, (err: any) => (!err ? resolve(null) : reject(err))),
-      )
+    const parent = previousPath.exec(this.path)
+    if (parent) {
+      await fs.mkdir(parent[1], { recursive: true })
     }
 
-    await new Promise((resolve, reject) =>
-      fs.writeFile(this.path, this.writeData(data), (err: any) =>
-        !err ? resolve(null) : reject(err),
-      ),
-    )
+    await fs.writeFile(this.path, this.writeData(data))
   }
   async read(effects: T.Effects) {
-    if (!fs.existsSync(this.path)) {
+    if (
+      !(await fs.access(this.path).then(
+        () => true,
+        () => false,
+      ))
+    ) {
       return null
     }
     return this.readData(
-      await new Promise((resolve, reject) =>
-        fs.readFile(this.path, (err: any, data: any) =>
-          !err ? resolve(data.toString("utf-8")) : reject(err),
-        ),
-      ),
+      await fs.readFile(this.path).then((data) => data.toString("utf-8")),
     )
   }
+
+  async merge(data: A, effects: T.Effects) {
+    const fileData = (await this.read(effects).catch(() => ({}))) || {}
+    const mergeData = _.merge({}, fileData, data)
+    return await this.write(mergeData, effects)
+  }
   /**
    * Create a File Helper for an arbitrary file type.
    *
@@ -135,7 +138,7 @@ export class FileHelper<A> {
     return new FileHelper<A>(
       path,
       (inData) => {
-        return JSON.stringify(inData, null, 2)
+        return YAML.stringify(inData, null, 2)
       },
       (inString) => {
         return shape.unsafeCast(YAML.parse(inString))

sdk/lib/util/getServiceInterface.ts

@@ -1,10 +1,15 @@
 import { ServiceInterfaceType } from "../StartSdk"
+import { knownProtocols } from "../interfaces/Host"
 import {
   AddressInfo,
   Effects,
-  HostInfo,
+  Host,
+  HostAddress,
   Hostname,
   HostnameInfo,
+  HostnameInfoIp,
+  HostnameInfoOnion,
+  IpInfo,
 } from "../types"
 
 export type UrlString = string
@@ -20,13 +25,13 @@ export const getHostname = (url: string): Hostname | null => {
 }
 
 export type Filled = {
-  hostnames: Hostname[]
-  onionHostnames: Hostname[]
-  localHostnames: Hostname[]
-  ipHostnames: Hostname[]
-  ipv4Hostnames: Hostname[]
-  ipv6Hostnames: Hostname[]
-  nonIpHostnames: Hostname[]
+  hostnames: HostnameInfo[]
+  onionHostnames: HostnameInfo[]
+  localHostnames: HostnameInfo[]
+  ipHostnames: HostnameInfo[]
+  ipv4Hostnames: HostnameInfo[]
+  ipv6Hostnames: HostnameInfo[]
+  nonIpHostnames: HostnameInfo[]
 
   urls: UrlString[]
   onionUrls: UrlString[]
@@ -50,7 +55,7 @@ export type ServiceInterfaceFilled = {
   /** Whether or not to mask the URIs for this interface. Useful if the URIs contain sensitive information, such as a password, macaroon, or API key */
   masked: boolean
   /** Information about the host for this binding */
-  hostInfo: HostInfo
+  host: Host
   /** URI information */
   addressInfo: FilledAddressInfo
   /** Indicates if we are a ui/p2p/api for the kind of interface that this is representing */
@@ -69,110 +74,103 @@ const negate =
   (a: A) =>
     !fn(a)
 const unique = <A>(values: A[]) => Array.from(new Set(values))
-function stringifyHostname(info: HostnameInfo): Hostname {
-  let base: string
-  if ("kind" in info.hostname && info.hostname.kind === "domain") {
-    base = info.hostname.subdomain
-      ? `${info.hostname.subdomain}.${info.hostname.domain}`
-      : info.hostname.domain
-  } else {
-    base = info.hostname.value
+export const addressHostToUrl = (
+  { scheme, sslScheme, username, suffix }: AddressInfo,
+  host: HostnameInfo,
+): UrlString[] => {
+  const res = []
+  const fmt = (scheme: string | null, host: HostnameInfo, port: number) => {
+    const includePort =
+      scheme &&
+      scheme in knownProtocols &&
+      port === knownProtocols[scheme as keyof typeof knownProtocols].defaultPort
+    let hostname
+    if (host.kind === "onion") {
+      hostname = host.hostname.value
+    } else if (host.kind === "ip") {
+      if (host.hostname.kind === "domain") {
+        hostname = `${host.hostname.subdomain ? `${host.hostname.subdomain}.` : ""}${host.hostname.domain}`
+      } else {
+        hostname = host.hostname.value
+      }
+    }
+    return `${scheme ? `${scheme}://` : ""}${
+      username ? `${username}@` : ""
+    }${hostname}${includePort ? `:${port}` : ""}${suffix}`
   }
-  if (info.hostname.port && info.hostname.sslPort) {
-    return `${base}:${info.hostname.port}` as Hostname
-  } else if (info.hostname.sslPort) {
-    return `${base}:${info.hostname.sslPort}` as Hostname
-  } else if (info.hostname.port) {
-    return `${base}:${info.hostname.port}` as Hostname
+  if (host.hostname.sslPort !== null) {
+    res.push(fmt(sslScheme, host, host.hostname.sslPort))
   }
-  return base as Hostname
-}
-const addressHostToUrl = (
-  { bindOptions, username, suffix }: AddressInfo,
-  host: Hostname,
-): UrlString => {
-  const scheme = host.endsWith(".onion")
-    ? bindOptions.scheme
-    : bindOptions.addSsl
-      ? bindOptions.addSsl.scheme
-      : bindOptions.scheme // TODO: encode whether hostname transport is "secure"?
-  return `${scheme ? `${scheme}//` : ""}${
-    username ? `${username}@` : ""
-  }${host}${suffix}`
+  if (host.hostname.port !== null) {
+    res.push(fmt(scheme, host, host.hostname.port))
+  }
+
+  return res
 }
+
 export const filledAddress = (
-  hostInfo: HostInfo,
+  host: Host,
   addressInfo: AddressInfo,
 ): FilledAddressInfo => {
   const toUrl = addressHostToUrl.bind(null, addressInfo)
-  const hostnameInfo =
-    hostInfo.kind == "multi"
-      ? hostInfo.hostnames
-      : hostInfo.hostname
-        ? [hostInfo.hostname]
-        : []
+  const hostnames = host.hostnameInfo[addressInfo.internalPort]
+
   return {
     ...addressInfo,
-    hostnames: hostnameInfo.flatMap((h) => stringifyHostname(h)),
+    hostnames,
     get onionHostnames() {
-      return hostnameInfo
-        .filter((h) => h.kind === "onion")
-        .map((h) => stringifyHostname(h))
+      return hostnames.filter((h) => h.kind === "onion")
     },
     get localHostnames() {
-      return hostnameInfo
-        .filter((h) => h.kind === "ip" && h.hostname.kind === "local")
-        .map((h) => stringifyHostname(h))
+      return hostnames.filter(
+        (h) => h.kind === "ip" && h.hostname.kind === "local",
+      )
     },
     get ipHostnames() {
-      return hostnameInfo
-        .filter(
-          (h) =>
-            h.kind === "ip" &&
-            (h.hostname.kind === "ipv4" || h.hostname.kind === "ipv6"),
-        )
-        .map((h) => stringifyHostname(h))
+      return hostnames.filter(
+        (h) =>
+          h.kind === "ip" &&
+          (h.hostname.kind === "ipv4" || h.hostname.kind === "ipv6"),
+      )
     },
     get ipv4Hostnames() {
-      return hostnameInfo
-        .filter((h) => h.kind === "ip" && h.hostname.kind === "ipv4")
-        .map((h) => stringifyHostname(h))
+      return hostnames.filter(
+        (h) => h.kind === "ip" && h.hostname.kind === "ipv4",
+      )
     },
     get ipv6Hostnames() {
-      return hostnameInfo
-        .filter((h) => h.kind === "ip" && h.hostname.kind === "ipv6")
-        .map((h) => stringifyHostname(h))
+      return hostnames.filter(
+        (h) => h.kind === "ip" && h.hostname.kind === "ipv6",
+      )
     },
     get nonIpHostnames() {
-      return hostnameInfo
-        .filter(
-          (h) =>
-            h.kind === "ip" &&
-            h.hostname.kind !== "ipv4" &&
-            h.hostname.kind !== "ipv6",
-        )
-        .map((h) => stringifyHostname(h))
+      return hostnames.filter(
+        (h) =>
+          h.kind === "ip" &&
+          h.hostname.kind !== "ipv4" &&
+          h.hostname.kind !== "ipv6",
+      )
     },
     get urls() {
-      return this.hostnames.map(toUrl)
+      return this.hostnames.flatMap(toUrl)
     },
     get onionUrls() {
-      return this.onionHostnames.map(toUrl)
+      return this.onionHostnames.flatMap(toUrl)
     },
     get localUrls() {
-      return this.localHostnames.map(toUrl)
+      return this.localHostnames.flatMap(toUrl)
     },
     get ipUrls() {
-      return this.ipHostnames.map(toUrl)
+      return this.ipHostnames.flatMap(toUrl)
     },
     get ipv4Urls() {
-      return this.ipv4Hostnames.map(toUrl)
+      return this.ipv4Hostnames.flatMap(toUrl)
     },
     get ipv6Urls() {
-      return this.ipv6Hostnames.map(toUrl)
+      return this.ipv6Hostnames.flatMap(toUrl)
     },
     get nonIpUrls() {
-      return this.nonIpHostnames.map(toUrl)
+      return this.nonIpHostnames.flatMap(toUrl)
     },
   }
 }
@@ -193,23 +191,25 @@ const makeInterfaceFilled = async ({
     packageId,
     callback,
   })
-  const hostInfo = await effects.getHostInfo({
-    packageId,
-    kind: null,
-    serviceInterfaceId: serviceInterfaceValue.id,
-    callback,
-  })
-  const primaryUrl = await effects.getPrimaryUrl({
-    serviceInterfaceId: id,
+  const hostId = serviceInterfaceValue.addressInfo.hostId
+  const host = await effects.getHostInfo({
     packageId,
+    hostId,
     callback,
   })
+  const primaryUrl = await effects
+    .getPrimaryUrl({
+      serviceInterfaceId: id,
+      packageId,
+      callback,
+    })
+    .catch((e) => null)
 
   const interfaceFilled: ServiceInterfaceFilled = {
     ...serviceInterfaceValue,
     primaryUrl: primaryUrl,
-    hostInfo,
-    addressInfo: filledAddress(hostInfo, serviceInterfaceValue.addressInfo),
+    host,
+    addressInfo: filledAddress(host, serviceInterfaceValue.addressInfo),
     get primaryHostname() {
       if (primaryUrl == null) return null
       return getHostname(primaryUrl)

sdk/lib/util/getServiceInterfaces.ts

@@ -18,41 +18,27 @@ const makeManyInterfaceFilled = async ({
     packageId,
     callback,
   })
-  const hostIdsRecord = Object.fromEntries(
-    await Promise.all(
-      Array.from(new Set(serviceInterfaceValues.map((x) => x.id))).map(
-        async (id) =>
-          [
-            id,
-            await effects.getHostInfo({
-              kind: null,
-              packageId,
-              serviceInterfaceId: id,
-              callback,
-            }),
-          ] as const,
-      ),
-    ),
-  )
 
   const serviceInterfacesFilled: ServiceInterfaceFilled[] = await Promise.all(
-    serviceInterfaceValues.map(async (serviceInterfaceValue) => {
-      const hostInfo = await effects.getHostInfo({
-        kind: null,
-        packageId,
-        serviceInterfaceId: serviceInterfaceValue.id,
-        callback,
-      })
-      const primaryUrl = await effects.getPrimaryUrl({
-        serviceInterfaceId: serviceInterfaceValue.id,
+    Object.values(serviceInterfaceValues).map(async (serviceInterfaceValue) => {
+      const hostId = serviceInterfaceValue.addressInfo.hostId
+      const host = await effects.getHostInfo({
         packageId,
+        hostId,
         callback,
       })
+      const primaryUrl = await effects
+        .getPrimaryUrl({
+          serviceInterfaceId: serviceInterfaceValue.id,
+          packageId,
+          callback,
+        })
+        .catch(() => null)
       return {
         ...serviceInterfaceValue,
         primaryUrl: primaryUrl,
-        hostInfo,
-        addressInfo: filledAddress(hostInfo, serviceInterfaceValue.addressInfo),
+        host,
+        addressInfo: filledAddress(host, serviceInterfaceValue.addressInfo),
         get primaryHostname() {
           if (primaryUrl == null) return null
           return getHostname(primaryUrl)

sdk/lib/util/index.ts

@@ -10,6 +10,8 @@ import "./once"
 
 export { GetServiceInterface, getServiceInterface } from "./getServiceInterface"
 export { getServiceInterfaces } from "./getServiceInterfaces"
+export { addressHostToUrl } from "./getServiceInterface"
+export { hostnameInfoToAddress } from "./Hostname"
 // prettier-ignore
 export type FlattenIntersection<T> = 
 T extends ArrayLike<any> ? T :

sdk/package-lock.json

@@ -1,20 +1,22 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-rev0.lib0.rc8.beta10",
+  "version": "0.3.6-alpha1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@start9labs/start-sdk",
-      "version": "0.4.0-rev0.lib0.rc8.beta10",
+      "version": "0.3.6-alpha1",
       "license": "MIT",
       "dependencies": {
         "isomorphic-fetch": "^3.0.0",
+        "lodash": "^4.17.21",
         "ts-matches": "^5.4.1"
       },
       "devDependencies": {
         "@iarna/toml": "^2.2.5",
         "@types/jest": "^29.4.0",
+        "@types/lodash": "^4.17.5",
         "jest": "^29.4.3",
         "prettier": "^3.2.5",
         "ts-jest": "^29.0.5",
@@ -1136,6 +1138,12 @@
         "pretty-format": "^29.0.0"
       }
     },
+    "node_modules/@types/lodash": {
+      "version": "4.17.5",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.5.tgz",
+      "integrity": "sha512-MBIOHVZqVqgfro1euRDWX7OO0fBVUUMrN6Pwm8LQsz8cWhEpihlvR70ENj3f40j58TNxZaWv2ndSkInykNBBJw==",
+      "dev": true
+    },
     "node_modules/@types/node": {
       "version": "18.15.10",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.10.tgz",
@@ -2841,6 +2849,11 @@
         "node": ">=8"
       }
     },
+    "node_modules/lodash": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
+    },
     "node_modules/lodash.memoize": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",

sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.3.6-alpha1",
+  "version": "0.3.6-alpha5",
   "description": "Software development kit to facilitate packaging services for StartOS",
   "main": "./cjs/lib/index.js",
   "types": "./cjs/lib/index.d.ts",
@@ -31,7 +31,10 @@
   "homepage": "https://github.com/Start9Labs/start-sdk#readme",
   "dependencies": {
     "isomorphic-fetch": "^3.0.0",
-    "ts-matches": "^5.4.1"
+    "lodash": "^4.17.21",
+    "ts-matches": "^5.4.1",
+    "yaml": "^2.2.2",
+    "@iarna/toml": "^2.2.5"
   },
   "prettier": {
     "trailingComma": "all",
@@ -40,9 +43,8 @@
     "singleQuote": false
   },
   "devDependencies": {
-    "@iarna/toml": "^2.2.5",
-    "yaml": "^2.2.2",
     "@types/jest": "^29.4.0",
+    "@types/lodash": "^4.17.5",
     "jest": "^29.4.3",
     "prettier": "^3.2.5",
     "ts-jest": "^29.0.5",