Feature/network (#2622)

* Feature: Add in the clear bindings

* wip: Working on network

* fix: Make it so the config gives the url

* chore: Remove the repeated types

* chore: Add in the todo's here

* chore: UPdate and remove some poorly name var

* chore: Remove the clear-bindings impl

* chore: Remove the wrapper

* handle HostnameInfo for Host bindings

Co-authored-by: Jade <Blu-J@users.noreply.github.com>

* ??

* chore: Make the install work

* Fix: Url's not being created

* chore: Fix the local onion in url

* include port in hostname

* Chore of adding a comment just to modify.

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: Jade <Blu-J@users.noreply.github.com>

container-runtime/src/Adapters/HostSystemStartOs.ts

@@ -96,7 +96,10 @@ export class HostSystemStartOs implements Effects {
   }
 
   bind(...[options]: Parameters<T.Effects["bind"]>) {
-    return this.rpcRound("bind", options) as ReturnType<T.Effects["bind"]>
+    return this.rpcRound("bind", {
+      ...options,
+      stack: new Error().stack,
+    }) as ReturnType<T.Effects["bind"]>
   }
   clearBindings(...[]: Parameters<T.Effects["clearBindings"]>) {
     return this.rpcRound("clearBindings", null) as ReturnType<
@@ -228,11 +231,6 @@ export class HostSystemStartOs implements Effects {
   restart(...[]: Parameters<T.Effects["restart"]>) {
     return this.rpcRound("restart", null)
   }
-  reverseProxy(...[options]: Parameters<T.Effects["reverseProxy"]>) {
-    return this.rpcRound("reverseProxy", options) as ReturnType<
-      T.Effects["reverseProxy"]
-    >
-  }
   running(...[packageId]: Parameters<T.Effects["running"]>) {
     return this.rpcRound("running", { packageId }) as ReturnType<
       T.Effects["running"]

container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts

@@ -97,11 +97,9 @@ export class MainLoop {
           id: interfaceId,
           internalPort,
           preferredExternalPort: torConf?.external || internalPort,
-          scheme: "http",
           secure: null,
           addSsl: lanConf?.ssl
             ? {
-                scheme: "https",
                 preferredExternalPort: lanConf.external,
                 alpn: { specified: ["http/1.1"] },
               }

container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts

@@ -31,6 +31,16 @@ import { HostSystemStartOs } from "../../HostSystemStartOs"
 import { JsonPath, unNestPath } from "../../../Models/JsonPath"
 import { RpcResult, matchRpcResult } from "../../RpcListener"
 import { CT } from "@start9labs/start-sdk"
+import {
+  AddSslOptions,
+  BindOptions,
+} from "@start9labs/start-sdk/cjs/lib/osBindings"
+import {
+  BindOptionsByProtocol,
+  Host,
+  MultiHost,
+} from "@start9labs/start-sdk/cjs/lib/interfaces/Host"
+import { ServiceInterfaceBuilder } from "@start9labs/start-sdk/cjs/lib/interfaces/ServiceInterfaceBuilder"
 
 type Optional<A> = A | undefined | null
 function todo(): never {
@@ -335,6 +345,85 @@ export class SystemForEmbassy implements System {
       await this.migration(effects, previousVersion, timeoutMs)
     await effects.setMainStatus({ status: "stopped" })
     await this.exportActions(effects)
+    await this.exportNetwork(effects)
+  }
+  async exportNetwork(effects: HostSystemStartOs) {
+    for (const [id, interfaceValue] of Object.entries(
+      this.manifest.interfaces,
+    )) {
+      const host = new MultiHost({ effects, id })
+      const internalPorts = new Set(
+        Object.values(interfaceValue["tor-config"]?.["port-mapping"] ?? {})
+          .map(Number.parseInt)
+          .concat(
+            ...Object.values(interfaceValue["lan-config"] ?? {}).map(
+              (c) => c.internal,
+            ),
+          )
+          .filter(Boolean),
+      )
+      const bindings = Array.from(internalPorts).map<
+        [number, BindOptionsByProtocol]
+      >((port) => {
+        const lanPort = Object.entries(interfaceValue["lan-config"] ?? {}).find(
+          ([external, internal]) => internal.internal === port,
+        )?.[0]
+        const torPort = Object.entries(
+          interfaceValue["tor-config"]?.["port-mapping"] ?? {},
+        ).find(
+          ([external, internal]) => Number.parseInt(internal) === port,
+        )?.[0]
+        let addSsl: AddSslOptions | null = null
+        if (lanPort) {
+          const lanPortNum = Number.parseInt(lanPort)
+          if (lanPortNum === 443) {
+            return [port, { protocol: "http", preferredExternalPort: 80 }]
+          }
+          addSsl = {
+            preferredExternalPort: lanPortNum,
+            alpn: { specified: [] },
+          }
+        }
+        return [
+          port,
+          {
+            secure: null,
+            preferredExternalPort: Number.parseInt(
+              torPort || lanPort || String(port),
+            ),
+            addSsl,
+          },
+        ]
+      })
+
+      await Promise.all(
+        bindings.map(async ([internal, options]) => {
+          if (internal == null) {
+            return
+          }
+          if (options?.preferredExternalPort == null) {
+            return
+          }
+          const origin = await host.bindPort(internal, options)
+          await origin.export([
+            new ServiceInterfaceBuilder({
+              effects,
+              name: interfaceValue.name,
+              id: `${id}-${internal}`,
+              description: interfaceValue.description,
+              hasPrimary: false,
+              disabled: false,
+              type: "api",
+              masked: false,
+              path: "",
+              schemeOverride: null,
+              search: {},
+              username: null,
+            }),
+          ])
+        }),
+      )
+    }
   }
   async exportActions(effects: HostSystemStartOs) {
     const manifest = this.manifest
@@ -486,6 +575,7 @@ export class SystemForEmbassy implements System {
     const newConfig = structuredClone(newConfigWithoutPointers)
     await updateConfig(
       effects,
+      this.manifest,
       await this.getConfigUncleaned(effects, timeoutMs).then((x) => x.spec),
       newConfig,
     )
@@ -866,6 +956,7 @@ function cleanConfigFromPointers<C, S>(
 
 async function updateConfig(
   effects: HostSystemStartOs,
+  manifest: Manifest,
   spec: unknown,
   mutConfigValue: unknown,
 ) {
@@ -877,7 +968,12 @@ async function updateConfig(
     const newConfigValue = mutConfigValue[key]
     if (matchSpec.test(specValue)) {
       const updateObject = { spec: null }
-      await updateConfig(effects, { spec: specValue.spec }, updateObject)
+      await updateConfig(
+        effects,
+        manifest,
+        { spec: specValue.spec },
+        updateObject,
+      )
       mutConfigValue[key] = updateObject.spec
     }
     if (
@@ -899,20 +995,48 @@ async function updateConfig(
     if (matchPointerPackage.test(specValue)) {
       if (specValue.target === "tor-key")
         throw new Error("This service uses an unsupported target TorKey")
+
+      const specInterface = specValue.interface
+      const serviceInterfaceId = extractServiceInterfaceId(
+        manifest,
+        specInterface,
+      )
       const filled = await utils
         .getServiceInterface(effects, {
           packageId: specValue["package-id"],
-          id: specValue.interface,
+          id: serviceInterfaceId,
         })
         .once()
-        .catch(() => null)
-
-      mutConfigValue[key] =
+        .catch((x) => {
+          console.error("Could not get the service interface", x)
+          return null
+        })
+      const catchFn = <X>(fn: () => X) => {
+        try {
+          return fn()
+        } catch (e) {
+          return undefined
+        }
+      }
+      const url: string =
         filled === null
           ? ""
-          : specValue.target === "lan-address"
-            ? filled.addressInfo.localHostnames[0]
-            : filled.addressInfo.onionHostnames[0]
+          : catchFn(() =>
+              utils.hostnameInfoToAddress(
+                specValue.target === "lan-address"
+                  ? filled.addressInfo.localHostnames[0] ||
+                      filled.addressInfo.onionHostnames[0]
+                  : filled.addressInfo.onionHostnames[0] ||
+                      filled.addressInfo.localHostnames[0],
+              ),
+            ) || ""
+      mutConfigValue[key] = url
     }
   }
 }
+function extractServiceInterfaceId(manifest: Manifest, specInterface: string) {
+  let serviceInterfaceId
+  const lanConfig = manifest.interfaces[specInterface]?.["lan-config"] || {}
+  serviceInterfaceId = `${specInterface}-${Object.entries(lanConfig)[0]?.[1]?.internal}`
+  return serviceInterfaceId
+}

container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts

@@ -72,6 +72,7 @@ export const matchManifest = object(
       object(
         {
           name: string,
+          description: string,
           "tor-config": object({
             "port-mapping": dictionary([string, string]),
           }),

container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts

@@ -99,6 +99,7 @@ export type Effects = {
   /** Sandbox mode lets us read but not write */
   is_sandboxed(): boolean
 
+  // Does a volume and path exist?
   exists(input: { volumeId: string; path: string }): Promise<boolean>
 
   fetch(