Feature/lxc container runtime (#2562)

* wip(fix): Dependencies

* wip: context

* wip(fix) Sorta auth

* wip: warnings

* wip(fix): registry/admin

* wip(fix) marketplace

* wip(fix) Some more converted and fixed with the linter and config

* wip: Working on the static server

* wip(fix)static server

* wip: Remove some asynnc

* wip: Something about the request and regular rpc

* wip: gut install

Co-authored-by: J H <Blu-J@users.noreply.github.com>

* wip: Convert the static server into the new system

* wip delete file

* test

* wip(fix) vhost does not need the with safe defaults

* wip: Adding in the wifi

* wip: Fix the developer and the verify

* wip: new install flow

Co-authored-by: J H <Blu-J@users.noreply.github.com>

* fix middleware

* wip

* wip: Fix the auth

* wip

* continue service refactor

* feature: Service get_config

* feat: Action

* wip: Fighting the great fight against the borrow checker

* wip: Remove an error in a file that I just need to deel with later

* chore: Add in some more lifetime stuff to the services

* wip: Install fix on lifetime

* cleanup

* wip: Deal with the borrow later

* more cleanup

* resolve borrowchecker errors

* wip(feat): add in the handler for the socket, for now

* wip(feat): Update the service_effect_handler::action

* chore: Add in the changes to make sure the from_service goes to context

* chore: Change the

* refactor service map

* fix references to service map

* fill out restore

* wip: Before I work on the store stuff

* fix backup module

* handle some warnings

* feat: add in the ui components on the rust side

* feature: Update the procedures

* chore: Update the js side of the main and a few of the others

* chore: Update the rpc listener to match the persistant container

* wip: Working on updating some things to have a better name

* wip(feat): Try and get the rpc to return the correct shape?

* lxc wip

* wip(feat): Try and get the rpc to return the correct shape?

* build for container runtime wip

* remove container-init

* fix build

* fix error

* chore: Update to work I suppose

* lxc wip

* remove docker module and feature

* download alpine squashfs automatically

* overlays effect

Co-authored-by: Jade <Blu-J@users.noreply.github.com>

* chore: Add the overlay effect

* feat: Add the mounter in the main

* chore: Convert to use the mounts, still need to work with the sandbox

* install fixes

* fix ssl

* fixes from testing

* implement tmpfile for upload

* wip

* misc fixes

* cleanup

* cleanup

* better progress reporting

* progress for sideload

* return real guid

* add devmode script

* fix lxc rootfs path

* fix percentage bar

* fix progress bar styling

* fix build for unstable

* tweaks

* label progress

* tweaks

* update progress more often

* make symlink in rpc_client

* make socket dir

* fix parent path

* add start-cli to container

* add echo and gitInfo commands

* wip: Add the init + errors

* chore: Add in the exit effect for the system

* chore: Change the type to null for failure to parse

* move sigterm timeout to stopping status

* update order

* chore: Update the return type

* remove dbg

* change the map error

* chore: Update the thing to capture id

* chore add some life changes

* chore: Update the loging

* chore: Update the package to run module

* us From for RpcError

* chore: Update to use import instead

* chore: update

* chore: Use require for the backup

* fix a default

* update the type that is wrong

* chore: Update the type of the manifest

* chore: Update to make null

* only symlink if not exists

* get rid of double result

* better debug info for ErrorCollection

* chore: Update effects

* chore: fix

* mount assets and volumes

* add exec instead of spawn

* fix mounting in image

* fix overlay mounts

Co-authored-by: Jade <Blu-J@users.noreply.github.com>

* misc fixes

* feat: Fix two

* fix: systemForEmbassy main

* chore: Fix small part of main loop

* chore: Modify the bundle

* merge

* fixMain loop"

* move tsc to makefile

* chore: Update the return types of the health check

* fix client

* chore: Convert the todo to use tsmatches

* add in the fixes for the seen and create the hack to allow demo

* chore: Update to include the systemForStartOs

* chore UPdate to the latest types from the expected outout

* fixes

* fix typo

* Don't emit if failure on tsc

* wip

Co-authored-by: Jade <Blu-J@users.noreply.github.com>

* add s9pk api

* add inspection

* add inspect manifest

* newline after display serializable

* fix squashfs in image name

* edit manifest

Co-authored-by: Jade <Blu-J@users.noreply.github.com>

* wait for response on repl

* ignore sig for now

* ignore sig for now

* re-enable sig verification

* fix

* wip

* env and chroot

* add profiling logs

* set uid & gid in squashfs to 100000

* set uid of sqfs to 100000

* fix mksquashfs args

* add env to compat

* fix

* re-add docker feature flag

* fix docker output format being stupid

* here be dragons

* chore: Add in the cross compiling for something

* fix npm link

* extract logs from container on exit

* chore: Update for testing

* add log capture to drop trait

* chore: add in the modifications that I make

* chore: Update small things for no updates

* chore: Update the types of something

* chore: Make main not complain

* idmapped mounts

* idmapped volumes

* re-enable kiosk

* chore: Add in some logging for the new system

* bring in start-sdk

* remove avahi

* chore: Update the deps

* switch to musl

* chore: Update the version of prettier

* chore: Organize'

* chore: Update some of the headers back to the standard of fetch

* fix musl build

* fix idmapped mounts

* fix cross build

* use cross compiler for correct arch

* feat: Add in the faked ssl stuff for the effects

* @dr_bonez Did a solution here

* chore: Something that DrBonez

* chore: up

* wip: We have a working server!!!

* wip

* uninstall

* wip

* tes

* misc fixes

* fix cli

* replace interface with host

* chore: Fix the types in some ts files

* chore: quick update for the system for embassy to update the types

* replace br-start9 with lxcbr0

* split patchdb into public/private

* chore: Add changes for config set

* Feat: Adding some debugging for the errors

* wip: Working on getting the set config to work

* chore: Update and fix the small issue with the deserialization

* lightning, masked, schemeOverride, invert host-iface relationship

* feat: Add in the changes for just the sdk

* feat: Add in the changes for the new effects I suppose for now

---------

Co-authored-by: J H <2364004+Blu-J@users.noreply.github.com>
Co-authored-by: J H <Blu-J@users.noreply.github.com>
Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Matt Hill <mattnine@protonmail.com>

sdk/lib/interfaces/Host.ts

@@ -35,6 +35,11 @@ const knownProtocols = {
     ssl: false,
     defaultPort: 8333,
   },
+  lightning: {
+    secure: true,
+    ssl: true,
+    defaultPort: 9735,
+  },
   grpc: {
     secure: true,
     ssl: true,
@@ -47,11 +52,11 @@ const knownProtocols = {
   },
 } as const
 
-type Scheme = string | null
+export type Scheme = string | null
 
 type AddSslOptions = {
-  preferredExternalPort: number
   scheme: Scheme
+  preferredExternalPort: number
   addXForwardedHeaders?: boolean /** default: false */
 }
 type Security = { secure: false; ssl: false } | { secure: true; ssl: boolean }
@@ -73,7 +78,7 @@ type NotProtocolsWithSslVariants = Exclude<
   ProtocolsWithSslVariants
 >
 
-type PortOptionsByKnownProtocol =
+type BindOptionsByKnownProtocol =
   | ({
       protocol: ProtocolsWithSslVariants
       preferredExternalPort?: number
@@ -85,7 +90,7 @@ type PortOptionsByKnownProtocol =
       scheme?: Scheme
       addSsl?: AddSslOptions | null
     }
-type PortOptionsByProtocol = PortOptionsByKnownProtocol | BindOptions
+type BindOptionsByProtocol = BindOptionsByKnownProtocol | BindOptions
 
 export type HostKind = "static" | "single" | "multi"
 
@@ -104,7 +109,7 @@ export class Host {
 
   async bindPort(
     internalPort: number,
-    options: PortOptionsByProtocol,
+    options: BindOptionsByProtocol,
   ): Promise<Origin<this>> {
     if (hasStringProtocol(options)) {
       return await this.bindPortForKnown(options, internalPort)
@@ -138,7 +143,7 @@ export class Host {
   }
 
   private async bindPortForKnown(
-    options: PortOptionsByKnownProtocol,
+    options: BindOptionsByKnownProtocol,
     internalPort: number,
   ) {
     const scheme =
@@ -174,7 +179,7 @@ export class Host {
   }
 
   private getAddSsl(
-    options: PortOptionsByKnownProtocol,
+    options: BindOptionsByKnownProtocol,
     protoInfo: KnownProtocols[keyof KnownProtocols],
   ): AddSslOptions | null {
     if ("noAddSsl" in options && options.noAddSsl) return null

sdk/lib/interfaces/Origin.ts

@@ -1,5 +1,7 @@
 import { AddressInfo } from "../types"
-import { Host, BindOptions } from "./Host"
+import { AddressReceipt } from "./AddressReceipt"
+import { Host, BindOptions, Scheme } from "./Host"
+import { ServiceInterfaceBuilder } from "./ServiceInterfaceBuilder"
 
 export class Origin<T extends Host> {
   constructor(
@@ -7,7 +9,7 @@ export class Origin<T extends Host> {
     readonly options: BindOptions,
   ) {}
 
-  build({ username, path, search }: BuildOptions): AddressInfo {
+  build({ username, path, search, schemeOverride }: BuildOptions): AddressInfo {
     const qpEntries = Object.entries(search)
       .map(
         ([key, val]) => `${encodeURIComponent(key)}=${encodeURIComponent(val)}`,
@@ -18,15 +20,77 @@ export class Origin<T extends Host> {
 
     return {
       hostId: this.host.options.id,
-      options: this.options,
+      bindOptions: {
+        ...this.options,
+        scheme: schemeOverride ? schemeOverride.noSsl : this.options.scheme,
+        addSsl: this.options.addSsl
+          ? {
+              ...this.options.addSsl,
+              scheme: schemeOverride
+                ? schemeOverride.ssl
+                : this.options.addSsl.scheme,
+            }
+          : null,
+      },
       suffix: `${path}${qp}`,
       username,
     }
   }
+
+  /**
+   * A function to register a group of origins (<PROTOCOL> :// <HOSTNAME> : <PORT>) with StartOS
+   *
+   * The returned addressReceipt serves as proof that the addresses were registered
+   *
+   * @param addressInfo
+   * @returns
+   */
+  async export(
+    serviceInterfaces: ServiceInterfaceBuilder[],
+  ): Promise<AddressInfo[] & AddressReceipt> {
+    const addressesInfo = []
+    for (let serviceInterface of serviceInterfaces) {
+      const {
+        name,
+        description,
+        hasPrimary,
+        disabled,
+        id,
+        type,
+        username,
+        path,
+        search,
+        schemeOverride,
+        masked,
+      } = serviceInterface.options
+
+      const addressInfo = this.build({
+        username,
+        path,
+        search,
+        schemeOverride,
+      })
+
+      await serviceInterface.options.effects.exportServiceInterface({
+        id,
+        name,
+        description,
+        hasPrimary,
+        disabled,
+        addressInfo,
+        type,
+        masked,
+      })
+
+      addressesInfo.push(addressInfo)
+    }
+
+    return addressesInfo as AddressInfo[] & AddressReceipt
+  }
 }
 
 type BuildOptions = {
-  scheme: string | null
+  schemeOverride: { ssl: Scheme; noSsl: Scheme } | null
   username: string | null
   path: string
   search: Record<string, string>

sdk/lib/interfaces/ServiceInterfaceBuilder.ts

@@ -1,8 +1,6 @@
-import { AddressInfo, Effects } from "../types"
+import { Effects } from "../types"
 import { ServiceInterfaceType } from "../util/utils"
-import { AddressReceipt } from "./AddressReceipt"
-import { Host } from "./Host"
-import { Origin } from "./Origin"
+import { Scheme } from "./Host"
 
 /**
  * A helper class for creating a Network Interface
@@ -25,47 +23,11 @@ export class ServiceInterfaceBuilder {
       hasPrimary: boolean
       disabled: boolean
       type: ServiceInterfaceType
-      username: null | string
+      username: string | null
       path: string
       search: Record<string, string>
+      schemeOverride: { ssl: Scheme; noSsl: Scheme } | null
+      masked: boolean
     },
   ) {}
-
-  /**
-   * A function to register a group of origins (<PROTOCOL> :// <HOSTNAME> : <PORT>) with StartOS
-   *
-   * The returned addressReceipt serves as proof that the addresses were registered
-   *
-   * @param addressInfo
-   * @returns
-   */
-  async export<OriginForHost extends Origin<Host>>(
-    origin: OriginForHost,
-  ): Promise<AddressInfo & AddressReceipt> {
-    const {
-      name,
-      description,
-      hasPrimary,
-      disabled,
-      id,
-      type,
-      username,
-      path,
-      search,
-    } = this.options
-
-    const addressInfo = origin.build({ username, path, search, scheme: null })
-
-    await this.options.effects.exportServiceInterface({
-      id,
-      name,
-      description,
-      hasPrimary,
-      disabled,
-      addressInfo,
-      type,
-    })
-
-    return addressInfo as AddressInfo & AddressReceipt
-  }
 }