feature: Include the start init files.

This includes the docker commands to get things compressed.
And this is the start of the rpc, but needs lots of work, or very little, not sure yet anymore.
I beleive that the things that are missing are the rpc, and the effects. So, lots of work, but is still good to have I suppose.

libs/start_init/.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+dist/
+bundle.js
+startInit.js
+service/
+service.js

libs/start_init/initSrc/CallbackHolder.ts

@@ -0,0 +1,22 @@
+
+
+export class CallbackHolder {
+    constructor() {
+
+    }
+    private root = (Math.random() + 1).toString(36).substring(7);
+    private inc = 0
+    private callbacks = new Map<string, Function>()
+    private newId() {
+        return this.root + (this.inc++).toString(36)
+    }
+    addCallback(callback: Function) {
+        return this.callbacks.set(this.newId(), callback);
+    }
+    callCallback(index: string, args: any[]): Promise<unknown> {
+        const callback = this.callbacks.get(index)
+        if (!callback) throw new Error(`Callback ${index} does not exist`)        
+        this.callbacks.delete(index)
+        return Promise.resolve().then(() => callback(...args))
+    }
+}

libs/start_init/initSrc/Effects.ts

@@ -0,0 +1,184 @@
+import * as T from "@start9labs/start-sdk/lib/types"
+import * as net from "net"
+import { CallbackHolder } from "./CallbackHolder"
+
+const path = "/start9/sockets/rpcOut.sock"
+const MAIN = "main" as const
+export class Effects implements T.Effects {
+  constructor(readonly method: string, readonly callbackHolder: CallbackHolder) {}
+  id = 0
+  rpcRound(method: string, params: unknown) {
+    const id = this.id++;
+    const client = net.createConnection(path, () => {
+        client.write(JSON.stringify({
+          id,
+          method,
+          params
+        }));
+    });
+    return new Promise((resolve, reject) => {
+      client.on('data', (data) => {
+        try {
+          resolve(JSON.parse(data.toString())?.result)
+        } catch (error) {
+          reject(error)
+        }
+        client.end();
+      });
+    })
+  }
+  started= this.method !== MAIN ? null : ()=> {
+    return this.rpcRound('started', null)
+  }
+  bind(...[options]: Parameters<T.Effects["bind"]>) {
+    return this.rpcRound('bind', (options)) as ReturnType<T.Effects["bind"]>
+  }
+  clearBindings(...[]: Parameters<T.Effects["clearBindings"]>) {
+    return this.rpcRound('clearBindings', null) as ReturnType<T.Effects["clearBindings"]>
+  }
+  clearNetworkInterfaces(
+    ...[]: Parameters<T.Effects["clearNetworkInterfaces"]>
+  ) {
+    return this.rpcRound('clearNetworkInterfaces', null) as ReturnType<T.Effects["clearNetworkInterfaces"]>
+  }
+  executeAction(...[options]: Parameters<T.Effects["executeAction"]>) {
+    return this.rpcRound('executeAction', options) as ReturnType<T.Effects["executeAction"]>
+  }
+  exists(...[packageId]: Parameters<T.Effects["exists"]>) {
+    return this.rpcRound('exists', packageId) as ReturnType<T.Effects["exists"]>
+  }
+  exportAction(...[options]: Parameters<T.Effects["exportAction"]>) {
+    return this.rpcRound('exportAction', (options)) as ReturnType<T.Effects["exportAction"]>
+  }
+  exportNetworkInterface(
+    ...[options]: Parameters<T.Effects["exportNetworkInterface"]>
+  ) {
+    return this.rpcRound('exportNetworkInterface', (options)) as ReturnType<T.Effects["exportNetworkInterface"]>
+  }
+  exposeForDependents(...[options]: any) {
+    
+    return this.rpcRound('exposeForDependents', (null)) as ReturnType<T.Effects["exposeForDependents"]>
+  }
+  exposeUi(...[options]: Parameters<T.Effects["exposeUi"]>) {
+    
+    return this.rpcRound('exposeUi', (options)) as ReturnType<T.Effects["exposeUi"]>
+  }
+  getConfigured(...[]: Parameters<T.Effects["getConfigured"]>) {
+    
+    return this.rpcRound('getConfigured',null) as ReturnType<T.Effects["getConfigured"]>
+  }
+  getContainerIp(...[]: Parameters<T.Effects["getContainerIp"]>) {
+    
+    return this.rpcRound('getContainerIp', null) as ReturnType<T.Effects["getContainerIp"]>
+  }
+  getHostnames: any = (...[allOptions]: any[]) => {
+    const options = {
+      ...allOptions,
+      callback: this.callbackHolder.addCallback(allOptions.callback)
+    }
+    return this.rpcRound('getHostnames', options) as ReturnType<T.Effects["getHostnames"]>
+  }
+  getInterface(...[options]: Parameters<T.Effects["getInterface"]>) {
+    
+    return this.rpcRound('getInterface', {...options, callback: this.callbackHolder.addCallback(options.callback)}) as ReturnType<T.Effects["getInterface"]>
+  }
+  getIPHostname(...[]: Parameters<T.Effects["getIPHostname"]>) {
+    
+    return this.rpcRound('getIPHostname', (null)) as ReturnType<T.Effects["getIPHostname"]>
+  }
+  getLocalHostname(...[]: Parameters<T.Effects["getLocalHostname"]>) {
+    
+    return this.rpcRound('getLocalHostname', null) as ReturnType<T.Effects["getLocalHostname"]>
+  }
+  getPrimaryUrl(...[options]: Parameters<T.Effects["getPrimaryUrl"]>) {
+    
+    return this.rpcRound('getPrimaryUrl', {...options, callback: this.callbackHolder.addCallback(options.callback)}) as ReturnType<T.Effects["getPrimaryUrl"]>
+  }
+  getServicePortForward(
+    ...[options]: Parameters<T.Effects["getServicePortForward"]>
+  ) {
+    
+    return this.rpcRound('getServicePortForward', (options)) as ReturnType<T.Effects["getServicePortForward"]>
+  }
+  getServiceTorHostname(
+    ...[interfaceId, packageId]: Parameters<T.Effects["getServiceTorHostname"]>
+  ) {
+    
+    return this.rpcRound('getServiceTorHostname', ({interfaceId, packageId})) as ReturnType<T.Effects["getServiceTorHostname"]>
+  }
+  getSslCertificate(...[packageId, algorithm]: Parameters<T.Effects["getSslCertificate"]>) {
+    
+    return this.rpcRound('getSslCertificate', ({packageId, algorithm})) as ReturnType<T.Effects["getSslCertificate"]>
+  }
+  getSslKey(...[packageId, algorithm]: Parameters<T.Effects["getSslKey"]>) {
+    
+    return this.rpcRound('getSslKey', ({packageId, algorithm})) as ReturnType<T.Effects["getSslKey"]>
+  }
+  getSystemSmtp(...[options]: Parameters<T.Effects["getSystemSmtp"]>) {
+    
+    return this.rpcRound('getSystemSmtp', {...options, callback: this.callbackHolder.addCallback(options.callback)}) as ReturnType<T.Effects["getSystemSmtp"]>
+  }
+  is_sandboxed(...[]: Parameters<T.Effects["is_sandboxed"]>) {
+    
+    return this.rpcRound('is_sandboxed', (null)) as ReturnType<T.Effects["is_sandboxed"]>
+  }
+  listInterface(...[options]: Parameters<T.Effects["listInterface"]>) {
+    
+    return this.rpcRound('listInterface', {...options, callback: this.callbackHolder.addCallback(options.callback)}) as ReturnType<T.Effects["listInterface"]>
+  }
+  mount(...[options]: Parameters<T.Effects["mount"]>) {
+    
+    return this.rpcRound('mount', options) as ReturnType<T.Effects["mount"]>
+  }
+  removeAction(...[options]: Parameters<T.Effects["removeAction"]>) {
+    
+    return this.rpcRound('removeAction', options) as ReturnType<T.Effects["removeAction"]>
+  }
+  removeAddress(...[options]: Parameters<T.Effects["removeAddress"]>) {
+    
+    return this.rpcRound('removeAddress', options) as ReturnType<T.Effects["removeAddress"]>
+  }
+  restart(...[]: Parameters<T.Effects["restart"]>) {
+    
+    this.rpcRound('restart', null) 
+  }
+  reverseProxy(...[options]: Parameters<T.Effects["reverseProxy"]>) {
+    
+    return this.rpcRound('reverseProxy', options) as ReturnType<T.Effects["reverseProxy"]>
+  }
+  running(...[packageId]: Parameters<T.Effects["running"]>) {
+    
+    return this.rpcRound('running', {packageId}) as ReturnType<T.Effects["running"]>
+  }
+  // runRsync(...[options]: Parameters<T.Effects[""]>) {
+  //   
+    // return this.rpcRound('executeAction', options) as ReturnType<T.Effects["executeAction"]>
+  //   
+    // return this.rpcRound('executeAction', options) as ReturnType<T.Effects["executeAction"]>
+  // }
+  setConfigured(...[configured]: Parameters<T.Effects["setConfigured"]>) {
+    
+    return this.rpcRound('setConfigured', {configured}) as ReturnType<T.Effects["setConfigured"]>
+  }
+  setDependencies(...[dependencies]: Parameters<T.Effects["setDependencies"]>) {
+    
+    return this.rpcRound('setDependencies', {dependencies}) as ReturnType<T.Effects["setDependencies"]>
+  }
+  setHealth(...[options]: Parameters<T.Effects["setHealth"]>) {
+    
+    return this.rpcRound('setHealth', options) as ReturnType<T.Effects["setHealth"]>
+  }
+  shutdown(...[]: Parameters<T.Effects["shutdown"]>) {
+    
+    return this.rpcRound('shutdown', null) 
+  }
+  stopped(...[packageId]: Parameters<T.Effects["stopped"]>) {
+    
+    return this.rpcRound('stopped', {packageId}) as ReturnType<T.Effects["stopped"]>
+  }
+  store: T.Effects['store'] = {
+    get:(options) => this.rpcRound('getStore', {...options, callback: this.callbackHolder.addCallback(options.callback)}) as ReturnType<T.Effects["store"]['get']>,
+    set:(options) => this.rpcRound('setStore', options) as ReturnType<T.Effects["store"]['set']>
+    
+  }
+}

libs/start_init/initSrc/Runtime.ts

@@ -0,0 +1,174 @@
+// @ts-check
+
+import * as net from "net"
+import {
+  object,
+  some,
+  string,
+  literal,
+  array,
+  number,
+  matches,
+} from "ts-matches"
+import { Effects } from "./Effects"
+import { CallbackHolder } from "./CallbackHolder"
+
+import * as CP from "child_process"
+import * as Mod from "module"
+
+const childProcesses = new Map<number, CP.ChildProcess[]>()
+let childProcessIndex = 0
+const require = Mod.prototype.require
+const setupRequire = () => {
+  const requireChildProcessIndex = childProcessIndex++
+  // @ts-ignore
+  Mod.prototype.require = (name, ...rest) => {
+    if (["child_process", "node:child_process"].indexOf(name) !== -1) {
+      return {
+        exec(...args: any[]) {
+          const returning = CP.exec.apply(null, args as any)
+          const childProcessArray =
+            childProcesses.get(requireChildProcessIndex) ?? []
+          childProcessArray.push(returning)
+          childProcesses.set(requireChildProcessIndex, childProcessArray)
+          return returning
+        },
+        execFile(...args: any[]) {
+          const returning = CP.execFile.apply(null, args as any)
+          const childProcessArray =
+            childProcesses.get(requireChildProcessIndex) ?? []
+          childProcessArray.push(returning)
+          childProcesses.set(requireChildProcessIndex, childProcessArray)
+          return returning
+        },
+        execFileSync: CP.execFileSync,
+        execSync: CP.execSync,
+        fork(...args: any[]) {
+          const returning = CP.fork.apply(null, args as any)
+          const childProcessArray =
+            childProcesses.get(requireChildProcessIndex) ?? []
+          childProcessArray.push(returning)
+          childProcesses.set(requireChildProcessIndex, childProcessArray)
+          return returning
+        },
+        spawn(...args: any[]) {
+          const returning = CP.spawn.apply(null, args as any)
+          const childProcessArray =
+            childProcesses.get(requireChildProcessIndex) ?? []
+          childProcessArray.push(returning)
+          childProcesses.set(requireChildProcessIndex, childProcessArray)
+          return returning
+        },
+        spawnSync: CP.spawnSync,
+      } as typeof CP
+    }
+    console.log("require", name)
+    return require(name, ...rest)
+  }
+  return requireChildProcessIndex
+}
+
+const cleanupRequire = (requireChildProcessIndex: number) => {
+  const foundChildren = childProcesses.get(requireChildProcessIndex)
+  if (!foundChildren) return
+  childProcesses.delete(requireChildProcessIndex)
+  foundChildren.forEach((x) => x.kill())
+}
+
+const idType = some(string, number)
+const path = "/start9/sockets/rpc.sock"
+const runType = object({
+  id: idType,
+  method: literal("run"),
+  params: object({
+    methodName: string.map((x) => {
+      const splitValue = x.split("/")
+      if (splitValue.length === 1)
+        throw new Error(`X (${x}) is not a valid path`)
+      return splitValue.slice(1)
+    }),
+    methodArgs: object,
+  }),
+})
+const callbackType = object({
+  id: idType,
+  method: literal("callback"),
+  params: object({
+    callback: string,
+    args: array,
+  }),
+})
+const dealWithInput = async (callbackHolder: CallbackHolder, input: unknown) =>
+  matches(input)
+    .when(runType, async ({ id, params: { methodName, methodArgs } }) => {
+      const index = setupRequire()
+      const effects = new Effects(`/${methodName.join("/")}`, callbackHolder)
+      // @ts-ignore
+      return import("/services/service.js")
+        .then((x) => methodName.reduce(reduceMethod(methodArgs, effects), x))
+        .then()
+        .then((result) => ({ id, result }))
+        .catch((error) => ({
+          id,
+          error: { message: error?.message ?? String(error) },
+        }))
+        .finally(() => cleanupRequire(index))
+    })
+    .when(callbackType, async ({ id, params: { callback, args } }) =>
+      Promise.resolve(callbackHolder.callCallback(callback, args))
+        .then((result) => ({ id, result }))
+        .catch((error) => ({
+          id,
+          error: { message: error?.message ?? String(error) },
+        })),
+    )
+
+    .defaultToLazy(() => {
+      console.warn(`Coudln't parse the following input ${input}`)
+      return {
+        error: { message: "Could not figure out shape" },
+      }
+    })
+
+const jsonParse = (x: Buffer) => JSON.parse(x.toString())
+export class Runtime {
+  unixSocketServer = net.createServer(async (server) => {})
+  private callbacks = new CallbackHolder()
+  constructor() {
+    this.unixSocketServer.listen(path)
+
+    this.unixSocketServer.on("connection", (s) => {
+      s.on("data", (a) =>
+        Promise.resolve(a)
+          .then(jsonParse)
+          .then(dealWithInput.bind(null, this.callbacks))
+          .then((x) => {
+            console.log("x", JSON.stringify(x), typeof x)
+            return x
+          })
+          .catch((error) => ({
+            error: { message: error?.message ?? String(error) },
+          }))
+          .then(JSON.stringify)
+          .then((x) => new Promise((resolve) => s.write("" + x, resolve)))
+          .finally(() => void s.end()),
+      )
+    })
+  }
+}
+function reduceMethod(
+  methodArgs: object,
+  effects: Effects,
+): (previousValue: any, currentValue: string) => any {
+  return (x: any, method: string) =>
+    Promise.resolve(x)
+      .then((x) => x[method])
+      .then((x) =>
+        typeof x !== "function"
+          ? x
+          : x({
+              ...methodArgs,
+              effects,
+            }),
+      )
+}

libs/start_init/initSrc/index.ts

@@ -0,0 +1,35 @@
+import { Runtime } from "./Runtime"
+
+new Runtime()
+
+/**
+
+So, this is going to be sent into a running comtainer along with any of the other node modules that are going to be needed and used.
+
+Once the container is started, we will go into a loading/ await state.
+This is the init system, and it will always be running, and it will be waiting for a command to be sent to it.
+
+Each command will be a stopable promise. And an example is going to be something like an action/ main/ or just a query into the types.
+
+A command will be sent an object which are the effects, and the effects will be things like the file system, the network, the process, and the os.
+
+
+ */
+// So OS Adapter
+// ==============
+
+/**
+* Why: So when the we call from the os we enter or leave here? 
+    
+ */
+
+/**
+Command: This is a command that the 
+
+There are 
+ */
+
+/**
+TODO:
+Should I seperate those adapter in/out?
+ */

libs/start_init/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "start-init",
+  "version": "0.0.0",
+  "description": "We want to be the sdk intermitent for the system",
+  "scripts": {
+    "bundle:esbuild": "esbuild initSrc/index.ts --platform=node --bundle --outfile=startInit.js",
+    "bundle:service": "esbuild /service/startos/procedures/index.ts --platform=node --bundle --outfile=service.js",
+    "run:manifest": "esbuild /service/startos/procedures/index.ts --platform=node --bundle --outfile=service.js"
+  },
+  "author": "",
+  "prettier": {
+    "trailingComma": "all",
+    "tabWidth": 2,
+    "semi": false,
+    "singleQuote": false
+  },
+  "dependencies": {
+    "@iarna/toml": "^2.2.5",
+    "@start9labs/start-sdk": "=0.4.0-rev0.lib0.rc8.alpha3",
+    "esbuild": "0.18.4",
+    "esbuild-plugin-resolve": "^2.0.0",
+    "filebrowser": "^1.0.0",
+    "isomorphic-fetch": "^3.0.0",
+    "ts-matches": "^5.4.1",
+    "tslib": "^2.5.3",
+    "typescript": "^5.1.3",
+    "yaml": "^2.3.1"
+  },
+  "devDependencies": {
+    "@swc/cli": "^0.1.62",
+    "@swc/core": "^1.3.65",
+    "@types/node": "^20.2.5",
+    "prettier": "^2.8.8",
+    "rollup": "^3.25.1"
+  }
+}

libs/start_init/readme.md

@@ -0,0 +1,86 @@
+## Testing
+
+So, we are going to
+
+1. create a fake server
+2. pretend socket server os (while the fake server is running)
+3. Run a fake effects system (while 1/2 are running)
+
+In order to simulate that we created a server like the start-os and
+a fake server (in this case I am using syncthing-wrapper)
+
+### TODO
+
+Undo the packing that I have done earlier, and hijack the embassy.js to use the bundle service + code
+
+Converting embassy.js -> service.js
+
+```sequence {theme="hand"}
+startOs ->> startInit.js: Rpc Call
+startInit.js ->> service.js: Rpc Converted into js code
+```
+
+### Create a fake server
+
+```bash
+run_test () {
+    (
+        set -e
+        libs=/home/jh/Projects/start-os/libs/start_init
+        sockets=/tmp/start9
+        service=/home/jh/Projects/syncthing-wrapper
+
+        docker run  \
+            -v $libs:/libs \
+            -v $service:/service \
+            -w /libs \
+            --rm node:18-alpine \
+            sh -c "
+                npm i &&
+                npm run bundle:esbuild  &&
+                npm run bundle:service
+            "
+
+
+
+        docker run  \
+            -v ./libs/start_init/:/libs \
+            -w /libs \
+            --rm node:18-alpine \
+            sh -c "
+                npm i &&
+                npm run bundle:esbuild
+            "
+
+
+
+        rm -rf $sockets  || true
+        mkdir -p $sockets/sockets
+        cd $service
+        docker run \
+            -v $libs:/start-init \
+            -v $sockets:/start9 \
+            --rm -it $(docker build -q .) sh -c "
+                apk add nodejs &&
+                node /start-init/bundleEs.js
+            "
+    )
+}
+run_test
+```
+
+### Pretend Socket Server OS
+
+First we are going to create our fake server client with the bash then send it the json possible data
+
+```bash
+sudo socat - unix-client:/tmp/start9/sockets/rpc.sock
+```
+
+<!-- prettier-ignore -->
+```json
+{"id":"a","method":"run","params":{"methodName":"/dependencyMounts","methodArgs":[]}}
+{"id":"a","method":"run","params":{"methodName":"/actions/test","methodArgs":{"input":{"id": 1}}}}
+{"id":"b","method":"run","params":{"methodName":"/actions/test","methodArgs":{"id": 1}}}
+
+```

libs/start_init/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "include": [
+    "./**/*.mjs",
+    "./**/*.js",
+    "initSrc/Runtime.ts",
+    "initSrc/index.ts",
+    "effects.ts"
+  ],
+  "exclude": [],
+  "inputs": ["./lib/index.ts"],
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "es2022",
+    "moduleResolution": "node",
+    "allowJs": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true
+  },
+  "ts-node": {
+    "compilerOptions": {
+      "module": "commonjs"
+    }
+  }
+}