Merge branch 'feature/start_init' into chore/removing-non-long-running

libs/start_init/.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+dist/
+bundle.js
+startInit.js
+service/
+service.js

libs/start_init/initSrc/CallbackHolder.ts

@@ -0,0 +1,22 @@
+
+
+export class CallbackHolder {
+    constructor() {
+
+    }
+    private root = (Math.random() + 1).toString(36).substring(7);
+    private inc = 0
+    private callbacks = new Map<string, Function>()
+    private newId() {
+        return this.root + (this.inc++).toString(36)
+    }
+    addCallback(callback: Function) {
+        return this.callbacks.set(this.newId(), callback);
+    }
+    callCallback(index: string, args: any[]): Promise<unknown> {
+        const callback = this.callbacks.get(index)
+        if (!callback) throw new Error(`Callback ${index} does not exist`)        
+        this.callbacks.delete(index)
+        return Promise.resolve().then(() => callback(...args))
+    }
+}

libs/start_init/initSrc/Effects.ts

@@ -0,0 +1,184 @@
+import * as T from "@start9labs/start-sdk/lib/types"
+import * as net from "net"
+import { CallbackHolder } from "./CallbackHolder"
+
+const path = "/start9/sockets/rpcOut.sock"
+const MAIN = "main" as const
+export class Effects implements T.Effects {
+  constructor(readonly method: string, readonly callbackHolder: CallbackHolder) {}
+  id = 0
+  rpcRound(method: string, params: unknown) {
+    const id = this.id++;
+    const client = net.createConnection(path, () => {
+        client.write(JSON.stringify({
+          id,
+          method,
+          params
+        }));
+    });
+    return new Promise((resolve, reject) => {
+      client.on('data', (data) => {
+        try {
+          resolve(JSON.parse(data.toString())?.result)
+        } catch (error) {
+          reject(error)
+        }
+        client.end();
+      });
+    })
+  }
+  started= this.method !== MAIN ? null : ()=> {
+    return this.rpcRound('started', null)
+  }
+  bind(...[options]: Parameters<T.Effects["bind"]>) {
+    return this.rpcRound('bind', (options)) as ReturnType<T.Effects["bind"]>
+  }
+  clearBindings(...[]: Parameters<T.Effects["clearBindings"]>) {
+    return this.rpcRound('clearBindings', null) as ReturnType<T.Effects["clearBindings"]>
+  }
+  clearNetworkInterfaces(
+    ...[]: Parameters<T.Effects["clearNetworkInterfaces"]>
+  ) {
+    return this.rpcRound('clearNetworkInterfaces', null) as ReturnType<T.Effects["clearNetworkInterfaces"]>
+  }
+  executeAction(...[options]: Parameters<T.Effects["executeAction"]>) {
+    return this.rpcRound('executeAction', options) as ReturnType<T.Effects["executeAction"]>
+  }
+  exists(...[packageId]: Parameters<T.Effects["exists"]>) {
+    return this.rpcRound('exists', packageId) as ReturnType<T.Effects["exists"]>
+  }
+  exportAction(...[options]: Parameters<T.Effects["exportAction"]>) {
+    return this.rpcRound('exportAction', (options)) as ReturnType<T.Effects["exportAction"]>
+  }
+  exportNetworkInterface(
+    ...[options]: Parameters<T.Effects["exportNetworkInterface"]>
+  ) {
+    return this.rpcRound('exportNetworkInterface', (options)) as ReturnType<T.Effects["exportNetworkInterface"]>
+  }
+  exposeForDependents(...[options]: any) {
+    
+    return this.rpcRound('exposeForDependents', (null)) as ReturnType<T.Effects["exposeForDependents"]>
+  }
+  exposeUi(...[options]: Parameters<T.Effects["exposeUi"]>) {
+    
+    return this.rpcRound('exposeUi', (options)) as ReturnType<T.Effects["exposeUi"]>
+  }
+  getConfigured(...[]: Parameters<T.Effects["getConfigured"]>) {
+    
+    return this.rpcRound('getConfigured',null) as ReturnType<T.Effects["getConfigured"]>
+  }
+  getContainerIp(...[]: Parameters<T.Effects["getContainerIp"]>) {
+    
+    return this.rpcRound('getContainerIp', null) as ReturnType<T.Effects["getContainerIp"]>
+  }
+  getHostnames: any = (...[allOptions]: any[]) => {
+    const options = {
+      ...allOptions,
+      callback: this.callbackHolder.addCallback(allOptions.callback)
+    }
+    return this.rpcRound('getHostnames', options) as ReturnType<T.Effects["getHostnames"]>
+  }
+  getInterface(...[options]: Parameters<T.Effects["getInterface"]>) {
+    
+    return this.rpcRound('getInterface', {...options, callback: this.callbackHolder.addCallback(options.callback)}) as ReturnType<T.Effects["getInterface"]>
+  }
+  getIPHostname(...[]: Parameters<T.Effects["getIPHostname"]>) {
+    
+    return this.rpcRound('getIPHostname', (null)) as ReturnType<T.Effects["getIPHostname"]>
+  }
+  getLocalHostname(...[]: Parameters<T.Effects["getLocalHostname"]>) {
+    
+    return this.rpcRound('getLocalHostname', null) as ReturnType<T.Effects["getLocalHostname"]>
+  }
+  getPrimaryUrl(...[options]: Parameters<T.Effects["getPrimaryUrl"]>) {
+    
+    return this.rpcRound('getPrimaryUrl', {...options, callback: this.callbackHolder.addCallback(options.callback)}) as ReturnType<T.Effects["getPrimaryUrl"]>
+  }
+  getServicePortForward(
+    ...[options]: Parameters<T.Effects["getServicePortForward"]>
+  ) {
+    
+    return this.rpcRound('getServicePortForward', (options)) as ReturnType<T.Effects["getServicePortForward"]>
+  }
+  getServiceTorHostname(
+    ...[interfaceId, packageId]: Parameters<T.Effects["getServiceTorHostname"]>
+  ) {
+    
+    return this.rpcRound('getServiceTorHostname', ({interfaceId, packageId})) as ReturnType<T.Effects["getServiceTorHostname"]>
+  }
+  getSslCertificate(...[packageId, algorithm]: Parameters<T.Effects["getSslCertificate"]>) {
+    
+    return this.rpcRound('getSslCertificate', ({packageId, algorithm})) as ReturnType<T.Effects["getSslCertificate"]>
+  }
+  getSslKey(...[packageId, algorithm]: Parameters<T.Effects["getSslKey"]>) {
+    
+    return this.rpcRound('getSslKey', ({packageId, algorithm})) as ReturnType<T.Effects["getSslKey"]>
+  }
+  getSystemSmtp(...[options]: Parameters<T.Effects["getSystemSmtp"]>) {
+    
+    return this.rpcRound('getSystemSmtp', {...options, callback: this.callbackHolder.addCallback(options.callback)}) as ReturnType<T.Effects["getSystemSmtp"]>
+  }
+  is_sandboxed(...[]: Parameters<T.Effects["is_sandboxed"]>) {
+    
+    return this.rpcRound('is_sandboxed', (null)) as ReturnType<T.Effects["is_sandboxed"]>
+  }
+  listInterface(...[options]: Parameters<T.Effects["listInterface"]>) {
+    
+    return this.rpcRound('listInterface', {...options, callback: this.callbackHolder.addCallback(options.callback)}) as ReturnType<T.Effects["listInterface"]>
+  }
+  mount(...[options]: Parameters<T.Effects["mount"]>) {
+    
+    return this.rpcRound('mount', options) as ReturnType<T.Effects["mount"]>
+  }
+  removeAction(...[options]: Parameters<T.Effects["removeAction"]>) {
+    
+    return this.rpcRound('removeAction', options) as ReturnType<T.Effects["removeAction"]>
+  }
+  removeAddress(...[options]: Parameters<T.Effects["removeAddress"]>) {
+    
+    return this.rpcRound('removeAddress', options) as ReturnType<T.Effects["removeAddress"]>
+  }
+  restart(...[]: Parameters<T.Effects["restart"]>) {
+    
+    this.rpcRound('restart', null) 
+  }
+  reverseProxy(...[options]: Parameters<T.Effects["reverseProxy"]>) {
+    
+    return this.rpcRound('reverseProxy', options) as ReturnType<T.Effects["reverseProxy"]>
+  }
+  running(...[packageId]: Parameters<T.Effects["running"]>) {
+    
+    return this.rpcRound('running', {packageId}) as ReturnType<T.Effects["running"]>
+  }
+  // runRsync(...[options]: Parameters<T.Effects[""]>) {
+  //   
+    // return this.rpcRound('executeAction', options) as ReturnType<T.Effects["executeAction"]>
+  //   
+    // return this.rpcRound('executeAction', options) as ReturnType<T.Effects["executeAction"]>
+  // }
+  setConfigured(...[configured]: Parameters<T.Effects["setConfigured"]>) {
+    
+    return this.rpcRound('setConfigured', {configured}) as ReturnType<T.Effects["setConfigured"]>
+  }
+  setDependencies(...[dependencies]: Parameters<T.Effects["setDependencies"]>) {
+    
+    return this.rpcRound('setDependencies', {dependencies}) as ReturnType<T.Effects["setDependencies"]>
+  }
+  setHealth(...[options]: Parameters<T.Effects["setHealth"]>) {
+    
+    return this.rpcRound('setHealth', options) as ReturnType<T.Effects["setHealth"]>
+  }
+  shutdown(...[]: Parameters<T.Effects["shutdown"]>) {
+    
+    return this.rpcRound('shutdown', null) 
+  }
+  stopped(...[packageId]: Parameters<T.Effects["stopped"]>) {
+    
+    return this.rpcRound('stopped', {packageId}) as ReturnType<T.Effects["stopped"]>
+  }
+  store: T.Effects['store'] = {
+    get:(options) => this.rpcRound('getStore', {...options, callback: this.callbackHolder.addCallback(options.callback)}) as ReturnType<T.Effects["store"]['get']>,
+    set:(options) => this.rpcRound('setStore', options) as ReturnType<T.Effects["store"]['set']>
+    
+  }
+}

libs/start_init/initSrc/Runtime.ts

@@ -0,0 +1,174 @@
+// @ts-check
+
+import * as net from "net"
+import {
+  object,
+  some,
+  string,
+  literal,
+  array,
+  number,
+  matches,
+} from "ts-matches"
+import { Effects } from "./Effects"
+import { CallbackHolder } from "./CallbackHolder"
+
+import * as CP from "child_process"
+import * as Mod from "module"
+
+const childProcesses = new Map<number, CP.ChildProcess[]>()
+let childProcessIndex = 0
+const require = Mod.prototype.require
+const setupRequire = () => {
+  const requireChildProcessIndex = childProcessIndex++
+  // @ts-ignore
+  Mod.prototype.require = (name, ...rest) => {
+    if (["child_process", "node:child_process"].indexOf(name) !== -1) {
+      return {
+        exec(...args: any[]) {
+          const returning = CP.exec.apply(null, args as any)
+          const childProcessArray =
+            childProcesses.get(requireChildProcessIndex) ?? []
+          childProcessArray.push(returning)
+          childProcesses.set(requireChildProcessIndex, childProcessArray)
+          return returning
+        },
+        execFile(...args: any[]) {
+          const returning = CP.execFile.apply(null, args as any)
+          const childProcessArray =
+            childProcesses.get(requireChildProcessIndex) ?? []
+          childProcessArray.push(returning)
+          childProcesses.set(requireChildProcessIndex, childProcessArray)
+          return returning
+        },
+        execFileSync: CP.execFileSync,
+        execSync: CP.execSync,
+        fork(...args: any[]) {
+          const returning = CP.fork.apply(null, args as any)
+          const childProcessArray =
+            childProcesses.get(requireChildProcessIndex) ?? []
+          childProcessArray.push(returning)
+          childProcesses.set(requireChildProcessIndex, childProcessArray)
+          return returning
+        },
+        spawn(...args: any[]) {
+          const returning = CP.spawn.apply(null, args as any)
+          const childProcessArray =
+            childProcesses.get(requireChildProcessIndex) ?? []
+          childProcessArray.push(returning)
+          childProcesses.set(requireChildProcessIndex, childProcessArray)
+          return returning
+        },
+        spawnSync: CP.spawnSync,
+      } as typeof CP
+    }
+    console.log("require", name)
+    return require(name, ...rest)
+  }
+  return requireChildProcessIndex
+}
+
+const cleanupRequire = (requireChildProcessIndex: number) => {
+  const foundChildren = childProcesses.get(requireChildProcessIndex)
+  if (!foundChildren) return
+  childProcesses.delete(requireChildProcessIndex)
+  foundChildren.forEach((x) => x.kill())
+}
+
+const idType = some(string, number)
+const path = "/start9/sockets/rpc.sock"
+const runType = object({
+  id: idType,
+  method: literal("run"),
+  params: object({
+    methodName: string.map((x) => {
+      const splitValue = x.split("/")
+      if (splitValue.length === 1)
+        throw new Error(`X (${x}) is not a valid path`)
+      return splitValue.slice(1)
+    }),
+    methodArgs: object,
+  }),
+})
+const callbackType = object({
+  id: idType,
+  method: literal("callback"),
+  params: object({
+    callback: string,
+    args: array,
+  }),
+})
+const dealWithInput = async (callbackHolder: CallbackHolder, input: unknown) =>
+  matches(input)
+    .when(runType, async ({ id, params: { methodName, methodArgs } }) => {
+      const index = setupRequire()
+      const effects = new Effects(`/${methodName.join("/")}`, callbackHolder)
+      // @ts-ignore
+      return import("/services/service.js")
+        .then((x) => methodName.reduce(reduceMethod(methodArgs, effects), x))
+        .then()
+        .then((result) => ({ id, result }))
+        .catch((error) => ({
+          id,
+          error: { message: error?.message ?? String(error) },
+        }))
+        .finally(() => cleanupRequire(index))
+    })
+    .when(callbackType, async ({ id, params: { callback, args } }) =>
+      Promise.resolve(callbackHolder.callCallback(callback, args))
+        .then((result) => ({ id, result }))
+        .catch((error) => ({
+          id,
+          error: { message: error?.message ?? String(error) },
+        })),
+    )
+
+    .defaultToLazy(() => {
+      console.warn(`Coudln't parse the following input ${input}`)
+      return {
+        error: { message: "Could not figure out shape" },
+      }
+    })
+
+const jsonParse = (x: Buffer) => JSON.parse(x.toString())
+export class Runtime {
+  unixSocketServer = net.createServer(async (server) => {})
+  private callbacks = new CallbackHolder()
+  constructor() {
+    this.unixSocketServer.listen(path)
+
+    this.unixSocketServer.on("connection", (s) => {
+      s.on("data", (a) =>
+        Promise.resolve(a)
+          .then(jsonParse)
+          .then(dealWithInput.bind(null, this.callbacks))
+          .then((x) => {
+            console.log("x", JSON.stringify(x), typeof x)
+            return x
+          })
+          .catch((error) => ({
+            error: { message: error?.message ?? String(error) },
+          }))
+          .then(JSON.stringify)
+          .then((x) => new Promise((resolve) => s.write("" + x, resolve)))
+          .finally(() => void s.end()),
+      )
+    })
+  }
+}
+function reduceMethod(
+  methodArgs: object,
+  effects: Effects,
+): (previousValue: any, currentValue: string) => any {
+  return (x: any, method: string) =>
+    Promise.resolve(x)
+      .then((x) => x[method])
+      .then((x) =>
+        typeof x !== "function"
+          ? x
+          : x({
+              ...methodArgs,
+              effects,
+            }),
+      )
+}

libs/start_init/initSrc/index.ts

@@ -0,0 +1,35 @@
+import { Runtime } from "./Runtime"
+
+new Runtime()
+
+/**
+
+So, this is going to be sent into a running comtainer along with any of the other node modules that are going to be needed and used.
+
+Once the container is started, we will go into a loading/ await state.
+This is the init system, and it will always be running, and it will be waiting for a command to be sent to it.
+
+Each command will be a stopable promise. And an example is going to be something like an action/ main/ or just a query into the types.
+
+A command will be sent an object which are the effects, and the effects will be things like the file system, the network, the process, and the os.
+
+
+ */
+// So OS Adapter
+// ==============
+
+/**
+* Why: So when the we call from the os we enter or leave here? 
+    
+ */
+
+/**
+Command: This is a command that the 
+
+There are 
+ */
+
+/**
+TODO:
+Should I seperate those adapter in/out?
+ */

libs/start_init/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "start-init",
+  "version": "0.0.0",
+  "description": "We want to be the sdk intermitent for the system",
+  "scripts": {
+    "bundle:esbuild": "esbuild initSrc/index.ts --platform=node --bundle --outfile=startInit.js",
+    "bundle:service": "esbuild /service/startos/procedures/index.ts --platform=node --bundle --outfile=service.js",
+    "run:manifest": "esbuild /service/startos/procedures/index.ts --platform=node --bundle --outfile=service.js"
+  },
+  "author": "",
+  "prettier": {
+    "trailingComma": "all",
+    "tabWidth": 2,
+    "semi": false,
+    "singleQuote": false
+  },
+  "dependencies": {
+    "@iarna/toml": "^2.2.5",
+    "@start9labs/start-sdk": "=0.4.0-rev0.lib0.rc8.alpha3",
+    "esbuild": "0.18.4",
+    "esbuild-plugin-resolve": "^2.0.0",
+    "filebrowser": "^1.0.0",
+    "isomorphic-fetch": "^3.0.0",
+    "ts-matches": "^5.4.1",
+    "tslib": "^2.5.3",
+    "typescript": "^5.1.3",
+    "yaml": "^2.3.1"
+  },
+  "devDependencies": {
+    "@swc/cli": "^0.1.62",
+    "@swc/core": "^1.3.65",
+    "@types/node": "^20.2.5",
+    "prettier": "^2.8.8",
+    "rollup": "^3.25.1"
+  }
+}

libs/start_init/readme.md

@@ -0,0 +1,86 @@
+## Testing
+
+So, we are going to
+
+1. create a fake server
+2. pretend socket server os (while the fake server is running)
+3. Run a fake effects system (while 1/2 are running)
+
+In order to simulate that we created a server like the start-os and
+a fake server (in this case I am using syncthing-wrapper)
+
+### TODO
+
+Undo the packing that I have done earlier, and hijack the embassy.js to use the bundle service + code
+
+Converting embassy.js -> service.js
+
+```sequence {theme="hand"}
+startOs ->> startInit.js: Rpc Call
+startInit.js ->> service.js: Rpc Converted into js code
+```
+
+### Create a fake server
+
+```bash
+run_test () {
+    (
+        set -e
+        libs=/home/jh/Projects/start-os/libs/start_init
+        sockets=/tmp/start9
+        service=/home/jh/Projects/syncthing-wrapper
+
+        docker run  \
+            -v $libs:/libs \
+            -v $service:/service \
+            -w /libs \
+            --rm node:18-alpine \
+            sh -c "
+                npm i &&
+                npm run bundle:esbuild  &&
+                npm run bundle:service
+            "
+
+
+
+        docker run  \
+            -v ./libs/start_init/:/libs \
+            -w /libs \
+            --rm node:18-alpine \
+            sh -c "
+                npm i &&
+                npm run bundle:esbuild
+            "
+
+
+
+        rm -rf $sockets  || true
+        mkdir -p $sockets/sockets
+        cd $service
+        docker run \
+            -v $libs:/start-init \
+            -v $sockets:/start9 \
+            --rm -it $(docker build -q .) sh -c "
+                apk add nodejs &&
+                node /start-init/bundleEs.js
+            "
+    )
+}
+run_test
+```
+
+### Pretend Socket Server OS
+
+First we are going to create our fake server client with the bash then send it the json possible data
+
+```bash
+sudo socat - unix-client:/tmp/start9/sockets/rpc.sock
+```
+
+<!-- prettier-ignore -->
+```json
+{"id":"a","method":"run","params":{"methodName":"/dependencyMounts","methodArgs":[]}}
+{"id":"a","method":"run","params":{"methodName":"/actions/test","methodArgs":{"input":{"id": 1}}}}
+{"id":"b","method":"run","params":{"methodName":"/actions/test","methodArgs":{"id": 1}}}
+
+```

libs/start_init/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "include": [
+    "./**/*.mjs",
+    "./**/*.js",
+    "initSrc/Runtime.ts",
+    "initSrc/index.ts",
+    "effects.ts"
+  ],
+  "exclude": [],
+  "inputs": ["./lib/index.ts"],
+  "compilerOptions": {
+    "target": "es2022",
+    "module": "es2022",
+    "moduleResolution": "node",
+    "allowJs": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true
+  },
+  "ts-node": {
+    "compilerOptions": {
+      "module": "commonjs"
+    }
+  }
+}