Feature/callbacks (#2678)

* wip

* initialize callbacks

* wip

* smtp

* list_service_interfaces

* wip

* wip

* fix domains

* fix hostname handling in NetService

* misc fixes

* getInstalledPackages

* misc fixes

* publish v6 lib

* refactor service effects

* fix import

* fix container runtime

* fix tests

* apply suggestions from review
This commit is contained in:
Aiden McClelland
2024-07-25 11:44:51 -06:00
committed by GitHub
parent ab465a755e
commit b36b62c68e
113 changed files with 4853 additions and 2517 deletions

View File

@@ -0,0 +1,301 @@
import { types as T } from "@start9labs/start-sdk"
import * as net from "net"
import { object, string, number, literals, some, unknown } from "ts-matches"
import { Effects } from "../Models/Effects"
import { CallbackHolder } from "../Models/CallbackHolder"
import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
const matchRpcError = object({
error: object(
{
code: number,
message: string,
data: some(
string,
object(
{
details: string,
debug: string,
},
["debug"],
),
),
},
["data"],
),
})
const testRpcError = matchRpcError.test
const testRpcResult = object({
result: unknown,
}).test
type RpcError = typeof matchRpcError._TYPE
const SOCKET_PATH = "/media/startos/rpc/host.sock"
let hostSystemId = 0
export type EffectContext = {
procedureId: string | null
callbacks: CallbackHolder | null
}
const rpcRoundFor =
(procedureId: string | null) =>
<K extends keyof Effects | "getStore" | "setStore" | "clearCallbacks">(
method: K,
params: Record<string, unknown>,
) => {
const id = hostSystemId++
const client = net.createConnection({ path: SOCKET_PATH }, () => {
client.write(
JSON.stringify({
id,
method,
params: { ...params, procedureId },
}) + "\n",
)
})
let bufs: Buffer[] = []
return new Promise((resolve, reject) => {
client.on("data", (data) => {
try {
bufs.push(data)
if (data.reduce((acc, x) => acc || x == 10, false)) {
const res: unknown = JSON.parse(
Buffer.concat(bufs).toString().split("\n")[0],
)
if (testRpcError(res)) {
let message = res.error.message
console.error("Error in host RPC:", { method, params })
if (string.test(res.error.data)) {
message += ": " + res.error.data
console.error(`Details: ${res.error.data}`)
} else {
if (res.error.data?.details) {
message += ": " + res.error.data.details
console.error(`Details: ${res.error.data.details}`)
}
if (res.error.data?.debug) {
message += "\n" + res.error.data.debug
console.error(`Debug: ${res.error.data.debug}`)
}
}
reject(new Error(`${message}@${method}`))
} else if (testRpcResult(res)) {
resolve(res.result)
} else {
reject(new Error(`malformed response ${JSON.stringify(res)}`))
}
}
} catch (error) {
reject(error)
}
client.end()
})
client.on("error", (error) => {
reject(error)
})
})
}
function makeEffects(context: EffectContext): Effects {
const rpcRound = rpcRoundFor(context.procedureId)
const self: Effects = {
bind(...[options]: Parameters<T.Effects["bind"]>) {
return rpcRound("bind", {
...options,
stack: new Error().stack,
}) as ReturnType<T.Effects["bind"]>
},
clearBindings(...[]: Parameters<T.Effects["clearBindings"]>) {
return rpcRound("clearBindings", {}) as ReturnType<
T.Effects["clearBindings"]
>
},
clearServiceInterfaces(
...[]: Parameters<T.Effects["clearServiceInterfaces"]>
) {
return rpcRound("clearServiceInterfaces", {}) as ReturnType<
T.Effects["clearServiceInterfaces"]
>
},
getInstalledPackages(...[]: Parameters<T.Effects["getInstalledPackages"]>) {
return rpcRound("getInstalledPackages", {}) as ReturnType<
T.Effects["getInstalledPackages"]
>
},
createOverlayedImage(options: {
imageId: string
}): Promise<[string, string]> {
return rpcRound("createOverlayedImage", options) as ReturnType<
T.Effects["createOverlayedImage"]
>
},
destroyOverlayedImage(options: { guid: string }): Promise<void> {
return rpcRound("destroyOverlayedImage", options) as ReturnType<
T.Effects["destroyOverlayedImage"]
>
},
executeAction(...[options]: Parameters<T.Effects["executeAction"]>) {
return rpcRound("executeAction", options) as ReturnType<
T.Effects["executeAction"]
>
},
exportAction(...[options]: Parameters<T.Effects["exportAction"]>) {
return rpcRound("exportAction", options) as ReturnType<
T.Effects["exportAction"]
>
},
exportServiceInterface: ((
...[options]: Parameters<Effects["exportServiceInterface"]>
) => {
return rpcRound("exportServiceInterface", options) as ReturnType<
T.Effects["exportServiceInterface"]
>
}) as Effects["exportServiceInterface"],
exposeForDependents(
...[options]: Parameters<T.Effects["exposeForDependents"]>
) {
return rpcRound("exposeForDependents", options) as ReturnType<
T.Effects["exposeForDependents"]
>
},
getConfigured(...[]: Parameters<T.Effects["getConfigured"]>) {
return rpcRound("getConfigured", {}) as ReturnType<
T.Effects["getConfigured"]
>
},
getContainerIp(...[]: Parameters<T.Effects["getContainerIp"]>) {
return rpcRound("getContainerIp", {}) as ReturnType<
T.Effects["getContainerIp"]
>
},
getHostInfo: ((...[allOptions]: Parameters<T.Effects["getHostInfo"]>) => {
const options = {
...allOptions,
callback: context.callbacks?.addCallback(allOptions.callback) || null,
}
return rpcRound("getHostInfo", options) as ReturnType<
T.Effects["getHostInfo"]
> as any
}) as Effects["getHostInfo"],
getServiceInterface(
...[options]: Parameters<T.Effects["getServiceInterface"]>
) {
return rpcRound("getServiceInterface", {
...options,
callback: context.callbacks?.addCallback(options.callback) || null,
}) as ReturnType<T.Effects["getServiceInterface"]>
},
getPrimaryUrl(...[options]: Parameters<T.Effects["getPrimaryUrl"]>) {
return rpcRound("getPrimaryUrl", {
...options,
callback: context.callbacks?.addCallback(options.callback) || null,
}) as ReturnType<T.Effects["getPrimaryUrl"]>
},
getServicePortForward(
...[options]: Parameters<T.Effects["getServicePortForward"]>
) {
return rpcRound("getServicePortForward", options) as ReturnType<
T.Effects["getServicePortForward"]
>
},
getSslCertificate(options: Parameters<T.Effects["getSslCertificate"]>[0]) {
return rpcRound("getSslCertificate", options) as ReturnType<
T.Effects["getSslCertificate"]
>
},
getSslKey(options: Parameters<T.Effects["getSslKey"]>[0]) {
return rpcRound("getSslKey", options) as ReturnType<
T.Effects["getSslKey"]
>
},
getSystemSmtp(...[options]: Parameters<T.Effects["getSystemSmtp"]>) {
return rpcRound("getSystemSmtp", {
...options,
callback: context.callbacks?.addCallback(options.callback) || null,
}) as ReturnType<T.Effects["getSystemSmtp"]>
},
listServiceInterfaces(
...[options]: Parameters<T.Effects["listServiceInterfaces"]>
) {
return rpcRound("listServiceInterfaces", {
...options,
callback: context.callbacks?.addCallback(options.callback) || null,
}) as ReturnType<T.Effects["listServiceInterfaces"]>
},
mount(...[options]: Parameters<T.Effects["mount"]>) {
return rpcRound("mount", options) as ReturnType<T.Effects["mount"]>
},
clearActions(...[]: Parameters<T.Effects["clearActions"]>) {
return rpcRound("clearActions", {}) as ReturnType<
T.Effects["clearActions"]
>
},
restart(...[]: Parameters<T.Effects["restart"]>) {
return rpcRound("restart", {}) as ReturnType<T.Effects["restart"]>
},
setConfigured(...[configured]: Parameters<T.Effects["setConfigured"]>) {
return rpcRound("setConfigured", { configured }) as ReturnType<
T.Effects["setConfigured"]
>
},
setDependencies(
dependencies: Parameters<T.Effects["setDependencies"]>[0],
): ReturnType<T.Effects["setDependencies"]> {
return rpcRound("setDependencies", dependencies) as ReturnType<
T.Effects["setDependencies"]
>
},
checkDependencies(
options: Parameters<T.Effects["checkDependencies"]>[0],
): ReturnType<T.Effects["checkDependencies"]> {
return rpcRound("checkDependencies", options) as ReturnType<
T.Effects["checkDependencies"]
>
},
getDependencies(): ReturnType<T.Effects["getDependencies"]> {
return rpcRound("getDependencies", {}) as ReturnType<
T.Effects["getDependencies"]
>
},
setHealth(...[options]: Parameters<T.Effects["setHealth"]>) {
return rpcRound("setHealth", options) as ReturnType<
T.Effects["setHealth"]
>
},
setMainStatus(o: { status: "running" | "stopped" }): Promise<void> {
return rpcRound("setMainStatus", o) as ReturnType<T.Effects["setHealth"]>
},
shutdown(...[]: Parameters<T.Effects["shutdown"]>) {
return rpcRound("shutdown", {}) as ReturnType<T.Effects["shutdown"]>
},
store: {
get: async (options: any) =>
rpcRound("getStore", {
...options,
callback: context.callbacks?.addCallback(options.callback) || null,
}) as any,
set: async (options: any) =>
rpcRound("setStore", options) as ReturnType<T.Effects["store"]["set"]>,
} as T.Effects["store"],
}
return self
}
export function makeProcedureEffects(procedureId: string): Effects {
return makeEffects({ procedureId, callbacks: null })
}
export function makeMainEffects(): MainEffects {
const rpcRound = rpcRoundFor(null)
return {
_type: "main",
clearCallbacks: () => {
return rpcRound("clearCallbacks", {}) as Promise<void>
},
...makeEffects({ procedureId: null, callbacks: new CallbackHolder() }),
}
}

View File

@@ -1,303 +0,0 @@
import { types as T } from "@start9labs/start-sdk"
import * as net from "net"
import { object, string, number, literals, some, unknown } from "ts-matches"
import { Effects } from "../Models/Effects"
import { CallbackHolder } from "../Models/CallbackHolder"
const matchRpcError = object({
error: object(
{
code: number,
message: string,
data: some(
string,
object(
{
details: string,
debug: string,
},
["debug"],
),
),
},
["data"],
),
})
const testRpcError = matchRpcError.test
const testRpcResult = object({
result: unknown,
}).test
type RpcError = typeof matchRpcError._TYPE
const SOCKET_PATH = "/media/startos/rpc/host.sock"
const MAIN = "/main" as const
let hostSystemId = 0
export const hostSystemStartOs =
(callbackHolder: CallbackHolder) =>
(procedureId: null | string): Effects => {
const rpcRound = <K extends keyof Effects | "getStore" | "setStore">(
method: K,
params: Record<string, unknown>,
) => {
const id = hostSystemId++
const client = net.createConnection({ path: SOCKET_PATH }, () => {
client.write(
JSON.stringify({
id,
method,
params: { ...params, procedureId: procedureId },
}) + "\n",
)
})
let bufs: Buffer[] = []
return new Promise((resolve, reject) => {
client.on("data", (data) => {
try {
bufs.push(data)
if (data.reduce((acc, x) => acc || x == 10, false)) {
const res: unknown = JSON.parse(
Buffer.concat(bufs).toString().split("\n")[0],
)
if (testRpcError(res)) {
let message = res.error.message
console.error({ method, params, hostSystemStartOs: true })
if (string.test(res.error.data)) {
message += ": " + res.error.data
console.error(res.error.data)
} else {
if (res.error.data?.details) {
message += ": " + res.error.data.details
console.error(res.error.data.details)
}
if (res.error.data?.debug) {
message += "\n" + res.error.data.debug
console.error("Debug: " + res.error.data.debug)
}
}
reject(new Error(`${message}@${method}`))
} else if (testRpcResult(res)) {
resolve(res.result)
} else {
reject(new Error(`malformed response ${JSON.stringify(res)}`))
}
}
} catch (error) {
reject(error)
}
client.end()
})
client.on("error", (error) => {
reject(error)
})
})
}
const self: Effects = {
bind(...[options]: Parameters<T.Effects["bind"]>) {
return rpcRound("bind", {
...options,
stack: new Error().stack,
}) as ReturnType<T.Effects["bind"]>
},
clearBindings(...[]: Parameters<T.Effects["clearBindings"]>) {
return rpcRound("clearBindings", {}) as ReturnType<
T.Effects["clearBindings"]
>
},
clearServiceInterfaces(
...[]: Parameters<T.Effects["clearServiceInterfaces"]>
) {
return rpcRound("clearServiceInterfaces", {}) as ReturnType<
T.Effects["clearServiceInterfaces"]
>
},
createOverlayedImage(options: {
imageId: string
}): Promise<[string, string]> {
return rpcRound("createOverlayedImage", options) as ReturnType<
T.Effects["createOverlayedImage"]
>
},
destroyOverlayedImage(options: { guid: string }): Promise<void> {
return rpcRound("destroyOverlayedImage", options) as ReturnType<
T.Effects["destroyOverlayedImage"]
>
},
executeAction(...[options]: Parameters<T.Effects["executeAction"]>) {
return rpcRound("executeAction", options) as ReturnType<
T.Effects["executeAction"]
>
},
exists(...[packageId]: Parameters<T.Effects["exists"]>) {
return rpcRound("exists", packageId) as ReturnType<T.Effects["exists"]>
},
exportAction(...[options]: Parameters<T.Effects["exportAction"]>) {
return rpcRound("exportAction", options) as ReturnType<
T.Effects["exportAction"]
>
},
exportServiceInterface: ((
...[options]: Parameters<Effects["exportServiceInterface"]>
) => {
return rpcRound("exportServiceInterface", options) as ReturnType<
T.Effects["exportServiceInterface"]
>
}) as Effects["exportServiceInterface"],
exposeForDependents(
...[options]: Parameters<T.Effects["exposeForDependents"]>
) {
return rpcRound("exposeForDependents", options) as ReturnType<
T.Effects["exposeForDependents"]
>
},
getConfigured(...[]: Parameters<T.Effects["getConfigured"]>) {
return rpcRound("getConfigured", {}) as ReturnType<
T.Effects["getConfigured"]
>
},
getContainerIp(...[]: Parameters<T.Effects["getContainerIp"]>) {
return rpcRound("getContainerIp", {}) as ReturnType<
T.Effects["getContainerIp"]
>
},
getHostInfo: ((...[allOptions]: any[]) => {
const options = {
...allOptions,
callback: callbackHolder.addCallback(allOptions.callback),
}
return rpcRound("getHostInfo", options) as ReturnType<
T.Effects["getHostInfo"]
> as any
}) as Effects["getHostInfo"],
getServiceInterface(
...[options]: Parameters<T.Effects["getServiceInterface"]>
) {
return rpcRound("getServiceInterface", {
...options,
callback: callbackHolder.addCallback(options.callback),
}) as ReturnType<T.Effects["getServiceInterface"]>
},
getPrimaryUrl(...[options]: Parameters<T.Effects["getPrimaryUrl"]>) {
return rpcRound("getPrimaryUrl", {
...options,
callback: callbackHolder.addCallback(options.callback),
}) as ReturnType<T.Effects["getPrimaryUrl"]>
},
getServicePortForward(
...[options]: Parameters<T.Effects["getServicePortForward"]>
) {
return rpcRound("getServicePortForward", options) as ReturnType<
T.Effects["getServicePortForward"]
>
},
getSslCertificate(
options: Parameters<T.Effects["getSslCertificate"]>[0],
) {
return rpcRound("getSslCertificate", options) as ReturnType<
T.Effects["getSslCertificate"]
>
},
getSslKey(options: Parameters<T.Effects["getSslKey"]>[0]) {
return rpcRound("getSslKey", options) as ReturnType<
T.Effects["getSslKey"]
>
},
getSystemSmtp(...[options]: Parameters<T.Effects["getSystemSmtp"]>) {
return rpcRound("getSystemSmtp", {
...options,
callback: callbackHolder.addCallback(options.callback),
}) as ReturnType<T.Effects["getSystemSmtp"]>
},
listServiceInterfaces(
...[options]: Parameters<T.Effects["listServiceInterfaces"]>
) {
return rpcRound("listServiceInterfaces", {
...options,
callback: callbackHolder.addCallback(options.callback),
}) as ReturnType<T.Effects["listServiceInterfaces"]>
},
mount(...[options]: Parameters<T.Effects["mount"]>) {
return rpcRound("mount", options) as ReturnType<T.Effects["mount"]>
},
removeAction(...[options]: Parameters<T.Effects["removeAction"]>) {
return rpcRound("removeAction", options) as ReturnType<
T.Effects["removeAction"]
>
},
removeAddress(...[options]: Parameters<T.Effects["removeAddress"]>) {
return rpcRound("removeAddress", options) as ReturnType<
T.Effects["removeAddress"]
>
},
restart(...[]: Parameters<T.Effects["restart"]>) {
return rpcRound("restart", {}) as ReturnType<T.Effects["restart"]>
},
running(...[packageId]: Parameters<T.Effects["running"]>) {
return rpcRound("running", { packageId }) as ReturnType<
T.Effects["running"]
>
},
// runRsync(...[options]: Parameters<T.Effects[""]>) {
//
// return rpcRound('executeAction', options) as ReturnType<T.Effects["executeAction"]>
//
// return rpcRound('executeAction', options) as ReturnType<T.Effects["executeAction"]>
// }
setConfigured(...[configured]: Parameters<T.Effects["setConfigured"]>) {
return rpcRound("setConfigured", { configured }) as ReturnType<
T.Effects["setConfigured"]
>
},
setDependencies(
dependencies: Parameters<T.Effects["setDependencies"]>[0],
): ReturnType<T.Effects["setDependencies"]> {
return rpcRound("setDependencies", dependencies) as ReturnType<
T.Effects["setDependencies"]
>
},
checkDependencies(
options: Parameters<T.Effects["checkDependencies"]>[0],
): ReturnType<T.Effects["checkDependencies"]> {
return rpcRound("checkDependencies", options) as ReturnType<
T.Effects["checkDependencies"]
>
},
getDependencies(): ReturnType<T.Effects["getDependencies"]> {
return rpcRound("getDependencies", {}) as ReturnType<
T.Effects["getDependencies"]
>
},
setHealth(...[options]: Parameters<T.Effects["setHealth"]>) {
return rpcRound("setHealth", options) as ReturnType<
T.Effects["setHealth"]
>
},
setMainStatus(o: { status: "running" | "stopped" }): Promise<void> {
return rpcRound("setMainStatus", o) as ReturnType<
T.Effects["setHealth"]
>
},
shutdown(...[]: Parameters<T.Effects["shutdown"]>) {
return rpcRound("shutdown", {}) as ReturnType<T.Effects["shutdown"]>
},
stopped(...[packageId]: Parameters<T.Effects["stopped"]>) {
return rpcRound("stopped", { packageId }) as ReturnType<
T.Effects["stopped"]
>
},
store: {
get: async (options: any) =>
rpcRound("getStore", {
...options,
callback: callbackHolder.addCallback(options.callback),
}) as any,
set: async (options: any) =>
rpcRound("setStore", options) as ReturnType<
T.Effects["store"]["set"]
>,
} as T.Effects["store"],
}
return self
}

View File

@@ -15,15 +15,16 @@ import {
} from "ts-matches"
import { types as T } from "@start9labs/start-sdk"
import * as CP from "child_process"
import * as Mod from "module"
import * as fs from "fs"
import { CallbackHolder } from "../Models/CallbackHolder"
import { AllGetDependencies } from "../Interfaces/AllGetDependencies"
import { HostSystem } from "../Interfaces/HostSystem"
import { jsonPath } from "../Models/JsonPath"
import { System } from "../Interfaces/System"
import { RunningMain, System } from "../Interfaces/System"
import {
MakeMainEffects,
MakeProcedureEffects,
} from "../Interfaces/MakeEffects"
type MaybePromise<T> = T | Promise<T>
export const matchRpcResult = anyOf(
object({ result: any }),
@@ -45,7 +46,7 @@ export const matchRpcResult = anyOf(
}),
)
export type RpcResult = typeof matchRpcResult._TYPE
type SocketResponse = { jsonrpc: "2.0"; id: IdType } & RpcResult
type SocketResponse = ({ jsonrpc: "2.0"; id: IdType } & RpcResult) | null
const SOCKET_PARENT = "/media/startos/rpc"
const SOCKET_PATH = "/media/startos/rpc/service.sock"
@@ -80,7 +81,6 @@ const sandboxRunType = object({
),
})
const callbackType = object({
id: idType,
method: literal("callback"),
params: object({
callback: number,
@@ -91,6 +91,14 @@ const initType = object({
id: idType,
method: literal("init"),
})
const startType = object({
id: idType,
method: literal("start"),
})
const stopType = object({
id: idType,
method: literal("stop"),
})
const exitType = object({
id: idType,
method: literal("exit"),
@@ -104,33 +112,40 @@ const evalType = object({
})
const jsonParse = (x: string) => JSON.parse(x)
function reduceMethod(
methodArgs: object,
effects: HostSystem,
): (previousValue: any, currentValue: string) => any {
return (x: any, method: string) =>
Promise.resolve(x)
.then((x) => x[method])
.then((x) =>
typeof x !== "function"
? x
: x({
...methodArgs,
effects,
}),
const handleRpc = (id: IdType, result: Promise<RpcResult>) =>
result
.then((result) => ({
jsonrpc,
id,
...result,
}))
.then((x) => {
if (
("result" in x && x.result === undefined) ||
!("error" in x || "result" in x)
)
}
(x as any).result = null
return x
})
.catch((error) => ({
jsonrpc,
id,
error: {
code: 0,
message: typeof error,
data: { details: "" + error, debug: error?.stack },
},
}))
const hasId = object({ id: idType }).test
export class RpcListener {
unixSocketServer = net.createServer(async (server) => {})
private _system: System | undefined
private _effects: HostSystem | undefined
private _makeProcedureEffects: MakeProcedureEffects | undefined
private _makeMainEffects: MakeMainEffects | undefined
constructor(
readonly getDependencies: AllGetDependencies,
private callbacks = new CallbackHolder(),
) {
constructor(readonly getDependencies: AllGetDependencies) {
if (!fs.existsSync(SOCKET_PARENT)) {
fs.mkdirSync(SOCKET_PARENT, { recursive: true })
}
@@ -165,8 +180,13 @@ export class RpcListener {
code: 1,
},
})
const writeDataToSocket = (x: SocketResponse) =>
new Promise((resolve) => s.write(JSON.stringify(x) + "\n", resolve))
const writeDataToSocket = (x: SocketResponse) => {
if (x != null) {
return new Promise((resolve) =>
s.write(JSON.stringify(x) + "\n", resolve),
)
}
}
s.on("data", (a) =>
Promise.resolve(a)
.then((b) => b.toString())
@@ -181,107 +201,116 @@ export class RpcListener {
})
}
private get effects() {
return this.getDependencies.hostSystem()(this.callbacks)
}
private get system() {
if (!this._system) throw new Error("System not initialized")
return this._system
}
private get makeProcedureEffects() {
if (!this._makeProcedureEffects) {
this._makeProcedureEffects = this.getDependencies.makeProcedureEffects()
}
return this._makeProcedureEffects
}
private get makeMainEffects() {
if (!this._makeMainEffects) {
this._makeMainEffects = this.getDependencies.makeMainEffects()
}
return this._makeMainEffects
}
private dealWithInput(input: unknown): MaybePromise<SocketResponse> {
return matches(input)
.when(some(runType, sandboxRunType), async ({ id, params }) => {
.when(runType, async ({ id, params }) => {
const system = this.system
const procedure = jsonPath.unsafeCast(params.procedure)
return system
.execute(this.effects, {
id: params.id,
const effects = this.getDependencies.makeProcedureEffects()(params.id)
return handleRpc(
id,
system.execute(effects, {
procedure,
input: params.input,
timeout: params.timeout,
})
.then((result) => ({
jsonrpc,
id,
...result,
}))
.then((x) => {
if (
("result" in x && x.result === undefined) ||
!("error" in x || "result" in x)
)
(x as any).result = null
return x
})
.catch((error) => ({
jsonrpc,
id,
error: {
code: 0,
message: typeof error,
data: { details: "" + error, debug: error?.stack },
},
}))
}),
)
})
.when(callbackType, async ({ id, params: { callback, args } }) =>
Promise.resolve(this.callbacks.callCallback(callback, args))
.then((result) => ({
jsonrpc,
id,
result,
}))
.catch((error) => ({
jsonrpc,
id,
error: {
code: 0,
message: typeof error,
data: {
details: error?.message ?? String(error),
debug: error?.stack,
},
},
})),
)
.when(exitType, async ({ id }) => {
if (this._system) await this._system.exit(this.effects(null))
delete this._system
delete this._effects
return {
jsonrpc,
.when(sandboxRunType, async ({ id, params }) => {
const system = this.system
const procedure = jsonPath.unsafeCast(params.procedure)
const effects = this.makeProcedureEffects(params.id)
return handleRpc(
id,
result: null,
}
system.sandbox(effects, {
procedure,
input: params.input,
timeout: params.timeout,
}),
)
})
.when(callbackType, async ({ params: { callback, args } }) => {
this.system.callCallback(callback, args)
return null
})
.when(startType, async ({ id }) => {
return handleRpc(
id,
this.system
.start(this.makeMainEffects())
.then((result) => ({ result })),
)
})
.when(stopType, async ({ id }) => {
return handleRpc(
id,
this.system.stop().then((result) => ({ result })),
)
})
.when(exitType, async ({ id }) => {
return handleRpc(
id,
(async () => {
if (this._system) await this._system.exit()
})().then((result) => ({ result })),
)
})
.when(initType, async ({ id }) => {
this._system = await this.getDependencies.system()
return {
jsonrpc,
return handleRpc(
id,
result: null,
}
(async () => {
if (!this._system) {
const system = await this.getDependencies.system()
await system.init()
this._system = system
}
})().then((result) => ({ result })),
)
})
.when(evalType, async ({ id, params }) => {
const result = await new Function(
`return (async () => { return (${params.script}) }).call(this)`,
).call({
listener: this,
require: require,
})
return {
jsonrpc,
return handleRpc(
id,
result: !["string", "number", "boolean", "null", "object"].includes(
typeof result,
)
? null
: result,
}
(async () => {
const result = await new Function(
`return (async () => { return (${params.script}) }).call(this)`,
).call({
listener: this,
require: require,
})
return {
jsonrpc,
id,
result: ![
"string",
"number",
"boolean",
"null",
"object",
].includes(typeof result)
? null
: result,
}
})(),
)
})
.when(shape({ id: idType, method: string }), ({ id, method }) => ({
jsonrpc,

View File

@@ -14,6 +14,7 @@ export class DockerProcedureContainer {
// }
static async of(
effects: T.Effects,
packageId: string,
data: DockerProcedure,
volumes: { [id: VolumeId]: Volume },
) {
@@ -38,16 +39,25 @@ export class DockerProcedureContainer {
mounts[mount],
)
} else if (volumeMount.type === "certificate") {
volumeMount
const hostnames = [
`${packageId}.embassy`,
...new Set(
Object.values(
(
await effects.getHostInfo({
hostId: volumeMount["interface-id"],
})
)?.hostnameInfo || {},
)
.flatMap((h) => h)
.flatMap((h) => (h.kind === "onion" ? [h.hostname.value] : [])),
).values(),
]
const certChain = await effects.getSslCertificate({
packageId: null,
hostId: volumeMount["interface-id"],
algorithm: null,
hostnames,
})
const key = await effects.getSslKey({
packageId: null,
hostId: volumeMount["interface-id"],
algorithm: null,
hostnames,
})
await fs.writeFile(
`${path}/${volumeMount["interface-id"]}.cert.pem`,

View File

@@ -1,10 +1,10 @@
import { polyfillEffects } from "./polyfillEffects"
import { DockerProcedureContainer } from "./DockerProcedureContainer"
import { SystemForEmbassy } from "."
import { hostSystemStartOs } from "../../HostSystemStartOs"
import { Daemons, T, daemons, utils } from "@start9labs/start-sdk"
import { T, utils } from "@start9labs/start-sdk"
import { Daemon } from "@start9labs/start-sdk/cjs/lib/mainFn/Daemon"
import { Effects } from "../../../Models/Effects"
import { off } from "node:process"
const EMBASSY_HEALTH_INTERVAL = 15 * 1000
const EMBASSY_PROPERTIES_LOOP = 30 * 1000
@@ -14,24 +14,28 @@ const EMBASSY_PROPERTIES_LOOP = 30 * 1000
* Also, this has an ability to clean itself up too if need be.
*/
export class MainLoop {
private healthLoops:
| {
name: string
interval: NodeJS.Timeout
}[]
| undefined
private healthLoops?: {
name: string
interval: NodeJS.Timeout
}[]
private mainEvent:
| Promise<{
daemon: Daemon
}>
| undefined
constructor(
private mainEvent?: {
daemon: Daemon
}
private constructor(
readonly system: SystemForEmbassy,
readonly effects: Effects,
) {
this.healthLoops = this.constructHealthLoops()
this.mainEvent = this.constructMainEvent()
) {}
static async of(
system: SystemForEmbassy,
effects: Effects,
): Promise<MainLoop> {
const res = new MainLoop(system, effects)
res.healthLoops = res.constructHealthLoops()
res.mainEvent = await res.constructMainEvent()
return res
}
private async constructMainEvent() {
@@ -46,6 +50,7 @@ export class MainLoop {
const jsMain = (this.system.moduleCode as any)?.jsMain
const dockerProcedureContainer = await DockerProcedureContainer.of(
effects,
this.system.manifest.id,
this.system.manifest.main,
this.system.manifest.volumes,
)
@@ -135,6 +140,7 @@ export class MainLoop {
if (actionProcedure.type === "docker") {
const container = await DockerProcedureContainer.of(
effects,
manifest.id,
actionProcedure,
manifest.volumes,
)

View File

@@ -3,8 +3,8 @@ import * as fs from "fs/promises"
import { polyfillEffects } from "./polyfillEffects"
import { Duration, duration, fromDuration } from "../../../Models/Duration"
import { System } from "../../../Interfaces/System"
import { matchManifest, Manifest, Procedure } from "./matchManifest"
import { System, Procedure } from "../../../Interfaces/System"
import { matchManifest, Manifest } from "./matchManifest"
import * as childProcess from "node:child_process"
import { DockerProcedureContainer } from "./DockerProcedureContainer"
import { promisify } from "node:util"
@@ -27,7 +27,6 @@ import {
Parser,
array,
} from "ts-matches"
import { hostSystemStartOs } from "../../HostSystemStartOs"
import { JsonPath, unNestPath } from "../../../Models/JsonPath"
import { RpcResult, matchRpcResult } from "../../RpcListener"
import { CT } from "@start9labs/start-sdk"
@@ -48,6 +47,7 @@ import {
transformConfigSpec,
transformOldConfigToNew,
} from "./transformConfigSpec"
import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
type Optional<A> = A | undefined | null
function todo(): never {
@@ -203,16 +203,39 @@ export class SystemForEmbassy implements System {
readonly manifest: Manifest,
readonly moduleCode: Partial<U.ExpectedExports>,
) {}
async init(): Promise<void> {}
async exit(): Promise<void> {
if (this.currentRunning) await this.currentRunning.clean()
delete this.currentRunning
}
async start(effects: MainEffects): Promise<void> {
if (!!this.currentRunning) return
this.currentRunning = await MainLoop.of(this, effects)
}
callCallback(_callback: number, _args: any[]): void {}
async stop(): Promise<void> {
const { currentRunning } = this
this.currentRunning?.clean()
delete this.currentRunning
if (currentRunning) {
await currentRunning.clean({
timeout: fromDuration(this.manifest.main["sigterm-timeout"] || "30s"),
})
}
}
async execute(
effectCreator: ReturnType<typeof hostSystemStartOs>,
effects: Effects,
options: {
id: string
procedure: JsonPath
input: unknown
timeout?: number | undefined
},
): Promise<RpcResult> {
const effects = effectCreator(options.id)
return this._execute(effects, options)
.then((x) =>
matches(x)
@@ -267,10 +290,6 @@ export class SystemForEmbassy implements System {
}
})
}
async exit(): Promise<void> {
if (this.currentRunning) await this.currentRunning.clean()
delete this.currentRunning
}
async _execute(
effects: Effects,
options: {
@@ -294,7 +313,7 @@ export class SystemForEmbassy implements System {
case "/actions/metadata":
return todo()
case "/init":
return this.init(
return this.initProcedure(
effects,
string.optional().unsafeCast(input),
options.timeout || null,
@@ -305,10 +324,6 @@ export class SystemForEmbassy implements System {
string.optional().unsafeCast(input),
options.timeout || null,
)
case "/main/start":
return this.mainStart(effects, options.timeout || null)
case "/main/stop":
return this.mainStop(effects, options.timeout || null)
default:
const procedures = unNestPath(options.procedure)
switch (true) {
@@ -345,7 +360,14 @@ export class SystemForEmbassy implements System {
}
throw new Error(`Could not find the path for ${options.procedure}`)
}
private async init(
async sandbox(
effects: Effects,
options: { procedure: Procedure; input: unknown; timeout?: number },
): Promise<RpcResult> {
return this.execute(effects, options)
}
private async initProcedure(
effects: Effects,
previousVersion: Optional<string>,
timeoutMs: number | null,
@@ -470,42 +492,22 @@ export class SystemForEmbassy implements System {
// TODO Do a migration down if the version exists
await effects.setMainStatus({ status: "stopped" })
}
private async mainStart(
effects: Effects,
timeoutMs: number | null,
): Promise<void> {
if (!!this.currentRunning) return
this.currentRunning = new MainLoop(this, effects)
}
private async mainStop(
effects: Effects,
timeoutMs: number | null,
): Promise<void> {
try {
const { currentRunning } = this
this.currentRunning?.clean()
delete this.currentRunning
if (currentRunning) {
await currentRunning.clean({
timeout: utils.inMs(this.manifest.main["sigterm-timeout"]),
})
}
return
} finally {
await effects.setMainStatus({ status: "stopped" })
}
}
private async createBackup(
effects: Effects,
timeoutMs: number | null,
): Promise<void> {
const backup = this.manifest.backup.create
if (backup.type === "docker") {
const container = await DockerProcedureContainer.of(effects, backup, {
...this.manifest.volumes,
BACKUP: { type: "backup", readonly: false },
})
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
backup,
{
...this.manifest.volumes,
BACKUP: { type: "backup", readonly: false },
},
)
await container.execFail([backup.entrypoint, ...backup.args], timeoutMs)
} else {
const moduleCode = await this.moduleCode
@@ -520,6 +522,7 @@ export class SystemForEmbassy implements System {
if (restoreBackup.type === "docker") {
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
restoreBackup,
{
...this.manifest.volumes,
@@ -552,6 +555,7 @@ export class SystemForEmbassy implements System {
if (config.type === "docker") {
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
config,
this.manifest.volumes,
)
@@ -594,6 +598,7 @@ export class SystemForEmbassy implements System {
if (setConfigValue.type === "docker") {
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
setConfigValue,
this.manifest.volumes,
)
@@ -702,6 +707,7 @@ export class SystemForEmbassy implements System {
if (procedure.type === "docker") {
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
procedure,
this.manifest.volumes,
)
@@ -744,6 +750,7 @@ export class SystemForEmbassy implements System {
if (setConfigValue.type === "docker") {
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
setConfigValue,
this.manifest.volumes,
)
@@ -785,6 +792,7 @@ export class SystemForEmbassy implements System {
if (actionProcedure.type === "docker") {
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
actionProcedure,
this.manifest.volumes,
)
@@ -825,6 +833,7 @@ export class SystemForEmbassy implements System {
if (actionProcedure.type === "docker") {
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
actionProcedure,
this.manifest.volumes,
)
@@ -1039,15 +1048,15 @@ async function updateConfig(
}
}
const url: string =
filled === null
filled === null || filled.addressInfo === null
? ""
: catchFn(() =>
utils.hostnameInfoToAddress(
specValue.target === "lan-address"
? filled.addressInfo.localHostnames[0] ||
filled.addressInfo.onionHostnames[0]
: filled.addressInfo.onionHostnames[0] ||
filled.addressInfo.localHostnames[0],
? filled.addressInfo!.localHostnames[0] ||
filled.addressInfo!.onionHostnames[0]
: filled.addressInfo!.onionHostnames[0] ||
filled.addressInfo!.localHostnames[0],
),
) || ""
mutConfigValue[key] = url

View File

@@ -126,6 +126,7 @@ export const polyfillEffects = (
} {
const dockerProcedureContainer = DockerProcedureContainer.of(
effects,
manifest.id,
manifest.main,
manifest.volumes,
)

View File

@@ -1,43 +1,84 @@
import { ExecuteResult, System } from "../../Interfaces/System"
import { ExecuteResult, Procedure, System } from "../../Interfaces/System"
import { unNestPath } from "../../Models/JsonPath"
import matches, { any, number, object, string, tuple } from "ts-matches"
import { hostSystemStartOs } from "../HostSystemStartOs"
import { Effects } from "../../Models/Effects"
import { RpcResult, matchRpcResult } from "../RpcListener"
import { duration } from "../../Models/Duration"
import { T } from "@start9labs/start-sdk"
import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
import { Volume } from "../../Models/Volume"
import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
import { CallbackHolder } from "../../Models/CallbackHolder"
export const STARTOS_JS_LOCATION = "/usr/lib/startos/package/index.js"
type RunningMain = {
effects: MainEffects
stop: () => Promise<void>
callbacks: CallbackHolder
}
export class SystemForStartOs implements System {
private onTerm: (() => Promise<void>) | undefined
private runningMain: RunningMain | undefined
static of() {
return new SystemForStartOs(require(STARTOS_JS_LOCATION))
}
constructor(readonly abi: T.ABI) {}
async init(): Promise<void> {}
async exit(): Promise<void> {}
async start(effects: MainEffects): Promise<void> {
if (this.runningMain) await this.stop()
let mainOnTerm: () => Promise<void> | undefined
const started = async (onTerm: () => Promise<void>) => {
await effects.setMainStatus({ status: "running" })
mainOnTerm = onTerm
}
const daemons = await (
await this.abi.main({
effects: effects as MainEffects,
started,
})
).build()
this.runningMain = {
effects,
stop: async () => {
if (mainOnTerm) await mainOnTerm()
await daemons.term()
},
callbacks: new CallbackHolder(),
}
}
callCallback(callback: number, args: any[]): void {
if (this.runningMain) {
this.runningMain.callbacks
.callCallback(callback, args)
.catch((error) => console.error(`callback ${callback} failed`, error))
} else {
console.warn(`callback ${callback} ignored because system is not running`)
}
}
async stop(): Promise<void> {
if (this.runningMain) {
await this.runningMain.stop()
await this.runningMain.effects.clearCallbacks()
this.runningMain = undefined
}
}
async execute(
effectCreator: ReturnType<typeof hostSystemStartOs>,
effects: Effects,
options: {
id: string
procedure:
| "/init"
| "/uninit"
| "/main/start"
| "/main/stop"
| "/config/set"
| "/config/get"
| "/backup/create"
| "/backup/restore"
| "/actions/metadata"
| `/actions/${string}/get`
| `/actions/${string}/run`
| `/dependencies/${string}/query`
| `/dependencies/${string}/update`
procedure: Procedure
input: unknown
timeout?: number | undefined
},
): Promise<RpcResult> {
const effects = effectCreator(options.id)
return this._execute(effects, options)
.then((x) =>
matches(x)
@@ -93,22 +134,9 @@ export class SystemForStartOs implements System {
})
}
async _execute(
effects: Effects,
effects: Effects | MainEffects,
options: {
procedure:
| "/init"
| "/uninit"
| "/main/start"
| "/main/stop"
| "/config/set"
| "/config/get"
| "/backup/create"
| "/backup/restore"
| "/actions/metadata"
| `/actions/${string}/get`
| `/actions/${string}/run`
| `/dependencies/${string}/query`
| `/dependencies/${string}/update`
procedure: Procedure
input: unknown
timeout?: number | undefined
},
@@ -123,30 +151,15 @@ export class SystemForStartOs implements System {
const nextVersion = string.optional().unsafeCast(options.input) || null
return this.abi.uninit({ effects, nextVersion })
}
case "/main/start": {
if (this.onTerm) await this.onTerm()
const started = async (onTerm: () => Promise<void>) => {
await effects.setMainStatus({ status: "running" })
this.onTerm = onTerm
}
const daemons = await (
await this.abi.main({
effects: { ...effects, _type: "main" },
started,
})
).build()
this.onTerm = daemons.term
return
}
case "/main/stop": {
try {
if (this.onTerm) await this.onTerm()
delete this.onTerm
return
} finally {
await effects.setMainStatus({ status: "stopped" })
}
}
// case "/main/start": {
//
// }
// case "/main/stop": {
// if (this.onTerm) await this.onTerm()
// await effects.setMainStatus({ status: "stopped" })
// delete this.onTerm
// return duration(30, "s")
// }
case "/config/set": {
const input = options.input as any // TODO
return this.abi.setConfig({ effects, input })
@@ -169,6 +182,9 @@ export class SystemForStartOs implements System {
case "/actions/metadata": {
return this.abi.actionsMetadata({ effects })
}
case "/properties": {
throw new Error("TODO")
}
default:
const procedures = unNestPath(options.procedure)
const id = procedures[2]
@@ -199,9 +215,12 @@ export class SystemForStartOs implements System {
}
return
}
throw new Error(`Method ${options.procedure} not implemented.`)
}
async exit(effects: Effects): Promise<void> {
return void null
async sandbox(
effects: Effects,
options: { procedure: Procedure; input: unknown; timeout?: number },
): Promise<RpcResult> {
return this.execute(effects, options)
}
}

View File

@@ -1,6 +1,7 @@
import { GetDependency } from "./GetDependency"
import { System } from "./System"
import { GetHostSystem, HostSystem } from "./HostSystem"
import { MakeMainEffects, MakeProcedureEffects } from "./MakeEffects"
export type AllGetDependencies = GetDependency<"system", Promise<System>> &
GetDependency<"hostSystem", GetHostSystem>
GetDependency<"makeProcedureEffects", MakeProcedureEffects> &
GetDependency<"makeMainEffects", MakeMainEffects>

View File

@@ -1,8 +0,0 @@
import { types as T } from "@start9labs/start-sdk"
import { CallbackHolder } from "../Models/CallbackHolder"
import { Effects } from "../Models/Effects"
export type HostSystem = Effects
export type GetHostSystem = (
callbackHolder: CallbackHolder,
) => (procedureId: null | string) => Effects

View File

@@ -0,0 +1,4 @@
import { Effects } from "../Models/Effects"
import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
export type MakeProcedureEffects = (procedureId: string) => Effects
export type MakeMainEffects = () => MainEffects

View File

@@ -1,33 +1,54 @@
import { types as T } from "@start9labs/start-sdk"
import { JsonPath } from "../Models/JsonPath"
import { RpcResult } from "../Adapters/RpcListener"
import { hostSystemStartOs } from "../Adapters/HostSystemStartOs"
import { Effects } from "../Models/Effects"
import { CallbackHolder } from "../Models/CallbackHolder"
import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
export type Procedure =
| "/init"
| "/uninit"
| "/config/set"
| "/config/get"
| "/backup/create"
| "/backup/restore"
| "/actions/metadata"
| "/properties"
| `/actions/${string}/get`
| `/actions/${string}/run`
| `/dependencies/${string}/query`
| `/dependencies/${string}/update`
export type ExecuteResult =
| { ok: unknown }
| { err: { code: number; message: string } }
export type System = {
// init(effects: Effects): Promise<void>
// exit(effects: Effects): Promise<void>
// start(effects: Effects): Promise<void>
// stop(effects: Effects, options: { timeout: number, signal?: number }): Promise<void>
init(): Promise<void>
start(effects: MainEffects): Promise<void>
callCallback(callback: number, args: any[]): void
stop(): Promise<void>
execute(
effectCreator: ReturnType<typeof hostSystemStartOs>,
effects: Effects,
options: {
id: string
procedure: JsonPath
procedure: Procedure
input: unknown
timeout?: number
},
): Promise<RpcResult>
sandbox(
effects: Effects,
options: {
procedure: Procedure
input: unknown
timeout?: number
},
): Promise<RpcResult>
// sandbox(
// effects: Effects,
// options: {
// procedure: JsonPath
// input: unknown
// timeout?: number
// },
// ): Promise<unknown>
exit(effects: T.Effects): Promise<void>
exit(): Promise<void>
}
export type RunningMain = {
callbacks: CallbackHolder
stop(): Promise<void>
}

View File

@@ -1,12 +1,14 @@
export class CallbackHolder {
constructor() {}
private root = (Math.random() + 1).toString(36).substring(7)
private inc = 0
private callbacks = new Map<number, Function>()
private newId() {
return this.inc++
}
addCallback(callback: Function) {
addCallback(callback?: Function) {
if (!callback) {
return
}
const id = this.newId()
this.callbacks.set(id, callback)
return id

View File

@@ -28,8 +28,6 @@ export const jsonPath = some(
literals(
"/init",
"/uninit",
"/main/start",
"/main/stop",
"/config/set",
"/config/get",
"/backup/create",

View File

@@ -1,12 +1,13 @@
import { RpcListener } from "./Adapters/RpcListener"
import { SystemForEmbassy } from "./Adapters/Systems/SystemForEmbassy"
import { hostSystemStartOs } from "./Adapters/HostSystemStartOs"
import { makeMainEffects, makeProcedureEffects } from "./Adapters/EffectCreator"
import { AllGetDependencies } from "./Interfaces/AllGetDependencies"
import { getSystem } from "./Adapters/Systems"
const getDependencies: AllGetDependencies = {
system: getSystem,
hostSystem: () => hostSystemStartOs,
makeProcedureEffects: () => makeProcedureEffects,
makeMainEffects: () => makeMainEffects,
}
new RpcListener(getDependencies)