Merge branch 'next/minor' of github.com:Start9Labs/start-os into next/major

This commit is contained in:
Matt Hill
2024-11-25 19:02:07 -07:00
712 changed files with 83068 additions and 9240 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -1,86 +0,0 @@
## Testing
So, we are going to
1. create a fake server
2. pretend socket server os (while the fake server is running)
3. Run a fake effects system (while 1/2 are running)
In order to simulate that we created a server like the start-os and
a fake server (in this case I am using syncthing-wrapper)
### TODO
Undo the packing that I have done earlier, and hijack the embassy.js to use the bundle service + code
Converting embassy.js -> service.js
```sequence {theme="hand"}
startOs ->> startInit.js: Rpc Call
startInit.js ->> service.js: Rpc Converted into js code
```
### Create a fake server
```bash
run_test () {
(
set -e
libs=/home/jh/Projects/start-os/libs/start_init
sockets=/tmp/start9
service=/home/jh/Projects/syncthing-wrapper
docker run \
-v $libs:/libs \
-v $service:/service \
-w /libs \
--rm node:18-alpine \
sh -c "
npm i &&
npm run bundle:esbuild &&
npm run bundle:service
"
docker run \
-v ./libs/start_init/:/libs \
-w /libs \
--rm node:18-alpine \
sh -c "
npm i &&
npm run bundle:esbuild
"
rm -rf $sockets || true
mkdir -p $sockets/sockets
cd $service
docker run \
-v $libs:/start-init \
-v $sockets:/start9 \
--rm -it $(docker build -q .) sh -c "
apk add nodejs &&
node /start-init/bundleEs.js
"
)
}
run_test
```
### Pretend Socket Server OS
First we are going to create our fake server client with the bash then send it the json possible data
```bash
sudo socat - unix-client:/tmp/start9/sockets/rpc.sock
```
<!-- prettier-ignore -->
```json
{"id":"a","method":"run","params":{"methodName":"/dependencyMounts","methodArgs":[]}}
{"id":"a","method":"run","params":{"methodName":"/actions/test","methodArgs":{"input":{"id": 1}}}}
{"id":"b","method":"run","params":{"methodName":"/actions/test","methodArgs":{"id": 1}}}
```

View File

@@ -4,7 +4,7 @@ import { object, string, number, literals, some, unknown } from "ts-matches"
import { Effects } from "../Models/Effects"
import { CallbackHolder } from "../Models/CallbackHolder"
import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
import { asError } from "@start9labs/start-sdk/base/lib/util"
const matchRpcError = object({
error: object(
{
@@ -35,7 +35,8 @@ let hostSystemId = 0
export type EffectContext = {
procedureId: string | null
callbacks: CallbackHolder | null
callbacks?: CallbackHolder
constRetry: () => void
}
const rpcRoundFor =
@@ -50,7 +51,7 @@ const rpcRoundFor =
JSON.stringify({
id,
method,
params: { ...params, procedureId },
params: { ...params, procedureId: procedureId || undefined },
}) + "\n",
)
})
@@ -67,7 +68,7 @@ const rpcRoundFor =
let message = res.error.message
console.error(
"Error in host RPC:",
utils.asError({ method, params }),
utils.asError({ method, params, error: res.error }),
)
if (string.test(res.error.data)) {
message += ": " + res.error.data
@@ -100,24 +101,64 @@ const rpcRoundFor =
})
}
function makeEffects(context: EffectContext): Effects {
export function makeEffects(context: EffectContext): Effects {
const rpcRound = rpcRoundFor(context.procedureId)
const self: Effects = {
constRetry: context.constRetry,
clearCallbacks(...[options]: Parameters<T.Effects["clearCallbacks"]>) {
return rpcRound("clear-callbacks", {
...options,
}) as ReturnType<T.Effects["clearCallbacks"]>
},
action: {
clear(...[options]: Parameters<T.Effects["action"]["clear"]>) {
return rpcRound("action.clear", {
...options,
}) as ReturnType<T.Effects["action"]["clear"]>
},
export(...[options]: Parameters<T.Effects["action"]["export"]>) {
return rpcRound("action.export", {
...options,
}) as ReturnType<T.Effects["action"]["export"]>
},
getInput(...[options]: Parameters<T.Effects["action"]["getInput"]>) {
return rpcRound("action.get-input", {
...options,
}) as ReturnType<T.Effects["action"]["getInput"]>
},
request(...[options]: Parameters<T.Effects["action"]["request"]>) {
return rpcRound("action.request", {
...options,
}) as ReturnType<T.Effects["action"]["request"]>
},
run(...[options]: Parameters<T.Effects["action"]["run"]>) {
return rpcRound("action.run", {
...options,
}) as ReturnType<T.Effects["action"]["run"]>
},
clearRequests(
...[options]: Parameters<T.Effects["action"]["clearRequests"]>
) {
return rpcRound("action.clear-requests", {
...options,
}) as ReturnType<T.Effects["action"]["clearRequests"]>
},
},
bind(...[options]: Parameters<T.Effects["bind"]>) {
return rpcRound("bind", {
...options,
stack: new Error().stack,
}) as ReturnType<T.Effects["bind"]>
},
clearBindings(...[]: Parameters<T.Effects["clearBindings"]>) {
return rpcRound("clear-bindings", {}) as ReturnType<
clearBindings(...[options]: Parameters<T.Effects["clearBindings"]>) {
return rpcRound("clear-bindings", { ...options }) as ReturnType<
T.Effects["clearBindings"]
>
},
clearServiceInterfaces(
...[]: Parameters<T.Effects["clearServiceInterfaces"]>
...[options]: Parameters<T.Effects["clearServiceInterfaces"]>
) {
return rpcRound("clear-service-interfaces", {}) as ReturnType<
return rpcRound("clear-service-interfaces", { ...options }) as ReturnType<
T.Effects["clearServiceInterfaces"]
>
},
@@ -127,27 +168,17 @@ function makeEffects(context: EffectContext): Effects {
>
},
subcontainer: {
createFs(options: { imageId: string }) {
createFs(options: { imageId: string; name: string }) {
return rpcRound("subcontainer.create-fs", options) as ReturnType<
T.Effects["subcontainer"]["createFs"]
>
},
destroyFs(options: { guid: string }): Promise<void> {
destroyFs(options: { guid: string }): Promise<null> {
return rpcRound("subcontainer.destroy-fs", options) as ReturnType<
T.Effects["subcontainer"]["destroyFs"]
>
},
},
executeAction(...[options]: Parameters<T.Effects["executeAction"]>) {
return rpcRound("execute-action", options) as ReturnType<
T.Effects["executeAction"]
>
},
exportAction(...[options]: Parameters<T.Effects["exportAction"]>) {
return rpcRound("export-action", options) as ReturnType<
T.Effects["exportAction"]
>
},
exportServiceInterface: ((
...[options]: Parameters<Effects["exportServiceInterface"]>
) => {
@@ -162,11 +193,6 @@ function makeEffects(context: EffectContext): Effects {
T.Effects["exposeForDependents"]
>
},
getConfigured(...[]: Parameters<T.Effects["getConfigured"]>) {
return rpcRound("get-configured", {}) as ReturnType<
T.Effects["getConfigured"]
>
},
getContainerIp(...[]: Parameters<T.Effects["getContainerIp"]>) {
return rpcRound("get-container-ip", {}) as ReturnType<
T.Effects["getContainerIp"]
@@ -230,19 +256,9 @@ function makeEffects(context: EffectContext): Effects {
mount(...[options]: Parameters<T.Effects["mount"]>) {
return rpcRound("mount", options) as ReturnType<T.Effects["mount"]>
},
clearActions(...[]: Parameters<T.Effects["clearActions"]>) {
return rpcRound("clear-actions", {}) as ReturnType<
T.Effects["clearActions"]
>
},
restart(...[]: Parameters<T.Effects["restart"]>) {
return rpcRound("restart", {}) as ReturnType<T.Effects["restart"]>
},
setConfigured(...[configured]: Parameters<T.Effects["setConfigured"]>) {
return rpcRound("set-configured", { configured }) as ReturnType<
T.Effects["setConfigured"]
>
},
setDependencies(
dependencies: Parameters<T.Effects["setDependencies"]>[0],
): ReturnType<T.Effects["setDependencies"]> {
@@ -268,7 +284,10 @@ function makeEffects(context: EffectContext): Effects {
>
},
setMainStatus(o: { status: "running" | "stopped" }): Promise<void> {
getStatus(...[o]: Parameters<T.Effects["getStatus"]>) {
return rpcRound("get-status", o) as ReturnType<T.Effects["getStatus"]>
},
setMainStatus(o: { status: "running" | "stopped" }): Promise<null> {
return rpcRound("set-main-status", o) as ReturnType<
T.Effects["setHealth"]
>
@@ -299,18 +318,3 @@ function makeEffects(context: EffectContext): Effects {
}
return self
}
export function makeProcedureEffects(procedureId: string): Effects {
return makeEffects({ procedureId, callbacks: null })
}
export function makeMainEffects(): MainEffects {
const rpcRound = rpcRoundFor(null)
return {
_type: "main",
clearCallbacks: () => {
return rpcRound("clearCallbacks", {}) as Promise<void>
},
...makeEffects({ procedureId: null, callbacks: new CallbackHolder() }),
}
}

View File

@@ -14,17 +14,14 @@ import {
anyOf,
} from "ts-matches"
import { types as T } from "@start9labs/start-sdk"
import { types as T, utils } from "@start9labs/start-sdk"
import * as fs from "fs"
import { CallbackHolder } from "../Models/CallbackHolder"
import { AllGetDependencies } from "../Interfaces/AllGetDependencies"
import { jsonPath, unNestPath } from "../Models/JsonPath"
import { RunningMain, System } from "../Interfaces/System"
import {
MakeMainEffects,
MakeProcedureEffects,
} from "../Interfaces/MakeEffects"
import { System } from "../Interfaces/System"
import { makeEffects } from "./EffectCreator"
type MaybePromise<T> = T | Promise<T>
export const matchRpcResult = anyOf(
object({ result: any }),
@@ -45,6 +42,7 @@ export const matchRpcResult = anyOf(
),
}),
)
export type RpcResult = typeof matchRpcResult._TYPE
type SocketResponse = ({ jsonrpc: "2.0"; id: IdType } & RpcResult) | null
@@ -55,73 +53,96 @@ const jsonrpc = "2.0" as const
const isResult = object({ result: any }).test
const idType = some(string, number, literal(null))
type IdType = null | string | number
const runType = object({
id: idType,
method: literal("execute"),
params: object(
{
id: string,
procedure: string,
input: any,
timeout: number,
},
["timeout"],
),
})
const sandboxRunType = object({
id: idType,
method: literal("sandbox"),
params: object(
{
id: string,
procedure: string,
input: any,
timeout: number,
},
["timeout"],
),
})
type IdType = null | string | number | undefined
const runType = object(
{
id: idType,
method: literal("execute"),
params: object(
{
id: string,
procedure: string,
input: any,
timeout: number,
},
["timeout"],
),
},
["id"],
)
const sandboxRunType = object(
{
id: idType,
method: literal("sandbox"),
params: object(
{
id: string,
procedure: string,
input: any,
timeout: number,
},
["timeout"],
),
},
["id"],
)
const callbackType = object({
method: literal("callback"),
params: object({
callback: number,
id: number,
args: array,
}),
})
const initType = object({
id: idType,
method: literal("init"),
})
const startType = object({
id: idType,
method: literal("start"),
})
const stopType = object({
id: idType,
method: literal("stop"),
})
const exitType = object({
id: idType,
method: literal("exit"),
})
const evalType = object({
id: idType,
method: literal("eval"),
params: object({
script: string,
}),
})
const initType = object(
{
id: idType,
method: literal("init"),
},
["id"],
)
const startType = object(
{
id: idType,
method: literal("start"),
},
["id"],
)
const stopType = object(
{
id: idType,
method: literal("stop"),
},
["id"],
)
const exitType = object(
{
id: idType,
method: literal("exit"),
},
["id"],
)
const evalType = object(
{
id: idType,
method: literal("eval"),
params: object({
script: string,
}),
},
["id"],
)
const jsonParse = (x: string) => JSON.parse(x)
const handleRpc = (id: IdType, result: Promise<RpcResult>) =>
result
.then((result) => ({
jsonrpc,
id,
...result,
}))
.then((result) => {
return {
jsonrpc,
id,
...result,
}
})
.then((x) => {
if (
("result" in x && x.result === undefined) ||
@@ -144,8 +165,7 @@ const hasId = object({ id: idType }).test
export class RpcListener {
unixSocketServer = net.createServer(async (server) => {})
private _system: System | undefined
private _makeProcedureEffects: MakeProcedureEffects | undefined
private _makeMainEffects: MakeMainEffects | undefined
private callbacks: CallbackHolder | undefined
constructor(readonly getDependencies: AllGetDependencies) {
if (!fs.existsSync(SOCKET_PARENT)) {
@@ -198,7 +218,11 @@ export class RpcListener {
.then((x) => this.dealWithInput(x))
.catch(mapError)
.then(logData("response"))
.then(writeDataToSocket),
.then(writeDataToSocket)
.catch((e) => {
console.error(`Major error in socket handling: ${e}`)
console.debug(`Data in: ${a.toString()}`)
}),
)
})
}
@@ -208,18 +232,33 @@ export class RpcListener {
return this._system
}
private get makeProcedureEffects() {
if (!this._makeProcedureEffects) {
this._makeProcedureEffects = this.getDependencies.makeProcedureEffects()
private callbackHolders: Map<string, CallbackHolder> = new Map()
private removeCallbackHolderFor(procedure: string) {
const prev = this.callbackHolders.get(procedure)
if (prev) {
this.callbackHolders.delete(procedure)
this.callbacks?.removeChild(prev)
}
return this._makeProcedureEffects
}
private callbackHolderFor(procedure: string): CallbackHolder {
this.removeCallbackHolderFor(procedure)
const callbackHolder = this.callbacks!.child()
this.callbackHolders.set(procedure, callbackHolder)
return callbackHolder
}
private get makeMainEffects() {
if (!this._makeMainEffects) {
this._makeMainEffects = this.getDependencies.makeMainEffects()
callCallback(callback: number, args: any[]): void {
if (this.callbacks) {
this.callbacks
.callCallback(callback, args)
.catch((error) =>
console.error(`callback ${callback} failed`, utils.asError(error)),
)
} else {
console.warn(
`callback ${callback} ignored because system is not initialized`,
)
}
return this._makeMainEffects
}
private dealWithInput(input: unknown): MaybePromise<SocketResponse> {
@@ -227,40 +266,49 @@ export class RpcListener {
.when(runType, async ({ id, params }) => {
const system = this.system
const procedure = jsonPath.unsafeCast(params.procedure)
const effects = this.getDependencies.makeProcedureEffects()(params.id)
const input = params.input
const timeout = params.timeout
const result = getResult(procedure, system, effects, timeout, input)
const { input, timeout, id: procedureId } = params
const result = this.getResult(
procedure,
system,
procedureId,
timeout,
input,
)
return handleRpc(id, result)
})
.when(sandboxRunType, async ({ id, params }) => {
const system = this.system
const procedure = jsonPath.unsafeCast(params.procedure)
const effects = this.makeProcedureEffects(params.id)
const result = getResult(
const { input, timeout, id: procedureId } = params
const result = this.getResult(
procedure,
system,
effects,
params.input,
params.input,
procedureId,
timeout,
input,
)
return handleRpc(id, result)
})
.when(callbackType, async ({ params: { callback, args } }) => {
this.system.callCallback(callback, args)
.when(callbackType, async ({ params: { id, args } }) => {
this.callCallback(id, args)
return null
})
.when(startType, async ({ id }) => {
const callbacks = this.callbackHolderFor("main")
const effects = makeEffects({
procedureId: null,
callbacks,
constRetry: () => {},
})
return handleRpc(
id,
this.system
.start(this.makeMainEffects())
.then((result) => ({ result })),
this.system.start(effects).then((result) => ({ result })),
)
})
.when(stopType, async ({ id }) => {
this.removeCallbackHolderFor("main")
return handleRpc(
id,
this.system.stop().then((result) => ({ result })),
@@ -280,7 +328,20 @@ export class RpcListener {
(async () => {
if (!this._system) {
const system = await this.getDependencies.system()
await system.containerInit()
this.callbacks = new CallbackHolder(
makeEffects({
procedureId: null,
constRetry: () => {},
}),
)
const callbacks = this.callbackHolderFor("containerInit")
await system.containerInit(
makeEffects({
procedureId: null,
callbacks,
constRetry: () => {},
}),
)
this._system = system
}
})().then((result) => ({ result })),
@@ -312,17 +373,20 @@ export class RpcListener {
})(),
)
})
.when(shape({ id: idType, method: string }), ({ id, method }) => ({
jsonrpc,
id,
error: {
code: -32601,
message: `Method not found`,
data: {
details: method,
.when(
shape({ id: idType, method: string }, ["id"]),
({ id, method }) => ({
jsonrpc,
id,
error: {
code: -32601,
message: `Method not found`,
data: {
details: method,
},
},
},
}))
}),
)
.defaultToLazy(() => {
console.warn(
@@ -341,98 +405,81 @@ export class RpcListener {
}
})
}
}
function getResult(
procedure: typeof jsonPath._TYPE,
system: System,
effects: T.Effects,
timeout: number | undefined,
input: any,
) {
const ensureResultTypeShape = (
result:
| void
| T.ConfigRes
| T.PropertiesReturn
| T.ActionMetadata[]
| T.ActionResult,
): { result: any } => {
if (isResult(result)) return result
return { result }
}
return (async () => {
switch (procedure) {
case "/backup/create":
return system.createBackup(effects, timeout || null)
case "/backup/restore":
return system.restoreBackup(effects, timeout || null)
case "/config/get":
return system.getConfig(effects, timeout || null)
case "/config/set":
return system.setConfig(effects, input, timeout || null)
case "/properties":
return system.properties(effects, timeout || null)
case "/actions/metadata":
return system.actionsMetadata(effects)
case "/init":
return system.packageInit(
effects,
string.optional().unsafeCast(input),
timeout || null,
)
case "/uninit":
return system.packageUninit(
effects,
string.optional().unsafeCast(input),
timeout || null,
)
default:
const procedures = unNestPath(procedure)
switch (true) {
case procedures[1] === "actions" && procedures[3] === "get":
return system.action(effects, procedures[2], input, timeout || null)
case procedures[1] === "actions" && procedures[3] === "run":
return system.action(effects, procedures[2], input, timeout || null)
case procedures[1] === "dependencies" && procedures[3] === "query":
return system.dependenciesAutoconfig(
effects,
procedures[2],
input,
timeout || null,
)
case procedures[1] === "dependencies" && procedures[3] === "update":
return system.dependenciesAutoconfig(
effects,
procedures[2],
input,
timeout || null,
)
}
private getResult(
procedure: typeof jsonPath._TYPE,
system: System,
procedureId: string,
timeout: number | undefined,
input: any,
) {
const ensureResultTypeShape = (
result: void | T.ActionInput | T.ActionResult | null,
): { result: any } => {
return { result }
}
})().then(ensureResultTypeShape, (error) =>
matches(error)
.when(
object(
{
error: string,
code: number,
},
["code"],
{ code: 0 },
),
(error) => ({
const callbacks = this.callbackHolderFor(procedure)
const effects = makeEffects({
procedureId,
callbacks,
constRetry: () => {},
})
return (async () => {
switch (procedure) {
case "/backup/create":
return system.createBackup(effects, timeout || null)
case "/backup/restore":
return system.restoreBackup(effects, timeout || null)
case "/packageInit":
return system.packageInit(effects, timeout || null)
case "/packageUninit":
return system.packageUninit(
effects,
string.optional().unsafeCast(input),
timeout || null,
)
default:
const procedures = unNestPath(procedure)
switch (true) {
case procedures[1] === "actions" && procedures[3] === "getInput":
return system.getActionInput(
effects,
procedures[2],
timeout || null,
)
case procedures[1] === "actions" && procedures[3] === "run":
return system.runAction(
effects,
procedures[2],
input.input,
timeout || null,
)
}
}
})().then(ensureResultTypeShape, (error) =>
matches(error)
.when(
object(
{
error: string,
code: number,
},
["code"],
{ code: 0 },
),
(error) => ({
error: {
code: error.code,
message: error.error,
},
}),
)
.defaultToLazy(() => ({
error: {
code: error.code,
message: error.error,
code: 0,
message: String(error),
},
}),
)
.defaultToLazy(() => ({
error: {
code: 0,
message: String(error),
},
})),
)
})),
)
}
}

View File

@@ -8,7 +8,7 @@ import {
CommandOptions,
ExecOptions,
ExecSpawnable,
} from "@start9labs/start-sdk/cjs/lib/util/SubContainer"
} from "@start9labs/start-sdk/package/lib/util/SubContainer"
export const exec = promisify(cp.exec)
export const execFile = promisify(cp.execFile)
@@ -20,6 +20,7 @@ export class DockerProcedureContainer {
packageId: string,
data: DockerProcedure,
volumes: { [id: VolumeId]: Volume },
name: string,
options: { subcontainer?: ExecSpawnable } = {},
) {
const subcontainer =
@@ -29,6 +30,7 @@ export class DockerProcedureContainer {
packageId,
data,
volumes,
name,
))
return new DockerProcedureContainer(subcontainer)
}
@@ -37,8 +39,13 @@ export class DockerProcedureContainer {
packageId: string,
data: DockerProcedure,
volumes: { [id: VolumeId]: Volume },
name: string,
) {
const subcontainer = await SubContainer.of(effects, { id: data.image })
const subcontainer = await SubContainer.of(
effects,
{ id: data.image },
name,
)
if (data.mounts) {
const mounts = data.mounts
@@ -144,7 +151,7 @@ export class DockerProcedureContainer {
}
}
async spawn(commands: string[]): Promise<cp.ChildProcessWithoutNullStreams> {
async spawn(commands: string[]): Promise<cp.ChildProcess> {
return await this.subcontainer.spawn(commands)
}
}

View File

@@ -2,11 +2,10 @@ import { polyfillEffects } from "./polyfillEffects"
import { DockerProcedureContainer } from "./DockerProcedureContainer"
import { SystemForEmbassy } from "."
import { T, utils } from "@start9labs/start-sdk"
import { Daemon } from "@start9labs/start-sdk/cjs/lib/mainFn/Daemon"
import { Daemon } from "@start9labs/start-sdk/package/lib/mainFn/Daemon"
import { Effects } from "../../../Models/Effects"
import { off } from "node:process"
import { CommandController } from "@start9labs/start-sdk/cjs/lib/mainFn/CommandController"
import { asError } from "@start9labs/start-sdk/cjs/lib/util"
import { CommandController } from "@start9labs/start-sdk/package/lib/mainFn/CommandController"
const EMBASSY_HEALTH_INTERVAL = 15 * 1000
const EMBASSY_PROPERTIES_LOOP = 30 * 1000
@@ -62,6 +61,7 @@ export class MainLoop {
this.system.manifest.id,
this.system.manifest.main,
this.system.manifest.volumes,
`Main - ${currentCommand.join(" ")}`,
)
return CommandController.of()(
this.effects,
@@ -136,7 +136,7 @@ export class MainLoop {
delete this.healthLoops
await main?.daemon
.stop()
.catch((e) => console.error(`Main loop error`, utils.asError(e)))
.catch((e: unknown) => console.error(`Main loop error`, utils.asError(e)))
this.effects.setMainStatus({ status: "stopped" })
if (healthLoops) healthLoops.forEach((x) => clearInterval(x.interval))
}
@@ -154,7 +154,7 @@ export class MainLoop {
result: "starting",
message: null,
})
.catch((e) => console.error(asError(e)))
.catch((e) => console.error(utils.asError(e)))
const interval = setInterval(async () => {
const actionProcedure = value
const timeChanged = Date.now() - start
@@ -162,21 +162,30 @@ export class MainLoop {
const subcontainer = actionProcedure.inject
? this.mainSubContainerHandle
: undefined
// prettier-ignore
const container =
await DockerProcedureContainer.of(
effects,
manifest.id,
actionProcedure,
manifest.volumes,
{
subcontainer,
}
)
const executed = await container.exec(
[actionProcedure.entrypoint, ...actionProcedure.args],
{ input: JSON.stringify(timeChanged) },
const commands = [
actionProcedure.entrypoint,
...actionProcedure.args,
]
const container = await DockerProcedureContainer.of(
effects,
manifest.id,
actionProcedure,
manifest.volumes,
`Health Check - ${commands.join(" ")}`,
{
subcontainer,
},
)
const env: Record<string, string> = actionProcedure.inject
? {
HOME: "/root",
}
: {}
const executed = await container.exec(commands, {
input: JSON.stringify(timeChanged),
env,
})
if (executed.exitCode === 0) {
await effects.setHealth({
id: healthId,
@@ -227,6 +236,18 @@ export class MainLoop {
})
return
}
if (executed.exitCode && executed.exitCode > 0) {
await effects.setHealth({
id: healthId,
name: value.name,
result: "failure",
message:
executed.stderr.toString() ||
executed.stdout.toString() ||
`Program exited with code ${executed.exitCode}:`,
})
return
}
await effects.setHealth({
id: healthId,
name: value.name,

View File

@@ -264,7 +264,6 @@ exports[`transformConfigSpec transformConfigSpec(bitcoind) 1`] = `
"disabled": false,
"immutable": false,
"name": "Pruning Mode",
"required": true,
"type": "union",
"variants": {
"automatic": {
@@ -524,7 +523,6 @@ exports[`transformConfigSpec transformConfigSpec(embassyPages) 1`] = `
"disabled": false,
"immutable": false,
"name": "Type",
"required": true,
"type": "union",
"variants": {
"index": {
@@ -589,7 +587,6 @@ exports[`transformConfigSpec transformConfigSpec(embassyPages) 1`] = `
"disabled": false,
"immutable": false,
"name": "Folder Location",
"required": false,
"type": "select",
"values": {
"filebrowser": "filebrowser",
@@ -644,7 +641,6 @@ exports[`transformConfigSpec transformConfigSpec(embassyPages) 1`] = `
"disabled": false,
"immutable": false,
"name": "Type",
"required": true,
"type": "union",
"variants": {
"redirect": {
@@ -705,7 +701,6 @@ exports[`transformConfigSpec transformConfigSpec(embassyPages) 1`] = `
"disabled": false,
"immutable": false,
"name": "Folder Location",
"required": false,
"type": "select",
"values": {
"filebrowser": "filebrowser",
@@ -758,7 +753,6 @@ exports[`transformConfigSpec transformConfigSpec(nostr2) 1`] = `
"disabled": false,
"immutable": false,
"name": "Relay Type",
"required": true,
"type": "union",
"variants": {
"private": {

View File

@@ -2,8 +2,8 @@ import { ExtendedVersion, types as T, utils } from "@start9labs/start-sdk"
import * as fs from "fs/promises"
import { polyfillEffects } from "./polyfillEffects"
import { Duration, duration, fromDuration } from "../../../Models/Duration"
import { System, Procedure } from "../../../Interfaces/System"
import { fromDuration } from "../../../Models/Duration"
import { System } from "../../../Interfaces/System"
import { matchManifest, Manifest } from "./matchManifest"
import * as childProcess from "node:child_process"
import { DockerProcedureContainer } from "./DockerProcedureContainer"
@@ -27,19 +27,12 @@ import {
Parser,
array,
} from "ts-matches"
import { JsonPath, unNestPath } from "../../../Models/JsonPath"
import { RpcResult, matchRpcResult } from "../../RpcListener"
import { CT } from "@start9labs/start-sdk"
import {
AddSslOptions,
BindOptions,
} from "@start9labs/start-sdk/cjs/lib/osBindings"
import { AddSslOptions } from "@start9labs/start-sdk/base/lib/osBindings"
import {
BindOptionsByProtocol,
Host,
MultiHost,
} from "@start9labs/start-sdk/cjs/lib/interfaces/Host"
import { ServiceInterfaceBuilder } from "@start9labs/start-sdk/cjs/lib/interfaces/ServiceInterfaceBuilder"
} from "@start9labs/start-sdk/base/lib/interfaces/Host"
import { ServiceInterfaceBuilder } from "@start9labs/start-sdk/base/lib/interfaces/ServiceInterfaceBuilder"
import { Effects } from "../../../Models/Effects"
import {
OldConfigSpec,
@@ -48,18 +41,16 @@ import {
transformNewConfigToOld,
transformOldConfigToNew,
} from "./transformConfigSpec"
import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
import { StorePath } from "@start9labs/start-sdk/cjs/lib/store/PathBuilder"
import { partialDiff } from "@start9labs/start-sdk/base/lib/util"
type Optional<A> = A | undefined | null
function todo(): never {
throw new Error("Not implemented")
}
const execFile = promisify(childProcess.execFile)
const MANIFEST_LOCATION = "/usr/lib/startos/package/embassyManifest.json"
export const EMBASSY_JS_LOCATION = "/usr/lib/startos/package/embassy.js"
const EMBASSY_POINTER_PATH_PREFIX = "/embassyConfig" as StorePath
const EMBASSY_POINTER_PATH_PREFIX = "/embassyConfig" as utils.StorePath
const matchResult = object({
result: any,
@@ -144,6 +135,34 @@ type OldGetConfigRes = {
spec: OldConfigSpec
}
export type PropertiesValue =
| {
/** The type of this value, either "string" or "object" */
type: "object"
/** A nested mapping of values. The user will experience this as a nested page with back button */
value: { [k: string]: PropertiesValue }
/** (optional) A human readable description of the new set of values */
description: string | null
}
| {
/** The type of this value, either "string" or "object" */
type: "string"
/** The value to display to the user */
value: string
/** A human readable description of the value */
description: string | null
/** Whether or not to mask the value, for example, when displaying a password */
masked: boolean | null
/** Whether or not to include a button for copying the value to clipboard */
copyable: boolean | null
/** Whether or not to include a button for displaying the value as a QR code */
qr: boolean | null
}
export type PropertiesReturn = {
[key: string]: PropertiesValue
}
export type PackagePropertiesV2 = {
[name: string]: PackagePropertyObject | PackagePropertyString
}
@@ -166,7 +185,7 @@ export type PackagePropertyObject = {
const asProperty_ = (
x: PackagePropertyString | PackagePropertyObject,
): T.PropertiesValue => {
): PropertiesValue => {
if (x.type === "object") {
return {
...x,
@@ -186,7 +205,7 @@ const asProperty_ = (
...x,
}
}
const asProperty = (x: PackagePropertiesV2): T.PropertiesReturn =>
const asProperty = (x: PackagePropertiesV2): PropertiesReturn =>
Object.fromEntries(
Object.entries(x).map(([key, value]) => [key, asProperty_(value)]),
)
@@ -223,6 +242,31 @@ const matchProperties = object({
data: matchPackageProperties,
})
function convertProperties(
name: string,
value: PropertiesValue,
): T.ActionResultMember {
if (value.type === "string") {
return {
type: "single",
name,
description: value.description,
copyable: value.copyable || false,
masked: value.masked || false,
qr: value.qr || false,
value: value.value,
}
}
return {
type: "group",
name,
description: value.description,
value: Object.entries(value.value).map(([name, value]) =>
convertProperties(name, value),
),
}
}
const DEFAULT_REGISTRY = "https://registry.start9.com"
export class SystemForEmbassy implements System {
currentRunning: MainLoop | undefined
@@ -248,50 +292,38 @@ export class SystemForEmbassy implements System {
readonly moduleCode: Partial<U.ExpectedExports>,
) {}
async actionsMetadata(effects: T.Effects): Promise<T.ActionMetadata[]> {
const actions = Object.entries(this.manifest.actions ?? {})
return Promise.all(
actions.map(async ([actionId, action]): Promise<T.ActionMetadata> => {
const name = action.name ?? actionId
const description = action.description
const warning = action.warning ?? null
const disabled = false
const input = (await convertToNewConfig(action["input-spec"] as any))
.spec
const hasRunning = !!action["allowed-statuses"].find(
(x) => x === "running",
)
const hasStopped = !!action["allowed-statuses"].find(
(x) => x === "stopped",
)
// prettier-ignore
const allowedStatuses =
hasRunning && hasStopped ? "any":
hasRunning ? "onlyRunning" :
"onlyStopped"
const group = null
return {
name,
description,
warning,
disabled,
allowedStatuses,
group,
input,
}
}),
)
async containerInit(effects: Effects): Promise<void> {
for (let depId in this.manifest.dependencies) {
if (this.manifest.dependencies[depId].config) {
await this.dependenciesAutoconfig(effects, depId, null)
}
}
await effects.setMainStatus({ status: "stopped" })
await this.exportActions(effects)
await this.exportNetwork(effects)
await this.containerSetDependencies(effects)
}
async containerSetDependencies(effects: T.Effects) {
const oldDeps: Record<string, string[]> = Object.fromEntries(
await effects
.getDependencies()
.then((x) =>
x.flatMap((x) =>
x.kind === "running" ? [[x.id, x?.healthChecks || []]] : [],
),
)
.catch(() => []),
)
await this.setDependencies(effects, oldDeps)
}
async containerInit(): Promise<void> {}
async exit(): Promise<void> {
if (this.currentRunning) await this.currentRunning.clean()
delete this.currentRunning
}
async start(effects: MainEffects): Promise<void> {
async start(effects: T.Effects): Promise<void> {
effects.constRetry = utils.once(() => effects.restart())
if (!!this.currentRunning) return
this.currentRunning = await MainLoop.of(this, effects)
@@ -308,16 +340,26 @@ export class SystemForEmbassy implements System {
}
}
async packageInit(
effects: Effects,
previousVersion: Optional<string>,
timeoutMs: number | null,
): Promise<void> {
if (previousVersion)
await this.migration(effects, previousVersion, timeoutMs)
await effects.setMainStatus({ status: "stopped" })
await this.exportActions(effects)
await this.exportNetwork(effects)
async packageInit(effects: Effects, timeoutMs: number | null): Promise<void> {
const previousVersion = await effects.getDataVersion()
if (previousVersion) {
if (
(await this.migration(effects, previousVersion, timeoutMs)).configured
) {
await effects.action.clearRequests({ only: ["needs-config"] })
}
await effects.setDataVersion({
version: ExtendedVersion.parseEmver(this.manifest.version).toString(),
})
} else if (this.manifest.config) {
await effects.action.request({
packageId: this.manifest.id,
actionId: "config",
severity: "critical",
replayId: "needs-config",
reason: "This service must be configured before it can be run",
})
}
}
async exportNetwork(effects: Effects) {
for (const [id, interfaceValue] of Object.entries(
@@ -400,10 +442,75 @@ export class SystemForEmbassy implements System {
)
}
}
async getActionInput(
effects: Effects,
actionId: string,
timeoutMs: number | null,
): Promise<T.ActionInput | null> {
if (actionId === "config") {
const config = await this.getConfig(effects, timeoutMs)
return { spec: config.spec, value: config.config }
} else if (actionId === "properties") {
return null
} else {
const oldSpec = this.manifest.actions?.[actionId]?.["input-spec"]
if (!oldSpec) return null
return {
spec: transformConfigSpec(oldSpec as OldConfigSpec),
value: null,
}
}
}
async runAction(
effects: Effects,
actionId: string,
input: unknown,
timeoutMs: number | null,
): Promise<T.ActionResult | null> {
if (actionId === "config") {
await this.setConfig(effects, input, timeoutMs)
return null
} else if (actionId === "properties") {
return {
version: "1",
title: "Properties",
message: null,
result: {
type: "group",
value: Object.entries(await this.properties(effects, timeoutMs)).map(
([name, value]) => convertProperties(name, value),
),
},
}
} else {
return this.action(effects, actionId, input, timeoutMs)
}
}
async exportActions(effects: Effects) {
const manifest = this.manifest
if (!manifest.actions) return
for (const [actionId, action] of Object.entries(manifest.actions)) {
const actions = {
...manifest.actions,
}
if (manifest.config) {
actions.config = {
name: "Configure",
description: `Customize ${manifest.title}`,
"allowed-statuses": ["running", "stopped"],
"input-spec": {},
implementation: { type: "script", args: [] },
}
}
if (manifest.properties) {
actions.properties = {
name: "Properties",
description:
"Runtime information, credentials, and other values of interest",
"allowed-statuses": ["running", "stopped"],
"input-spec": null,
implementation: { type: "script", args: [] },
}
}
for (const [actionId, action] of Object.entries(actions)) {
const hasRunning = !!action["allowed-statuses"].find(
(x) => x === "running",
)
@@ -412,21 +519,22 @@ export class SystemForEmbassy implements System {
)
// prettier-ignore
const allowedStatuses = hasRunning && hasStopped ? "any":
hasRunning ? "onlyRunning" :
"onlyStopped"
await effects.exportAction({
hasRunning ? "only-running" :
"only-stopped"
await effects.action.export({
id: actionId,
metadata: {
name: action.name,
description: action.description,
warning: action.warning || null,
input: action["input-spec"] as CT.InputSpec,
disabled: false,
visibility: "enabled",
allowedStatuses,
hasInput: !!action["input-spec"],
group: null,
},
})
}
await effects.action.clear({ except: Object.keys(actions) })
}
async packageUninit(
effects: Effects,
@@ -443,6 +551,7 @@ export class SystemForEmbassy implements System {
): Promise<void> {
const backup = this.manifest.backup.create
if (backup.type === "docker") {
const commands = [backup.entrypoint, ...backup.args]
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
@@ -451,8 +560,9 @@ export class SystemForEmbassy implements System {
...this.manifest.volumes,
BACKUP: { type: "backup", readonly: false },
},
`Backup - ${commands.join(" ")}`,
)
await container.execFail([backup.entrypoint, ...backup.args], timeoutMs)
await container.execFail(commands, timeoutMs)
} else {
const moduleCode = await this.moduleCode
await moduleCode.createBackup?.(polyfillEffects(effects, this.manifest))
@@ -464,6 +574,7 @@ export class SystemForEmbassy implements System {
): Promise<void> {
const restoreBackup = this.manifest.backup.restore
if (restoreBackup.type === "docker") {
const commands = [restoreBackup.entrypoint, ...restoreBackup.args]
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
@@ -472,20 +583,15 @@ export class SystemForEmbassy implements System {
...this.manifest.volumes,
BACKUP: { type: "backup", readonly: true },
},
`Restore Backup - ${commands.join(" ")}`,
)
await container.execFail(
[restoreBackup.entrypoint, ...restoreBackup.args],
timeoutMs,
)
await container.execFail(commands, timeoutMs)
} else {
const moduleCode = await this.moduleCode
await moduleCode.restoreBackup?.(polyfillEffects(effects, this.manifest))
}
}
async getConfig(
effects: Effects,
timeoutMs: number | null,
): Promise<T.ConfigRes> {
async getConfig(effects: Effects, timeoutMs: number | null) {
return this.getConfigUncleaned(effects, timeoutMs).then(convertToNewConfig)
}
private async getConfigUncleaned(
@@ -495,20 +601,17 @@ export class SystemForEmbassy implements System {
const config = this.manifest.config?.get
if (!config) return { spec: {} }
if (config.type === "docker") {
const commands = [config.entrypoint, ...config.args]
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
config,
this.manifest.volumes,
`Get Config - ${commands.join(" ")}`,
)
// TODO: yaml
return JSON.parse(
(
await container.execFail(
[config.entrypoint, ...config.args],
timeoutMs,
)
).stdout.toString(),
(await container.execFail(commands, timeoutMs)).stdout.toString(),
)
} else {
const moduleCode = await this.moduleCode
@@ -543,28 +646,25 @@ export class SystemForEmbassy implements System {
const setConfigValue = this.manifest.config?.set
if (!setConfigValue) return
if (setConfigValue.type === "docker") {
const commands = [
setConfigValue.entrypoint,
...setConfigValue.args,
JSON.stringify(newConfig),
]
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
setConfigValue,
this.manifest.volumes,
`Set Config - ${commands.join(" ")}`,
)
const answer = matchSetResult.unsafeCast(
JSON.parse(
(
await container.execFail(
[
setConfigValue.entrypoint,
...setConfigValue.args,
JSON.stringify(newConfig),
],
timeoutMs,
)
).stdout.toString(),
(await container.execFail(commands, timeoutMs)).stdout.toString(),
),
)
const dependsOn = answer["depends-on"] ?? answer.dependsOn ?? {}
await this.setConfigSetConfig(effects, dependsOn)
await this.setDependencies(effects, dependsOn)
return
} else if (setConfigValue.type === "script") {
const moduleCode = await this.moduleCode
@@ -587,31 +687,60 @@ export class SystemForEmbassy implements System {
}),
)
const dependsOn = answer["depends-on"] ?? answer.dependsOn ?? {}
await this.setConfigSetConfig(effects, dependsOn)
await this.setDependencies(effects, dependsOn)
return
}
}
private async setConfigSetConfig(
private async setDependencies(
effects: Effects,
dependsOn: { [x: string]: readonly string[] },
rawDepends: { [x: string]: readonly string[] },
) {
const dependsOn: Record<string, readonly string[] | null> = {
...Object.fromEntries(
Object.entries(this.manifest.dependencies || {})?.map((x) => [
x[0],
null,
]) || [],
),
...rawDepends,
}
await effects.setDependencies({
dependencies: Object.entries(dependsOn).flatMap(([key, value]) => {
const dependency = this.manifest.dependencies?.[key]
if (!dependency) return []
const versionRange = dependency.version
const registryUrl = DEFAULT_REGISTRY
const kind = "running"
return [
{
id: key,
versionRange,
registryUrl,
kind,
healthChecks: [...value],
},
]
}),
dependencies: Object.entries(dependsOn).flatMap(
([key, value]): T.Dependencies => {
const dependency = this.manifest.dependencies?.[key]
if (!dependency) return []
if (value == null) {
const versionRange = dependency.version
if (dependency.requirement.type === "required") {
return [
{
id: key,
versionRange,
kind: "running",
healthChecks: [],
},
]
}
return [
{
kind: "exists",
id: key,
versionRange,
},
]
}
const versionRange = dependency.version
const kind = "running"
return [
{
id: key,
versionRange,
kind,
healthChecks: [...value],
},
]
},
),
})
}
@@ -619,7 +748,7 @@ export class SystemForEmbassy implements System {
effects: Effects,
fromVersion: string,
timeoutMs: number | null,
): Promise<T.MigrationRes> {
): Promise<{ configured: boolean }> {
const fromEmver = ExtendedVersion.parseEmver(fromVersion)
const currentEmver = ExtendedVersion.parseEmver(this.manifest.version)
if (!this.manifest.migrations) return { configured: true }
@@ -652,23 +781,20 @@ export class SystemForEmbassy implements System {
if (migration) {
const [version, procedure] = migration
if (procedure.type === "docker") {
const commands = [
procedure.entrypoint,
...procedure.args,
JSON.stringify(fromVersion),
]
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
procedure,
this.manifest.volumes,
`Migration - ${commands.join(" ")}`,
)
return JSON.parse(
(
await container.execFail(
[
procedure.entrypoint,
...procedure.args,
JSON.stringify(fromVersion),
],
timeoutMs,
)
).stdout.toString(),
(await container.execFail(commands, timeoutMs)).stdout.toString(),
)
} else if (procedure.type === "script") {
const moduleCode = await this.moduleCode
@@ -690,25 +816,22 @@ export class SystemForEmbassy implements System {
async properties(
effects: Effects,
timeoutMs: number | null,
): Promise<T.PropertiesReturn> {
): Promise<PropertiesReturn> {
// TODO BLU-J set the properties ever so often
const setConfigValue = this.manifest.properties
if (!setConfigValue) throw new Error("There is no properties")
if (setConfigValue.type === "docker") {
const commands = [setConfigValue.entrypoint, ...setConfigValue.args]
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
setConfigValue,
this.manifest.volumes,
`Properties - ${commands.join(" ")}`,
)
const properties = matchProperties.unsafeCast(
JSON.parse(
(
await container.execFail(
[setConfigValue.entrypoint, ...setConfigValue.args],
timeoutMs,
)
).stdout.toString(),
(await container.execFail(commands, timeoutMs)).stdout.toString(),
),
)
return asProperty(properties.data)
@@ -735,13 +858,13 @@ export class SystemForEmbassy implements System {
const actionProcedure = this.manifest.actions?.[actionId]?.implementation
const toActionResult = ({
message,
value = "",
value,
copyable,
qr,
}: U.ActionResult): T.ActionResult => ({
version: "0",
message,
value,
value: value ?? null,
copyable,
qr,
})
@@ -750,11 +873,18 @@ export class SystemForEmbassy implements System {
const subcontainer = actionProcedure.inject
? this.currentRunning?.mainSubContainerHandle
: undefined
const env: Record<string, string> = actionProcedure.inject
? {
HOME: "/root",
}
: {}
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
actionProcedure,
this.manifest.volumes,
`Action ${actionId}`,
{
subcontainer,
},
@@ -769,6 +899,7 @@ export class SystemForEmbassy implements System {
JSON.stringify(formData),
],
timeoutMs,
{ env },
)
).stdout.toString(),
),
@@ -794,23 +925,20 @@ export class SystemForEmbassy implements System {
const actionProcedure = this.manifest.dependencies?.[id]?.config?.check
if (!actionProcedure) return { message: "Action not found", value: null }
if (actionProcedure.type === "docker") {
const commands = [
actionProcedure.entrypoint,
...actionProcedure.args,
JSON.stringify(oldConfig),
]
const container = await DockerProcedureContainer.of(
effects,
this.manifest.id,
actionProcedure,
this.manifest.volumes,
`Dependencies Check - ${commands.join(" ")}`,
)
return JSON.parse(
(
await container.execFail(
[
actionProcedure.entrypoint,
...actionProcedure.args,
JSON.stringify(oldConfig),
],
timeoutMs,
)
).stdout.toString(),
(await container.execFail(commands, timeoutMs)).stdout.toString(),
)
} else if (actionProcedure.type === "script") {
const moduleCode = await this.moduleCode
@@ -834,24 +962,46 @@ export class SystemForEmbassy implements System {
async dependenciesAutoconfig(
effects: Effects,
id: string,
input: unknown,
timeoutMs: number | null,
): Promise<void> {
const oldConfig = object({ remoteConfig: any }).unsafeCast(
input,
).remoteConfig
// TODO: docker
const oldConfig = (await effects.store.get({
packageId: id,
path: EMBASSY_POINTER_PATH_PREFIX,
callback: () => {
this.dependenciesAutoconfig(effects, id, timeoutMs)
},
})) as U.Config
if (!oldConfig) return
const moduleCode = await this.moduleCode
const method = moduleCode.dependencies?.[id]?.autoConfigure
if (!method) return
return (await method(
const newConfig = (await method(
polyfillEffects(effects, this.manifest),
oldConfig,
JSON.parse(JSON.stringify(oldConfig)),
).then((x) => {
if ("result" in x) return x.result
if ("error" in x) throw new Error("Error getting config: " + x.error)
throw new Error("Error getting config: " + x["error-code"][1])
})) as any
const diff = partialDiff(oldConfig, newConfig)
if (diff) {
await effects.action.request({
actionId: "config",
packageId: id,
replayId: `${id}/config`,
severity: "important",
reason: `Configure this dependency for the needs of ${this.manifest.title}`,
input: {
kind: "partial",
value: diff.diff,
},
when: {
condition: "input-not-matches",
once: false,
},
})
}
}
}
@@ -1026,9 +1176,7 @@ function extractServiceInterfaceId(manifest: Manifest, specInterface: string) {
const serviceInterfaceId = `${specInterface}-${internalPort}`
return serviceInterfaceId
}
async function convertToNewConfig(
value: OldGetConfigRes,
): Promise<T.ConfigRes> {
async function convertToNewConfig(value: OldGetConfigRes) {
const valueSpec: OldConfigSpec = matchOldConfigSpec.unsafeCast(value.spec)
const spec = transformConfigSpec(valueSpec)
if (!value.config) return { spec, config: null }

View File

@@ -42,6 +42,7 @@ const matchAction = object(
export const matchManifest = object(
{
id: string,
title: string,
version: string,
main: matchDockerProcedure,
assets: object(

View File

@@ -105,12 +105,14 @@ export const polyfillEffects = (
args?: string[] | undefined
timeoutMillis?: number | undefined
}): Promise<oet.ResultType<string>> {
const commands: [string, ...string[]] = [command, ...(args || [])]
return startSdk
.runCommand(
effects,
{ id: manifest.main.image },
[command, ...(args || [])],
commands,
{},
commands.join(" "),
)
.then((x: any) => ({
stderr: x.stderr.toString(),
@@ -129,6 +131,7 @@ export const polyfillEffects = (
manifest.id,
manifest.main,
manifest.volumes,
[input.command, ...(input.args || [])].join(" "),
)
const daemon = promiseSubcontainer.then((subcontainer) =>
daemons.runCommand()(
@@ -153,11 +156,17 @@ export const polyfillEffects = (
path: string
uid: string
}): Promise<null> {
const commands: [string, ...string[]] = [
"chown",
"--recursive",
input.uid,
`/drive/${input.path}`,
]
await startSdk
.runCommand(
effects,
{ id: manifest.main.image },
["chown", "--recursive", input.uid, `/drive/${input.path}`],
commands,
{
mounts: [
{
@@ -171,6 +180,7 @@ export const polyfillEffects = (
},
],
},
commands.join(" "),
)
.then((x: any) => ({
stderr: x.stderr.toString(),
@@ -188,11 +198,17 @@ export const polyfillEffects = (
path: string
mode: string
}): Promise<null> {
const commands: [string, ...string[]] = [
"chmod",
"--recursive",
input.mode,
`/drive/${input.path}`,
]
await startSdk
.runCommand(
effects,
{ id: manifest.main.image },
["chmod", "--recursive", input.mode, `/drive/${input.path}`],
commands,
{
mounts: [
{
@@ -206,6 +222,7 @@ export const polyfillEffects = (
},
],
},
commands.join(" "),
)
.then((x: any) => ({
stderr: x.stderr.toString(),

View File

@@ -1,4 +1,4 @@
import { CT } from "@start9labs/start-sdk"
import { IST } from "@start9labs/start-sdk"
import {
dictionary,
object,
@@ -15,9 +15,9 @@ import {
literal,
} from "ts-matches"
export function transformConfigSpec(oldSpec: OldConfigSpec): CT.InputSpec {
export function transformConfigSpec(oldSpec: OldConfigSpec): IST.InputSpec {
return Object.entries(oldSpec).reduce((inputSpec, [key, oldVal]) => {
let newVal: CT.ValueSpec
let newVal: IST.ValueSpec
if (oldVal.type === "boolean") {
newVal = {
@@ -43,7 +43,6 @@ export function transformConfigSpec(oldSpec: OldConfigSpec): CT.InputSpec {
}),
{},
),
required: false,
disabled: false,
immutable: false,
}
@@ -124,10 +123,9 @@ export function transformConfigSpec(oldSpec: OldConfigSpec): CT.InputSpec {
spec: transformConfigSpec(matchOldConfigSpec.unsafeCast(spec)),
},
}),
{} as Record<string, { name: string; spec: CT.InputSpec }>,
{} as Record<string, { name: string; spec: IST.InputSpec }>,
),
disabled: false,
required: true,
default: oldVal.default,
immutable: false,
}
@@ -141,7 +139,7 @@ export function transformConfigSpec(oldSpec: OldConfigSpec): CT.InputSpec {
...inputSpec,
[key]: newVal,
}
}, {} as CT.InputSpec)
}, {} as IST.InputSpec)
}
export function transformOldConfigToNew(
@@ -233,10 +231,10 @@ export function transformNewConfigToOld(
function getListSpec(
oldVal: OldValueSpecList,
): CT.ValueSpecMultiselect | CT.ValueSpecList {
): IST.ValueSpecMultiselect | IST.ValueSpecList {
const range = Range.from(oldVal.range)
let partial: Omit<CT.ValueSpecList, "type" | "spec" | "default"> = {
let partial: Omit<IST.ValueSpecList, "type" | "spec" | "default"> = {
name: oldVal.name,
description: oldVal.description || null,
warning: oldVal.warning || null,

View File

@@ -1,21 +1,12 @@
import { ExecuteResult, Procedure, System } from "../../Interfaces/System"
import { unNestPath } from "../../Models/JsonPath"
import matches, { any, number, object, string, tuple } from "ts-matches"
import { System } from "../../Interfaces/System"
import { Effects } from "../../Models/Effects"
import { RpcResult, matchRpcResult } from "../RpcListener"
import { duration } from "../../Models/Duration"
import { T, utils } from "@start9labs/start-sdk"
import { Volume } from "../../Models/Volume"
import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
import { CallbackHolder } from "../../Models/CallbackHolder"
import { Optional } from "ts-matches/lib/parsers/interfaces"
export const STARTOS_JS_LOCATION = "/usr/lib/startos/package/index.js"
type RunningMain = {
effects: MainEffects
stop: () => Promise<void>
callbacks: CallbackHolder
}
export class SystemForStartOs implements System {
@@ -25,23 +16,24 @@ export class SystemForStartOs implements System {
return new SystemForStartOs(require(STARTOS_JS_LOCATION))
}
constructor(readonly abi: T.ABI) {}
containerInit(): Promise<void> {
throw new Error("Method not implemented.")
constructor(readonly abi: T.ABI) {
this
}
async containerInit(effects: Effects): Promise<void> {
return void (await this.abi.containerInit({ effects }))
}
async packageInit(
effects: Effects,
previousVersion: Optional<string> = null,
timeoutMs: number | null = null,
): Promise<void> {
return void (await this.abi.init({ effects }))
return void (await this.abi.packageInit({ effects }))
}
async packageUninit(
effects: Effects,
nextVersion: Optional<string> = null,
timeoutMs: number | null = null,
): Promise<void> {
return void (await this.abi.uninit({ effects, nextVersion }))
return void (await this.abi.packageUninit({ effects, nextVersion }))
}
async createBackup(
effects: T.Effects,
@@ -49,8 +41,6 @@ export class SystemForStartOs implements System {
): Promise<void> {
return void (await this.abi.createBackup({
effects,
pathMaker: ((options) =>
new Volume(options.volume, options.path).path) as T.PathMaker,
}))
}
async restoreBackup(
@@ -59,118 +49,56 @@ export class SystemForStartOs implements System {
): Promise<void> {
return void (await this.abi.restoreBackup({
effects,
pathMaker: ((options) =>
new Volume(options.volume, options.path).path) as T.PathMaker,
}))
}
getConfig(
effects: T.Effects,
timeoutMs: number | null,
): Promise<T.ConfigRes> {
return this.abi.getConfig({ effects })
}
async setConfig(
effects: Effects,
input: { effects: Effects; input: Record<string, unknown> },
timeoutMs: number | null,
): Promise<void> {
const _: unknown = await this.abi.setConfig({ effects, input })
return
}
migration(
effects: Effects,
fromVersion: string,
timeoutMs: number | null,
): Promise<T.MigrationRes> {
throw new Error("Method not implemented.")
}
properties(
effects: Effects,
timeoutMs: number | null,
): Promise<T.PropertiesReturn> {
throw new Error("Method not implemented.")
}
async action(
getActionInput(
effects: Effects,
id: string,
formData: unknown,
timeoutMs: number | null,
): Promise<T.ActionResult> {
const action = (await this.abi.actions({ effects }))[id]
): Promise<T.ActionInput | null> {
const action = this.abi.actions.get(id)
if (!action) throw new Error(`Action ${id} not found`)
return action.run({ effects })
return action.getInput({ effects })
}
dependenciesCheck(
runAction(
effects: Effects,
id: string,
oldConfig: unknown,
input: unknown,
timeoutMs: number | null,
): Promise<any> {
const dependencyConfig = this.abi.dependencyConfig[id]
if (!dependencyConfig) throw new Error(`dependencyConfig ${id} not found`)
return dependencyConfig.query({ effects })
): Promise<T.ActionResult | null> {
const action = this.abi.actions.get(id)
if (!action) throw new Error(`Action ${id} not found`)
return action.run({ effects, input })
}
async dependenciesAutoconfig(
effects: Effects,
id: string,
remoteConfig: unknown,
timeoutMs: number | null,
): Promise<void> {
const dependencyConfig = this.abi.dependencyConfig[id]
if (!dependencyConfig) throw new Error(`dependencyConfig ${id} not found`)
const queryResults = await this.getConfig(effects, timeoutMs)
return void (await dependencyConfig.update({
queryResults,
remoteConfig,
})) // TODO
}
async actionsMetadata(effects: T.Effects): Promise<T.ActionMetadata[]> {
return this.abi.actionsMetadata({ effects })
}
async init(): Promise<void> {}
async exit(): Promise<void> {}
async start(effects: MainEffects): Promise<void> {
async start(effects: Effects): Promise<void> {
effects.constRetry = utils.once(() => effects.restart())
if (this.runningMain) await this.stop()
let mainOnTerm: () => Promise<void> | undefined
const started = async (onTerm: () => Promise<void>) => {
await effects.setMainStatus({ status: "running" })
mainOnTerm = onTerm
return null
}
const daemons = await (
await this.abi.main({
effects: effects as MainEffects,
effects,
started,
})
).build()
this.runningMain = {
effects,
stop: async () => {
if (mainOnTerm) await mainOnTerm()
await daemons.term()
},
callbacks: new CallbackHolder(),
}
}
callCallback(callback: number, args: any[]): void {
if (this.runningMain) {
this.runningMain.callbacks
.callCallback(callback, args)
.catch((error) =>
console.error(`callback ${callback} failed`, utils.asError(error)),
)
} else {
console.warn(`callback ${callback} ignored because system is not running`)
}
}
async stop(): Promise<void> {
if (this.runningMain) {
await this.runningMain.stop()
await this.runningMain.effects.clearCallbacks()
this.runningMain = undefined
}
}

View File

@@ -1,7 +1,4 @@
import { GetDependency } from "./GetDependency"
import { System } from "./System"
import { MakeMainEffects, MakeProcedureEffects } from "./MakeEffects"
export type AllGetDependencies = GetDependency<"system", Promise<System>> &
GetDependency<"makeProcedureEffects", MakeProcedureEffects> &
GetDependency<"makeMainEffects", MakeMainEffects>
export type AllGetDependencies = GetDependency<"system", Promise<System>>

View File

@@ -1,4 +0,0 @@
import { Effects } from "../Models/Effects"
import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
export type MakeProcedureEffects = (procedureId: string) => Effects
export type MakeMainEffects = () => MainEffects

View File

@@ -1,39 +1,26 @@
import { types as T } from "@start9labs/start-sdk"
import { RpcResult } from "../Adapters/RpcListener"
import { Effects } from "../Models/Effects"
import { CallbackHolder } from "../Models/CallbackHolder"
import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
import { Optional } from "ts-matches/lib/parsers/interfaces"
export type Procedure =
| "/init"
| "/uninit"
| "/config/set"
| "/config/get"
| "/packageInit"
| "/packageUninit"
| "/backup/create"
| "/backup/restore"
| "/actions/metadata"
| "/properties"
| `/actions/${string}/get`
| `/actions/${string}/getInput`
| `/actions/${string}/run`
| `/dependencies/${string}/query`
| `/dependencies/${string}/update`
export type ExecuteResult =
| { ok: unknown }
| { err: { code: number; message: string } }
export type System = {
containerInit(): Promise<void>
containerInit(effects: T.Effects): Promise<void>
start(effects: MainEffects): Promise<void>
callCallback(callback: number, args: any[]): void
start(effects: T.Effects): Promise<void>
stop(): Promise<void>
packageInit(
effects: Effects,
previousVersion: Optional<string>,
timeoutMs: number | null,
): Promise<void>
packageInit(effects: Effects, timeoutMs: number | null): Promise<void>
packageUninit(
effects: Effects,
nextVersion: Optional<string>,
@@ -42,41 +29,17 @@ export type System = {
createBackup(effects: T.Effects, timeoutMs: number | null): Promise<void>
restoreBackup(effects: T.Effects, timeoutMs: number | null): Promise<void>
getConfig(effects: T.Effects, timeoutMs: number | null): Promise<T.ConfigRes>
setConfig(
effects: Effects,
input: { effects: Effects; input: Record<string, unknown> },
timeoutMs: number | null,
): Promise<void>
migration(
effects: Effects,
fromVersion: string,
timeoutMs: number | null,
): Promise<T.MigrationRes>
properties(
effects: Effects,
timeoutMs: number | null,
): Promise<T.PropertiesReturn>
action(
runAction(
effects: Effects,
actionId: string,
formData: unknown,
input: unknown,
timeoutMs: number | null,
): Promise<T.ActionResult>
dependenciesCheck(
): Promise<T.ActionResult | null>
getActionInput(
effects: Effects,
id: string,
oldConfig: unknown,
actionId: string,
timeoutMs: number | null,
): Promise<any>
dependenciesAutoconfig(
effects: Effects,
id: string,
oldConfig: unknown,
timeoutMs: number | null,
): Promise<void>
actionsMetadata(effects: T.Effects): Promise<T.ActionMetadata[]>
): Promise<T.ActionInput | null>
exit(): Promise<void>
}

View File

@@ -1,22 +1,62 @@
import { T } from "@start9labs/start-sdk"
const CallbackIdCell = { inc: 1 }
const callbackRegistry = new FinalizationRegistry(
async (options: { cbs: Map<number, Function>; effects: T.Effects }) => {
await options.effects.clearCallbacks({
only: Array.from(options.cbs.keys()),
})
},
)
export class CallbackHolder {
constructor() {}
private inc = 0
constructor(private effects?: T.Effects) {}
private callbacks = new Map<number, Function>()
private children: WeakRef<CallbackHolder>[] = []
private newId() {
return this.inc++
return CallbackIdCell.inc++
}
addCallback(callback?: Function) {
if (!callback) {
return
}
const id = this.newId()
console.error("adding callback", id)
this.callbacks.set(id, callback)
if (this.effects)
callbackRegistry.register(this, {
cbs: this.callbacks,
effects: this.effects,
})
return id
}
child(): CallbackHolder {
const child = new CallbackHolder()
this.children.push(new WeakRef(child))
return child
}
removeChild(child: CallbackHolder) {
this.children = this.children.filter((c) => {
const ref = c.deref()
return ref && ref !== child
})
}
private getCallback(index: number): Function | undefined {
let callback = this.callbacks.get(index)
if (callback) this.callbacks.delete(index)
else {
for (let i = 0; i < this.children.length; i++) {
callback = this.children[i].deref()?.getCallback(index)
if (callback) return callback
}
}
return callback
}
callCallback(index: number, args: any[]): Promise<unknown> {
const callback = this.callbacks.get(index)
if (!callback) throw new Error(`Callback ${index} does not exist`)
this.callbacks.delete(index)
const callback = this.getCallback(index)
if (!callback) return Promise.resolve()
return Promise.resolve().then(() => callback(...args))
}
}

View File

@@ -1,9 +1,7 @@
import { literals, some, string } from "ts-matches"
type NestedPath<A extends string, B extends string> = `/${A}/${string}/${B}`
type NestedPaths =
| NestedPath<"actions", "run" | "get">
| NestedPath<"dependencies", "query" | "update">
type NestedPaths = NestedPath<"actions", "run" | "getInput">
// prettier-ignore
type UnNestPaths<A> =
A extends `${infer A}/${infer B}` ? [...UnNestPaths<A>, ... UnNestPaths<B>] :
@@ -15,25 +13,16 @@ export function unNestPath<A extends string>(a: A): UnNestPaths<A> {
function isNestedPath(path: string): path is NestedPaths {
const paths = path.split("/")
if (paths.length !== 4) return false
if (paths[1] === "actions" && (paths[3] === "run" || paths[3] === "get"))
return true
if (
paths[1] === "dependencies" &&
(paths[3] === "query" || paths[3] === "update")
)
if (paths[1] === "actions" && (paths[3] === "run" || paths[3] === "getInput"))
return true
return false
}
export const jsonPath = some(
literals(
"/init",
"/uninit",
"/config/set",
"/config/get",
"/packageInit",
"/packageUninit",
"/backup/create",
"/backup/restore",
"/actions/metadata",
"/properties",
),
string.refine(isNestedPath, "isNestedPath"),
)

View File

@@ -1,13 +1,10 @@
import { RpcListener } from "./Adapters/RpcListener"
import { SystemForEmbassy } from "./Adapters/Systems/SystemForEmbassy"
import { makeMainEffects, makeProcedureEffects } from "./Adapters/EffectCreator"
import { AllGetDependencies } from "./Interfaces/AllGetDependencies"
import { getSystem } from "./Adapters/Systems"
const getDependencies: AllGetDependencies = {
system: getSystem,
makeProcedureEffects: () => makeProcedureEffects,
makeMainEffects: () => makeMainEffects,
}
new RpcListener(getDependencies)