Files
start-os/sdk/lib/util/utils.ts
Aiden McClelland 089199e7c2 Feature/lxc container runtime (#2562)
* wip(fix): Dependencies

* wip: context

* wip(fix) Sorta auth

* wip: warnings

* wip(fix): registry/admin

* wip(fix) marketplace

* wip(fix) Some more converted and fixed with the linter and config

* wip: Working on the static server

* wip(fix)static server

* wip: Remove some asynnc

* wip: Something about the request and regular rpc

* wip: gut install

Co-authored-by: J H <Blu-J@users.noreply.github.com>

* wip: Convert the static server into the new system

* wip delete file

* test

* wip(fix) vhost does not need the with safe defaults

* wip: Adding in the wifi

* wip: Fix the developer and the verify

* wip: new install flow

Co-authored-by: J H <Blu-J@users.noreply.github.com>

* fix middleware

* wip

* wip: Fix the auth

* wip

* continue service refactor

* feature: Service get_config

* feat: Action

* wip: Fighting the great fight against the borrow checker

* wip: Remove an error in a file that I just need to deel with later

* chore: Add in some more lifetime stuff to the services

* wip: Install fix on lifetime

* cleanup

* wip: Deal with the borrow later

* more cleanup

* resolve borrowchecker errors

* wip(feat): add in the handler for the socket, for now

* wip(feat): Update the service_effect_handler::action

* chore: Add in the changes to make sure the from_service goes to context

* chore: Change the

* refactor service map

* fix references to service map

* fill out restore

* wip: Before I work on the store stuff

* fix backup module

* handle some warnings

* feat: add in the ui components on the rust side

* feature: Update the procedures

* chore: Update the js side of the main and a few of the others

* chore: Update the rpc listener to match the persistant container

* wip: Working on updating some things to have a better name

* wip(feat): Try and get the rpc to return the correct shape?

* lxc wip

* wip(feat): Try and get the rpc to return the correct shape?

* build for container runtime wip

* remove container-init

* fix build

* fix error

* chore: Update to work I suppose

* lxc wip

* remove docker module and feature

* download alpine squashfs automatically

* overlays effect

Co-authored-by: Jade <Blu-J@users.noreply.github.com>

* chore: Add the overlay effect

* feat: Add the mounter in the main

* chore: Convert to use the mounts, still need to work with the sandbox

* install fixes

* fix ssl

* fixes from testing

* implement tmpfile for upload

* wip

* misc fixes

* cleanup

* cleanup

* better progress reporting

* progress for sideload

* return real guid

* add devmode script

* fix lxc rootfs path

* fix percentage bar

* fix progress bar styling

* fix build for unstable

* tweaks

* label progress

* tweaks

* update progress more often

* make symlink in rpc_client

* make socket dir

* fix parent path

* add start-cli to container

* add echo and gitInfo commands

* wip: Add the init + errors

* chore: Add in the exit effect for the system

* chore: Change the type to null for failure to parse

* move sigterm timeout to stopping status

* update order

* chore: Update the return type

* remove dbg

* change the map error

* chore: Update the thing to capture id

* chore add some life changes

* chore: Update the loging

* chore: Update the package to run module

* us From for RpcError

* chore: Update to use import instead

* chore: update

* chore: Use require for the backup

* fix a default

* update the type that is wrong

* chore: Update the type of the manifest

* chore: Update to make null

* only symlink if not exists

* get rid of double result

* better debug info for ErrorCollection

* chore: Update effects

* chore: fix

* mount assets and volumes

* add exec instead of spawn

* fix mounting in image

* fix overlay mounts

Co-authored-by: Jade <Blu-J@users.noreply.github.com>

* misc fixes

* feat: Fix two

* fix: systemForEmbassy main

* chore: Fix small part of main loop

* chore: Modify the bundle

* merge

* fixMain loop"

* move tsc to makefile

* chore: Update the return types of the health check

* fix client

* chore: Convert the todo to use tsmatches

* add in the fixes for the seen and create the hack to allow demo

* chore: Update to include the systemForStartOs

* chore UPdate to the latest types from the expected outout

* fixes

* fix typo

* Don't emit if failure on tsc

* wip

Co-authored-by: Jade <Blu-J@users.noreply.github.com>

* add s9pk api

* add inspection

* add inspect manifest

* newline after display serializable

* fix squashfs in image name

* edit manifest

Co-authored-by: Jade <Blu-J@users.noreply.github.com>

* wait for response on repl

* ignore sig for now

* ignore sig for now

* re-enable sig verification

* fix

* wip

* env and chroot

* add profiling logs

* set uid & gid in squashfs to 100000

* set uid of sqfs to 100000

* fix mksquashfs args

* add env to compat

* fix

* re-add docker feature flag

* fix docker output format being stupid

* here be dragons

* chore: Add in the cross compiling for something

* fix npm link

* extract logs from container on exit

* chore: Update for testing

* add log capture to drop trait

* chore: add in the modifications that I make

* chore: Update small things for no updates

* chore: Update the types of something

* chore: Make main not complain

* idmapped mounts

* idmapped volumes

* re-enable kiosk

* chore: Add in some logging for the new system

* bring in start-sdk

* remove avahi

* chore: Update the deps

* switch to musl

* chore: Update the version of prettier

* chore: Organize'

* chore: Update some of the headers back to the standard of fetch

* fix musl build

* fix idmapped mounts

* fix cross build

* use cross compiler for correct arch

* feat: Add in the faked ssl stuff for the effects

* @dr_bonez Did a solution here

* chore: Something that DrBonez

* chore: up

* wip: We have a working server!!!

* wip

* uninstall

* wip

* tes

* misc fixes

* fix cli

* replace interface with host

* chore: Fix the types in some ts files

* chore: quick update for the system for embassy to update the types

* replace br-start9 with lxcbr0

* split patchdb into public/private

* chore: Add changes for config set

* Feat: Adding some debugging for the errors

* wip: Working on getting the set config to work

* chore: Update and fix the small issue with the deserialization

* lightning, masked, schemeOverride, invert host-iface relationship

* feat: Add in the changes for just the sdk

* feat: Add in the changes for the new effects I suppose for now

---------

Co-authored-by: J H <2364004+Blu-J@users.noreply.github.com>
Co-authored-by: J H <Blu-J@users.noreply.github.com>
Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Matt Hill <mattnine@protonmail.com>
2024-02-22 21:00:49 +00:00

296 lines
9.0 KiB
TypeScript

import nullIfEmpty from "./nullIfEmpty"
import {
CheckResult,
checkPortListening,
checkWebUrl,
} from "../health/checkFns"
import {
DaemonReturned,
Effects,
EnsureStorePath,
ExtractStore,
ServiceInterfaceId,
PackageId,
ValidIfNoStupidEscape,
} from "../types"
import { GetSystemSmtp } from "./GetSystemSmtp"
import { GetStore, getStore } from "../store/getStore"
import {
MountDependenciesOut,
mountDependencies,
} from "../dependency/mountDependencies"
import {
ManifestId,
VolumeName,
NamedPath,
Path,
} from "../dependency/setupDependencyMounts"
import { MultiHost, Scheme, SingleHost, StaticHost } from "../interfaces/Host"
import { ServiceInterfaceBuilder } from "../interfaces/ServiceInterfaceBuilder"
import { GetServiceInterface, getServiceInterface } from "./getServiceInterface"
import {
GetServiceInterfaces,
getServiceInterfaces,
} from "./getServiceInterfaces"
import * as CP from "node:child_process"
import { promisify } from "node:util"
import { splitCommand } from "./splitCommand"
import { SDKManifest } from "../manifest/ManifestTypes"
import { MountOptions, Overlay, CommandOptions } from "./Overlay"
export type Signals = NodeJS.Signals
export const SIGTERM: Signals = "SIGTERM"
export const SIGKILL: Signals = "SIGTERM"
export const NO_TIMEOUT = -1
const childProcess = {
exec: promisify(CP.exec),
execFile: promisify(CP.execFile),
}
export type ServiceInterfaceType = "ui" | "p2p" | "api"
export type Utils<
Manifest extends SDKManifest,
Store,
WrapperOverWrite = { const: never },
> = {
checkPortListening(
port: number,
options: {
errorMessage: string
successMessage: string
timeoutMessage?: string
timeout?: number
},
): Promise<CheckResult>
checkWebUrl(
url: string,
options?: {
timeout?: number
successMessage?: string
errorMessage?: string
},
): Promise<CheckResult>
childProcess: typeof childProcess
createInterface: (options: {
name: string
id: string
description: string
hasPrimary: boolean
disabled: boolean
type: ServiceInterfaceType
username: null | string
path: string
search: Record<string, string>
schemeOverride: { ssl: Scheme; noSsl: Scheme } | null
masked: boolean
}) => ServiceInterfaceBuilder
getSystemSmtp: () => GetSystemSmtp & WrapperOverWrite
host: {
static: (id: string) => StaticHost
single: (id: string) => SingleHost
multi: (id: string) => MultiHost
}
mountDependencies: <
In extends
| Record<ManifestId, Record<VolumeName, Record<NamedPath, Path>>>
| Record<VolumeName, Record<NamedPath, Path>>
| Record<NamedPath, Path>
| Path,
>(
value: In,
) => Promise<MountDependenciesOut<In>>
serviceInterface: {
getOwn: (id: ServiceInterfaceId) => GetServiceInterface & WrapperOverWrite
get: (opts: {
id: ServiceInterfaceId
packageId: PackageId
}) => GetServiceInterface & WrapperOverWrite
getAllOwn: () => GetServiceInterfaces & WrapperOverWrite
getAll: (opts: {
packageId: PackageId
}) => GetServiceInterfaces & WrapperOverWrite
}
nullIfEmpty: typeof nullIfEmpty
runCommand: <A extends string>(
imageId: Manifest["images"][number],
command: ValidIfNoStupidEscape<A> | [string, ...string[]],
options: CommandOptions & {
mounts?: { path: string; options: MountOptions }[]
},
) => Promise<{ stdout: string | Buffer; stderr: string | Buffer }>
runDaemon: <A extends string>(
imageId: Manifest["images"][number],
command: ValidIfNoStupidEscape<A> | [string, ...string[]],
options: CommandOptions & {
mounts?: { path: string; options: MountOptions }[]
overlay?: Overlay
},
) => Promise<DaemonReturned>
store: {
get: <Path extends string>(
packageId: string,
path: EnsureStorePath<Store, Path>,
) => GetStore<Store, Path> & WrapperOverWrite
getOwn: <Path extends string>(
path: EnsureStorePath<Store, Path>,
) => GetStore<Store, Path> & WrapperOverWrite
setOwn: <Path extends string | never>(
path: EnsureStorePath<Store, Path>,
value: ExtractStore<Store, Path>,
) => Promise<void>
}
}
export const createUtils = <
Manifest extends SDKManifest,
Store = never,
WrapperOverWrite = { const: never },
>(
effects: Effects,
): Utils<Manifest, Store, WrapperOverWrite> => {
return {
createInterface: (options: {
name: string
id: string
description: string
hasPrimary: boolean
disabled: boolean
type: ServiceInterfaceType
username: null | string
path: string
search: Record<string, string>
schemeOverride: { ssl: Scheme; noSsl: Scheme } | null
masked: boolean
}) => new ServiceInterfaceBuilder({ ...options, effects }),
childProcess,
getSystemSmtp: () =>
new GetSystemSmtp(effects) as GetSystemSmtp & WrapperOverWrite,
host: {
static: (id: string) => new StaticHost({ id, effects }),
single: (id: string) => new SingleHost({ id, effects }),
multi: (id: string) => new MultiHost({ id, effects }),
},
nullIfEmpty,
serviceInterface: {
getOwn: (id: ServiceInterfaceId) =>
getServiceInterface(effects, { id }) as GetServiceInterface &
WrapperOverWrite,
get: (opts: { id: ServiceInterfaceId; packageId: PackageId }) =>
getServiceInterface(effects, opts) as GetServiceInterface &
WrapperOverWrite,
getAllOwn: () =>
getServiceInterfaces(effects, {}) as GetServiceInterfaces &
WrapperOverWrite,
getAll: (opts: { packageId: PackageId }) =>
getServiceInterfaces(effects, opts) as GetServiceInterfaces &
WrapperOverWrite,
},
store: {
get: <Path extends string = never>(
packageId: string,
path: EnsureStorePath<Store, Path>,
) =>
getStore<Store, Path>(effects, path as any, {
packageId,
}) as any,
getOwn: <Path extends string>(path: EnsureStorePath<Store, Path>) =>
getStore<Store, Path>(effects, path as any) as any,
setOwn: <Path extends string | never>(
path: EnsureStorePath<Store, Path>,
value: ExtractStore<Store, Path>,
) => effects.store.set<Store, Path>({ value, path: path as any }),
},
runCommand: async <A extends string>(
imageId: Manifest["images"][number],
command: ValidIfNoStupidEscape<A> | [string, ...string[]],
options: CommandOptions & {
mounts?: { path: string; options: MountOptions }[]
},
): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> => {
const commands = splitCommand(command)
const overlay = await Overlay.of(effects, imageId)
try {
for (let mount of options.mounts || []) {
await overlay.mount(mount.options, mount.path)
}
return await overlay.exec(commands)
} finally {
await overlay.destroy()
}
},
runDaemon: async <A extends string>(
imageId: Manifest["images"][number],
command: ValidIfNoStupidEscape<A> | [string, ...string[]],
options: CommandOptions & {
mounts?: { path: string; options: MountOptions }[]
overlay?: Overlay
},
): Promise<DaemonReturned> => {
const commands = splitCommand(command)
const overlay = options.overlay || (await Overlay.of(effects, imageId))
for (let mount of options.mounts || []) {
await overlay.mount(mount.options, mount.path)
}
const childProcess = await overlay.spawn(commands, {
env: options.env,
})
const answer = new Promise<null>((resolve, reject) => {
childProcess.stdout.on("data", (data: any) => {
console.log(data.toString())
})
childProcess.stderr.on("data", (data: any) => {
console.error(data.toString())
})
childProcess.on("close", (code: any) => {
if (code === 0) {
return resolve(null)
}
return reject(new Error(`${commands[0]} exited with code ${code}`))
})
})
return {
wait() {
return answer
},
async term({ signal = SIGTERM, timeout = NO_TIMEOUT } = {}) {
try {
childProcess.kill(signal)
if (timeout <= NO_TIMEOUT) {
const didTimeout = await Promise.race([
new Promise((resolve) => setTimeout(resolve, timeout)).then(
() => true,
),
answer.then(() => false),
])
if (didTimeout) childProcess.kill(SIGKILL)
}
await answer
} finally {
await overlay.destroy()
}
},
}
},
checkPortListening: checkPortListening.bind(null, effects),
checkWebUrl: checkWebUrl.bind(null, effects),
mountDependencies: <
In extends
| Record<ManifestId, Record<VolumeName, Record<NamedPath, Path>>>
| Record<VolumeName, Record<NamedPath, Path>>
| Record<NamedPath, Path>
| Path,
>(
value: In,
) => mountDependencies(effects, value),
}
}
function noop(): void {}