Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-26 02:11:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feature: pack s9pk (#2642)

Browse Source 
* TODO: images

* wip

* pack s9pk images

* include path in packsource error

* debug info

* add cmd as context to invoke

* filehelper bugfix

* fix file helper

* fix exposeForDependents

* misc fixes

* force image removal

* fix filtering

* fix deadlock

* fix api

* chore: Up the version of the package.json

* always allow concurrency within same call stack

* Update core/startos/src/s9pk/merkle_archive/expected.rs

Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

---------

Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>
This commit is contained in:
Aiden McClelland
2024-06-12 11:46:59 -06:00
committed by GitHub
parent 5aefb707fa
commit 3f380fa0da
84 changed files with 2552 additions and 2108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@@ -160,6 +160,10 @@ wormhole-deb: results/$(BASENAME).deb
@echo "Paste the following command into the shell of your start-os server:"
@wormhole send results/$(BASENAME).deb 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo /usr/lib/startos/scripts/chroot-and-upgrade '"'"'cd $$(mktemp -d) && wormhole receive --accept-file %s && apt-get install -y --reinstall ./$(BASENAME).deb'"'"'\n", $$3 }'
wormhole-cli: core/target/$(ARCH)-unknown-linux-musl/release/start-cli
@echo "Paste the following command into the shell of your start-os server:"
@wormhole send results/$(BASENAME).deb 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo /usr/lib/startos/scripts/chroot-and-upgrade '"'"'cd $$(mktemp -d) && wormhole receive --accept-file %s && apt-get install -y --reinstall ./$(BASENAME).deb'"'"'\n", $$3 }'
update: $(ALL_TARGETS)
@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
$(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create')

2
container-runtime/deb-install.sh
View File

@@ -6,7 +6,7 @@ mkdir -p /run/systemd/resolve
echo "nameserver 8.8.8.8" > /run/systemd/resolve/stub-resolv.conf
apt-get update
apt-get install -y curl rsync
apt-get install -y curl rsync qemu-user-static
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
source ~/.bashrc

1797
container-runtime/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

1
container-runtime/package.json
View File

@@ -21,6 +21,7 @@
"esbuild-plugin-resolve": "^2.0.0",
"filebrowser": "^1.0.0",
"isomorphic-fetch": "^3.0.0",
"lodash": "^4.17.21",
"node-fetch": "^3.1.0",
"ts-matches": "^5.5.1",
"tslib": "^2.5.3",

26
container-runtime/src/Adapters/HostSystemStartOs.ts
View File

@@ -32,6 +32,8 @@ type RpcError = typeof matchRpcError._TYPE
const SOCKET_PATH = "/media/startos/rpc/host.sock"
const MAIN = "/main" as const
export class HostSystemStartOs implements Effects {
procedureId: string | null = null
static of(callbackHolder: CallbackHolder) {
return new HostSystemStartOs(callbackHolder)
}
@@ -40,7 +42,7 @@ export class HostSystemStartOs implements Effects {
id = 0
rpcRound<K extends keyof Effects | "getStore" | "setStore">(
method: K,
params: unknown,
params: Record<string, unknown>,
) {
const id = this.id++
const client = net.createConnection({ path: SOCKET_PATH }, () => {
@@ -48,7 +50,7 @@ export class HostSystemStartOs implements Effects {
JSON.stringify({
id,
method,
params,
params: { ...params, procedureId: this.procedureId },
}) + "\n",
)
})
@@ -102,14 +104,14 @@ export class HostSystemStartOs implements Effects {
}) as ReturnType<T.Effects["bind"]>
}
clearBindings(...[]: Parameters<T.Effects["clearBindings"]>) {
return this.rpcRound("clearBindings", null) as ReturnType<
return this.rpcRound("clearBindings", {}) as ReturnType<
T.Effects["clearBindings"]
>
}
clearServiceInterfaces(
...[]: Parameters<T.Effects["clearServiceInterfaces"]>
) {
return this.rpcRound("clearServiceInterfaces", null) as ReturnType<
return this.rpcRound("clearServiceInterfaces", {}) as ReturnType<
T.Effects["clearServiceInterfaces"]
>
}
@@ -145,18 +147,20 @@ export class HostSystemStartOs implements Effects {
T.Effects["exportServiceInterface"]
>
}
exposeForDependents(...[options]: any) {
return this.rpcRound("exposeForDependents", null) as ReturnType<
exposeForDependents(
...[options]: Parameters<T.Effects["exposeForDependents"]>
) {
return this.rpcRound("exposeForDependents", options) as ReturnType<
T.Effects["exposeForDependents"]
>
}
getConfigured(...[]: Parameters<T.Effects["getConfigured"]>) {
return this.rpcRound("getConfigured", null) as ReturnType<
return this.rpcRound("getConfigured", {}) as ReturnType<
T.Effects["getConfigured"]
>
}
getContainerIp(...[]: Parameters<T.Effects["getContainerIp"]>) {
return this.rpcRound("getContainerIp", null) as ReturnType<
return this.rpcRound("getContainerIp", {}) as ReturnType<
T.Effects["getContainerIp"]
>
}
@@ -229,7 +233,7 @@ export class HostSystemStartOs implements Effects {
>
}
restart(...[]: Parameters<T.Effects["restart"]>) {
return this.rpcRound("restart", null)
return this.rpcRound("restart", {}) as ReturnType<T.Effects["restart"]>
}
running(...[packageId]: Parameters<T.Effects["running"]>) {
return this.rpcRound("running", { packageId }) as ReturnType<
@@ -262,7 +266,7 @@ export class HostSystemStartOs implements Effects {
>
}
getDependencies(): ReturnType<T.Effects["getDependencies"]> {
return this.rpcRound("getDependencies", null) as ReturnType<
return this.rpcRound("getDependencies", {}) as ReturnType<
T.Effects["getDependencies"]
>
}
@@ -279,7 +283,7 @@ export class HostSystemStartOs implements Effects {
}
shutdown(...[]: Parameters<T.Effects["shutdown"]>) {
return this.rpcRound("shutdown", null)
return this.rpcRound("shutdown", {}) as ReturnType<T.Effects["shutdown"]>
}
stopped(...[packageId]: Parameters<T.Effects["stopped"]>) {
return this.rpcRound("stopped", { packageId }) as ReturnType<

3
container-runtime/src/Adapters/RpcListener.ts
View File

@@ -58,6 +58,7 @@ const runType = object({
method: literal("execute"),
params: object(
{
id: string,
procedure: string,
input: any,
timeout: number,
@@ -70,6 +71,7 @@ const sandboxRunType = object({
method: literal("sandbox"),
params: object(
{
id: string,
procedure: string,
input: any,
timeout: number,
@@ -195,6 +197,7 @@ export class RpcListener {
const procedure = jsonPath.unsafeCast(params.procedure)
return system
.execute(this.effects, {
id: params.id,
procedure,
input: params.input,
timeout: params.timeout,

7
container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts
View File

@@ -49,7 +49,7 @@ function todo(): never {
const execFile = promisify(childProcess.execFile)
const MANIFEST_LOCATION = "/usr/lib/startos/package/embassyManifest.json"
const EMBASSY_JS_LOCATION = "/usr/lib/startos/package/embassy.js"
export const EMBASSY_JS_LOCATION = "/usr/lib/startos/package/embassy.js"
const EMBASSY_POINTER_PATH_PREFIX = "/embassyConfig"
const matchSetResult = object(
@@ -199,11 +199,14 @@ export class SystemForEmbassy implements System {
async execute(
effects: HostSystemStartOs,
options: {
id: string
procedure: JsonPath
input: unknown
timeout?: number | undefined
},
): Promise<RpcResult> {
effects = Object.create(effects)
effects.procedureId = options.id
return this._execute(effects, options)
.then((x) =>
matches(x)
@@ -724,7 +727,7 @@ export class SystemForEmbassy implements System {
private async properties(
effects: HostSystemStartOs,
timeoutMs: number | null,
): Promise<ReturnType<T.ExpectedExports.Properties>> {
): Promise<ReturnType<T.ExpectedExports.properties>> {
// TODO BLU-J set the properties ever so often
const setConfigValue = this.manifest.properties
if (!setConfigValue) throw new Error("There is no properties")

139
container-runtime/src/Adapters/Systems/SystemForStartOs.ts
View File

@@ -1,20 +1,23 @@
import { ExecuteResult, System } from "../../Interfaces/System"
import { unNestPath } from "../../Models/JsonPath"
import { string } from "ts-matches"
import matches, { any, number, object, string, tuple } from "ts-matches"
import { HostSystemStartOs } from "../HostSystemStartOs"
import { Effects } from "../../Models/Effects"
import { RpcResult } from "../RpcListener"
import { RpcResult, matchRpcResult } from "../RpcListener"
import { duration } from "../../Models/Duration"
const LOCATION = "/usr/lib/startos/package/startos"
import { T } from "@start9labs/start-sdk"
import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk"
export const STARTOS_JS_LOCATION = "/usr/lib/startos/package/index.js"
export class SystemForStartOs implements System {
private onTerm: (() => Promise<void>) | undefined
static of() {
return new SystemForStartOs()
return new SystemForStartOs(require(STARTOS_JS_LOCATION))
}
constructor() {}
constructor(readonly abi: T.ABI) {}
async execute(
effects: HostSystemStartOs,
options: {
id: string
procedure:
| "/init"
| "/uninit"
@@ -33,7 +36,61 @@ export class SystemForStartOs implements System {
timeout?: number | undefined
},
): Promise<RpcResult> {
return { result: await this._execute(effects, options) }
effects = Object.create(effects)
effects.procedureId = options.id
return this._execute(effects, options)
.then((x) =>
matches(x)
.when(
object({
result: any,
}),
(x) => x,
)
.when(
object({
error: string,
}),
(x) => ({
error: {
code: 0,
message: x.error,
},
}),
)
.when(
object({
"error-code": tuple(number, string),
}),
({ "error-code": [code, message] }) => ({
error: {
code,
message,
},
}),
)
.defaultTo({ result: x }),
)
.catch((error: unknown) => {
if (error instanceof Error)
return {
error: {
code: 0,
message: error.name,
data: {
details: error.message,
debug: `${error?.cause ?? "[noCause]"}:${error?.stack ?? "[noStack]"}`,
},
},
}
if (matchRpcResult.test(error)) return error
return {
error: {
code: 0,
message: String(error),
},
}
})
}
async _execute(
effects: Effects,
@@ -58,26 +115,27 @@ export class SystemForStartOs implements System {
): Promise<unknown> {
switch (options.procedure) {
case "/init": {
const path = `${LOCATION}/procedures/init`
const procedure: any = await import(path).catch(() => require(path))
const previousVersion = string.optional().unsafeCast(options)
return procedure.init({ effects, previousVersion })
const previousVersion =
string.optional().unsafeCast(options.input) || null
return this.abi.init({ effects, previousVersion })
}
case "/uninit": {
const path = `${LOCATION}/procedures/init`
const procedure: any = await import(path).catch(() => require(path))
const nextVersion = string.optional().unsafeCast(options)
return procedure.uninit({ effects, nextVersion })
const nextVersion = string.optional().unsafeCast(options.input) || null
return this.abi.uninit({ effects, nextVersion })
}
case "/main/start": {
const path = `${LOCATION}/procedures/main`
const procedure: any = await import(path).catch(() => require(path))
const started = async (onTerm: () => Promise<void>) => {
await effects.setMainStatus({ status: "running" })
if (this.onTerm) await this.onTerm()
this.onTerm = onTerm
}
return procedure.main({ effects, started })
const daemons = await (
await this.abi.main({
effects: { ...effects, _type: "main" },
started,
})
).build()
this.onTerm = daemons.term
}
case "/main/stop": {
await effects.setMainStatus({ status: "stopped" })
@@ -86,67 +144,50 @@ export class SystemForStartOs implements System {
return duration(30, "s")
}
case "/config/set": {
const path = `${LOCATION}/procedures/config`
const procedure: any = await import(path).catch(() => require(path))
const input = options.input
return procedure.setConfig({ effects, input })
const input = options.input as any // TODO
return this.abi.setConfig({ effects, input })
}
case "/config/get": {
const path = `${LOCATION}/procedures/config`
const procedure: any = await import(path).catch(() => require(path))
return procedure.getConfig({ effects })
return this.abi.getConfig({ effects })
}
case "/backup/create":
case "/backup/restore":
throw new Error("this should be called with the init/unit")
case "/actions/metadata": {
const path = `${LOCATION}/procedures/actions`
const procedure: any = await import(path).catch(() => require(path))
return procedure.actionsMetadata({ effects })
return this.abi.actionsMetadata({ effects })
}
default:
const procedures = unNestPath(options.procedure)
const id = procedures[2]
switch (true) {
case procedures[1] === "actions" && procedures[3] === "get": {
const path = `${LOCATION}/procedures/actions`
const action: any = (await import(path).catch(() => require(path)))
.actions[id]
const action = (await this.abi.actions({ effects }))[id]
if (!action) throw new Error(`Action ${id} not found`)
return action.get({ effects })
return action.getConfig({ effects })
}
case procedures[1] === "actions" && procedures[3] === "run": {
const path = `${LOCATION}/procedures/actions`
const action: any = (await import(path).catch(() => require(path)))
.actions[id]
const action = (await this.abi.actions({ effects }))[id]
if (!action) throw new Error(`Action ${id} not found`)
const input = options.input
return action.run({ effects, input })
return action.run({ effects, input: options.input as any }) // TODO
}
case procedures[1] === "dependencies" && procedures[3] === "query": {
const path = `${LOCATION}/procedures/dependencies`
const dependencyConfig: any = (
await import(path).catch(() => require(path))
).dependencyConfig[id]
const dependencyConfig = this.abi.dependencyConfig[id]
if (!dependencyConfig)
throw new Error(`dependencyConfig ${id} not found`)
const localConfig = options.input
return dependencyConfig.query({ effects, localConfig })
return dependencyConfig.query({ effects })
}
case procedures[1] === "dependencies" && procedures[3] === "update": {
const path = `${LOCATION}/procedures/dependencies`
const dependencyConfig: any = (
await import(path).catch(() => require(path))
).dependencyConfig[id]
const dependencyConfig = this.abi.dependencyConfig[id]
if (!dependencyConfig)
throw new Error(`dependencyConfig ${id} not found`)
return dependencyConfig.update(options.input)
return dependencyConfig.update(options.input as any) // TODO
}
}
}
throw new Error("Method not implemented.")
throw new Error(`Method ${options.procedure} not implemented.`)
}
exit(effects: Effects): Promise<void> {
throw new Error("Method not implemented.")
async exit(effects: Effects): Promise<void> {
return void null
}
}

22
container-runtime/src/Adapters/Systems/index.ts
View File

@@ -1,6 +1,22 @@
import * as fs from "node:fs/promises"
import { System } from "../../Interfaces/System"
import { SystemForEmbassy } from "./SystemForEmbassy"
import { SystemForStartOs } from "./SystemForStartOs"
import { EMBASSY_JS_LOCATION, SystemForEmbassy } from "./SystemForEmbassy"
import { STARTOS_JS_LOCATION, SystemForStartOs } from "./SystemForStartOs"
export async function getSystem(): Promise<System> {
return SystemForEmbassy.of()
if (
await fs.access(STARTOS_JS_LOCATION).then(
() => true,
() => false,
)
) {
return SystemForStartOs.of()
} else if (
await fs.access(EMBASSY_JS_LOCATION).then(
() => true,
() => false,
)
) {
return SystemForEmbassy.of()
}
throw new Error(`${STARTOS_JS_LOCATION} not found`)
}

1
container-runtime/src/Interfaces/System.ts
View File

@@ -14,6 +14,7 @@ export interface System {
execute(
effects: T.Effects,
options: {
id: string
procedure: JsonPath
input: unknown
timeout?: number

172
core/Cargo.lock generated
View File

@@ -387,6 +387,22 @@ dependencies = [
"tower-service",
]
[[package]]
name = "backhand"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f2fc1bc7bb7fd449e02000cc1592cc63dcdcd61710f8b9efe32bab2d1784603"
dependencies = [
"deku",
"flate2",
"rustc-hash",
"thiserror",
"tracing",
"xz2",
"zstd",
"zstd-safe",
]
[[package]]
name = "backtrace"
version = "0.3.71"
@@ -588,6 +604,11 @@ name = "cc"
version = "1.0.96"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "065a29261d53ba54260972629f9ca6bffa69bac13cd1fed61420f7fa68b9f8bd"
dependencies = [
"jobserver",
"libc",
"once_cell",
]
[[package]]
name = "cfg-if"
@@ -1058,12 +1079,6 @@ dependencies = [
"cipher 0.3.0",
]
[[package]]
name = "current_platform"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a74858bcfe44b22016cb49337d7b6f04618c58e5dbfdef61b06b8c434324a0bc"
[[package]]
name = "curve25519-dalek"
version = "3.2.0"
@@ -1146,6 +1161,31 @@ version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
[[package]]
name = "deku"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "709ade444d53896e60f6265660eb50480dd08b77bfc822e5dcc233b88b0b2fba"
dependencies = [
"bitvec",
"deku_derive",
"no_std_io",
"rustversion",
]
[[package]]
name = "deku_derive"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7534973f93f9de83203e41c8ddd32d230599fa73fa889f3deb1580ccd186913"
dependencies = [
"darling",
"proc-macro-crate",
"proc-macro2",
"quote",
"syn 2.0.60",
]
[[package]]
name = "der"
version = "0.7.9"
@@ -2405,6 +2445,15 @@ dependencies = [
"jaq-parse",
]
[[package]]
name = "jobserver"
version = "0.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e"
dependencies = [
"libc",
]
[[package]]
name = "josekit"
version = "0.8.6"
@@ -2561,6 +2610,12 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "libyml"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e281a65eeba3d4503a2839252f86374528f9ceafe6fed97c1d3b52e1fb625c1"
[[package]]
name = "linux-raw-sys"
version = "0.4.13"
@@ -2583,6 +2638,17 @@ version = "0.4.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
[[package]]
name = "lzma-sys"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
name = "matchers"
version = "0.1.0"
@@ -2788,6 +2854,15 @@ dependencies = [
"libc",
]
[[package]]
name = "no_std_io"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fa5f306a6f2c01b4fd172f29bb46195b1764061bf926c75e96ff55df3178208"
dependencies = [
"memchr",
]
[[package]]
name = "nom"
version = "7.1.3"
@@ -3800,6 +3875,12 @@ version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc_version"
version = "0.4.0"
@@ -3895,9 +3976,9 @@ dependencies = [
[[package]]
name = "rustversion"
version = "1.0.15"
version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47"
checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6"
[[package]]
name = "rusty-fork"
@@ -3929,9 +4010,9 @@ dependencies = [
[[package]]
name = "ryu"
version = "1.0.17"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "same-file"
@@ -4015,9 +4096,9 @@ dependencies = [
[[package]]
name = "serde"
version = "1.0.200"
version = "1.0.203"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f"
checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094"
dependencies = [
"serde_derive",
]
@@ -4041,9 +4122,9 @@ dependencies = [
[[package]]
name = "serde_derive"
version = "1.0.200"
version = "1.0.203"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb"
checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba"
dependencies = [
"proc-macro2",
"quote",
@@ -4052,9 +4133,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.116"
version = "1.0.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813"
checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3"
dependencies = [
"indexmap 2.2.6",
"itoa",
@@ -4124,16 +4205,20 @@ dependencies = [
]
[[package]]
name = "serde_yaml"
version = "0.9.34+deprecated"
name = "serde_yml"
version = "0.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
checksum = "78ce6afeda22f0b55dde2c34897bce76a629587348480384231205c14b59a01f"
dependencies = [
"indexmap 2.2.6",
"itoa",
"libyml",
"log",
"memchr",
"ryu",
"serde",
"unsafe-libyaml",
"serde_json",
"tempfile",
]
[[package]]
@@ -4600,6 +4685,7 @@ dependencies = [
"async-trait",
"axum 0.7.5",
"axum-server",
"backhand",
"base32",
"base64 0.21.7",
"base64ct",
@@ -4614,7 +4700,6 @@ dependencies = [
"console-subscriber",
"cookie 0.18.1",
"cookie_store",
"current_platform",
"der",
"digest 0.10.7",
"divrem",
@@ -4681,7 +4766,7 @@ dependencies = [
"serde_json",
"serde_urlencoded",
"serde_with",
"serde_yaml",
"serde_yml",
"sha2 0.10.8",
"shell-words",
"simple-logging",
@@ -5537,12 +5622,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
[[package]]
name = "unsafe-libyaml"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
[[package]]
name = "untrusted"
version = "0.9.0"
@@ -5993,6 +6072,15 @@ dependencies = [
"rustix",
]
[[package]]
name = "xz2"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
dependencies = [
"lzma-sys",
]
[[package]]
name = "yajrc"
version = "0.1.3"
@@ -6056,3 +6144,31 @@ dependencies = [
"quote",
"syn 2.0.60",
]
[[package]]
name = "zstd"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.10+zstd.1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa"
dependencies = [
"cc",
"pkg-config",
]

4
core/startos/Cargo.toml
View File

@@ -59,6 +59,7 @@ async-stream = "0.3.5"
async-trait = "0.1.74"
axum = { version = "0.7.3", features = ["ws"] }
axum-server = "0.6.0"
backhand = "0.18.0"
base32 = "0.4.0"
base64 = "0.21.4"
base64ct = "1.6.0"
@@ -72,7 +73,6 @@ console = "0.15.7"
console-subscriber = { version = "0.2", optional = true }
cookie = "0.18.0"
cookie_store = "0.20.0"
current_platform = "0.2.0"
der = { version = "0.7.9", features = ["derive", "pem"] }
digest = "0.10.7"
divrem = "1.0.0"
@@ -154,7 +154,7 @@ serde_json = "1.0"
serde_toml = { package = "toml", version = "0.8.2" }
serde_urlencoded = "0.7"
serde_with = { version = "3.4.0", features = ["macros", "json"] }
serde_yaml = "0.9.25"
serde_yaml = { package = "serde_yml", version = "0.0.10" }
sha2 = "0.10.2"
shell-words = "1"
simple-logging = "2.0.2"

2
core/startos/src/action.rs
View File

@@ -8,6 +8,7 @@ use ts_rs::TS;
use crate::config::Config;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::util::serde::{display_serializable, StdinDeserializable, WithIoFormat};
#[derive(Debug, Serialize, Deserialize)]
@@ -77,6 +78,7 @@ pub async fn action(
.as_ref()
.or_not_found(lazy_format!("Manager for {}", package_id))?
.action(
Guid::new(),
action_id,
input.map(|c| to_value(&c)).transpose()?.unwrap_or_default(),
)

1
core/startos/src/auth.rs
View File

@@ -178,6 +178,7 @@ pub fn check_password_against_db(db: &DatabaseModel, password: &str) -> Result<(
#[derive(Deserialize, Serialize, Parser, TS)]
#[serde(rename_all = "camelCase")]
#[command(rename_all = "kebab-case")]
#[ts(export)]
pub struct LoginParams {
password: Option<PasswordType>,
#[ts(skip)]

1
core/startos/src/backup/restore.rs
View File

@@ -149,7 +149,6 @@ async fn restore_packages(
S9pk::open(
backup_dir.path().join(&id).with_extension("s9pk"),
Some(&id),
true,
)
.await?,
Some(backup_dir),

5
core/startos/src/config/mod.rs
View File

@@ -16,6 +16,7 @@ use ts_rs::TS;
use crate::context::{CliContext, RpcContext};
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::util::serde::{HandlerExtSerde, StdinDeserializable};
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
@@ -156,7 +157,7 @@ pub async fn get(ctx: RpcContext, _: Empty, id: PackageId) -> Result<ConfigRes,
.await
.as_ref()
.or_not_found(lazy_format!("Manager for {id}"))?
.get_config()
.get_config(Guid::new())
.await
}
@@ -218,7 +219,7 @@ pub async fn set_impl(
ErrorKind::Unknown,
)
})?
.configure(configure_context)
.configure(Guid::new(), configure_context)
.await?;
Ok(())
}

21
core/startos/src/context/config.rs
View File

@@ -93,26 +93,28 @@ impl ClientConfig {
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct ServerConfig {
#[arg(short = 'c', long = "config")]
#[arg(short, long)]
pub config: Option<PathBuf>,
#[arg(long = "ethernet-interface")]
#[arg(long)]
pub ethernet_interface: Option<String>,
#[arg(skip)]
pub os_partitions: Option<OsPartitionInfo>,
#[arg(long = "bind-rpc")]
#[arg(long)]
pub bind_rpc: Option<SocketAddr>,
#[arg(long = "tor-control")]
#[arg(long)]
pub tor_control: Option<SocketAddr>,
#[arg(long = "tor-socks")]
#[arg(long)]
pub tor_socks: Option<SocketAddr>,
#[arg(long = "dns-bind")]
#[arg(long)]
pub dns_bind: Option<Vec<SocketAddr>>,
#[arg(long = "revision-cache-size")]
#[arg(long)]
pub revision_cache_size: Option<usize>,
#[arg(short = 'd', long = "datadir")]
#[arg(short, long)]
pub datadir: Option<PathBuf>,
#[arg(long = "disable-encryption")]
#[arg(long)]
pub disable_encryption: Option<bool>,
#[arg(long)]
pub multi_arch_s9pks: Option<bool>,
}
impl ContextConfig for ServerConfig {
fn next(&mut self) -> Option<PathBuf> {
@@ -131,6 +133,7 @@ impl ContextConfig for ServerConfig {
.or(other.revision_cache_size);
self.datadir = self.datadir.take().or(other.datadir);
self.disable_encryption = self.disable_encryption.take().or(other.disable_encryption);
self.multi_arch_s9pks = self.multi_arch_s9pks.take().or(other.multi_arch_s9pks);
}
}

6
core/startos/src/context/rpc.rs
View File

@@ -43,6 +43,7 @@ pub struct RpcContextSeed {
pub db: TypedPatchDb<Database>,
pub account: RwLock<AccountInfo>,
pub net_controller: Arc<NetController>,
pub s9pk_arch: Option<&'static str>,
pub services: ServiceMap,
pub metrics_cache: RwLock<Option<crate::system::Metrics>>,
pub shutdown: broadcast::Sender<Option<Shutdown>>,
@@ -152,6 +153,11 @@ impl RpcContext {
db,
account: RwLock::new(account),
net_controller,
s9pk_arch: if config.multi_arch_s9pks.unwrap_or(false) {
None
} else {
Some(crate::ARCH)
},
services,
metrics_cache,
shutdown,

7
core/startos/src/control.rs
View File

@@ -7,6 +7,7 @@ use ts_rs::TS;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::Error;
#[derive(Deserialize, Serialize, Parser, TS)]
@@ -23,7 +24,7 @@ pub async fn start(ctx: RpcContext, ControlParams { id }: ControlParams) -> Resu
.await
.as_ref()
.or_not_found(lazy_format!("Manager for {id}"))?
.start()
.start(Guid::new())
.await?;
Ok(())
@@ -36,7 +37,7 @@ pub async fn stop(ctx: RpcContext, ControlParams { id }: ControlParams) -> Resul
.await
.as_ref()
.ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
.stop()
.stop(Guid::new())
.await?;
Ok(())
@@ -48,7 +49,7 @@ pub async fn restart(ctx: RpcContext, ControlParams { id }: ControlParams) -> Re
.await
.as_ref()
.ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
.restart()
.restart(Guid::new())
.await?;
Ok(())

13
core/startos/src/dependencies.rs
View File

@@ -13,6 +13,7 @@ use crate::config::{Config, ConfigSpec, ConfigureContext};
use crate::context::RpcContext;
use crate::db::model::package::CurrentDependencies;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::Error;
pub fn dependency<C: Context>() -> ParentHandler<C> {
@@ -86,7 +87,7 @@ pub async fn configure_impl(
ErrorKind::Unknown,
)
})?
.configure(configure_context)
.configure(Guid::new(), configure_context)
.await?;
Ok(())
}
@@ -103,14 +104,15 @@ pub async fn configure_logic(
ctx: RpcContext,
(dependent_id, dependency_id): (PackageId, PackageId),
) -> Result<ConfigDryRes, Error> {
let procedure_id = Guid::new();
let dependency_guard = ctx.services.get(&dependency_id).await;
let dependency = dependency_guard.as_ref().or_not_found(&dependency_id)?;
let dependent_guard = ctx.services.get(&dependent_id).await;
let dependent = dependent_guard.as_ref().or_not_found(&dependent_id)?;
let config_res = dependency.get_config().await?;
let config_res = dependency.get_config(procedure_id.clone()).await?;
let diff = Value::Object(
dependent
.dependency_config(dependency_id, config_res.config.clone())
.dependency_config(procedure_id, dependency_id, config_res.config.clone())
.await?
.unwrap_or_default(),
);
@@ -129,6 +131,7 @@ pub async fn compute_dependency_config_errs(
id: &PackageId,
current_dependencies: &mut CurrentDependencies,
) -> Result<(), Error> {
let procedure_id = Guid::new();
let service_guard = ctx.services.get(id).await;
let service = service_guard.as_ref().or_not_found(id)?;
for (dep_id, dep_info) in current_dependencies.0.iter_mut() {
@@ -137,10 +140,10 @@ pub async fn compute_dependency_config_errs(
continue;
};
let dep_config = dependency.get_config().await?.config;
let dep_config = dependency.get_config(procedure_id.clone()).await?.config;
dep_info.config_satisfied = service
.dependency_config(dep_id.clone(), dep_config)
.dependency_config(procedure_id.clone(), dep_id.clone(), dep_config)
.await?
.is_none();
}

1
core/startos/src/disk/mount/backup.rs
View File

@@ -178,7 +178,6 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
Ok(())
}
}
#[async_trait::async_trait]
impl<G: GenericMountGuard> GenericMountGuard for BackupMountGuard<G> {
fn path(&self) -> &Path {
if let Some(guard) = &self.encrypted_guard {

23
core/startos/src/disk/mount/filesystem/overlayfs.rs
View File

@@ -6,8 +6,8 @@ use digest::generic_array::GenericArray;
use digest::{Digest, OutputSizeUser};
use sha2::Sha256;
use crate::disk::mount::filesystem::{FileSystem, ReadOnly, ReadWrite};
use crate::disk::mount::guard::{GenericMountGuard, MountGuard, TmpMountGuard};
use crate::disk::mount::filesystem::{FileSystem, ReadWrite};
use crate::disk::mount::guard::{GenericMountGuard, MountGuard};
use crate::prelude::*;
use crate::util::io::TmpDir;
@@ -94,17 +94,13 @@ impl<
}
#[derive(Debug)]
pub struct OverlayGuard {
lower: Option<TmpMountGuard>,
pub struct OverlayGuard<G: GenericMountGuard> {
lower: Option<G>,
upper: Option<TmpDir>,
inner_guard: MountGuard,
}
impl OverlayGuard {
pub async fn mount(
base: &impl FileSystem,
mountpoint: impl AsRef<Path>,
) -> Result<Self, Error> {
let lower = TmpMountGuard::mount(base, ReadOnly).await?;
impl<G: GenericMountGuard> OverlayGuard<G> {
pub async fn mount(lower: G, mountpoint: impl AsRef<Path>) -> Result<Self, Error> {
let upper = TmpDir::new().await?;
let inner_guard = MountGuard::mount(
&OverlayFs::new(
@@ -140,16 +136,15 @@ impl OverlayGuard {
}
}
}
#[async_trait::async_trait]
impl GenericMountGuard for OverlayGuard {
impl<G: GenericMountGuard> GenericMountGuard for OverlayGuard<G> {
fn path(&self) -> &Path {
self.inner_guard.path()
}
async fn unmount(mut self) -> Result<(), Error> {
async fn unmount(self) -> Result<(), Error> {
self.unmount(false).await
}
}
impl Drop for OverlayGuard {
impl<G: GenericMountGuard> Drop for OverlayGuard<G> {
fn drop(&mut self) {
let lower = self.lower.take();
let upper = self.upper.take();

19
core/startos/src/disk/mount/guard.rs
View File

@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Weak};
use futures::Future;
use lazy_static::lazy_static;
use models::ResultExt;
use tokio::sync::Mutex;
@@ -14,23 +15,20 @@ use crate::Error;
pub const TMP_MOUNTPOINT: &'static str = "/media/startos/tmp";
#[async_trait::async_trait]
pub trait GenericMountGuard: std::fmt::Debug + Send + Sync + 'static {
fn path(&self) -> &Path;
async fn unmount(mut self) -> Result<(), Error>;
fn unmount(self) -> impl Future<Output = Result<(), Error>> + Send;
}
#[async_trait::async_trait]
impl GenericMountGuard for Never {
fn path(&self) -> &Path {
match *self {}
}
async fn unmount(mut self) -> Result<(), Error> {
async fn unmount(self) -> Result<(), Error> {
match self {}
}
}
#[async_trait::async_trait]
impl<T> GenericMountGuard for Arc<T>
where
T: GenericMountGuard,
@@ -38,7 +36,7 @@ where
fn path(&self) -> &Path {
(&**self).path()
}
async fn unmount(mut self) -> Result<(), Error> {
async fn unmount(self) -> Result<(), Error> {
if let Ok(guard) = Arc::try_unwrap(self) {
guard.unmount().await?;
}
@@ -102,12 +100,11 @@ impl Drop for MountGuard {
}
}
}
#[async_trait::async_trait]
impl GenericMountGuard for MountGuard {
fn path(&self) -> &Path {
&self.mountpoint
}
async fn unmount(mut self) -> Result<(), Error> {
async fn unmount(self) -> Result<(), Error> {
MountGuard::unmount(self, false).await
}
}
@@ -165,12 +162,11 @@ impl TmpMountGuard {
std::mem::replace(self, unmounted)
}
}
#[async_trait::async_trait]
impl GenericMountGuard for TmpMountGuard {
fn path(&self) -> &Path {
self.guard.path()
}
async fn unmount(mut self) -> Result<(), Error> {
async fn unmount(self) -> Result<(), Error> {
self.guard.unmount().await
}
}
@@ -187,12 +183,11 @@ impl<G: GenericMountGuard> SubPath<G> {
Self { guard, path }
}
}
#[async_trait::async_trait]
impl<G: GenericMountGuard> GenericMountGuard for SubPath<G> {
fn path(&self) -> &Path {
self.path.as_path()
}
async fn unmount(mut self) -> Result<(), Error> {
async fn unmount(self) -> Result<(), Error> {
self.guard.unmount().await
}
}

2
core/startos/src/init.rs
View File

@@ -242,7 +242,7 @@ pub async fn init(cfg: &ServerConfig) -> Result<InitResult, Error> {
let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok()
|| &*server_info.version < &emver::Version::new(0, 3, 2, 0)
|| (*ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0));
|| (ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0));
let log_dir = cfg.datadir().join("main/logs");
if tokio::fs::metadata(&log_dir).await.is_err() {

2
core/startos/src/install/mod.rs
View File

@@ -152,7 +152,6 @@ pub async fn install(
.await?,
),
None, // TODO
true,
)
.await?;
@@ -262,7 +261,6 @@ pub async fn sideload(ctx: RpcContext) -> Result<SideloadResponse, Error> {
if let Err(e) = async {
let s9pk = S9pk::deserialize(
&file, None, // TODO
true,
)
.await?;
let _ = id_send.send(s9pk.as_manifest().id.clone());

6
core/startos/src/lib.rs
View File

@@ -4,12 +4,8 @@ pub const CAP_1_KiB: usize = 1024;
pub const CAP_1_MiB: usize = CAP_1_KiB * CAP_1_KiB;
pub const CAP_10_MiB: usize = 10 * CAP_1_MiB;
pub const HOST_IP: [u8; 4] = [172, 18, 0, 1];
pub const TARGET: &str = current_platform::CURRENT_PLATFORM;
pub use std::env::consts::ARCH;
lazy_static::lazy_static! {
pub static ref ARCH: &'static str = {
let (arch, _) = TARGET.split_once("-").unwrap();
arch
};
pub static ref PLATFORM: String = {
if let Ok(platform) = std::fs::read_to_string("/usr/lib/startos/PLATFORM.txt") {
platform

20
core/startos/src/lxc/mod.rs
View File

@@ -29,7 +29,7 @@ use crate::disk::mount::filesystem::bind::Bind;
use crate::disk::mount::filesystem::block_dev::BlockDev;
use crate::disk::mount::filesystem::idmapped::IdMapped;
use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
use crate::disk::mount::filesystem::{MountType, ReadWrite};
use crate::disk::mount::filesystem::{MountType, ReadOnly, ReadWrite};
use crate::disk::mount::guard::{GenericMountGuard, MountGuard, TmpMountGuard};
use crate::disk::mount::util::unmount;
use crate::prelude::*;
@@ -153,7 +153,7 @@ impl LxcManager {
pub struct LxcContainer {
manager: Weak<LxcManager>,
rootfs: OverlayGuard,
rootfs: OverlayGuard<TmpMountGuard>,
pub guid: Arc<ContainerId>,
rpc_bind: TmpMountGuard,
log_mount: Option<MountGuard>,
@@ -184,12 +184,16 @@ impl LxcContainer {
.invoke(ErrorKind::Filesystem)
.await?;
let rootfs = OverlayGuard::mount(
&IdMapped::new(
BlockDev::new("/usr/lib/startos/container-runtime/rootfs.squashfs"),
0,
100000,
65536,
),
TmpMountGuard::mount(
&IdMapped::new(
BlockDev::new("/usr/lib/startos/container-runtime/rootfs.squashfs"),
0,
100000,
65536,
),
ReadOnly,
)
.await?,
&rootfs_dir,
)
.await?;

2
core/startos/src/os_install/gpt.rs
View File

@@ -87,7 +87,7 @@ pub async fn partition(disk: &DiskInfo, overwrite: bool) -> Result<OsPartitionIn
gpt.add_partition(
"root",
15 * 1024 * 1024 * 1024,
match *crate::ARCH {
match crate::ARCH {
"x86_64" => gpt::partition_types::LINUX_ROOT_X64,
"aarch64" => gpt::partition_types::LINUX_ROOT_ARM_64,
_ => gpt::partition_types::LINUX_FS,

2
core/startos/src/os_install/mod.rs
View File

@@ -366,7 +366,7 @@ pub async fn execute<C: Context>(
if tokio::fs::metadata("/sys/firmware/efi").await.is_err() {
install.arg("--target=i386-pc");
} else {
match *ARCH {
match ARCH {
"x86_64" => install.arg("--target=x86_64-efi"),
"aarch64" => install.arg("--target=arm64-efi"),
_ => &mut install,

2
core/startos/src/registry/device_info.rs
View File

@@ -134,7 +134,7 @@ pub struct HardwareInfo {
impl From<&RpcContext> for HardwareInfo {
fn from(value: &RpcContext) -> Self {
Self {
arch: InternedString::intern(&**crate::ARCH),
arch: InternedString::intern(crate::ARCH),
ram: value.hardware.ram,
devices: value
.hardware

4
core/startos/src/registry/package/add.rs
View File

@@ -53,7 +53,6 @@ pub async fn add_package(
let s9pk = S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
Some(&commitment),
false,
)
.await?;
@@ -109,7 +108,7 @@ pub async fn cli_add_package(
..
}: HandlerArgs<CliContext, CliAddPackageParams>,
) -> Result<(), Error> {
let s9pk = S9pk::open(&file, None, false).await?;
let s9pk = S9pk::open(&file, None).await?;
let mut progress = FullProgressTracker::new();
let progress_handle = progress.handle();
@@ -143,7 +142,6 @@ pub async fn cli_add_package(
let mut src = S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
Some(&commitment),
false,
)
.await?;
src.serialize(&mut TrackingIO::new(0, tokio::io::sink()), true)

5
core/startos/src/rpc_continuations.rs
View File

@@ -39,6 +39,11 @@ impl Guid {
Some(Guid(InternedString::intern(r)))
}
}
impl Default for Guid {
fn default() -> Self {
Self::new()
}
}
impl AsRef<str> for Guid {
fn as_ref(&self) -> &str {
self.0.as_ref()

5
core/startos/src/s9pk/merkle_archive/directory_contents.rs
View File

@@ -211,7 +211,10 @@ impl<S: FileSource + Clone> DirectoryContents<S> {
if !filter(path) {
if v.hash.is_none() {
return Err(Error::new(
eyre!("cannot filter out unhashed file, run `update_hashes` first"),
eyre!(
"cannot filter out unhashed file {}, run `update_hashes` first",
path.display()
),
ErrorKind::InvalidRequest,
));
}

103
core/startos/src/s9pk/merkle_archive/expected.rs Normal file
View File

@@ -0,0 +1,103 @@
use std::ffi::OsStr;
use std::path::Path;
use crate::prelude::*;
use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
use crate::s9pk::merkle_archive::source::FileSource;
use crate::s9pk::merkle_archive::Entry;
/// An object for tracking the files expected to be in an s9pk
pub struct Expected<'a, T> {
keep: DirectoryContents<()>,
dir: &'a DirectoryContents<T>,
}
impl<'a, T> Expected<'a, T> {
pub fn new(dir: &'a DirectoryContents<T>,) -> Self {
Self {
keep: DirectoryContents::new(),
dir
}
}
}
impl<'a, T: Clone> Expected<'a, T> {
pub fn check_file(&mut self, path: impl AsRef<Path>) -> Result<(), Error> {
if self
.dir
.get_path(path.as_ref())
.and_then(|e| e.as_file())
.is_some()
{
self.keep.insert_path(path, Entry::file(()))?;
Ok(())
} else {
Err(Error::new(
eyre!("file {} missing from archive", path.as_ref().display()),
ErrorKind::ParseS9pk,
))
}
}
pub fn check_stem(
&mut self,
path: impl AsRef<Path>,
mut valid_extension: impl FnMut(Option<&OsStr>) -> bool,
) -> Result<(), Error> {
let (dir, stem) = if let Some(parent) = path.as_ref().parent().filter(|p| *p != Path::new("")) {
(
self.dir
.get_path(parent)
.and_then(|e| e.as_directory())
.ok_or_else(|| {
Error::new(
eyre!("directory {} missing from archive", parent.display()),
ErrorKind::ParseS9pk,
)
})?,
path.as_ref().strip_prefix(parent).unwrap(),
)
} else {
(self.dir, path.as_ref())
};
let name = dir
.with_stem(&stem.as_os_str().to_string_lossy())
.filter(|(_, e)| e.as_file().is_some())
.try_fold(
Err(Error::new(
eyre!(
"file {} with valid extension missing from archive",
path.as_ref().display()
),
ErrorKind::ParseS9pk,
)),
|acc, (name, _)|
if valid_extension(Path::new(&*name).extension()) {
match acc {
Ok(_) => Err(Error::new(
eyre!(
"more than one file matching {} with valid extension in archive",
path.as_ref().display()
),
ErrorKind::ParseS9pk,
)),
Err(_) => Ok(Ok(name))
}
} else {
Ok(acc)
}
)??;
self.keep
.insert_path(path.as_ref().with_file_name(name), Entry::file(()))?;
Ok(())
}
pub fn into_filter(self) -> Filter {
Filter(self.keep)
}
}
pub struct Filter(DirectoryContents<()>);
impl Filter {
pub fn keep_checked<T: FileSource + Clone>(&self, dir: &mut DirectoryContents<T>) -> Result<(), Error> {
dir.filter(|path| self.0.get_path(path).is_some())
}
}

4
core/startos/src/s9pk/merkle_archive/mod.rs
View File

@@ -19,6 +19,7 @@ use crate::util::serde::Base64;
use crate::CAP_1_MiB;
pub mod directory_contents;
pub mod expected;
pub mod file_contents;
pub mod hash;
pub mod sink;
@@ -217,6 +218,9 @@ impl<S> Entry<S> {
pub fn file(source: S) -> Self {
Self::new(EntryContents::File(FileContents::new(source)))
}
pub fn directory(directory: DirectoryContents<S>) -> Self {
Self::new(EntryContents::Directory(directory))
}
pub fn hash(&self) -> Option<(Hash, u64)> {
self.hash
}

5
core/startos/src/s9pk/merkle_archive/source/mod.rs
View File

@@ -280,3 +280,8 @@ impl<S: ArchiveSource> FileSource for Section<S> {
self.source.copy_to(self.position, self.size, w).await
}
}
pub type DynRead = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
pub fn into_dyn_read<R: AsyncRead + Unpin + Send + Sync + 'static>(r: R) -> DynRead {
Box::new(r)
}

2
core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs
View File

@@ -97,7 +97,7 @@ impl ArchiveSource for MultiCursorFile {
.ok()
.map(|m| m.len())
}
async fn fetch_all(&self) -> Result<impl AsyncRead + Unpin + Send, Error> {
async fn fetch_all(&self) -> Result<impl AsyncRead + Unpin + Send + 'static, Error> {
use tokio::io::AsyncSeekExt;
let mut file = self.cursor().await?;

130
core/startos/src/s9pk/rpc.rs
View File

@@ -1,32 +1,26 @@
use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::path::PathBuf;
use clap::Parser;
use itertools::Itertools;
use models::ImageId;
use rpc_toolkit::{from_fn_async, Empty, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use tokio::fs::File;
use tokio::process::Command;
use ts_rs::TS;
use crate::context::CliContext;
use crate::prelude::*;
use crate::s9pk::manifest::Manifest;
use crate::s9pk::merkle_archive::source::DynFileSource;
use crate::s9pk::merkle_archive::Entry;
use crate::s9pk::v2::compat::CONTAINER_TOOL;
use crate::s9pk::v2::pack::ImageConfig;
use crate::s9pk::v2::SIG_CONTEXT;
use crate::s9pk::S9pk;
use crate::util::io::TmpDir;
use crate::util::serde::{apply_expr, HandlerExtSerde};
use crate::util::Invoke;
pub const SKIP_ENV: &[&str] = &["TERM", "container", "HOME", "HOSTNAME"];
pub fn s9pk() -> ParentHandler<CliContext> {
ParentHandler::new()
.subcommand("pack", from_fn_async(super::v2::pack::pack).no_display())
.subcommand("edit", edit())
.subcommand("inspect", inspect())
}
@@ -77,117 +71,21 @@ fn inspect() -> ParentHandler<CliContext, S9pkPath> {
#[derive(Deserialize, Serialize, Parser, TS)]
struct AddImageParams {
id: ImageId,
image: String,
arches: Option<Vec<String>>,
#[command(flatten)]
config: ImageConfig,
}
async fn add_image(
ctx: CliContext,
AddImageParams { id, image, arches }: AddImageParams,
AddImageParams { id, config }: AddImageParams,
S9pkPath { s9pk: s9pk_path }: S9pkPath,
) -> Result<(), Error> {
let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?, false)
let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?)
.await?
.into_dyn();
let arches: BTreeSet<_> = arches
.unwrap_or_else(|| vec!["x86_64".to_owned(), "aarch64".to_owned()])
.into_iter()
.collect();
s9pk.as_manifest_mut().images.insert(id, config);
let tmpdir = TmpDir::new().await?;
for arch in arches {
let sqfs_path = tmpdir.join(format!("image.{arch}.squashfs"));
let docker_platform = if arch == "x86_64" {
"--platform=linux/amd64".to_owned()
} else if arch == "aarch64" {
"--platform=linux/arm64".to_owned()
} else {
format!("--platform=linux/{arch}")
};
let env = String::from_utf8(
Command::new(CONTAINER_TOOL)
.arg("run")
.arg("--rm")
.arg(&docker_platform)
.arg("--entrypoint")
.arg("env")
.arg(&image)
.invoke(ErrorKind::Docker)
.await?,
)?
.lines()
.filter(|l| {
l.trim()
.split_once("=")
.map_or(false, |(v, _)| !SKIP_ENV.contains(&v))
})
.join("\n")
+ "\n";
let workdir = Path::new(
String::from_utf8(
Command::new(CONTAINER_TOOL)
.arg("run")
.arg(&docker_platform)
.arg("--rm")
.arg("--entrypoint")
.arg("pwd")
.arg(&image)
.invoke(ErrorKind::Docker)
.await?,
)?
.trim(),
)
.to_owned();
let container_id = String::from_utf8(
Command::new(CONTAINER_TOOL)
.arg("create")
.arg(&docker_platform)
.arg(&image)
.invoke(ErrorKind::Docker)
.await?,
)?;
Command::new("bash")
.arg("-c")
.arg(format!(
"{CONTAINER_TOOL} export {container_id} | mksquashfs - {sqfs} -tar",
container_id = container_id.trim(),
sqfs = sqfs_path.display()
))
.invoke(ErrorKind::Docker)
.await?;
Command::new(CONTAINER_TOOL)
.arg("rm")
.arg(container_id.trim())
.invoke(ErrorKind::Docker)
.await?;
let archive = s9pk.as_archive_mut();
archive.set_signer(ctx.developer_key()?.clone(), SIG_CONTEXT);
archive.contents_mut().insert_path(
Path::new("images")
.join(&arch)
.join(&id)
.with_extension("squashfs"),
Entry::file(DynFileSource::new(sqfs_path)),
)?;
archive.contents_mut().insert_path(
Path::new("images")
.join(&arch)
.join(&id)
.with_extension("env"),
Entry::file(DynFileSource::new(Arc::<[u8]>::from(Vec::from(env)))),
)?;
archive.contents_mut().insert_path(
Path::new("images")
.join(&arch)
.join(&id)
.with_extension("json"),
Entry::file(DynFileSource::new(Arc::<[u8]>::from(
serde_json::to_vec(&serde_json::json!({
"workdir": workdir
}))
.with_kind(ErrorKind::Serialization)?,
))),
)?;
}
s9pk.as_manifest_mut().images.insert(id);
s9pk.load_images(&tmpdir).await?;
s9pk.validate_and_filter(None)?;
let tmp_path = s9pk_path.with_extension("s9pk.tmp");
let mut tmp_file = File::create(&tmp_path).await?;
s9pk.serialize(&mut tmp_file, true).await?;
@@ -206,7 +104,7 @@ async fn edit_manifest(
EditManifestParams { expression }: EditManifestParams,
S9pkPath { s9pk: s9pk_path }: S9pkPath,
) -> Result<Manifest, Error> {
let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?, false).await?;
let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?).await?;
let old = serde_json::to_value(s9pk.as_manifest()).with_kind(ErrorKind::Serialization)?;
*s9pk.as_manifest_mut() = serde_json::from_value(apply_expr(old.into(), &expression)?.into())
.with_kind(ErrorKind::Serialization)?;
@@ -227,7 +125,7 @@ async fn file_tree(
_: Empty,
S9pkPath { s9pk }: S9pkPath,
) -> Result<Vec<PathBuf>, Error> {
let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?, false).await?;
let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?).await?;
Ok(s9pk.as_archive().contents().file_paths(""))
}
@@ -244,7 +142,7 @@ async fn cat(
) -> Result<(), Error> {
use crate::s9pk::merkle_archive::source::FileSource;
let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?, false).await?;
let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?).await?;
tokio::io::copy(
&mut s9pk
.as_archive()
@@ -266,6 +164,6 @@ async fn inspect_manifest(
_: Empty,
S9pkPath { s9pk }: S9pkPath,
) -> Result<Manifest, Error> {
let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?, false).await?;
let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?).await?;
Ok(s9pk.as_manifest().clone())
}

67
core/startos/src/s9pk/v2/compat.rs
View File

@@ -1,6 +1,5 @@
use std::collections::{BTreeMap, BTreeSet};
use std::io::Cursor;
use std::path::{Path, PathBuf};
use std::collections::BTreeMap;
use std::path::Path;
use std::sync::Arc;
use itertools::Itertools;
@@ -14,49 +13,18 @@ use crate::prelude::*;
use crate::s9pk::manifest::Manifest;
use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
use crate::s9pk::merkle_archive::source::{FileSource, Section};
use crate::s9pk::merkle_archive::source::Section;
use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
use crate::s9pk::rpc::SKIP_ENV;
use crate::s9pk::v1::manifest::{Manifest as ManifestV1, PackageProcedure};
use crate::s9pk::v1::reader::S9pkReader;
use crate::s9pk::v2::pack::{PackSource, CONTAINER_TOOL};
use crate::s9pk::v2::{S9pk, SIG_CONTEXT};
use crate::util::io::TmpDir;
use crate::util::Invoke;
pub const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x01];
#[cfg(not(feature = "docker"))]
pub const CONTAINER_TOOL: &str = "podman";
#[cfg(feature = "docker")]
pub const CONTAINER_TOOL: &str = "docker";
type DynRead = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
fn into_dyn_read<R: AsyncRead + Unpin + Send + Sync + 'static>(r: R) -> DynRead {
Box::new(r)
}
#[derive(Clone)]
enum CompatSource {
Buffered(Arc<[u8]>),
File(PathBuf),
}
impl FileSource for CompatSource {
type Reader = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
async fn size(&self) -> Result<u64, Error> {
match self {
Self::Buffered(a) => Ok(a.len() as u64),
Self::File(f) => Ok(tokio::fs::metadata(f).await?.len()),
}
}
async fn reader(&self) -> Result<Self::Reader, Error> {
match self {
Self::Buffered(a) => Ok(into_dyn_read(Cursor::new(a.clone()))),
Self::File(f) => Ok(into_dyn_read(File::open(f).await?)),
}
}
}
impl S9pk<Section<MultiCursorFile>> {
#[instrument(skip_all)]
pub async fn from_v1<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
@@ -66,7 +34,7 @@ impl S9pk<Section<MultiCursorFile>> {
) -> Result<Self, Error> {
let scratch_dir = TmpDir::new().await?;
let mut archive = DirectoryContents::<CompatSource>::new();
let mut archive = DirectoryContents::<PackSource>::new();
// manifest.json
let manifest_raw = reader.manifest().await?;
@@ -88,21 +56,21 @@ impl S9pk<Section<MultiCursorFile>> {
let license: Arc<[u8]> = reader.license().await?.to_vec().await?.into();
archive.insert_path(
"LICENSE.md",
Entry::file(CompatSource::Buffered(license.into())),
Entry::file(PackSource::Buffered(license.into())),
)?;
// instructions.md
let instructions: Arc<[u8]> = reader.instructions().await?.to_vec().await?.into();
archive.insert_path(
"instructions.md",
Entry::file(CompatSource::Buffered(instructions.into())),
Entry::file(PackSource::Buffered(instructions.into())),
)?;
// icon.md
let icon: Arc<[u8]> = reader.icon().await?.to_vec().await?.into();
archive.insert_path(
format!("icon.{}", manifest.assets.icon_type()),
Entry::file(CompatSource::Buffered(icon.into())),
Entry::file(PackSource::Buffered(icon.into())),
)?;
// images
@@ -122,7 +90,9 @@ impl S9pk<Section<MultiCursorFile>> {
.invoke(ErrorKind::Docker)
.await?;
for (image, system) in &images {
new_manifest.images.insert(image.clone());
let mut image_config = new_manifest.images.remove(image).unwrap_or_default();
image_config.arch.insert(arch.as_str().into());
new_manifest.images.insert(image.clone(), image_config);
let sqfs_path = images_dir.join(image).with_extension("squashfs");
let image_name = if *system {
format!("start9/{}:latest", image)
@@ -190,21 +160,21 @@ impl S9pk<Section<MultiCursorFile>> {
.join(&arch)
.join(&image)
.with_extension("squashfs"),
Entry::file(CompatSource::File(sqfs_path)),
Entry::file(PackSource::File(sqfs_path)),
)?;
archive.insert_path(
Path::new("images")
.join(&arch)
.join(&image)
.with_extension("env"),
Entry::file(CompatSource::Buffered(Vec::from(env).into())),
Entry::file(PackSource::Buffered(Vec::from(env).into())),
)?;
archive.insert_path(
Path::new("images")
.join(&arch)
.join(&image)
.with_extension("json"),
Entry::file(CompatSource::Buffered(
Entry::file(PackSource::Buffered(
serde_json::to_vec(&serde_json::json!({
"workdir": workdir
}))
@@ -240,7 +210,7 @@ impl S9pk<Section<MultiCursorFile>> {
.await?;
archive.insert_path(
Path::new("assets").join(&asset_id),
Entry::file(CompatSource::File(sqfs_path)),
Entry::file(PackSource::File(sqfs_path)),
)?;
}
@@ -267,12 +237,12 @@ impl S9pk<Section<MultiCursorFile>> {
.await?;
archive.insert_path(
Path::new("javascript.squashfs"),
Entry::file(CompatSource::File(sqfs_path)),
Entry::file(PackSource::File(sqfs_path)),
)?;
archive.insert_path(
"manifest.json",
Entry::file(CompatSource::Buffered(
Entry::file(PackSource::Buffered(
serde_json::to_vec::<Manifest>(&new_manifest)
.with_kind(ErrorKind::Serialization)?
.into(),
@@ -289,7 +259,6 @@ impl S9pk<Section<MultiCursorFile>> {
Ok(S9pk::deserialize(
&MultiCursorFile::from(File::open(destination.as_ref()).await?),
None,
false,
)
.await?)
}
@@ -310,7 +279,7 @@ impl From<ManifestV1> for Manifest {
marketing_site: value.marketing_site.unwrap_or_else(|| default_url.clone()),
donation_url: value.donation_url,
description: value.description,
images: BTreeSet::new(),
images: BTreeMap::new(),
assets: value
.volumes
.iter()

85
core/startos/src/s9pk/v2/manifest.rs
View File

@@ -1,10 +1,11 @@
use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use color_eyre::eyre::eyre;
use helpers::const_true;
use imbl_value::InternedString;
pub use models::PackageId;
use models::{ImageId, VolumeId};
use models::{mime, ImageId, VolumeId};
use serde::{Deserialize, Serialize};
use ts_rs::TS;
use url::Url;
@@ -12,6 +13,9 @@ use url::Url;
use crate::dependencies::Dependencies;
use crate::prelude::*;
use crate::s9pk::git_hash::GitHash;
use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
use crate::s9pk::merkle_archive::expected::{Expected, Filter};
use crate::s9pk::v2::pack::ImageConfig;
use crate::util::serde::Regex;
use crate::util::VersionString;
use crate::version::{Current, VersionT};
@@ -42,7 +46,7 @@ pub struct Manifest {
#[ts(type = "string | null")]
pub donation_url: Option<Url>,
pub description: Description,
pub images: BTreeSet<ImageId>,
pub images: BTreeMap<ImageId, ImageConfig>,
pub assets: BTreeSet<VolumeId>, // TODO: AssetsId
pub volumes: BTreeSet<VolumeId>,
#[serde(default)]
@@ -59,6 +63,83 @@ pub struct Manifest {
#[serde(default = "const_true")]
pub has_config: bool,
}
impl Manifest {
pub fn validate_for<'a, T: Clone>(
&self,
arch: Option<&str>,
archive: &'a DirectoryContents<T>,
) -> Result<Filter, Error> {
let mut expected = Expected::new(archive);
expected.check_file("manifest.json")?;
expected.check_stem("icon", |ext| {
ext.and_then(|e| e.to_str())
.and_then(mime)
.map_or(false, |mime| mime.starts_with("image/"))
})?;
expected.check_file("LICENSE.md")?;
expected.check_file("instructions.md")?;
expected.check_file("javascript.squashfs")?;
for assets in &self.assets {
expected.check_file(Path::new("assets").join(assets).with_extension("squashfs"))?;
}
for (image_id, config) in &self.images {
let mut check_arch = |arch: &str| {
let mut arch = arch;
if let Err(e) = expected.check_file(
Path::new("images")
.join(arch)
.join(image_id)
.with_extension("squashfs"),
) {
if let Some(emulate_as) = &config.emulate_missing_as {
expected.check_file(
Path::new("images")
.join(arch)
.join(image_id)
.with_extension("squashfs"),
)?;
arch = &**emulate_as;
} else {
return Err(e);
}
}
expected.check_file(
Path::new("images")
.join(arch)
.join(image_id)
.with_extension("json"),
)?;
expected.check_file(
Path::new("images")
.join(arch)
.join(image_id)
.with_extension("env"),
)?;
Ok(())
};
if let Some(arch) = arch {
check_arch(arch)?;
} else if let Some(arches) = &self.hardware_requirements.arch {
for arch in arches {
check_arch(arch)?;
}
} else if let Some(arch) = config.emulate_missing_as.as_deref() {
if !config.arch.contains(arch) {
return Err(Error::new(
eyre!("`emulateMissingAs` must match an included `arch`"),
ErrorKind::ParseS9pk,
));
}
for arch in &config.arch {
check_arch(&arch)?;
}
} else {
return Err(Error::new(eyre!("`emulateMissingAs` required for all images if no `arch` specified in `hardwareRequirements`"), ErrorKind::ParseS9pk));
}
}
Ok(expected.into_filter())
}
}
#[derive(Clone, Debug, Default, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]

76
core/startos/src/s9pk/v2/mod.rs
View File

@@ -14,7 +14,8 @@ use crate::s9pk::merkle_archive::sink::Sink;
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section};
use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
use crate::ARCH;
use crate::s9pk::v2::pack::{ImageSource, PackSource};
use crate::util::io::TmpDir;
const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x02];
@@ -22,6 +23,7 @@ pub const SIG_CONTEXT: &str = "s9pk";
pub mod compat;
pub mod manifest;
pub mod pack;
/**
/
@@ -34,10 +36,14 @@ pub mod manifest;
│ └── <id>.squashfs (xN)
└── images
└── <arch>
├── <id>.json (xN)
├── <id>.env (xN)
└── <id>.squashfs (xN)
*/
// this sorts the s9pk to optimize such that the parts that are used first appear earlier in the s9pk
// this is useful for manipulating an s9pk while partially downloaded on a source that does not support
// random access
fn priority(s: &str) -> Option<usize> {
match s {
"manifest.json" => Some(0),
@@ -51,26 +57,6 @@ fn priority(s: &str) -> Option<usize> {
}
}
fn filter(p: &Path) -> bool {
match p.iter().count() {
1 if p.file_name() == Some(OsStr::new("manifest.json")) => true,
1 if p.file_stem() == Some(OsStr::new("icon")) => true,
1 if p.file_name() == Some(OsStr::new("LICENSE.md")) => true,
1 if p.file_name() == Some(OsStr::new("instructions.md")) => true,
1 if p.file_name() == Some(OsStr::new("javascript.squashfs")) => true,
1 if p.file_name() == Some(OsStr::new("assets")) => true,
1 if p.file_name() == Some(OsStr::new("images")) => true,
2 if p.parent() == Some(Path::new("assets")) => {
p.extension().map_or(false, |ext| ext == "squashfs")
}
2 if p.parent() == Some(Path::new("images")) => p.file_name() == Some(OsStr::new(&*ARCH)),
3 if p.parent() == Some(&*Path::new("images").join(&*ARCH)) => p
.extension()
.map_or(false, |ext| ext == "squashfs" || ext == "env"),
_ => false,
}
}
#[derive(Clone)]
pub struct S9pk<S = Section<MultiCursorFile>> {
pub manifest: Manifest,
@@ -108,6 +94,11 @@ impl<S: FileSource + Clone> S9pk<S> {
})
}
pub fn validate_and_filter(&mut self, arch: Option<&str>) -> Result<(), Error> {
let filter = self.manifest.validate_for(arch, self.archive.contents())?;
filter.keep_checked(self.archive.contents_mut())
}
pub async fn icon(&self) -> Result<(InternedString, FileContents<S>), Error> {
let mut best_icon = None;
for (path, icon) in self
@@ -174,12 +165,37 @@ impl<S: FileSource + Clone> S9pk<S> {
}
}
impl<S: From<PackSource> + FileSource + Clone> S9pk<S> {
pub async fn load_images(&mut self, tmpdir: &TmpDir) -> Result<(), Error> {
let id = &self.manifest.id;
let version = &self.manifest.version;
for (image_id, image_config) in &mut self.manifest.images {
self.manifest_dirty = true;
for arch in &image_config.arch {
image_config
.source
.load(
tmpdir,
id,
version,
image_id,
arch,
self.archive.contents_mut(),
)
.await?;
}
image_config.source = ImageSource::Packed;
}
Ok(())
}
}
impl<S: ArchiveSource + Clone> S9pk<Section<S>> {
#[instrument(skip_all)]
pub async fn deserialize(
source: &S,
commitment: Option<&MerkleArchiveCommitment>,
apply_filter: bool,
) -> Result<Self, Error> {
use tokio::io::AsyncReadExt;
@@ -201,10 +217,6 @@ impl<S: ArchiveSource + Clone> S9pk<Section<S>> {
let mut archive =
MerkleArchive::deserialize(source, SIG_CONTEXT, &mut header, commitment).await?;
if apply_filter {
archive.filter(filter)?;
}
archive.sort_by(|a, b| match (priority(a), priority(b)) {
(Some(a), Some(b)) => a.cmp(&b),
(Some(_), None) => std::cmp::Ordering::Less,
@@ -216,15 +228,11 @@ impl<S: ArchiveSource + Clone> S9pk<Section<S>> {
}
}
impl S9pk {
pub async fn from_file(file: File, apply_filter: bool) -> Result<Self, Error> {
Self::deserialize(&MultiCursorFile::from(file), None, apply_filter).await
pub async fn from_file(file: File) -> Result<Self, Error> {
Self::deserialize(&MultiCursorFile::from(file), None).await
}
pub async fn open(
path: impl AsRef<Path>,
id: Option<&PackageId>,
apply_filter: bool,
) -> Result<Self, Error> {
let res = Self::from_file(tokio::fs::File::open(path).await?, apply_filter).await?;
pub async fn open(path: impl AsRef<Path>, id: Option<&PackageId>) -> Result<Self, Error> {
let res = Self::from_file(tokio::fs::File::open(path).await?).await?;
if let Some(id) = id {
ensure_code!(
&res.as_manifest().id == id,

536
core/startos/src/s9pk/v2/pack.rs Normal file
View File

@@ -0,0 +1,536 @@
use std::collections::BTreeSet;
use std::ffi::OsStr;
use std::io::Cursor;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use clap::Parser;
use futures::future::{ready, BoxFuture};
use futures::{FutureExt, TryStreamExt};
use imbl_value::InternedString;
use models::{ImageId, PackageId, VersionString};
use serde::{Deserialize, Serialize};
use tokio::fs::File;
use tokio::io::AsyncRead;
use tokio::process::Command;
use tokio::sync::OnceCell;
use tokio_stream::wrappers::ReadDirStream;
use ts_rs::TS;
use crate::context::CliContext;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
use crate::s9pk::merkle_archive::source::{
into_dyn_read, ArchiveSource, DynFileSource, FileSource,
};
use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
use crate::s9pk::v2::SIG_CONTEXT;
use crate::s9pk::S9pk;
use crate::util::io::TmpDir;
use crate::util::Invoke;
#[cfg(not(feature = "docker"))]
pub const CONTAINER_TOOL: &str = "podman";
#[cfg(feature = "docker")]
pub const CONTAINER_TOOL: &str = "docker";
pub struct SqfsDir {
path: PathBuf,
tmpdir: Arc<TmpDir>,
sqfs: OnceCell<MultiCursorFile>,
}
impl SqfsDir {
pub fn new(path: PathBuf, tmpdir: Arc<TmpDir>) -> Self {
Self {
path,
tmpdir,
sqfs: OnceCell::new(),
}
}
async fn file(&self) -> Result<&MultiCursorFile, Error> {
self.sqfs
.get_or_try_init(|| async move {
let guid = Guid::new();
let path = self.tmpdir.join(guid.as_ref()).with_extension("squashfs");
let mut cmd = Command::new("mksquashfs");
if self.path.extension().and_then(|s| s.to_str()) == Some("tar") {
cmd.arg("-tar");
}
cmd.arg(&self.path)
.arg(&path)
.invoke(ErrorKind::Filesystem)
.await?;
Ok(MultiCursorFile::from(
File::open(&path)
.await
.with_ctx(|_| (ErrorKind::Filesystem, path.display()))?,
))
})
.await
}
}
#[derive(Clone)]
pub enum PackSource {
Buffered(Arc<[u8]>),
File(PathBuf),
Squashfs(Arc<SqfsDir>),
}
impl FileSource for PackSource {
type Reader = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
async fn size(&self) -> Result<u64, Error> {
match self {
Self::Buffered(a) => Ok(a.len() as u64),
Self::File(f) => Ok(tokio::fs::metadata(f)
.await
.with_ctx(|_| (ErrorKind::Filesystem, f.display()))?
.len()),
Self::Squashfs(dir) => dir
.file()
.await
.with_ctx(|_| (ErrorKind::Filesystem, dir.path.display()))?
.size()
.await
.or_not_found("file metadata"),
}
}
async fn reader(&self) -> Result<Self::Reader, Error> {
match self {
Self::Buffered(a) => Ok(into_dyn_read(Cursor::new(a.clone()))),
Self::File(f) => Ok(into_dyn_read(
File::open(f)
.await
.with_ctx(|_| (ErrorKind::Filesystem, f.display()))?,
)),
Self::Squashfs(dir) => dir.file().await?.fetch_all().await.map(into_dyn_read),
}
}
}
impl From<PackSource> for DynFileSource {
fn from(value: PackSource) -> Self {
DynFileSource::new(value)
}
}
#[derive(Deserialize, Serialize, Parser)]
pub struct PackParams {
pub path: Option<PathBuf>,
#[arg(short = 'o', long = "output")]
pub output: Option<PathBuf>,
#[arg(long = "javascript")]
pub javascript: Option<PathBuf>,
#[arg(long = "icon")]
pub icon: Option<PathBuf>,
#[arg(long = "license")]
pub license: Option<PathBuf>,
#[arg(long = "instructions")]
pub instructions: Option<PathBuf>,
#[arg(long = "assets")]
pub assets: Option<PathBuf>,
}
impl PackParams {
fn path(&self) -> &Path {
self.path.as_deref().unwrap_or(Path::new("."))
}
fn output(&self, id: &PackageId) -> PathBuf {
self.output
.as_ref()
.cloned()
.unwrap_or_else(|| self.path().join(id).with_extension("s9pk"))
}
fn javascript(&self) -> PathBuf {
self.javascript
.as_ref()
.cloned()
.unwrap_or_else(|| self.path().join("javascript"))
}
async fn icon(&self) -> Result<PathBuf, Error> {
if let Some(icon) = &self.icon {
Ok(icon.clone())
} else {
ReadDirStream::new(tokio::fs::read_dir(self.path()).await?).try_filter(|x| ready(x.path().file_stem() == Some(OsStr::new("icon")))).map_err(Error::from).try_fold(Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)), |acc, x| async move { match acc {
Ok(_) => Err(Error::new(eyre!("multiple icons found in working directory, please specify which to use with `--icon`"), ErrorKind::InvalidRequest)),
Err(e) => Ok({
let path = x.path();
if path.file_stem().and_then(|s| s.to_str()) == Some("icon") {
Ok(path)
} else {
Err(e)
}
})
}}).await?
}
}
fn license(&self) -> PathBuf {
self.license
.as_ref()
.cloned()
.unwrap_or_else(|| self.path().join("LICENSE.md"))
}
fn instructions(&self) -> PathBuf {
self.instructions
.as_ref()
.cloned()
.unwrap_or_else(|| self.path().join("instructions.md"))
}
fn assets(&self) -> PathBuf {
self.assets
.as_ref()
.cloned()
.unwrap_or_else(|| self.path().join("assets"))
}
}
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct ImageConfig {
pub source: ImageSource,
#[ts(type = "string[]")]
pub arch: BTreeSet<InternedString>,
#[ts(type = "string | null")]
pub emulate_missing_as: Option<InternedString>,
}
impl Default for ImageConfig {
fn default() -> Self {
Self {
source: ImageSource::Packed,
arch: BTreeSet::new(),
emulate_missing_as: None,
}
}
}
#[derive(Parser)]
struct CliImageConfig {
#[arg(long, conflicts_with("docker-tag"))]
docker_build: bool,
#[arg(long, requires("docker-build"))]
dockerfile: Option<PathBuf>,
#[arg(long, requires("docker-build"))]
workdir: Option<PathBuf>,
#[arg(long, conflicts_with_all(["dockerfile", "workdir"]))]
docker_tag: Option<String>,
#[arg(long)]
arch: Vec<InternedString>,
#[arg(long)]
emulate_missing_as: Option<InternedString>,
}
impl TryFrom<CliImageConfig> for ImageConfig {
type Error = clap::Error;
fn try_from(value: CliImageConfig) -> Result<Self, Self::Error> {
let res = Self {
source: if value.docker_build {
ImageSource::DockerBuild {
dockerfile: value.dockerfile,
workdir: value.workdir,
}
} else if let Some(tag) = value.docker_tag {
ImageSource::DockerTag(tag)
} else {
ImageSource::Packed
},
arch: value.arch.into_iter().collect(),
emulate_missing_as: value.emulate_missing_as,
};
res.emulate_missing_as
.as_ref()
.map(|a| {
if !res.arch.contains(a) {
Err(clap::Error::raw(
clap::error::ErrorKind::InvalidValue,
"`emulate-missing-as` must match one of the provided `arch`es",
))
} else {
Ok(())
}
})
.transpose()?;
Ok(res)
}
}
impl clap::Args for ImageConfig {
fn augment_args(cmd: clap::Command) -> clap::Command {
CliImageConfig::augment_args(cmd)
}
fn augment_args_for_update(cmd: clap::Command) -> clap::Command {
CliImageConfig::augment_args_for_update(cmd)
}
}
impl clap::FromArgMatches for ImageConfig {
fn from_arg_matches(matches: &clap::ArgMatches) -> Result<Self, clap::Error> {
Self::try_from(CliImageConfig::from_arg_matches(matches)?)
}
fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> {
*self = Self::try_from(CliImageConfig::from_arg_matches(matches)?)?;
Ok(())
}
}
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub enum ImageSource {
Packed,
#[serde(rename_all = "camelCase")]
DockerBuild {
workdir: Option<PathBuf>,
dockerfile: Option<PathBuf>,
},
DockerTag(String),
}
impl ImageSource {
#[instrument(skip_all)]
pub fn load<'a, S: From<PackSource> + FileSource + Clone>(
&'a self,
tmpdir: &'a TmpDir,
id: &'a PackageId,
version: &'a VersionString,
image_id: &'a ImageId,
arch: &'a str,
into: &'a mut DirectoryContents<S>,
) -> BoxFuture<'a, Result<(), Error>> {
#[derive(Deserialize)]
#[serde(rename_all = "PascalCase")]
struct DockerImageConfig {
env: Vec<String>,
#[serde(default)]
working_dir: PathBuf,
#[serde(default)]
user: String,
}
async move {
match self {
ImageSource::Packed => Ok(()),
ImageSource::DockerBuild {
workdir,
dockerfile,
} => {
let workdir = workdir.as_deref().unwrap_or(Path::new("."));
let dockerfile = dockerfile
.clone()
.unwrap_or_else(|| workdir.join("Dockerfile"));
let docker_platform = if arch == "x86_64" {
"--platform=linux/amd64".to_owned()
} else if arch == "aarch64" {
"--platform=linux/arm64".to_owned()
} else {
format!("--platform=linux/{arch}")
};
// docker buildx build ${path} -o type=image,name=start9/${id}
let tag = format!("start9/{id}/{image_id}:{version}");
Command::new(CONTAINER_TOOL)
.arg("build")
.arg(workdir)
.arg("-f")
.arg(dockerfile)
.arg("-t")
.arg(&tag)
.arg(&docker_platform)
.arg("-o")
.arg("type=image")
.capture(false)
.invoke(ErrorKind::Docker)
.await?;
ImageSource::DockerTag(tag.clone())
.load(tmpdir, id, version, image_id, arch, into)
.await?;
Command::new(CONTAINER_TOOL)
.arg("rmi")
.arg("-f")
.arg(&tag)
.invoke(ErrorKind::Docker)
.await?;
Ok(())
}
ImageSource::DockerTag(tag) => {
let docker_platform = if arch == "x86_64" {
"--platform=linux/amd64".to_owned()
} else if arch == "aarch64" {
"--platform=linux/arm64".to_owned()
} else {
format!("--platform=linux/{arch}")
};
let mut inspect_cmd = Command::new(CONTAINER_TOOL);
inspect_cmd
.arg("image")
.arg("inspect")
.arg("--format")
.arg("{{json .Config}}")
.arg(&tag);
let inspect_res = match inspect_cmd.invoke(ErrorKind::Docker).await {
Ok(a) => a,
Err(e)
if {
let msg = e.source.to_string();
#[cfg(feature = "docker")]
let matches = msg.contains("No such image:");
#[cfg(not(feature = "docker"))]
let matches = msg.contains(": image not known");
matches
} =>
{
Command::new(CONTAINER_TOOL)
.arg("pull")
.arg(&docker_platform)
.arg(tag)
.capture(false)
.invoke(ErrorKind::Docker)
.await?;
inspect_cmd.invoke(ErrorKind::Docker).await?
}
Err(e) => return Err(e),
};
let config = serde_json::from_slice::<DockerImageConfig>(&inspect_res)
.with_kind(ErrorKind::Deserialization)?;
let base_path = Path::new("images").join(arch).join(image_id);
into.insert_path(
base_path.with_extension("json"),
Entry::file(
PackSource::Buffered(
serde_json::to_vec(&ImageMetadata {
workdir: if config.working_dir == Path::new("") {
"/".into()
} else {
config.working_dir
},
user: if config.user.is_empty() {
"root".into()
} else {
config.user.into()
},
})
.with_kind(ErrorKind::Serialization)?
.into(),
)
.into(),
),
)?;
into.insert_path(
base_path.with_extension("env"),
Entry::file(
PackSource::Buffered(config.env.join("\n").into_bytes().into()).into(),
),
)?;
let dest = tmpdir.join(Guid::new().as_ref()).with_extension("squashfs");
let container = String::from_utf8(
Command::new(CONTAINER_TOOL)
.arg("create")
.arg(&docker_platform)
.arg(&tag)
.invoke(ErrorKind::Docker)
.await?,
)?;
Command::new(CONTAINER_TOOL)
.arg("export")
.arg(container.trim())
.pipe(Command::new("mksquashfs").arg("-").arg(&dest).arg("-tar"))
.capture(false)
.invoke(ErrorKind::Docker)
.await?;
Command::new(CONTAINER_TOOL)
.arg("rm")
.arg(container.trim())
.invoke(ErrorKind::Docker)
.await?;
into.insert_path(
base_path.with_extension("squashfs"),
Entry::file(PackSource::File(dest).into()),
)?;
Ok(())
}
}
}
.boxed()
}
}
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct ImageMetadata {
pub workdir: PathBuf,
#[ts(type = "string")]
pub user: InternedString,
}
#[instrument(skip_all)]
pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
let tmpdir = Arc::new(TmpDir::new().await?);
let mut files = DirectoryContents::<PackSource>::new();
let js_dir = params.javascript();
let manifest: Arc<[u8]> = Command::new("node")
.arg("-e")
.arg(format!(
"console.log(JSON.stringify(require('{}/index.js').manifest))",
js_dir.display()
))
.invoke(ErrorKind::Javascript)
.await?
.into();
files.insert(
"manifest.json".into(),
Entry::file(PackSource::Buffered(manifest.clone())),
);
let icon = params.icon().await?;
let icon_ext = icon
.extension()
.or_not_found("icon file extension")?
.to_string_lossy();
files.insert(
InternedString::from_display(&lazy_format!("icon.{}", icon_ext)),
Entry::file(PackSource::File(icon)),
);
files.insert(
"LICENSE.md".into(),
Entry::file(PackSource::File(params.license())),
);
files.insert(
"instructions.md".into(),
Entry::file(PackSource::File(params.instructions())),
);
files.insert(
"javascript.squashfs".into(),
Entry::file(PackSource::Squashfs(Arc::new(SqfsDir::new(
js_dir,
tmpdir.clone(),
)))),
);
let mut s9pk = S9pk::new(
MerkleArchive::new(files, ctx.developer_key()?.clone(), SIG_CONTEXT),
None,
)
.await?;
let assets_dir = params.assets();
for assets in s9pk.as_manifest().assets.clone() {
s9pk.as_archive_mut().contents_mut().insert_path(
Path::new("assets").join(&assets).with_extension("squashfs"),
Entry::file(PackSource::Squashfs(Arc::new(SqfsDir::new(
assets_dir.join(&assets),
tmpdir.clone(),
)))),
)?;
}
s9pk.load_images(&*tmpdir).await?;
s9pk.validate_and_filter(None)?;
s9pk.serialize(
&mut File::create(params.output(&s9pk.as_manifest().id)).await?,
false,
)
.await?;
drop(s9pk);
tmpdir.gc().await?;
Ok(())
}

27
core/startos/src/service/action.rs
View File

@@ -4,6 +4,7 @@ use models::{ActionId, ProcedureName};
use crate::action::ActionResult;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::config::GetConfig;
use crate::service::dependencies::DependencyConfig;
use crate::service::{Service, ServiceActor};
@@ -23,13 +24,18 @@ impl Handler<Action> for ServiceActor {
}
async fn handle(
&mut self,
Action { id, input }: Action,
id: Guid,
Action {
id: action_id,
input,
}: Action,
_: &BackgroundJobQueue,
) -> Self::Response {
let container = &self.0.persistent_container;
container
.execute::<ActionResult>(
ProcedureName::RunAction(id),
id,
ProcedureName::RunAction(action_id),
input,
Some(Duration::from_secs(30)),
)
@@ -39,7 +45,20 @@ impl Handler<Action> for ServiceActor {
}
impl Service {
pub async fn action(&self, id: ActionId, input: Value) -> Result<ActionResult, Error> {
self.actor.send(Action { id, input }).await?
pub async fn action(
&self,
id: Guid,
action_id: ActionId,
input: Value,
) -> Result<ActionResult, Error> {
self.actor
.send(
id,
Action {
id: action_id,
input,
},
)
.await?
}
}

15
core/startos/src/service/config.rs
View File

@@ -5,6 +5,7 @@ use models::ProcedureName;
use crate::config::action::ConfigRes;
use crate::config::ConfigureContext;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::dependencies::DependencyConfig;
use crate::service::{Service, ServiceActor};
use crate::util::actor::background::BackgroundJobQueue;
@@ -19,6 +20,7 @@ impl Handler<Configure> for ServiceActor {
}
async fn handle(
&mut self,
id: Guid,
Configure(ConfigureContext { timeout, config }): Configure,
_: &BackgroundJobQueue,
) -> Self::Response {
@@ -26,7 +28,7 @@ impl Handler<Configure> for ServiceActor {
let package_id = &self.0.id;
container
.execute::<NoOutput>(ProcedureName::SetConfig, to_value(&config)?, timeout)
.execute::<NoOutput>(id, ProcedureName::SetConfig, to_value(&config)?, timeout)
.await
.with_kind(ErrorKind::ConfigRulesViolation)?;
self.0
@@ -52,10 +54,11 @@ impl Handler<GetConfig> for ServiceActor {
fn conflicts_with(_: &GetConfig) -> ConflictBuilder<Self> {
ConflictBuilder::nothing().except::<Configure>()
}
async fn handle(&mut self, _: GetConfig, _: &BackgroundJobQueue) -> Self::Response {
async fn handle(&mut self, id: Guid, _: GetConfig, _: &BackgroundJobQueue) -> Self::Response {
let container = &self.0.persistent_container;
container
.execute::<ConfigRes>(
id,
ProcedureName::GetConfig,
Value::Null,
Some(Duration::from_secs(30)), // TODO timeout
@@ -66,10 +69,10 @@ impl Handler<GetConfig> for ServiceActor {
}
impl Service {
pub async fn configure(&self, ctx: ConfigureContext) -> Result<(), Error> {
self.actor.send(Configure(ctx)).await?
pub async fn configure(&self, id: Guid, ctx: ConfigureContext) -> Result<(), Error> {
self.actor.send(id, Configure(ctx)).await?
}
pub async fn get_config(&self) -> Result<ConfigRes, Error> {
self.actor.send(GetConfig).await?
pub async fn get_config(&self, id: Guid) -> Result<ConfigRes, Error> {
self.actor.send(id, GetConfig).await?
}
}

13
core/startos/src/service/control.rs
View File

@@ -1,4 +1,5 @@
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::config::GetConfig;
use crate::service::dependencies::DependencyConfig;
use crate::service::start_stop::StartStop;
@@ -15,7 +16,7 @@ impl Handler<Start> for ServiceActor {
.except::<GetConfig>()
.except::<DependencyConfig>()
}
async fn handle(&mut self, _: Start, _: &BackgroundJobQueue) -> Self::Response {
async fn handle(&mut self, _: Guid, _: Start, _: &BackgroundJobQueue) -> Self::Response {
self.0.persistent_container.state.send_modify(|x| {
x.desired_state = StartStop::Start;
});
@@ -23,8 +24,8 @@ impl Handler<Start> for ServiceActor {
}
}
impl Service {
pub async fn start(&self) -> Result<(), Error> {
self.actor.send(Start).await
pub async fn start(&self, id: Guid) -> Result<(), Error> {
self.actor.send(id, Start).await
}
}
@@ -36,7 +37,7 @@ impl Handler<Stop> for ServiceActor {
.except::<GetConfig>()
.except::<DependencyConfig>()
}
async fn handle(&mut self, _: Stop, _: &BackgroundJobQueue) -> Self::Response {
async fn handle(&mut self, _: Guid, _: Stop, _: &BackgroundJobQueue) -> Self::Response {
let mut transition_state = None;
self.0.persistent_container.state.send_modify(|x| {
x.desired_state = StartStop::Stop;
@@ -51,7 +52,7 @@ impl Handler<Stop> for ServiceActor {
}
}
impl Service {
pub async fn stop(&self) -> Result<(), Error> {
self.actor.send(Stop).await
pub async fn stop(&self, id: Guid) -> Result<(), Error> {
self.actor.send(id, Stop).await
}
}

67
core/startos/src/service/dependencies.rs
View File

@@ -4,35 +4,28 @@ use imbl_value::json;
use models::{PackageId, ProcedureName};
use crate::prelude::*;
use crate::service::{Service, ServiceActor};
use crate::rpc_continuations::Guid;
use crate::service::{Service, ServiceActor, ServiceActorSeed};
use crate::util::actor::background::BackgroundJobQueue;
use crate::util::actor::{ConflictBuilder, Handler};
use crate::Config;
pub(super) struct DependencyConfig {
dependency_id: PackageId,
remote_config: Option<Config>,
}
impl Handler<DependencyConfig> for ServiceActor {
type Response = Result<Option<Config>, Error>;
fn conflicts_with(_: &DependencyConfig) -> ConflictBuilder<Self> {
ConflictBuilder::nothing()
}
async fn handle(
&mut self,
DependencyConfig {
dependency_id,
remote_config,
}: DependencyConfig,
_: &BackgroundJobQueue,
) -> Self::Response {
let container = &self.0.persistent_container;
impl ServiceActorSeed {
async fn dependency_config(
&self,
id: Guid,
dependency_id: PackageId,
remote_config: Option<Config>,
) -> Result<Option<Config>, Error> {
let container = &self.persistent_container;
container
.sanboxed::<Option<Config>>(
id.clone(),
ProcedureName::UpdateDependency(dependency_id.clone()),
json!({
"queryResults": container
.execute::<Value>(
id,
ProcedureName::QueryDependency(dependency_id),
Value::Null,
Some(Duration::from_secs(30)),
@@ -49,17 +42,45 @@ impl Handler<DependencyConfig> for ServiceActor {
}
}
pub(super) struct DependencyConfig {
dependency_id: PackageId,
remote_config: Option<Config>,
}
impl Handler<DependencyConfig> for ServiceActor {
type Response = Result<Option<Config>, Error>;
fn conflicts_with(_: &DependencyConfig) -> ConflictBuilder<Self> {
ConflictBuilder::nothing()
}
async fn handle(
&mut self,
id: Guid,
DependencyConfig {
dependency_id,
remote_config,
}: DependencyConfig,
_: &BackgroundJobQueue,
) -> Self::Response {
self.0
.dependency_config(id, dependency_id, remote_config)
.await
}
}
impl Service {
pub async fn dependency_config(
&self,
id: Guid,
dependency_id: PackageId,
remote_config: Option<Config>,
) -> Result<Option<Config>, Error> {
self.actor
.send(DependencyConfig {
dependency_id,
remote_config,
})
.send(
id,
DependencyConfig {
dependency_id,
remote_config,
},
)
.await?
}
}

179
core/startos/src/service/mod.rs
View File

@@ -1,4 +1,5 @@
use std::sync::Arc;
use std::ops::Deref;
use std::sync::{Arc, Weak};
use std::time::Duration;
use chrono::{DateTime, Utc};
@@ -68,13 +69,87 @@ pub enum LoadDisposition {
Undo,
}
pub struct ServiceRef(Arc<Service>);
impl ServiceRef {
pub fn weak(&self) -> Weak<Service> {
Arc::downgrade(&self.0)
}
pub async fn uninstall(
self,
target_version: Option<models::VersionString>,
) -> Result<(), Error> {
self.seed
.persistent_container
.execute(
Guid::new(),
ProcedureName::Uninit,
to_value(&target_version)?,
None,
) // TODO timeout
.await?;
let id = self.seed.persistent_container.s9pk.as_manifest().id.clone();
let ctx = self.seed.ctx.clone();
self.shutdown().await?;
if target_version.is_none() {
ctx.db
.mutate(|d| d.as_public_mut().as_package_data_mut().remove(&id))
.await?;
}
Ok(())
}
pub async fn shutdown(self) -> Result<(), Error> {
if let Some((hdl, shutdown)) = self.seed.persistent_container.rpc_server.send_replace(None)
{
self.seed
.persistent_container
.rpc_client
.request(rpc::Exit, Empty {})
.await?;
shutdown.shutdown();
hdl.await.with_kind(ErrorKind::Cancelled)?;
}
let service = Arc::try_unwrap(self.0).map_err(|_| {
Error::new(
eyre!("ServiceActor held somewhere after actor shutdown"),
ErrorKind::Unknown,
)
})?;
service
.actor
.shutdown(crate::util::actor::PendingMessageStrategy::FinishAll { timeout: None }) // TODO timeout
.await;
Arc::try_unwrap(service.seed)
.map_err(|_| {
Error::new(
eyre!("ServiceActorSeed held somewhere after actor shutdown"),
ErrorKind::Unknown,
)
})?
.persistent_container
.exit()
.await?;
Ok(())
}
}
impl Deref for ServiceRef {
type Target = Service;
fn deref(&self) -> &Self::Target {
&*self.0
}
}
impl From<Service> for ServiceRef {
fn from(value: Service) -> Self {
Self(Arc::new(value))
}
}
pub struct Service {
actor: ConcurrentActor<ServiceActor>,
seed: Arc<ServiceActorSeed>,
}
impl Service {
#[instrument(skip_all)]
async fn new(ctx: RpcContext, s9pk: S9pk, start: StartStop) -> Result<Self, Error> {
async fn new(ctx: RpcContext, s9pk: S9pk, start: StartStop) -> Result<ServiceRef, Error> {
let id = s9pk.as_manifest().id.clone();
let persistent_container = PersistentContainer::new(
&ctx, s9pk,
@@ -89,13 +164,17 @@ impl Service {
ctx,
synchronized: Arc::new(Notify::new()),
});
seed.persistent_container
.init(Arc::downgrade(&seed))
.await?;
Ok(Self {
let service: ServiceRef = Self {
actor: ConcurrentActor::new(ServiceActor(seed.clone())),
seed,
})
}
.into();
service
.seed
.persistent_container
.init(service.weak())
.await?;
Ok(service)
}
#[instrument(skip_all)]
@@ -103,7 +182,7 @@ impl Service {
ctx: &RpcContext,
id: &PackageId,
disposition: LoadDisposition,
) -> Result<Option<Self>, Error> {
) -> Result<Option<ServiceRef>, Error> {
let handle_installed = {
let ctx = ctx.clone();
move |s9pk: S9pk, i: Model<PackageDataEntry>| async move {
@@ -137,7 +216,7 @@ impl Service {
match entry.as_state_info().as_match() {
PackageStateMatchModelRef::Installing(_) => {
if disposition == LoadDisposition::Retry {
if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id), true).await.map_err(|e| {
if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id)).await.map_err(|e| {
tracing::error!("Error opening s9pk for install: {e}");
tracing::debug!("{e:?}")
}) {
@@ -170,7 +249,7 @@ impl Service {
&& progress == &Progress::Complete(true)
})
{
if let Ok(s9pk) = S9pk::open(&s9pk_path, Some(id), true).await.map_err(|e| {
if let Ok(s9pk) = S9pk::open(&s9pk_path, Some(id)).await.map_err(|e| {
tracing::error!("Error opening s9pk for update: {e}");
tracing::debug!("{e:?}")
}) {
@@ -189,7 +268,7 @@ impl Service {
}
}
}
let s9pk = S9pk::open(s9pk_path, Some(id), true).await?;
let s9pk = S9pk::open(s9pk_path, Some(id)).await?;
ctx.db
.mutate({
|db| {
@@ -214,7 +293,7 @@ impl Service {
handle_installed(s9pk, entry).await
}
PackageStateMatchModelRef::Removing(_) | PackageStateMatchModelRef::Restoring(_) => {
if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id), true).await.map_err(|e| {
if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id)).await.map_err(|e| {
tracing::error!("Error opening s9pk for removal: {e}");
tracing::debug!("{e:?}")
}) {
@@ -225,7 +304,7 @@ impl Service {
tracing::debug!("{e:?}")
})
{
match service.uninstall(None).await {
match ServiceRef::from(service).uninstall(None).await {
Err(e) => {
tracing::error!("Error uninstalling service: {e}");
tracing::debug!("{e:?}")
@@ -242,7 +321,7 @@ impl Service {
Ok(None)
}
PackageStateMatchModelRef::Installed(_) => {
handle_installed(S9pk::open(s9pk_path, Some(id), true).await?, entry).await
handle_installed(S9pk::open(s9pk_path, Some(id)).await?, entry).await
}
PackageStateMatchModelRef::Error(e) => Err(Error::new(
eyre!("Failed to parse PackageDataEntry, found {e:?}"),
@@ -257,7 +336,7 @@ impl Service {
s9pk: S9pk,
src_version: Option<models::VersionString>,
progress: Option<InstallProgressHandles>,
) -> Result<Self, Error> {
) -> Result<ServiceRef, Error> {
let manifest = s9pk.as_manifest().clone();
let developer_key = s9pk.as_archive().signer();
let icon = s9pk.icon_data_url().await?;
@@ -265,7 +344,12 @@ impl Service {
service
.seed
.persistent_container
.execute(ProcedureName::Init, to_value(&src_version)?, None) // TODO timeout
.execute(
Guid::new(),
ProcedureName::Init,
to_value(&src_version)?,
None,
) // TODO timeout
.await
.with_kind(ErrorKind::MigrationFailed)?; // TODO: handle cancellation
if let Some(mut progress) = progress {
@@ -301,61 +385,21 @@ impl Service {
s9pk: S9pk,
backup_source: impl GenericMountGuard,
progress: Option<InstallProgressHandles>,
) -> Result<Self, Error> {
) -> Result<ServiceRef, Error> {
let service = Service::install(ctx.clone(), s9pk, None, progress).await?;
service
.actor
.send(transition::restore::Restore {
path: backup_source.path().to_path_buf(),
})
.send(
Guid::new(),
transition::restore::Restore {
path: backup_source.path().to_path_buf(),
},
)
.await??;
Ok(service)
}
pub async fn shutdown(self) -> Result<(), Error> {
self.actor
.shutdown(crate::util::actor::PendingMessageStrategy::FinishAll { timeout: None }) // TODO timeout
.await;
if let Some((hdl, shutdown)) = self.seed.persistent_container.rpc_server.send_replace(None)
{
self.seed
.persistent_container
.rpc_client
.request(rpc::Exit, Empty {})
.await?;
shutdown.shutdown();
hdl.await.with_kind(ErrorKind::Cancelled)?;
}
Arc::try_unwrap(self.seed)
.map_err(|_| {
Error::new(
eyre!("ServiceActorSeed held somewhere after actor shutdown"),
ErrorKind::Unknown,
)
})?
.persistent_container
.exit()
.await?;
Ok(())
}
pub async fn uninstall(self, target_version: Option<models::VersionString>) -> Result<(), Error> {
self.seed
.persistent_container
.execute(ProcedureName::Uninit, to_value(&target_version)?, None) // TODO timeout
.await?;
let id = self.seed.persistent_container.s9pk.as_manifest().id.clone();
let ctx = self.seed.ctx.clone();
self.shutdown().await?;
if target_version.is_none() {
ctx.db
.mutate(|d| d.as_public_mut().as_package_data_mut().remove(&id))
.await?;
}
Ok(())
}
#[instrument(skip_all)]
pub async fn backup(&self, guard: impl GenericMountGuard) -> Result<(), Error> {
let id = &self.seed.id;
@@ -368,9 +412,12 @@ impl Service {
.await?;
drop(file);
self.actor
.send(transition::backup::Backup {
path: guard.path().to_path_buf(),
})
.send(
Guid::new(),
transition::backup::Backup {
path: guard.path().to_path_buf(),
},
)
.await??;
Ok(())
}

97
core/startos/src/service/persistent_container.rs
View File

@@ -6,8 +6,7 @@ use std::time::Duration;
use futures::future::ready;
use futures::{Future, FutureExt};
use helpers::NonDetachingJoinHandle;
use imbl_value::InternedString;
use models::{ProcedureName, VolumeId};
use models::{ImageId, ProcedureName, VolumeId};
use rpc_toolkit::{Empty, Server, ShutdownHandle};
use serde::de::DeserializeOwned;
use tokio::fs::File;
@@ -24,14 +23,15 @@ use crate::disk::mount::filesystem::idmapped::IdMapped;
use crate::disk::mount::filesystem::loop_dev::LoopDev;
use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
use crate::disk::mount::filesystem::{MountType, ReadOnly};
use crate::disk::mount::guard::MountGuard;
use crate::disk::mount::guard::{GenericMountGuard, MountGuard};
use crate::lxc::{LxcConfig, LxcContainer, HOST_RPC_SERVER_SOCKET};
use crate::net::net_controller::NetService;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::s9pk::merkle_archive::source::FileSource;
use crate::s9pk::S9pk;
use crate::service::start_stop::StartStop;
use crate::service::{rpc, RunningStatus};
use crate::service::{rpc, RunningStatus, Service};
use crate::util::rpc_client::UnixRpcClient;
use crate::util::Invoke;
use crate::volume::{asset_dir, data_dir};
@@ -89,7 +89,8 @@ pub struct PersistentContainer {
js_mount: MountGuard,
volumes: BTreeMap<VolumeId, MountGuard>,
assets: BTreeMap<VolumeId, MountGuard>,
pub(super) overlays: Arc<Mutex<BTreeMap<InternedString, OverlayGuard>>>,
pub(super) images: BTreeMap<ImageId, Arc<MountGuard>>,
pub(super) overlays: Arc<Mutex<BTreeMap<Guid, OverlayGuard<Arc<MountGuard>>>>>,
pub(super) state: Arc<watch::Sender<ServiceState>>,
pub(super) net_service: Mutex<NetService>,
destroyed: bool,
@@ -178,14 +179,62 @@ impl PersistentContainer {
.await?,
);
}
let mut images = BTreeMap::new();
let image_path = lxc_container.rootfs_dir().join("media/startos/images");
tokio::fs::create_dir_all(&image_path).await?;
for image in &s9pk.as_manifest().images {
for (image, config) in &s9pk.as_manifest().images {
let mut arch = ARCH;
let mut sqfs_path = Path::new("images")
.join(arch)
.join(image)
.with_extension("squashfs");
if !s9pk
.as_archive()
.contents()
.get_path(&sqfs_path)
.and_then(|e| e.as_file())
.is_some()
{
arch = if let Some(arch) = config.emulate_missing_as.as_deref() {
arch
} else {
continue;
};
sqfs_path = Path::new("images")
.join(arch)
.join(image)
.with_extension("squashfs");
}
let sqfs = s9pk
.as_archive()
.contents()
.get_path(&sqfs_path)
.and_then(|e| e.as_file())
.or_not_found(sqfs_path.display())?;
let mountpoint = image_path.join(image);
tokio::fs::create_dir_all(&mountpoint).await?;
Command::new("chown")
.arg("100000:100000")
.arg(&mountpoint)
.invoke(ErrorKind::Filesystem)
.await?;
images.insert(
image.clone(),
Arc::new(
MountGuard::mount(
&IdMapped::new(LoopDev::from(&**sqfs), 0, 100000, 65536),
&mountpoint,
ReadOnly,
)
.await?,
),
);
let env_filename = Path::new(image.as_ref()).with_extension("env");
if let Some(env) = s9pk
.as_archive()
.contents()
.get_path(Path::new("images").join(*ARCH).join(&env_filename))
.get_path(Path::new("images").join(arch).join(&env_filename))
.and_then(|e| e.as_file())
{
env.copy(&mut File::create(image_path.join(&env_filename)).await?)
@@ -195,7 +244,7 @@ impl PersistentContainer {
if let Some(json) = s9pk
.as_archive()
.contents()
.get_path(Path::new("images").join(*ARCH).join(&json_filename))
.get_path(Path::new("images").join(arch).join(&json_filename))
.and_then(|e| e.as_file())
{
json.copy(&mut File::create(image_path.join(&json_filename)).await?)
@@ -215,6 +264,7 @@ impl PersistentContainer {
js_mount,
volumes,
assets,
images,
overlays: Arc::new(Mutex::new(BTreeMap::new())),
state: Arc::new(watch::channel(ServiceState::new(start)).0),
net_service: Mutex::new(net_service),
@@ -257,7 +307,7 @@ impl PersistentContainer {
}
#[instrument(skip_all)]
pub async fn init(&self, seed: Weak<ServiceActorSeed>) -> Result<(), Error> {
pub async fn init(&self, seed: Weak<Service>) -> Result<(), Error> {
let socket_server_context = EffectContext::new(seed);
let server = Server::new(
move || ready(Ok(socket_server_context.clone())),
@@ -330,6 +380,7 @@ impl PersistentContainer {
let js_mount = self.js_mount.take();
let volumes = std::mem::take(&mut self.volumes);
let assets = std::mem::take(&mut self.assets);
let images = std::mem::take(&mut self.images);
let overlays = self.overlays.clone();
let lxc_container = self.lxc_container.take();
self.destroyed = true;
@@ -352,6 +403,9 @@ impl PersistentContainer {
for (_, overlay) in std::mem::take(&mut *overlays.lock().await) {
errs.handle(overlay.unmount(true).await);
}
for (_, images) in images {
errs.handle(images.unmount().await);
}
errs.handle(js_mount.unmount(true).await);
if let Some(lxc_container) = lxc_container {
errs.handle(lxc_container.exit().await);
@@ -378,6 +432,7 @@ impl PersistentContainer {
#[instrument(skip_all)]
pub async fn start(&self) -> Result<(), Error> {
self.execute(
Guid::new(),
ProcedureName::StartMain,
Value::Null,
Some(Duration::from_secs(5)), // TODO
@@ -389,7 +444,7 @@ impl PersistentContainer {
#[instrument(skip_all)]
pub async fn stop(&self) -> Result<Duration, Error> {
let timeout: Option<crate::util::serde::Duration> = self
.execute(ProcedureName::StopMain, Value::Null, None)
.execute(Guid::new(), ProcedureName::StopMain, Value::Null, None)
.await?;
Ok(timeout.map(|a| *a).unwrap_or(Duration::from_secs(30)))
}
@@ -397,6 +452,7 @@ impl PersistentContainer {
#[instrument(skip_all)]
pub async fn execute<O>(
&self,
id: Guid,
name: ProcedureName,
input: Value,
timeout: Option<Duration>,
@@ -404,7 +460,7 @@ impl PersistentContainer {
where
O: DeserializeOwned,
{
self._execute(name, input, timeout)
self._execute(id, name, input, timeout)
.await
.and_then(from_value)
}
@@ -412,6 +468,7 @@ impl PersistentContainer {
#[instrument(skip_all)]
pub async fn sanboxed<O>(
&self,
id: Guid,
name: ProcedureName,
input: Value,
timeout: Option<Duration>,
@@ -419,7 +476,7 @@ impl PersistentContainer {
where
O: DeserializeOwned,
{
self._sandboxed(name, input, timeout)
self._sandboxed(id, name, input, timeout)
.await
.and_then(from_value)
}
@@ -427,13 +484,15 @@ impl PersistentContainer {
#[instrument(skip_all)]
async fn _execute(
&self,
id: Guid,
name: ProcedureName,
input: Value,
timeout: Option<Duration>,
) -> Result<Value, Error> {
let fut = self
.rpc_client
.request(rpc::Execute, rpc::ExecuteParams::new(name, input, timeout));
let fut = self.rpc_client.request(
rpc::Execute,
rpc::ExecuteParams::new(id, name, input, timeout),
);
Ok(if let Some(timeout) = timeout {
tokio::time::timeout(timeout, fut)
@@ -447,13 +506,15 @@ impl PersistentContainer {
#[instrument(skip_all)]
async fn _sandboxed(
&self,
id: Guid,
name: ProcedureName,
input: Value,
timeout: Option<Duration>,
) -> Result<Value, Error> {
let fut = self
.rpc_client
.request(rpc::Sandbox, rpc::ExecuteParams::new(name, input, timeout));
let fut = self.rpc_client.request(
rpc::Sandbox,
rpc::ExecuteParams::new(id, name, input, timeout),
);
Ok(if let Some(timeout) = timeout {
tokio::time::timeout(timeout, fut)

2
core/startos/src/service/properties.rs
View File

@@ -3,6 +3,7 @@ use std::time::Duration;
use models::ProcedureName;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::Service;
impl Service {
@@ -11,6 +12,7 @@ impl Service {
let container = &self.seed.persistent_container;
container
.execute::<Value>(
Guid::new(),
ProcedureName::Properties,
Value::Null,
Some(Duration::from_secs(30)),

10
core/startos/src/service/rpc.rs
View File

@@ -7,6 +7,7 @@ use rpc_toolkit::Empty;
use ts_rs::TS;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
#[derive(Clone)]
pub struct Init;
@@ -46,14 +47,21 @@ impl serde::Serialize for Exit {
#[derive(Clone, serde::Deserialize, serde::Serialize, TS)]
pub struct ExecuteParams {
id: Guid,
procedure: String,
#[ts(type = "any")]
input: Value,
timeout: Option<u128>,
}
impl ExecuteParams {
pub fn new(procedure: ProcedureName, input: Value, timeout: Option<Duration>) -> Self {
pub fn new(
id: Guid,
procedure: ProcedureName,
input: Value,
timeout: Option<Duration>,
) -> Self {
Self {
id,
procedure: procedure.js_function_name(),
input,
timeout: timeout.map(|d| d.as_millis()),

326
core/startos/src/service/service_effect_handler.rs
View File

@@ -7,9 +7,9 @@ use std::str::FromStr;
use std::sync::{Arc, Weak};
use clap::builder::ValueParserFactory;
use clap::Parser;
use clap::{CommandFactory, FromArgMatches, Parser};
use emver::VersionRange;
use imbl_value::{json, InternedString};
use imbl_value::json;
use itertools::Itertools;
use models::{
ActionId, DataUrl, HealthCheckId, HostId, ImageId, PackageId, ServiceInterfaceId, VolumeId,
@@ -25,35 +25,34 @@ use crate::db::model::package::{
ActionMetadata, CurrentDependencies, CurrentDependencyInfo, CurrentDependencyKind,
ManifestPreference,
};
use crate::disk::mount::filesystem::idmapped::IdMapped;
use crate::disk::mount::filesystem::loop_dev::LoopDev;
use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
use crate::echo;
use crate::net::host::address::HostAddress;
use crate::net::host::binding::{BindOptions, LanInfo};
use crate::net::host::{Host, HostKind};
use crate::net::service_interface::{AddressInfo, ServiceInterface, ServiceInterfaceType};
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::s9pk::merkle_archive::source::http::HttpSource;
use crate::s9pk::rpc::SKIP_ENV;
use crate::s9pk::S9pk;
use crate::service::cli::ContainerCliContext;
use crate::service::ServiceActorSeed;
use crate::service::Service;
use crate::status::health_check::HealthCheckResult;
use crate::status::MainStatus;
use crate::util::clap::FromStrParser;
use crate::util::{new_guid, Invoke};
use crate::{echo, ARCH};
use crate::util::Invoke;
#[derive(Clone)]
pub(super) struct EffectContext(Weak<ServiceActorSeed>);
pub(super) struct EffectContext(Weak<Service>);
impl EffectContext {
pub fn new(seed: Weak<ServiceActorSeed>) -> Self {
Self(seed)
pub fn new(service: Weak<Service>) -> Self {
Self(service)
}
}
impl Context for EffectContext {}
impl EffectContext {
fn deref(&self) -> Result<Arc<ServiceActorSeed>, Error> {
fn deref(&self) -> Result<Arc<Service>, Error> {
if let Some(seed) = Weak::upgrade(&self.0) {
Ok(seed)
} else {
@@ -66,11 +65,55 @@ impl EffectContext {
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
struct RpcData {
id: i64,
method: String,
params: Value,
#[serde(rename_all = "camelCase")]
pub struct WithProcedureId<T> {
#[serde(default)]
procedure_id: Guid,
#[serde(flatten)]
rest: T,
}
impl<T: FromArgMatches> FromArgMatches for WithProcedureId<T> {
fn from_arg_matches(matches: &clap::ArgMatches) -> Result<Self, clap::Error> {
let rest = T::from_arg_matches(matches)?;
Ok(Self {
procedure_id: matches.get_one("procedure-id").cloned().unwrap_or_default(),
rest,
})
}
fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result<Self, clap::Error> {
let rest = T::from_arg_matches_mut(matches)?;
Ok(Self {
procedure_id: matches.get_one("procedure-id").cloned().unwrap_or_default(),
rest,
})
}
fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> {
self.rest.update_from_arg_matches(matches)?;
self.procedure_id = matches.get_one("procedure-id").cloned().unwrap_or_default();
Ok(())
}
fn update_from_arg_matches_mut(
&mut self,
matches: &mut clap::ArgMatches,
) -> Result<(), clap::Error> {
self.rest.update_from_arg_matches_mut(matches)?;
self.procedure_id = matches.get_one("procedure-id").cloned().unwrap_or_default();
Ok(())
}
}
impl<T: CommandFactory> CommandFactory for WithProcedureId<T> {
fn command() -> clap::Command {
T::command_for_update().arg(
clap::Arg::new("procedure-id")
.action(clap::ArgAction::Set)
.value_parser(clap::value_parser!(Guid)),
)
}
fn command_for_update() -> clap::Command {
Self::command()
}
}
pub fn service_effect_handler<C: Context>() -> ParentHandler<C> {
ParentHandler::new()
.subcommand("gitInfo", from_fn(|_: C| crate::version::git_info()))
@@ -290,6 +333,7 @@ struct MountParams {
async fn set_system_smtp(context: EffectContext, data: SetSystemSmtpParams) -> Result<(), Error> {
let context = context.deref()?;
context
.seed
.ctx
.db
.mutate(|db| {
@@ -304,6 +348,7 @@ async fn get_system_smtp(
) -> Result<String, Error> {
let context = context.deref()?;
let res = context
.seed
.ctx
.db
.peek()
@@ -323,7 +368,7 @@ async fn get_system_smtp(
}
async fn get_container_ip(context: EffectContext, _: Empty) -> Result<Ipv4Addr, Error> {
let context = context.deref()?;
let net_service = context.persistent_container.net_service.lock().await;
let net_service = context.seed.persistent_container.net_service.lock().await;
Ok(net_service.get_ip())
}
async fn get_service_port_forward(
@@ -333,14 +378,15 @@ async fn get_service_port_forward(
let internal_port = data.internal_port as u16;
let context = context.deref()?;
let net_service = context.persistent_container.net_service.lock().await;
let net_service = context.seed.persistent_container.net_service.lock().await;
net_service.get_ext_port(data.host_id, internal_port)
}
async fn clear_network_interfaces(context: EffectContext, _: Empty) -> Result<(), Error> {
let context = context.deref()?;
let package_id = context.id.clone();
let package_id = context.seed.id.clone();
context
.seed
.ctx
.db
.mutate(|db| {
@@ -369,7 +415,7 @@ async fn export_service_interface(
}: ExportServiceInterfaceParams,
) -> Result<(), Error> {
let context = context.deref()?;
let package_id = context.id.clone();
let package_id = context.seed.id.clone();
let service_interface = ServiceInterface {
id: id.clone(),
@@ -384,6 +430,7 @@ async fn export_service_interface(
let svc_interface_with_host_info = service_interface;
context
.seed
.ctx
.db
.mutate(|db| {
@@ -407,7 +454,7 @@ async fn get_primary_url(
}: GetPrimaryUrlParams,
) -> Result<Option<HostAddress>, Error> {
let context = context.deref()?;
let package_id = package_id.unwrap_or_else(|| context.id.clone());
let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
Ok(None) // TODO
}
@@ -419,9 +466,10 @@ async fn list_service_interfaces(
}: ListServiceInterfacesParams,
) -> Result<BTreeMap<ServiceInterfaceId, ServiceInterface>, Error> {
let context = context.deref()?;
let package_id = package_id.unwrap_or_else(|| context.id.clone());
let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
context
.seed
.ctx
.db
.peek()
@@ -435,9 +483,10 @@ async fn list_service_interfaces(
}
async fn remove_address(context: EffectContext, data: RemoveAddressParams) -> Result<(), Error> {
let context = context.deref()?;
let package_id = context.id.clone();
let package_id = context.seed.id.clone();
context
.seed
.ctx
.db
.mutate(|db| {
@@ -454,8 +503,9 @@ async fn remove_address(context: EffectContext, data: RemoveAddressParams) -> Re
}
async fn export_action(context: EffectContext, data: ExportActionParams) -> Result<(), Error> {
let context = context.deref()?;
let package_id = context.id.clone();
let package_id = context.seed.id.clone();
context
.seed
.ctx
.db
.mutate(|db| {
@@ -477,8 +527,9 @@ async fn export_action(context: EffectContext, data: ExportActionParams) -> Resu
}
async fn remove_action(context: EffectContext, data: RemoveActionParams) -> Result<(), Error> {
let context = context.deref()?;
let package_id = context.id.clone();
let package_id = context.seed.id.clone();
context
.seed
.ctx
.db
.mutate(|db| {
@@ -514,16 +565,16 @@ struct GetHostInfoParams {
callback: Callback,
}
async fn get_host_info(
ctx: EffectContext,
context: EffectContext,
GetHostInfoParams {
callback,
package_id,
host_id,
}: GetHostInfoParams,
) -> Result<Host, Error> {
let ctx = ctx.deref()?;
let db = ctx.ctx.db.peek().await;
let package_id = package_id.unwrap_or_else(|| ctx.id.clone());
let context = context.deref()?;
let db = context.seed.ctx.db.peek().await;
let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
db.as_public()
.as_package_data()
@@ -536,8 +587,8 @@ async fn get_host_info(
}
async fn clear_bindings(context: EffectContext, _: Empty) -> Result<(), Error> {
let ctx = context.deref()?;
let mut svc = ctx.persistent_container.net_service.lock().await;
let context = context.deref()?;
let mut svc = context.seed.persistent_container.net_service.lock().await;
svc.clear_bindings().await?;
Ok(())
}
@@ -559,8 +610,8 @@ async fn bind(context: EffectContext, bind_params: Value) -> Result<(), Error> {
internal_port,
options,
} = from_value(bind_params)?;
let ctx = context.deref()?;
let mut svc = ctx.persistent_container.net_service.lock().await;
let context = context.deref()?;
let mut svc = context.seed.persistent_container.net_service.lock().await;
svc.bind(kind, id, internal_port, options).await
}
@@ -575,16 +626,16 @@ struct GetServiceInterfaceParams {
}
async fn get_service_interface(
ctx: EffectContext,
context: EffectContext,
GetServiceInterfaceParams {
callback,
package_id,
service_interface_id,
}: GetServiceInterfaceParams,
) -> Result<ServiceInterface, Error> {
let ctx = ctx.deref()?;
let package_id = package_id.unwrap_or_else(|| ctx.id.clone());
let db = ctx.ctx.db.peek().await;
let context = context.deref()?;
let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
let db = context.seed.ctx.db.peek().await;
let interface = db
.as_public()
@@ -729,8 +780,8 @@ async fn get_store(
GetStoreParams { package_id, path }: GetStoreParams,
) -> Result<Value, Error> {
let context = context.deref()?;
let peeked = context.ctx.db.peek().await;
let package_id = package_id.unwrap_or(context.id.clone());
let peeked = context.seed.ctx.db.peek().await;
let package_id = package_id.unwrap_or(context.seed.id.clone());
let value = peeked
.as_private()
.as_package_stores()
@@ -758,8 +809,9 @@ async fn set_store(
SetStoreParams { value, path }: SetStoreParams,
) -> Result<(), Error> {
let context = context.deref()?;
let package_id = context.id.clone();
let package_id = context.seed.id.clone();
context
.seed
.ctx
.db
.mutate(|db| {
@@ -812,7 +864,7 @@ struct ParamsMaybePackageId {
async fn exists(context: EffectContext, params: ParamsPackageId) -> Result<Value, Error> {
let context = context.deref()?;
let peeked = context.ctx.db.peek().await;
let peeked = context.seed.ctx.db.peek().await;
let package = peeked
.as_public()
.as_package_data()
@@ -834,31 +886,30 @@ struct ExecuteAction {
}
async fn execute_action(
context: EffectContext,
ExecuteAction {
action_id,
input,
service_id,
}: ExecuteAction,
WithProcedureId {
procedure_id,
rest:
ExecuteAction {
service_id,
action_id,
input,
},
}: WithProcedureId<ExecuteAction>,
) -> Result<Value, Error> {
let context = context.deref()?;
let package_id = service_id.clone().unwrap_or_else(|| context.id.clone());
let service = context.ctx.services.get(&package_id).await;
let service = service.as_ref().ok_or_else(|| {
Error::new(
eyre!("Could not find package {package_id}"),
ErrorKind::Unknown,
)
})?;
let package_id = service_id
.clone()
.unwrap_or_else(|| context.seed.id.clone());
Ok(json!(service.action(action_id, input).await?))
Ok(json!(context.action(procedure_id, action_id, input).await?))
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
struct FromService {}
async fn get_configured(context: EffectContext, _: Empty) -> Result<Value, Error> {
let context = context.deref()?;
let peeked = context.ctx.db.peek().await;
let package_id = &context.id;
let peeked = context.seed.ctx.db.peek().await;
let package_id = &context.seed.id;
let package = peeked
.as_public()
.as_package_data()
@@ -872,8 +923,8 @@ async fn get_configured(context: EffectContext, _: Empty) -> Result<Value, Error
async fn stopped(context: EffectContext, params: ParamsMaybePackageId) -> Result<Value, Error> {
let context = context.deref()?;
let peeked = context.ctx.db.peek().await;
let package_id = params.package_id.unwrap_or_else(|| context.id.clone());
let peeked = context.seed.ctx.db.peek().await;
let package_id = params.package_id.unwrap_or_else(|| context.seed.id.clone());
let package = peeked
.as_public()
.as_package_data()
@@ -887,7 +938,7 @@ async fn stopped(context: EffectContext, params: ParamsMaybePackageId) -> Result
async fn running(context: EffectContext, params: ParamsPackageId) -> Result<Value, Error> {
dbg!("Starting the running {params:?}");
let context = context.deref()?;
let peeked = context.ctx.db.peek().await;
let peeked = context.seed.ctx.db.peek().await;
let package_id = params.package_id;
let package = peeked
.as_public()
@@ -900,30 +951,24 @@ async fn running(context: EffectContext, params: ParamsPackageId) -> Result<Valu
Ok(json!(matches!(package, MainStatus::Running { .. })))
}
async fn restart(context: EffectContext, _: Empty) -> Result<Value, Error> {
async fn restart(
context: EffectContext,
WithProcedureId { procedure_id, .. }: WithProcedureId<Empty>,
) -> Result<(), Error> {
let context = context.deref()?;
let service = context.ctx.services.get(&context.id).await;
let service = service.as_ref().ok_or_else(|| {
Error::new(
eyre!("Could not find package {}", context.id),
ErrorKind::Unknown,
)
})?;
service.restart().await?;
Ok(json!(()))
dbg!("here");
context.restart(procedure_id).await?;
dbg!("here");
Ok(())
}
async fn shutdown(context: EffectContext, _: Empty) -> Result<Value, Error> {
async fn shutdown(
context: EffectContext,
WithProcedureId { procedure_id, .. }: WithProcedureId<Empty>,
) -> Result<(), Error> {
let context = context.deref()?;
let service = context.ctx.services.get(&context.id).await;
let service = service.as_ref().ok_or_else(|| {
Error::new(
eyre!("Could not find package {}", context.id),
ErrorKind::Unknown,
)
})?;
service.stop().await?;
Ok(json!(()))
context.stop(procedure_id).await?;
Ok(())
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Parser, TS)]
@@ -935,8 +980,9 @@ struct SetConfigured {
}
async fn set_configured(context: EffectContext, params: SetConfigured) -> Result<Value, Error> {
let context = context.deref()?;
let package_id = &context.id;
let package_id = &context.seed.id;
context
.seed
.ctx
.db
.mutate(|db| {
@@ -989,9 +1035,9 @@ async fn set_main_status(context: EffectContext, params: SetMainStatus) -> Resul
dbg!(format!("Status for main will be is {params:?}"));
let context = context.deref()?;
match params.status {
SetMainStatusStatus::Running => context.started(),
SetMainStatusStatus::Stopped => context.stopped(),
SetMainStatusStatus::Starting => context.stopped(),
SetMainStatusStatus::Running => context.seed.started(),
SetMainStatusStatus::Stopped => context.seed.stopped(),
SetMainStatusStatus::Starting => context.seed.stopped(),
}
Ok(Value::Null)
}
@@ -1011,8 +1057,9 @@ async fn set_health(
) -> Result<Value, Error> {
let context = context.deref()?;
let package_id = &context.id;
let package_id = &context.seed.id;
context
.seed
.ctx
.db
.mutate(move |db| {
@@ -1041,17 +1088,17 @@ async fn set_health(
#[command(rename_all = "camelCase")]
#[ts(export)]
pub struct DestroyOverlayedImageParams {
#[ts(type = "string")]
guid: InternedString,
guid: Guid,
}
#[instrument(skip_all)]
pub async fn destroy_overlayed_image(
ctx: EffectContext,
context: EffectContext,
DestroyOverlayedImageParams { guid }: DestroyOverlayedImageParams,
) -> Result<(), Error> {
let ctx = ctx.deref()?;
if ctx
let context = context.deref()?;
if context
.seed
.persistent_container
.overlays
.lock()
@@ -1068,30 +1115,25 @@ pub async fn destroy_overlayed_image(
#[command(rename_all = "camelCase")]
#[ts(export)]
pub struct CreateOverlayedImageParams {
#[ts(type = "string")]
image_id: ImageId,
}
#[instrument(skip_all)]
pub async fn create_overlayed_image(
ctx: EffectContext,
context: EffectContext,
CreateOverlayedImageParams { image_id }: CreateOverlayedImageParams,
) -> Result<(PathBuf, InternedString), Error> {
let ctx = ctx.deref()?;
let path = Path::new("images")
.join(*ARCH)
.join(&image_id)
.with_extension("squashfs");
if let Some(image) = ctx
) -> Result<(PathBuf, Guid), Error> {
let context = context.deref()?;
if let Some(image) = context
.seed
.persistent_container
.s9pk
.as_archive()
.contents()
.get_path(&path)
.and_then(|e| e.as_file())
.images
.get(&image_id)
.cloned()
{
let guid = new_guid();
let rootfs_dir = ctx
let guid = Guid::new();
let rootfs_dir = context
.seed
.persistent_container
.lxc_container
.get()
@@ -1102,7 +1144,9 @@ pub async fn create_overlayed_image(
)
})?
.rootfs_dir();
let mountpoint = rootfs_dir.join("media/startos/overlays").join(&*guid);
let mountpoint = rootfs_dir
.join("media/startos/overlays")
.join(guid.as_ref());
tokio::fs::create_dir_all(&mountpoint).await?;
let container_mountpoint = Path::new("/").join(
mountpoint
@@ -1110,18 +1154,16 @@ pub async fn create_overlayed_image(
.with_kind(ErrorKind::Incoherent)?,
);
tracing::info!("Mounting overlay {guid} for {image_id}");
let guard = OverlayGuard::mount(
&IdMapped::new(LoopDev::from(&**image), 0, 100000, 65536),
&mountpoint,
)
.await?;
let guard = OverlayGuard::mount(image, &mountpoint).await?;
Command::new("chown")
.arg("100000:100000")
.arg(&mountpoint)
.invoke(ErrorKind::Filesystem)
.await?;
tracing::info!("Mounted overlay {guid} for {image_id}");
ctx.persistent_container
context
.seed
.persistent_container
.overlays
.lock()
.await
@@ -1228,13 +1270,15 @@ struct SetDependenciesParams {
}
async fn set_dependencies(
ctx: EffectContext,
SetDependenciesParams { dependencies }: SetDependenciesParams,
context: EffectContext,
WithProcedureId {
procedure_id,
rest: SetDependenciesParams { dependencies },
}: WithProcedureId<SetDependenciesParams>,
) -> Result<(), Error> {
let ctx = ctx.deref()?;
let id = &ctx.id;
let service_guard = ctx.ctx.services.get(id).await;
let service = service_guard.as_ref().or_not_found(id)?;
let context = context.deref()?;
let id = &context.seed.id;
let mut deps = BTreeMap::new();
for dependency in dependencies {
let (dep_id, kind, registry_url, version_spec) = match dependency {
@@ -1264,14 +1308,13 @@ async fn set_dependencies(
let remote_s9pk = S9pk::deserialize(
&Arc::new(
HttpSource::new(
ctx.ctx.client.clone(),
context.seed.ctx.client.clone(),
registry_url
.join(&format!("package/v2/{}.s9pk?spec={}", dep_id, version_spec))?,
)
.await?,
),
None, // TODO
true,
)
.await?;
@@ -1291,14 +1334,19 @@ async fn set_dependencies(
)
}
};
let config_satisfied = if let Some(dep_service) = &*ctx.ctx.services.get(&dep_id).await {
service
.dependency_config(dep_id.clone(), dep_service.get_config().await?.config)
.await?
.is_none()
} else {
true
};
let config_satisfied =
if let Some(dep_service) = &*context.seed.ctx.services.get(&dep_id).await {
context
.dependency_config(
procedure_id.clone(),
dep_id.clone(),
dep_service.get_config(procedure_id.clone()).await?.config,
)
.await?
.is_none()
} else {
true
};
deps.insert(
dep_id,
CurrentDependencyInfo {
@@ -1311,7 +1359,9 @@ async fn set_dependencies(
},
);
}
ctx.ctx
context
.seed
.ctx
.db
.mutate(|db| {
db.as_public_mut()
@@ -1324,10 +1374,10 @@ async fn set_dependencies(
.await
}
async fn get_dependencies(ctx: EffectContext) -> Result<Vec<DependencyRequirement>, Error> {
let ctx = ctx.deref()?;
let id = &ctx.id;
let db = ctx.ctx.db.peek().await;
async fn get_dependencies(context: EffectContext) -> Result<Vec<DependencyRequirement>, Error> {
let context = context.deref()?;
let id = &context.seed.id;
let db = context.seed.ctx.db.peek().await;
let data = db
.as_public()
.as_package_data()
@@ -1384,16 +1434,16 @@ struct CheckDependenciesResult {
}
async fn check_dependencies(
ctx: EffectContext,
context: EffectContext,
CheckDependenciesParam { package_ids }: CheckDependenciesParam,
) -> Result<Vec<CheckDependenciesResult>, Error> {
let ctx = ctx.deref()?;
let db = ctx.ctx.db.peek().await;
let context = context.deref()?;
let db = context.seed.ctx.db.peek().await;
let current_dependencies = db
.as_public()
.as_package_data()
.as_idx(&ctx.id)
.or_not_found(&ctx.id)?
.as_idx(&context.seed.id)
.or_not_found(&context.seed.id)?
.as_current_dependencies()
.de()?;
let package_ids: Vec<_> = package_ids

38
core/startos/src/service/service_map.rs
View File

@@ -25,7 +25,7 @@ use crate::progress::{
use crate::s9pk::manifest::PackageId;
use crate::s9pk::merkle_archive::source::FileSource;
use crate::s9pk::S9pk;
use crate::service::{LoadDisposition, Service};
use crate::service::{LoadDisposition, Service, ServiceRef};
use crate::status::{MainStatus, Status};
use crate::util::serde::Pem;
@@ -39,23 +39,22 @@ pub struct InstallProgressHandles {
/// This is the structure to contain all the services
#[derive(Default)]
pub struct ServiceMap(Mutex<OrdMap<PackageId, Arc<RwLock<Option<Service>>>>>);
pub struct ServiceMap(Mutex<OrdMap<PackageId, Arc<RwLock<Option<ServiceRef>>>>>);
impl ServiceMap {
async fn entry(&self, id: &PackageId) -> Arc<RwLock<Option<Service>>> {
async fn entry(&self, id: &PackageId) -> Arc<RwLock<Option<ServiceRef>>> {
let mut lock = self.0.lock().await;
dbg!(lock.keys().collect::<Vec<_>>());
lock.entry(id.clone())
.or_insert_with(|| Arc::new(RwLock::new(None)))
.clone()
}
#[instrument(skip_all)]
pub async fn get(&self, id: &PackageId) -> OwnedRwLockReadGuard<Option<Service>> {
pub async fn get(&self, id: &PackageId) -> OwnedRwLockReadGuard<Option<ServiceRef>> {
self.entry(id).await.read_owned().await
}
#[instrument(skip_all)]
pub async fn get_mut(&self, id: &PackageId) -> OwnedRwLockWriteGuard<Option<Service>> {
pub async fn get_mut(&self, id: &PackageId) -> OwnedRwLockWriteGuard<Option<ServiceRef>> {
self.entry(id).await.write_owned().await
}
@@ -83,7 +82,7 @@ impl ServiceMap {
shutdown_err = service.shutdown().await;
}
// TODO: retry on error?
*service = Service::load(ctx, id, disposition).await?;
*service = Service::load(ctx, id, disposition).await?.map(From::from);
shutdown_err?;
Ok(())
}
@@ -95,6 +94,7 @@ impl ServiceMap {
mut s9pk: S9pk<S>,
recovery_source: Option<impl GenericMountGuard>,
) -> Result<DownloadInstallFuture, Error> {
s9pk.validate_and_filter(ctx.s9pk_arch)?;
let manifest = s9pk.as_manifest().clone();
let id = manifest.id.clone();
let icon = s9pk.icon_data_url().await?;
@@ -128,7 +128,7 @@ impl ServiceMap {
);
let restoring = recovery_source.is_some();
let mut reload_guard = ServiceReloadGuard::new(ctx.clone(), id.clone(), op_name);
let mut reload_guard = ServiceRefReloadGuard::new(ctx.clone(), id.clone(), op_name);
reload_guard
.handle(ctx.db.mutate({
@@ -231,7 +231,7 @@ impl ServiceMap {
Ok(reload_guard
.handle_last(async move {
finalization_progress.start();
let s9pk = S9pk::open(&installed_path, Some(&id), true).await?;
let s9pk = S9pk::open(&installed_path, Some(&id)).await?;
let prev = if let Some(service) = service.take() {
ensure_code!(
recovery_source.is_none(),
@@ -264,7 +264,8 @@ impl ServiceMap {
progress_handle,
}),
)
.await?,
.await?
.into(),
);
} else {
*service = Some(
@@ -277,7 +278,8 @@ impl ServiceMap {
progress_handle,
}),
)
.await?,
.await?
.into(),
);
}
sync_progress_task.await.map_err(|_| {
@@ -295,7 +297,7 @@ impl ServiceMap {
pub async fn uninstall(&self, ctx: &RpcContext, id: &PackageId) -> Result<(), Error> {
let mut guard = self.get_mut(id).await;
if let Some(service) = guard.take() {
ServiceReloadGuard::new(ctx.clone(), id.clone(), "Uninstall")
ServiceRefReloadGuard::new(ctx.clone(), id.clone(), "Uninstall")
.handle_last(async move {
let res = service.uninstall(None).await;
drop(guard);
@@ -326,17 +328,17 @@ impl ServiceMap {
}
}
pub struct ServiceReloadGuard(Option<ServiceReloadInfo>);
impl Drop for ServiceReloadGuard {
pub struct ServiceRefReloadGuard(Option<ServiceRefReloadInfo>);
impl Drop for ServiceRefReloadGuard {
fn drop(&mut self) {
if let Some(info) = self.0.take() {
tokio::spawn(info.reload(None));
}
}
}
impl ServiceReloadGuard {
impl ServiceRefReloadGuard {
pub fn new(ctx: RpcContext, id: PackageId, operation: &'static str) -> Self {
Self(Some(ServiceReloadInfo { ctx, id, operation }))
Self(Some(ServiceRefReloadInfo { ctx, id, operation }))
}
pub async fn handle<T>(
@@ -365,12 +367,12 @@ impl ServiceReloadGuard {
}
}
struct ServiceReloadInfo {
struct ServiceRefReloadInfo {
ctx: RpcContext,
id: PackageId,
operation: &'static str,
}
impl ServiceReloadInfo {
impl ServiceRefReloadInfo {
async fn reload(self, error: Option<Error>) -> Result<(), Error> {
self.ctx
.services

10
core/startos/src/service/transition/backup.rs
View File

@@ -6,6 +6,7 @@ use models::ProcedureName;
use super::TempDesiredRestore;
use crate::disk::mount::filesystem::ReadWrite;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::config::GetConfig;
use crate::service::dependencies::DependencyConfig;
use crate::service::transition::{TransitionKind, TransitionState};
@@ -24,7 +25,12 @@ impl Handler<Backup> for ServiceActor {
.except::<GetConfig>()
.except::<DependencyConfig>()
}
async fn handle(&mut self, backup: Backup, jobs: &BackgroundJobQueue) -> Self::Response {
async fn handle(
&mut self,
id: Guid,
backup: Backup,
jobs: &BackgroundJobQueue,
) -> Self::Response {
// So Need a handle to just a single field in the state
let temp: TempDesiredRestore = TempDesiredRestore::new(&self.0.persistent_container.state);
let mut current = self.0.persistent_container.state.subscribe();
@@ -45,7 +51,7 @@ impl Handler<Backup> for ServiceActor {
.mount_backup(path, ReadWrite)
.await?;
seed.persistent_container
.execute(ProcedureName::CreateBackup, Value::Null, None)
.execute(id, ProcedureName::CreateBackup, Value::Null, None)
.await?;
backup_guard.unmount(true).await?;

9
core/startos/src/service/transition/restart.rs
View File

@@ -2,6 +2,7 @@ use futures::FutureExt;
use super::TempDesiredRestore;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::config::GetConfig;
use crate::service::dependencies::DependencyConfig;
use crate::service::transition::{TransitionKind, TransitionState};
@@ -18,7 +19,8 @@ impl Handler<Restart> for ServiceActor {
.except::<GetConfig>()
.except::<DependencyConfig>()
}
async fn handle(&mut self, _: Restart, jobs: &BackgroundJobQueue) -> Self::Response {
async fn handle(&mut self, _: Guid, _: Restart, jobs: &BackgroundJobQueue) -> Self::Response {
dbg!("here");
// So Need a handle to just a single field in the state
let temp = TempDesiredRestore::new(&self.0.persistent_container.state);
let mut current = self.0.persistent_container.state.subscribe();
@@ -74,7 +76,8 @@ impl Handler<Restart> for ServiceActor {
}
impl Service {
#[instrument(skip_all)]
pub async fn restart(&self) -> Result<(), Error> {
self.actor.send(Restart).await
pub async fn restart(&self, id: Guid) -> Result<(), Error> {
dbg!("here");
self.actor.send(id, Restart).await
}
}

10
core/startos/src/service/transition/restore.rs
View File

@@ -5,6 +5,7 @@ use models::ProcedureName;
use crate::disk::mount::filesystem::ReadOnly;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::transition::{TransitionKind, TransitionState};
use crate::service::ServiceActor;
use crate::util::actor::background::BackgroundJobQueue;
@@ -19,7 +20,12 @@ impl Handler<Restore> for ServiceActor {
fn conflicts_with(_: &Restore) -> ConflictBuilder<Self> {
ConflictBuilder::everything()
}
async fn handle(&mut self, restore: Restore, jobs: &BackgroundJobQueue) -> Self::Response {
async fn handle(
&mut self,
id: Guid,
restore: Restore,
jobs: &BackgroundJobQueue,
) -> Self::Response {
// So Need a handle to just a single field in the state
let path = restore.path.clone();
let seed = self.0.clone();
@@ -32,7 +38,7 @@ impl Handler<Restore> for ServiceActor {
.mount_backup(path, ReadOnly)
.await?;
seed.persistent_container
.execute(ProcedureName::RestoreBackup, Value::Null, None)
.execute(id, ProcedureName::RestoreBackup, Value::Null, None)
.await?;
backup_guard.unmount(true).await?;

39
core/startos/src/util/actor/concurrent.rs
View File

@@ -8,6 +8,7 @@ use helpers::NonDetachingJoinHandle;
use tokio::sync::{mpsc, oneshot};
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::util::actor::background::{BackgroundJobQueue, BackgroundJobRunner};
use crate::util::actor::{Actor, ConflictFn, Handler, PendingMessageStrategy, Request};
@@ -18,6 +19,7 @@ struct ConcurrentRunner<A> {
waiting: Vec<Request<A>>,
recv: mpsc::UnboundedReceiver<Request<A>>,
handlers: Vec<(
Guid,
Arc<ConflictFn<A>>,
oneshot::Sender<Box<dyn Any + Send>>,
BoxFuture<'static, Box<dyn Any + Send>>,
@@ -41,16 +43,21 @@ impl<A: Actor + Clone> Future for ConcurrentRunner<A> {
}
});
if this.shutdown.is_some() {
while let std::task::Poll::Ready(Some((msg, reply))) = this.recv.poll_recv(cx) {
if this.handlers.iter().any(|(f, _, _)| f(&*msg)) {
this.waiting.push((msg, reply));
while let std::task::Poll::Ready(Some((id, msg, reply))) = this.recv.poll_recv(cx) {
if this
.handlers
.iter()
.any(|(hid, f, _, _)| &id != hid && f(&*msg))
{
this.waiting.push((id, msg, reply));
} else {
let mut actor = this.actor.clone();
let queue = this.queue.clone();
this.handlers.push((
id.clone(),
msg.conflicts_with(),
reply,
async move { msg.handle_with(&mut actor, &queue).await }.boxed(),
async move { msg.handle_with(id, &mut actor, &queue).await }.boxed(),
))
}
}
@@ -62,29 +69,34 @@ impl<A: Actor + Clone> Future for ConcurrentRunner<A> {
.handlers
.iter_mut()
.enumerate()
.filter_map(|(i, (_, _, f))| match f.poll_unpin(cx) {
.filter_map(|(i, (_, _, _, f))| match f.poll_unpin(cx) {
std::task::Poll::Pending => None,
std::task::Poll::Ready(res) => Some((i, res)),
})
.collect::<Vec<_>>();
for (idx, res) in complete.into_iter().rev() {
#[allow(clippy::let_underscore_future)]
let (f, reply, _) = this.handlers.swap_remove(idx);
let (_, f, reply, _) = this.handlers.swap_remove(idx);
let _ = reply.send(res);
// TODO: replace with Vec::extract_if once stable
if this.shutdown.is_some() {
let mut i = 0;
while i < this.waiting.len() {
if f(&*this.waiting[i].0)
&& !this.handlers.iter().any(|(f, _, _)| f(&*this.waiting[i].0))
if f(&*this.waiting[i].1)
&& !this
.handlers
.iter()
.any(|(_, f, _, _)| f(&*this.waiting[i].1))
{
let (msg, reply) = this.waiting.remove(i);
let (id, msg, reply) = this.waiting.remove(i);
let mut actor = this.actor.clone();
let queue = this.queue.clone();
this.handlers.push((
id.clone(),
msg.conflicts_with(),
reply,
async move { msg.handle_with(&mut actor, &queue).await }.boxed(),
async move { msg.handle_with(id, &mut actor, &queue).await }
.boxed(),
));
cont = true;
} else {
@@ -137,6 +149,7 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
/// Message is guaranteed to be queued immediately
pub fn queue<M: Send + 'static>(
&self,
id: Guid,
message: M,
) -> impl Future<Output = Result<A::Response, Error>>
where
@@ -150,7 +163,7 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
}
let (reply_send, reply_recv) = oneshot::channel();
self.messenger
.send((Box::new(message), reply_send))
.send((id, Box::new(message), reply_send))
.unwrap();
futures::future::Either::Right(
reply_recv
@@ -170,11 +183,11 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
)
}
pub async fn send<M: Send + 'static>(&self, message: M) -> Result<A::Response, Error>
pub async fn send<M: Send + 'static>(&self, id: Guid, message: M) -> Result<A::Response, Error>
where
A: Handler<M>,
{
self.queue(message).await
self.queue(id, message).await
}
pub async fn shutdown(self, strategy: PendingMessageStrategy) {

12
core/startos/src/util/actor/mod.rs
View File

@@ -9,6 +9,7 @@ use tokio::sync::oneshot;
#[allow(unused_imports)]
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::util::actor::background::BackgroundJobQueue;
pub mod background;
@@ -28,6 +29,7 @@ pub trait Handler<M: Any + Send>: Actor {
}
fn handle(
&mut self,
id: Guid,
msg: M,
jobs: &BackgroundJobQueue,
) -> impl Future<Output = Self::Response> + Send;
@@ -39,6 +41,7 @@ trait Message<A>: Send + Any {
fn conflicts_with(&self) -> Arc<ConflictFn<A>>;
fn handle_with<'a>(
self: Box<Self>,
id: Guid,
actor: &'a mut A,
jobs: &'a BackgroundJobQueue,
) -> BoxFuture<'a, Box<dyn Any + Send>>;
@@ -52,10 +55,11 @@ where
}
fn handle_with<'a>(
self: Box<Self>,
id: Guid,
actor: &'a mut A,
jobs: &'a BackgroundJobQueue,
) -> BoxFuture<'a, Box<dyn Any + Send>> {
async move { Box::new(actor.handle(*self, jobs).await) as Box<dyn Any + Send> }.boxed()
async move { Box::new(actor.handle(id, *self, jobs).await) as Box<dyn Any + Send> }.boxed()
}
}
impl<A: Actor> dyn Message<A> {
@@ -80,7 +84,11 @@ impl<A: Actor> dyn Message<A> {
}
}
type Request<A> = (Box<dyn Message<A>>, oneshot::Sender<Box<dyn Any + Send>>);
type Request<A> = (
Guid,
Box<dyn Message<A>>,
oneshot::Sender<Box<dyn Any + Send>>,
);
pub enum PendingMessageStrategy {
CancelAll,

7
core/startos/src/util/actor/simple.rs
View File

@@ -7,6 +7,7 @@ use tokio::sync::oneshot::error::TryRecvError;
use tokio::sync::{mpsc, oneshot};
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::util::actor::background::BackgroundJobQueue;
use crate::util::actor::{Actor, Handler, PendingMessageStrategy, Request};
@@ -26,9 +27,9 @@ impl<A: Actor> SimpleActor<A> {
tokio::select! {
_ = &mut runner => (),
msg = messenger_recv.recv() => match msg {
Some((msg, reply)) if shutdown_recv.try_recv() == Err(TryRecvError::Empty) => {
Some((id, msg, reply)) if shutdown_recv.try_recv() == Err(TryRecvError::Empty) => {
tokio::select! {
res = msg.handle_with(&mut actor, &queue) => { let _ = reply.send(res); },
res = msg.handle_with(id, &mut actor, &queue) => { let _ = reply.send(res); },
_ = &mut runner => (),
}
}
@@ -60,7 +61,7 @@ impl<A: Actor> SimpleActor<A> {
}
let (reply_send, reply_recv) = oneshot::channel();
self.messenger
.send((Box::new(message), reply_send))
.send((Guid::new(), Box::new(message), reply_send))
.unwrap();
futures::future::Either::Right(
reply_recv

10
core/startos/src/util/io.rs
View File

@@ -681,8 +681,6 @@ impl<S: AsyncRead + AsyncWrite> AsyncWrite for TimeoutStream<S> {
}
}
pub struct TmpFile {}
#[derive(Debug)]
pub struct TmpDir {
path: PathBuf,
@@ -707,6 +705,14 @@ impl TmpDir {
tokio::fs::remove_dir_all(&self.path).await?;
Ok(())
}
pub async fn gc(self: Arc<Self>) -> Result<(), Error> {
if let Ok(dir) = Arc::try_unwrap(self) {
dir.delete().await
} else {
Ok(())
}
}
}
impl std::ops::Deref for TmpDir {
type Target = Path;

248
core/startos/src/util/mod.rs
View File

@@ -1,4 +1,4 @@
use std::collections::BTreeMap;
use std::collections::{BTreeMap, VecDeque};
use std::future::Future;
use std::marker::PhantomData;
use std::path::{Path, PathBuf};
@@ -11,6 +11,8 @@ use std::time::Duration;
use async_trait::async_trait;
use color_eyre::eyre::{self, eyre};
use fd_lock_rs::FdLock;
use futures::future::BoxFuture;
use futures::FutureExt;
use helpers::canonicalize;
pub use helpers::NonDetachingJoinHandle;
use imbl_value::InternedString;
@@ -19,7 +21,8 @@ pub use models::VersionString;
use pin_project::pin_project;
use sha2::Digest;
use tokio::fs::File;
use tokio::sync::{Mutex, OwnedMutexGuard, RwLock};
use tokio::io::{AsyncRead, AsyncReadExt, BufReader};
use tokio::sync::{oneshot, Mutex, OwnedMutexGuard, RwLock};
use tracing::instrument;
use crate::shutdown::Shutdown;
@@ -62,11 +65,16 @@ pub trait Invoke<'a> {
where
Self: 'ext,
'ext: 'a;
fn pipe<'ext: 'a>(
&'ext mut self,
next: &'ext mut tokio::process::Command,
) -> Self::Extended<'ext>;
fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext>;
fn input<'ext: 'a, Input: tokio::io::AsyncRead + Unpin + Send>(
&'ext mut self,
input: Option<&'ext mut Input>,
) -> Self::Extended<'ext>;
fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext>;
fn invoke(
&mut self,
error_kind: crate::ErrorKind,
@@ -76,7 +84,20 @@ pub trait Invoke<'a> {
pub struct ExtendedCommand<'a> {
cmd: &'a mut tokio::process::Command,
timeout: Option<Duration>,
input: Option<&'a mut (dyn tokio::io::AsyncRead + Unpin + Send)>,
input: Option<&'a mut (dyn AsyncRead + Unpin + Send)>,
pipe: VecDeque<&'a mut tokio::process::Command>,
capture: bool,
}
impl<'a> From<&'a mut tokio::process::Command> for ExtendedCommand<'a> {
fn from(value: &'a mut tokio::process::Command) -> Self {
ExtendedCommand {
cmd: value,
timeout: None,
input: None,
pipe: VecDeque::new(),
capture: true,
}
}
}
impl<'a> std::ops::Deref for ExtendedCommand<'a> {
type Target = tokio::process::Command;
@@ -95,35 +116,38 @@ impl<'a> Invoke<'a> for tokio::process::Command {
where
Self: 'ext,
'ext: 'a;
fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext> {
ExtendedCommand {
cmd: self,
timeout,
input: None,
}
fn pipe<'ext: 'a>(
&'ext mut self,
next: &'ext mut tokio::process::Command,
) -> Self::Extended<'ext> {
let mut cmd = ExtendedCommand::from(self);
cmd.pipe.push_back(next);
cmd
}
fn input<'ext: 'a, Input: tokio::io::AsyncRead + Unpin + Send>(
fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext> {
let mut cmd = ExtendedCommand::from(self);
cmd.timeout = timeout;
cmd
}
fn input<'ext: 'a, Input: AsyncRead + Unpin + Send>(
&'ext mut self,
input: Option<&'ext mut Input>,
) -> Self::Extended<'ext> {
ExtendedCommand {
cmd: self,
timeout: None,
input: if let Some(input) = input {
Some(&mut *input)
} else {
None
},
}
let mut cmd = ExtendedCommand::from(self);
cmd.input = if let Some(input) = input {
Some(&mut *input)
} else {
None
};
cmd
}
fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext> {
let mut cmd = ExtendedCommand::from(self);
cmd.capture = capture;
cmd
}
async fn invoke(&mut self, error_kind: crate::ErrorKind) -> Result<Vec<u8>, Error> {
ExtendedCommand {
cmd: self,
timeout: None,
input: None,
}
.invoke(error_kind)
.await
ExtendedCommand::from(self).invoke(error_kind).await
}
}
@@ -132,6 +156,13 @@ impl<'a> Invoke<'a> for ExtendedCommand<'a> {
where
Self: 'ext,
'ext: 'a;
fn pipe<'ext: 'a>(
&'ext mut self,
next: &'ext mut tokio::process::Command,
) -> Self::Extended<'ext> {
self.pipe.push_back(next.kill_on_drop(true));
self
}
fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext> {
self.timeout = timeout;
self
@@ -147,39 +178,150 @@ impl<'a> Invoke<'a> for ExtendedCommand<'a> {
};
self
}
fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext> {
self.capture = capture;
self
}
#[instrument(skip_all)]
async fn invoke(&mut self, error_kind: crate::ErrorKind) -> Result<Vec<u8>, Error> {
let cmd_str = self
.cmd
.as_std()
.get_program()
.to_string_lossy()
.into_owned();
self.cmd.kill_on_drop(true);
if self.input.is_some() {
self.cmd.stdin(Stdio::piped());
}
self.cmd.stdout(Stdio::piped());
self.cmd.stderr(Stdio::piped());
let mut child = self.cmd.spawn().with_kind(error_kind)?;
if let (Some(mut stdin), Some(input)) = (child.stdin.take(), self.input.take()) {
use tokio::io::AsyncWriteExt;
tokio::io::copy(input, &mut stdin).await?;
stdin.flush().await?;
stdin.shutdown().await?;
drop(stdin);
if self.pipe.is_empty() {
if self.capture {
self.cmd.stdout(Stdio::piped());
self.cmd.stderr(Stdio::piped());
}
let mut child = self.cmd.spawn().with_ctx(|_| (error_kind, &cmd_str))?;
if let (Some(mut stdin), Some(input)) = (child.stdin.take(), self.input.take()) {
use tokio::io::AsyncWriteExt;
tokio::io::copy(input, &mut stdin).await?;
stdin.flush().await?;
stdin.shutdown().await?;
drop(stdin);
}
let res = match self.timeout {
None => child
.wait_with_output()
.await
.with_ctx(|_| (error_kind, &cmd_str))?,
Some(t) => tokio::time::timeout(t, child.wait_with_output())
.await
.with_kind(ErrorKind::Timeout)?
.with_ctx(|_| (error_kind, &cmd_str))?,
};
crate::ensure_code!(
res.status.success(),
error_kind,
"{}",
Some(&res.stderr)
.filter(|a| !a.is_empty())
.or(Some(&res.stdout))
.filter(|a| !a.is_empty())
.and_then(|a| std::str::from_utf8(a).ok())
.unwrap_or(&format!(
"{} exited with code {}",
self.cmd.as_std().get_program().to_string_lossy(),
res.status
))
);
Ok(res.stdout)
} else {
let mut futures = Vec::<BoxFuture<'_, Result<(), Error>>>::new(); // todo: predict capacity
let mut cmds = std::mem::take(&mut self.pipe);
cmds.push_front(&mut *self.cmd);
let len = cmds.len();
let timeout = self.timeout;
let mut prev = self
.input
.take()
.map(|i| Box::new(i) as Box<dyn AsyncRead + Unpin + Send>);
for (idx, cmd) in IntoIterator::into_iter(cmds).enumerate() {
let last = idx == len - 1;
if self.capture || !last {
cmd.stdout(Stdio::piped());
}
if self.capture {
cmd.stderr(Stdio::piped());
}
if prev.is_some() {
cmd.stdin(Stdio::piped());
}
let mut child = cmd.spawn().with_kind(error_kind)?;
let input = std::mem::replace(
&mut prev,
child
.stdout
.take()
.map(|i| Box::new(BufReader::new(i)) as Box<dyn AsyncRead + Unpin + Send>),
);
futures.push(
async move {
if let (Some(mut stdin), Some(mut input)) = (child.stdin.take(), input) {
use tokio::io::AsyncWriteExt;
tokio::io::copy(&mut input, &mut stdin).await?;
stdin.flush().await?;
stdin.shutdown().await?;
drop(stdin);
}
let res = match timeout {
None => child.wait_with_output().await?,
Some(t) => tokio::time::timeout(t, child.wait_with_output())
.await
.with_kind(ErrorKind::Timeout)??,
};
crate::ensure_code!(
res.status.success(),
error_kind,
"{}",
Some(&res.stderr)
.filter(|a| !a.is_empty())
.or(Some(&res.stdout))
.filter(|a| !a.is_empty())
.and_then(|a| std::str::from_utf8(a).ok())
.unwrap_or(&format!(
"{} exited with code {}",
cmd.as_std().get_program().to_string_lossy(),
res.status
))
);
Ok(())
}
.boxed(),
);
}
let (send, recv) = oneshot::channel();
futures.push(
async move {
if let Some(mut prev) = prev {
let mut res = Vec::new();
prev.read_to_end(&mut res).await?;
send.send(res).unwrap();
} else {
send.send(Vec::new()).unwrap();
}
Ok(())
}
.boxed(),
);
futures::future::try_join_all(futures).await?;
Ok(recv.await.unwrap())
}
let res = match self.timeout {
None => child.wait_with_output().await?,
Some(t) => tokio::time::timeout(t, child.wait_with_output())
.await
.with_kind(ErrorKind::Timeout)??,
};
crate::ensure_code!(
res.status.success(),
error_kind,
"{}",
Some(&res.stderr)
.filter(|a| !a.is_empty())
.or(Some(&res.stdout))
.filter(|a| !a.is_empty())
.and_then(|a| std::str::from_utf8(a).ok())
.unwrap_or(&format!("Unknown Error ({})", res.status))
);
Ok(res.stdout)
}
}

9
sdk/lib/StartSdk.ts
View File

@@ -187,7 +187,10 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
nullIfEmpty,
runCommand: async <A extends string>(
effects: Effects,
image: { id: Manifest["images"][number]; sharedRun?: boolean },
image: {
id: keyof Manifest["images"] & T.ImageId
sharedRun?: boolean
},
command: ValidIfNoStupidEscape<A> | [string, ...string[]],
options: CommandOptions & {
mounts?: { path: string; options: MountOptions }[]
@@ -396,7 +399,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
setupProperties:
(
fn: (options: { effects: Effects }) => Promise<T.SdkPropertiesReturn>,
): T.ExpectedExports.Properties =>
): T.ExpectedExports.properties =>
(options) =>
fn(options).then(nullifyProperties),
setupUninstall: (fn: UninstallFn<Manifest, Store>) =>
@@ -743,7 +746,7 @@ export class StartSdk<Manifest extends SDKManifest, Store> {
export async function runCommand<Manifest extends SDKManifest>(
effects: Effects,
image: { id: Manifest["images"][number]; sharedRun?: boolean },
image: { id: keyof Manifest["images"] & T.ImageId; sharedRun?: boolean },
command: string | [string, ...string[]],
options: CommandOptions & {
mounts?: { path: string; options: MountOptions }[]

3
sdk/lib/health/HealthCheck.ts
View File

@@ -8,12 +8,13 @@ import { defaultTrigger } from "../trigger/defaultTrigger"
import { once } from "../util/once"
import { Overlay } from "../util/Overlay"
import { object, unknown } from "ts-matches"
import { T } from ".."
export type HealthCheckParams<Manifest extends SDKManifest> = {
effects: Effects
name: string
image: {
id: Manifest["images"][number]
id: keyof Manifest["images"] & T.ImageId
sharedRun?: boolean
}
trigger?: Trigger

4
sdk/lib/interfaces/Host.ts
View File

@@ -69,12 +69,12 @@ type NotProtocolsWithSslVariants = Exclude<
type BindOptionsByKnownProtocol =
| {
protocol: ProtocolsWithSslVariants
preferredExternalPort: number
preferredExternalPort?: number
addSsl?: Partial<AddSslOptions>
}
| {
protocol: NotProtocolsWithSslVariants
preferredExternalPort: number
preferredExternalPort?: number
addSsl?: AddSslOptions
}
export type BindOptionsByProtocol = BindOptionsByKnownProtocol | BindOptions

4
sdk/lib/mainFn/CommandController.ts
View File

@@ -1,6 +1,6 @@
import { NO_TIMEOUT, SIGKILL, SIGTERM } from "../StartSdk"
import { SDKManifest } from "../manifest/ManifestTypes"
import { Effects, ValidIfNoStupidEscape } from "../types"
import { Effects, ImageId, ValidIfNoStupidEscape } from "../types"
import { MountOptions, Overlay } from "../util/Overlay"
import { splitCommand } from "../util/splitCommand"
import { cpExecFile, cpExec } from "./Daemons"
@@ -15,7 +15,7 @@ export class CommandController {
return async <A extends string>(
effects: Effects,
imageId: {
id: Manifest["images"][number]
id: keyof Manifest["images"] & ImageId
sharedRun?: boolean
},
command: ValidIfNoStupidEscape<A> | [string, ...string[]],

4
sdk/lib/mainFn/Daemon.ts
View File

@@ -1,5 +1,5 @@
import { SDKManifest } from "../manifest/ManifestTypes"
import { Effects, ValidIfNoStupidEscape } from "../types"
import { Effects, ImageId, ValidIfNoStupidEscape } from "../types"
import { MountOptions, Overlay } from "../util/Overlay"
import { CommandController } from "./CommandController"
@@ -18,7 +18,7 @@ export class Daemon {
return async <A extends string>(
effects: Effects,
imageId: {
id: Manifest["images"][number]
id: keyof Manifest["images"] & ImageId
sharedRun?: boolean
},
command: ValidIfNoStupidEscape<A> | [string, ...string[]],

21
sdk/lib/mainFn/Daemons.ts
View File

@@ -5,7 +5,12 @@ import { SDKManifest } from "../manifest/ManifestTypes"
import { Trigger } from "../trigger"
import { TriggerInput } from "../trigger/TriggerInput"
import { defaultTrigger } from "../trigger/defaultTrigger"
import { DaemonReturned, Effects, ValidIfNoStupidEscape } from "../types"
import {
DaemonReturned,
Effects,
ImageId,
ValidIfNoStupidEscape,
} from "../types"
import { Mounts } from "./Mounts"
import { CommandOptions, MountOptions, Overlay } from "../util/Overlay"
import { splitCommand } from "../util/splitCommand"
@@ -34,8 +39,8 @@ type DaemonsParams<
Id extends string,
> = {
command: ValidIfNoStupidEscape<Command> | [string, ...string[]]
image: { id: Manifest["images"][number]; sharedRun?: boolean }
mounts: { path: string; options: MountOptions }[]
image: { id: keyof Manifest["images"] & ImageId; sharedRun?: boolean }
mounts: Mounts<Manifest>
env?: Record<string, string>
ready: Ready
requires: Exclude<Ids, Id>[]
@@ -116,12 +121,10 @@ export class Daemons<Manifest extends SDKManifest, Ids extends string> {
options: DaemonsParams<Manifest, Ids, Command, Id>,
) {
const daemonIndex = this.daemons.length
const daemon = Daemon.of()(
this.effects,
options.image,
options.command,
options,
)
const daemon = Daemon.of()(this.effects, options.image, options.command, {
...options,
mounts: options.mounts.build(),
})
const healthDaemon = new HealthDaemon(
daemon,
daemonIndex,

6
sdk/lib/manifest/ManifestTypes.ts
View File

@@ -1,5 +1,5 @@
import { ValidEmVer } from "../emverLite/mod"
import { ActionMetadata } from "../types"
import { ActionMetadata, ImageConfig, ImageId } from "../types"
export interface Container {
/** This should be pointing to a docker container name */
@@ -28,8 +28,6 @@ export type SDKManifest = {
readonly releaseNotes: string
/** The type of license for the project. Include the LICENSE in the root of the project directory. A license is required for a Start9 package.*/
readonly license: string // name of license
/** A list of normie (hosted, SaaS, custodial, etc) services this services intends to replace */
readonly replaces: Readonly<string[]>
/** The Start9 wrapper repository URL for the package. This repo contains the manifest file (this),
* any scripts necessary for configuration, backups, actions, or health checks (more below). This key
* must exist. But could be embedded into the source repository
@@ -52,7 +50,7 @@ export type SDKManifest = {
}
/** Defines the os images needed to run the container processes */
readonly images: string[]
readonly images: Record<ImageId, ImageConfig>
/** This denotes readonly asset directories that should be available to mount to the container.
* Assuming that there will be three files with names along the lines:
* icon.* : the icon that will be this packages icon on the ui

9
sdk/lib/manifest/setupManifest.ts
View File

@@ -1,18 +1,19 @@
import { ImageConfig, ImageId, VolumeId } from "../osBindings"
import { SDKManifest, ManifestVersion } from "./ManifestTypes"
export function setupManifest<
Id extends string,
Version extends ManifestVersion,
Dependencies extends Record<string, unknown>,
VolumesTypes extends string,
AssetTypes extends string,
ImagesTypes extends string,
VolumesTypes extends VolumeId,
AssetTypes extends VolumeId,
ImagesTypes extends ImageId,
Manifest extends SDKManifest & {
dependencies: Dependencies
id: Id
version: Version
assets: AssetTypes[]
images: ImagesTypes[]
images: Record<ImagesTypes, ImageConfig>
volumes: VolumesTypes[]
},
>(manifest: Manifest): Manifest {

3
sdk/lib/osBindings/CreateOverlayedImageParams.ts
View File

@@ -1,3 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ImageId } from "./ImageId"
export type CreateOverlayedImageParams = { imageId: string }
export type CreateOverlayedImageParams = { imageId: ImageId }

3
sdk/lib/osBindings/DestroyOverlayedImageParams.ts
View File

@@ -1,3 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Guid } from "./Guid"
export type DestroyOverlayedImageParams = { guid: string }
export type DestroyOverlayedImageParams = { guid: Guid }

8
sdk/lib/osBindings/ImageConfig.ts Normal file
View File

@@ -0,0 +1,8 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ImageSource } from "./ImageSource"
export type ImageConfig = {
source: ImageSource
arch: string[]
emulateMissingAs: string | null
}

3
sdk/lib/osBindings/ImageMetadata.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type ImageMetadata = { workdir: string; user: string }

6
sdk/lib/osBindings/ImageSource.ts Normal file
View File

@@ -0,0 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type ImageSource =
| "packed"
| { dockerBuild: { workdir: string | null; dockerfile: string | null } }
| { dockerTag: string }

4
sdk/lib/osBindings/LoginParams.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { PasswordType } from "./PasswordType"
export type LoginParams = { password: PasswordType | null; metadata: any }

3
sdk/lib/osBindings/Manifest.ts
View File

@@ -3,6 +3,7 @@ import type { Alerts } from "./Alerts"
import type { Dependencies } from "./Dependencies"
import type { Description } from "./Description"
import type { HardwareRequirements } from "./HardwareRequirements"
import type { ImageConfig } from "./ImageConfig"
import type { ImageId } from "./ImageId"
import type { PackageId } from "./PackageId"
import type { Version } from "./Version"
@@ -20,7 +21,7 @@ export type Manifest = {
marketingSite: string
donationUrl: string | null
description: Description
images: Array<ImageId>
images: { [key: ImageId]: ImageConfig }
assets: Array<VolumeId>
volumes: Array<VolumeId>
alerts: Alerts

4
sdk/lib/osBindings/index.ts
View File

@@ -69,7 +69,10 @@ export { HostKind } from "./HostKind"
export { HostnameInfo } from "./HostnameInfo"
export { Hosts } from "./Hosts"
export { Host } from "./Host"
export { ImageConfig } from "./ImageConfig"
export { ImageId } from "./ImageId"
export { ImageMetadata } from "./ImageMetadata"
export { ImageSource } from "./ImageSource"
export { InstalledState } from "./InstalledState"
export { InstallingInfo } from "./InstallingInfo"
export { InstallingState } from "./InstallingState"
@@ -78,6 +81,7 @@ export { IpInfo } from "./IpInfo"
export { LanInfo } from "./LanInfo"
export { ListServiceInterfacesParams } from "./ListServiceInterfacesParams"
export { ListVersionSignersParams } from "./ListVersionSignersParams"
export { LoginParams } from "./LoginParams"
export { MainStatus } from "./MainStatus"
export { Manifest } from "./Manifest"
export { MaybeUtf8String } from "./MaybeUtf8String"

2
sdk/lib/test/configBuilder.test.ts
View File

@@ -400,7 +400,7 @@ describe("values", () => {
long: "",
},
containers: {},
images: [],
images: {},
volumes: [],
assets: [],
alerts: {

2
sdk/lib/test/output.sdk.ts
View File

@@ -21,7 +21,7 @@ export const sdk = StartSdk.of()
long: "",
},
containers: {},
images: [],
images: {},
volumes: [],
assets: [],
alerts: {

24
sdk/lib/types.ts
View File

@@ -11,6 +11,7 @@ import {
GetPrimaryUrlParams,
LanInfo,
BindParams,
Manifest,
} from "./osBindings"
import { MainEffects, ServiceInterfaceType, Signals } from "./StartSdk"
@@ -110,9 +111,26 @@ export namespace ExpectedExports {
*/
export type dependencyConfig = Record<PackageId, DependencyConfig | null>
export type Properties = (options: {
export type properties = (options: {
effects: Effects
}) => Promise<PropertiesReturn>
export type manifest = Manifest
}
export type ABI = {
setConfig: ExpectedExports.setConfig
getConfig: ExpectedExports.getConfig
createBackup: ExpectedExports.createBackup
restoreBackup: ExpectedExports.restoreBackup
actions: ExpectedExports.actions
actionsMetadata: ExpectedExports.actionsMetadata
main: ExpectedExports.main
afterShutdown: ExpectedExports.afterShutdown
init: ExpectedExports.init
uninit: ExpectedExports.uninit
dependencyConfig: ExpectedExports.dependencyConfig
properties: ExpectedExports.properties
manifest: ExpectedExports.manifest
}
export type TimeMs = number
export type VersionString = string
@@ -453,8 +471,8 @@ export type Effects = {
/** Exists could be useful during the runtime to know if some service is running, option dep */
running(options: { packageId: PackageId }): Promise<boolean>
restart(): void
shutdown(): void
restart(): Promise<void>
shutdown(): Promise<void>
mount(options: {
location: string

16
sdk/lib/util/Overlay.ts
View File

@@ -8,16 +8,18 @@ const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/`
export class Overlay {
private constructor(
readonly effects: T.Effects,
readonly imageId: string,
readonly imageId: T.ImageId,
readonly rootfs: string,
readonly guid: string,
readonly guid: T.Guid,
) {}
static async of(
effects: T.Effects,
image: { id: string; sharedRun?: boolean },
image: { id: T.ImageId; sharedRun?: boolean },
) {
const { id: imageId, sharedRun } = image
const [rootfs, guid] = await effects.createOverlayedImage({ imageId })
const { id, sharedRun } = image
const [rootfs, guid] = await effects.createOverlayedImage({
imageId: id as string,
})
const shared = ["dev", "sys", "proc"]
if (!!sharedRun) {
@@ -33,7 +35,7 @@ export class Overlay {
])
}
return new Overlay(effects, imageId, rootfs, guid)
return new Overlay(effects, id, rootfs, guid)
}
async mount(options: MountOptions, path: string): Promise<Overlay> {
@@ -97,7 +99,7 @@ export class Overlay {
stdout: string | Buffer
stderr: string | Buffer
}> {
const imageMeta: any = await fs
const imageMeta: T.ImageMetadata = await fs
.readFile(`/media/startos/images/${this.imageId}.json`, {
encoding: "utf8",
})

30
sdk/lib/util/fileHelper.ts
View File

@@ -3,7 +3,7 @@ import * as YAML from "yaml"
import * as TOML from "@iarna/toml"
import _ from "lodash"
import * as T from "../types"
import * as fs from "fs"
import * as fs from "node:fs/promises"
const previousPath = /(.+?)\/([^/]*)$/
@@ -59,28 +59,24 @@ export class FileHelper<A> {
readonly readData: (stringValue: string) => A,
) {}
async write(data: A, effects: T.Effects) {
if (previousPath.exec(this.path)) {
await new Promise((resolve, reject) =>
fs.mkdir(this.path, (err: any) => (!err ? resolve(null) : reject(err))),
)
const parent = previousPath.exec(this.path)
if (parent) {
await fs.mkdir(parent[1], { recursive: true })
}
await new Promise((resolve, reject) =>
fs.writeFile(this.path, this.writeData(data), (err: any) =>
!err ? resolve(null) : reject(err),
),
)
await fs.writeFile(this.path, this.writeData(data))
}
async read(effects: T.Effects) {
if (!fs.existsSync(this.path)) {
if (
!(await fs.access(this.path).then(
() => true,
() => false,
))
) {
return null
}
return this.readData(
await new Promise((resolve, reject) =>
fs.readFile(this.path, (err: any, data: any) =>
!err ? resolve(data.toString("utf-8")) : reject(err),
),
),
await fs.readFile(this.path).then((data) => data.toString("utf-8")),
)
}
@@ -142,7 +138,7 @@ export class FileHelper<A> {
return new FileHelper<A>(
path,
(inData) => {
return JSON.stringify(inData, null, 2)
return YAML.stringify(inData, null, 2)
},
(inString) => {
return shape.unsafeCast(YAML.parse(inString))

2
sdk/package-lock.json generated
View File

@@ -10,7 +10,7 @@
"license": "MIT",
"dependencies": {
"isomorphic-fetch": "^3.0.0",
"lodash": "4.*.*",
"lodash": "^4.17.21",
"ts-matches": "^5.4.1"
},
"devDependencies": {

12
sdk/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "@start9labs/start-sdk",
"version": "0.3.6-alpha1",
"version": "0.3.6-alpha5",
"description": "Software development kit to facilitate packaging services for StartOS",
"main": "./cjs/lib/index.js",
"types": "./cjs/lib/index.d.ts",
@@ -31,8 +31,10 @@
"homepage": "https://github.com/Start9Labs/start-sdk#readme",
"dependencies": {
"isomorphic-fetch": "^3.0.0",
"lodash": "4.*.*",
"ts-matches": "^5.4.1"
"lodash": "^4.17.21",
"ts-matches": "^5.4.1",
"yaml": "^2.2.2",
"@iarna/toml": "^2.2.5"
},
"prettier": {
"trailingComma": "all",
@@ -41,7 +43,6 @@
"singleQuote": false
},
"devDependencies": {
"@iarna/toml": "^2.2.5",
"@types/jest": "^29.4.0",
"@types/lodash": "^4.17.5",
"jest": "^29.4.3",
@@ -49,7 +50,6 @@
"ts-jest": "^29.0.5",
"ts-node": "^10.9.1",
"tsx": "^4.7.1",
"typescript": "^5.0.4",
"yaml": "^2.2.2"
"typescript": "^5.0.4"
}
}

24
web/projects/ui/src/app/services/api/api.fixures.ts
View File

@@ -69,7 +69,13 @@ export module Mock {
osVersion: '0.2.12',
dependencies: {},
hasConfig: true,
images: ['main'],
images: {
main: {
source: 'packed',
arch: ['x86_64', 'aarch64'],
emulateMissingAs: 'aarch64',
},
},
assets: [],
volumes: ['main'],
hardwareRequirements: {
@@ -116,7 +122,13 @@ export module Mock {
},
},
hasConfig: true,
images: ['main'],
images: {
main: {
source: 'packed',
arch: ['x86_64', 'aarch64'],
emulateMissingAs: 'aarch64',
},
},
assets: [],
volumes: ['main'],
hardwareRequirements: {
@@ -157,7 +169,13 @@ export module Mock {
},
},
hasConfig: false,
images: ['main'],
images: {
main: {
source: 'packed',
arch: ['x86_64', 'aarch64'],
emulateMissingAs: 'aarch64',
},
},
assets: [],
volumes: ['main'],
hardwareRequirements: {