From e9c9a67365e6157a9490e90dc662ac0ec8319481 Mon Sep 17 00:00:00 2001 From: Lucy <12953208+elvece@users.noreply.github.com> Date: Thu, 18 Apr 2024 13:35:38 -0400 Subject: [PATCH 01/17] update registry upload to take id for new admin permissions (#2605) --- core/startos/src/registry/admin.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/startos/src/registry/admin.rs b/core/startos/src/registry/admin.rs index 44b83d161..7cb734d8c 100644 --- a/core/startos/src/registry/admin.rs +++ b/core/startos/src/registry/admin.rs @@ -71,12 +71,14 @@ async fn do_upload( mut url: Url, user: &str, pass: &str, + pkg_id: &str, body: Body, ) -> Result<(), Error> { url.set_path("/admin/v0/upload"); let req = httpc .post(url) .header(header::ACCEPT, "text/plain") + .query(&["id", pkg_id]) .basic_auth(user, Some(pass)) .body(body) .build()?; @@ -178,6 +180,7 @@ pub async fn publish( registry.clone(), &user, &pass, + &pkg.id, Body::wrap_stream(file_stream), ) .await?; From df777c63fe651a6d91fb75f088bfaef0837aeb60 Mon Sep 17 00:00:00 2001 From: Lucy <12953208+elvece@users.noreply.github.com> Date: Tue, 23 Apr 2024 16:02:11 -0400 Subject: [PATCH 02/17] fix type for query params (#2611) --- core/startos/src/registry/admin.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/startos/src/registry/admin.rs b/core/startos/src/registry/admin.rs index 7cb734d8c..e994b5c53 100644 --- a/core/startos/src/registry/admin.rs +++ b/core/startos/src/registry/admin.rs @@ -78,7 +78,7 @@ async fn do_upload( let req = httpc .post(url) .header(header::ACCEPT, "text/plain") - .query(&["id", pkg_id]) + .query(&[("id", pkg_id)]) .basic_auth(user, Some(pass)) .body(body) .build()?; From c832b5d29eee5e13295b090395441c562c80f3c9 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Wed, 29 May 2024 14:07:36 -0600 Subject: [PATCH 03/17] Update README.md (#2630) * Update README.md * Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8383498b1..e9d71401a 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,8 @@
## Running StartOS -There are multiple ways to get started with StartOS: +> [!WARNING] +> StartOS is in beta. It lacks features. It doesn't always work perfectly. Start9 servers are not plug and play. Using them properly requires some effort and patience. Please do not use StartOS or purchase a server if you are unable or unwilling to follow instructions and learn new concepts. ### 💰 Buy a Start9 server This is the most convenient option. Simply [buy a server](https://store.start9.com) from Start9 and plug it in. From 2c12af5af8668cb82c2ffc85eb6a5bbc6e455d4c Mon Sep 17 00:00:00 2001 From: Jade <2364004+Blu-J@users.noreply.github.com> Date: Thu, 6 Jun 2024 15:39:54 -0600 Subject: [PATCH 04/17] Feature/network (#2622) * Feature: Add in the clear bindings * wip: Working on network * fix: Make it so the config gives the url * chore: Remove the repeated types * chore: Add in the todo's here * chore: UPdate and remove some poorly name var * chore: Remove the clear-bindings impl * chore: Remove the wrapper * handle HostnameInfo for Host bindings Co-authored-by: Jade * ?? * chore: Make the install work * Fix: Url's not being created * chore: Fix the local onion in url * include port in hostname * Chore of adding a comment just to modify. --------- Co-authored-by: Aiden McClelland Co-authored-by: Jade --- container-runtime/package-lock.json | 16 +- container-runtime/package.json | 2 +- .../src/Adapters/HostSystemStartOs.ts | 10 +- .../Systems/SystemForEmbassy/MainLoop.ts | 2 - .../Systems/SystemForEmbassy/index.ts | 140 +++++- .../Systems/SystemForEmbassy/matchManifest.ts | 1 + .../SystemForEmbassy/oldEmbassyTypes.ts | 1 + core/models/src/id/mod.rs | 2 +- core/startos/src/db/model/package.rs | 8 +- core/startos/src/disk/mod.rs | 4 +- core/startos/src/net/dhcp.rs | 2 +- core/startos/src/net/host/binding.rs | 65 ++- core/startos/src/net/host/mod.rs | 19 +- core/startos/src/net/net_controller.rs | 264 ++++++---- core/startos/src/net/service_interface.rs | 39 +- core/startos/src/net/vhost.rs | 2 +- core/startos/src/registry/auth.rs | 1 - core/startos/src/service/mod.rs | 7 +- .../src/service/persistent_container.rs | 2 +- .../src/service/service_effect_handler.rs | 208 +++----- sdk/lib/StartSdk.ts | 10 +- sdk/lib/interfaces/Host.ts | 86 ++-- sdk/lib/interfaces/Origin.ts | 19 +- sdk/lib/osBindings/AddAssetParams.ts | 1 - sdk/lib/osBindings/AddPackageParams.ts | 9 + sdk/lib/osBindings/AddSslOptions.ts | 3 +- sdk/lib/osBindings/AddressInfo.ts | 5 +- sdk/lib/osBindings/BindInfo.ts | 3 +- sdk/lib/osBindings/BindOptions.ts | 1 - sdk/lib/osBindings/BindParams.ts | 1 - .../ExportServiceInterfaceParams.ts | 4 - sdk/lib/osBindings/ExportedHostInfo.ts | 10 - sdk/lib/osBindings/ExportedHostnameInfo.ts | 12 - sdk/lib/osBindings/GetHostInfoParams.ts | 5 +- sdk/lib/osBindings/GetHostInfoParamsKind.ts | 3 - sdk/lib/osBindings/GetPackageResponse.ts | 1 + sdk/lib/osBindings/GetPackageResponseFull.ts | 1 + sdk/lib/osBindings/GetPrimaryUrlParams.ts | 5 +- .../osBindings/GetServiceInterfaceParams.ts | 3 +- sdk/lib/osBindings/HardwareRequirements.ts | 2 +- sdk/lib/osBindings/Host.ts | 6 +- sdk/lib/osBindings/HostnameInfo.ts | 12 + sdk/lib/osBindings/{HostInfo.ts => Hosts.ts} | 2 +- .../{ExportedIpHostname.ts => IpHostname.ts} | 2 +- ...{ReverseProxyDestination.ts => LanInfo.ts} | 7 +- ...ortedOnionHostname.ts => OnionHostname.ts} | 2 +- sdk/lib/osBindings/OsVersionInfo.ts | 2 +- sdk/lib/osBindings/PackageDataEntry.ts | 8 +- sdk/lib/osBindings/PackageInfo.ts | 3 +- sdk/lib/osBindings/PackageVersionInfo.ts | 1 - sdk/lib/osBindings/ReverseProxyBind.ts | 3 - sdk/lib/osBindings/ReverseProxyHttp.ts | 3 - sdk/lib/osBindings/ReverseProxyParams.ts | 10 - .../ServiceInterfaceWithHostInfo.ts | 17 - sdk/lib/osBindings/index.ts | 17 +- sdk/lib/test/host.test.ts | 1 + sdk/lib/test/startosTypeValidation.test.ts | 2 - sdk/lib/types.ts | 97 +--- sdk/lib/util/Hostname.ts | 25 + sdk/lib/util/getServiceInterface.ts | 178 +++---- sdk/lib/util/getServiceInterfaces.ts | 40 +- sdk/lib/util/index.ts | 2 + .../app-interfaces/app-interfaces.page.ts | 57 +-- .../ui/src/app/services/api/api.fixures.ts | 450 +----------------- .../ui/src/app/services/api/mock-patch.ts | 352 +------------- .../ui/src/app/services/config.service.ts | 31 +- .../src/app/services/ui-launcher.service.ts | 5 +- 67 files changed, 798 insertions(+), 1516 deletions(-) create mode 100644 sdk/lib/osBindings/AddPackageParams.ts delete mode 100644 sdk/lib/osBindings/ExportedHostInfo.ts delete mode 100644 sdk/lib/osBindings/ExportedHostnameInfo.ts delete mode 100644 sdk/lib/osBindings/GetHostInfoParamsKind.ts create mode 100644 sdk/lib/osBindings/HostnameInfo.ts rename sdk/lib/osBindings/{HostInfo.ts => Hosts.ts} (79%) rename sdk/lib/osBindings/{ExportedIpHostname.ts => IpHostname.ts} (94%) rename sdk/lib/osBindings/{ReverseProxyDestination.ts => LanInfo.ts} (55%) rename sdk/lib/osBindings/{ExportedOnionHostname.ts => OnionHostname.ts} (82%) delete mode 100644 sdk/lib/osBindings/ReverseProxyBind.ts delete mode 100644 sdk/lib/osBindings/ReverseProxyHttp.ts delete mode 100644 sdk/lib/osBindings/ReverseProxyParams.ts delete mode 100644 sdk/lib/osBindings/ServiceInterfaceWithHostInfo.ts create mode 100644 sdk/lib/util/Hostname.ts diff --git a/container-runtime/package-lock.json b/container-runtime/package-lock.json index dd8ba7855..838dcb769 100644 --- a/container-runtime/package-lock.json +++ b/container-runtime/package-lock.json @@ -14,7 +14,7 @@ "filebrowser": "^1.0.0", "isomorphic-fetch": "^3.0.0", "node-fetch": "^3.1.0", - "ts-matches": "^5.4.1", + "ts-matches": "^5.5.1", "tslib": "^2.5.3", "typescript": "^5.1.3", "yaml": "^2.3.1" @@ -29,7 +29,7 @@ }, "../sdk/dist": { "name": "@start9labs/start-sdk", - "version": "0.4.0-rev0.lib0.rc8.beta10", + "version": "0.3.6-alpha1", "license": "MIT", "dependencies": { "isomorphic-fetch": "^3.0.0", @@ -2428,9 +2428,9 @@ } }, "node_modules/ts-matches": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-5.4.1.tgz", - "integrity": "sha512-kXrY75F0s0WD15N2bWKDScKlKgwnusN6dTRzGs1N7LlxQRnazrsBISC1HL4sy2adsyk65Zbx3Ui3IGN8leAFOQ==" + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-5.5.1.tgz", + "integrity": "sha512-UFYaKgfqlg9FROK7bdpYqFwG1CJvP4kOJdjXuWoqxo9jCmANoDw1GxkSCpJgoTeIiSTaTH5Qr1klSspb8c+ydg==" }, "node_modules/tslib": { "version": "2.6.2", @@ -4195,9 +4195,9 @@ } }, "ts-matches": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-5.4.1.tgz", - "integrity": "sha512-kXrY75F0s0WD15N2bWKDScKlKgwnusN6dTRzGs1N7LlxQRnazrsBISC1HL4sy2adsyk65Zbx3Ui3IGN8leAFOQ==" + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-5.5.1.tgz", + "integrity": "sha512-UFYaKgfqlg9FROK7bdpYqFwG1CJvP4kOJdjXuWoqxo9jCmANoDw1GxkSCpJgoTeIiSTaTH5Qr1klSspb8c+ydg==" }, "tslib": { "version": "2.6.2", diff --git a/container-runtime/package.json b/container-runtime/package.json index f5e34a618..e2c56afff 100644 --- a/container-runtime/package.json +++ b/container-runtime/package.json @@ -22,7 +22,7 @@ "filebrowser": "^1.0.0", "isomorphic-fetch": "^3.0.0", "node-fetch": "^3.1.0", - "ts-matches": "^5.4.1", + "ts-matches": "^5.5.1", "tslib": "^2.5.3", "typescript": "^5.1.3", "yaml": "^2.3.1" diff --git a/container-runtime/src/Adapters/HostSystemStartOs.ts b/container-runtime/src/Adapters/HostSystemStartOs.ts index b745799e1..ba2076f52 100644 --- a/container-runtime/src/Adapters/HostSystemStartOs.ts +++ b/container-runtime/src/Adapters/HostSystemStartOs.ts @@ -96,7 +96,10 @@ export class HostSystemStartOs implements Effects { } bind(...[options]: Parameters) { - return this.rpcRound("bind", options) as ReturnType + return this.rpcRound("bind", { + ...options, + stack: new Error().stack, + }) as ReturnType } clearBindings(...[]: Parameters) { return this.rpcRound("clearBindings", null) as ReturnType< @@ -228,11 +231,6 @@ export class HostSystemStartOs implements Effects { restart(...[]: Parameters) { return this.rpcRound("restart", null) } - reverseProxy(...[options]: Parameters) { - return this.rpcRound("reverseProxy", options) as ReturnType< - T.Effects["reverseProxy"] - > - } running(...[packageId]: Parameters) { return this.rpcRound("running", { packageId }) as ReturnType< T.Effects["running"] diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts index 917bfff83..08bf944ed 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts @@ -97,11 +97,9 @@ export class MainLoop { id: interfaceId, internalPort, preferredExternalPort: torConf?.external || internalPort, - scheme: "http", secure: null, addSsl: lanConf?.ssl ? { - scheme: "https", preferredExternalPort: lanConf.external, alpn: { specified: ["http/1.1"] }, } diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index ada893b2b..8e0cb28c5 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -31,6 +31,16 @@ import { HostSystemStartOs } from "../../HostSystemStartOs" import { JsonPath, unNestPath } from "../../../Models/JsonPath" import { RpcResult, matchRpcResult } from "../../RpcListener" import { CT } from "@start9labs/start-sdk" +import { + AddSslOptions, + BindOptions, +} from "@start9labs/start-sdk/cjs/lib/osBindings" +import { + BindOptionsByProtocol, + Host, + MultiHost, +} from "@start9labs/start-sdk/cjs/lib/interfaces/Host" +import { ServiceInterfaceBuilder } from "@start9labs/start-sdk/cjs/lib/interfaces/ServiceInterfaceBuilder" type Optional = A | undefined | null function todo(): never { @@ -335,6 +345,85 @@ export class SystemForEmbassy implements System { await this.migration(effects, previousVersion, timeoutMs) await effects.setMainStatus({ status: "stopped" }) await this.exportActions(effects) + await this.exportNetwork(effects) + } + async exportNetwork(effects: HostSystemStartOs) { + for (const [id, interfaceValue] of Object.entries( + this.manifest.interfaces, + )) { + const host = new MultiHost({ effects, id }) + const internalPorts = new Set( + Object.values(interfaceValue["tor-config"]?.["port-mapping"] ?? {}) + .map(Number.parseInt) + .concat( + ...Object.values(interfaceValue["lan-config"] ?? {}).map( + (c) => c.internal, + ), + ) + .filter(Boolean), + ) + const bindings = Array.from(internalPorts).map< + [number, BindOptionsByProtocol] + >((port) => { + const lanPort = Object.entries(interfaceValue["lan-config"] ?? {}).find( + ([external, internal]) => internal.internal === port, + )?.[0] + const torPort = Object.entries( + interfaceValue["tor-config"]?.["port-mapping"] ?? {}, + ).find( + ([external, internal]) => Number.parseInt(internal) === port, + )?.[0] + let addSsl: AddSslOptions | null = null + if (lanPort) { + const lanPortNum = Number.parseInt(lanPort) + if (lanPortNum === 443) { + return [port, { protocol: "http", preferredExternalPort: 80 }] + } + addSsl = { + preferredExternalPort: lanPortNum, + alpn: { specified: [] }, + } + } + return [ + port, + { + secure: null, + preferredExternalPort: Number.parseInt( + torPort || lanPort || String(port), + ), + addSsl, + }, + ] + }) + + await Promise.all( + bindings.map(async ([internal, options]) => { + if (internal == null) { + return + } + if (options?.preferredExternalPort == null) { + return + } + const origin = await host.bindPort(internal, options) + await origin.export([ + new ServiceInterfaceBuilder({ + effects, + name: interfaceValue.name, + id: `${id}-${internal}`, + description: interfaceValue.description, + hasPrimary: false, + disabled: false, + type: "api", + masked: false, + path: "", + schemeOverride: null, + search: {}, + username: null, + }), + ]) + }), + ) + } } async exportActions(effects: HostSystemStartOs) { const manifest = this.manifest @@ -486,6 +575,7 @@ export class SystemForEmbassy implements System { const newConfig = structuredClone(newConfigWithoutPointers) await updateConfig( effects, + this.manifest, await this.getConfigUncleaned(effects, timeoutMs).then((x) => x.spec), newConfig, ) @@ -866,6 +956,7 @@ function cleanConfigFromPointers( async function updateConfig( effects: HostSystemStartOs, + manifest: Manifest, spec: unknown, mutConfigValue: unknown, ) { @@ -877,7 +968,12 @@ async function updateConfig( const newConfigValue = mutConfigValue[key] if (matchSpec.test(specValue)) { const updateObject = { spec: null } - await updateConfig(effects, { spec: specValue.spec }, updateObject) + await updateConfig( + effects, + manifest, + { spec: specValue.spec }, + updateObject, + ) mutConfigValue[key] = updateObject.spec } if ( @@ -899,20 +995,48 @@ async function updateConfig( if (matchPointerPackage.test(specValue)) { if (specValue.target === "tor-key") throw new Error("This service uses an unsupported target TorKey") + + const specInterface = specValue.interface + const serviceInterfaceId = extractServiceInterfaceId( + manifest, + specInterface, + ) const filled = await utils .getServiceInterface(effects, { packageId: specValue["package-id"], - id: specValue.interface, + id: serviceInterfaceId, }) .once() - .catch(() => null) - - mutConfigValue[key] = + .catch((x) => { + console.error("Could not get the service interface", x) + return null + }) + const catchFn = (fn: () => X) => { + try { + return fn() + } catch (e) { + return undefined + } + } + const url: string = filled === null ? "" - : specValue.target === "lan-address" - ? filled.addressInfo.localHostnames[0] - : filled.addressInfo.onionHostnames[0] + : catchFn(() => + utils.hostnameInfoToAddress( + specValue.target === "lan-address" + ? filled.addressInfo.localHostnames[0] || + filled.addressInfo.onionHostnames[0] + : filled.addressInfo.onionHostnames[0] || + filled.addressInfo.localHostnames[0], + ), + ) || "" + mutConfigValue[key] = url } } } +function extractServiceInterfaceId(manifest: Manifest, specInterface: string) { + let serviceInterfaceId + const lanConfig = manifest.interfaces[specInterface]?.["lan-config"] || {} + serviceInterfaceId = `${specInterface}-${Object.entries(lanConfig)[0]?.[1]?.internal}` + return serviceInterfaceId +} diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts index d4758669b..8ce6cabbc 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/matchManifest.ts @@ -72,6 +72,7 @@ export const matchManifest = object( object( { name: string, + description: string, "tor-config": object({ "port-mapping": dictionary([string, string]), }), diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts index 35b95e095..0d7521626 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts @@ -99,6 +99,7 @@ export type Effects = { /** Sandbox mode lets us read but not write */ is_sandboxed(): boolean + // Does a volume and path exist? exists(input: { volumeId: string; path: string }): Promise fetch( diff --git a/core/models/src/id/mod.rs b/core/models/src/id/mod.rs index 10882d4f1..85c9d8255 100644 --- a/core/models/src/id/mod.rs +++ b/core/models/src/id/mod.rs @@ -24,7 +24,7 @@ pub use service_interface::ServiceInterfaceId; pub use volume::VolumeId; lazy_static::lazy_static! { - static ref ID_REGEX: Regex = Regex::new("^[a-z]+(-[a-z]+)*$").unwrap(); + static ref ID_REGEX: Regex = Regex::new("^[a-z]+(-[a-z0-9]+)*$").unwrap(); pub static ref SYSTEM_ID: Id = Id(InternedString::intern("x_system")); } diff --git a/core/startos/src/db/model/package.rs b/core/startos/src/db/model/package.rs index 3fad0ad9f..8bb5a9517 100644 --- a/core/startos/src/db/model/package.rs +++ b/core/startos/src/db/model/package.rs @@ -10,8 +10,8 @@ use reqwest::Url; use serde::{Deserialize, Serialize}; use ts_rs::TS; -use crate::net::host::HostInfo; -use crate::net::service_interface::ServiceInterfaceWithHostInfo; +use crate::net::host::Hosts; +use crate::net::service_interface::ServiceInterface; use crate::prelude::*; use crate::progress::FullProgress; use crate::s9pk::manifest::Manifest; @@ -333,8 +333,8 @@ pub struct PackageDataEntry { pub last_backup: Option>, pub current_dependencies: CurrentDependencies, pub actions: BTreeMap, - pub service_interfaces: BTreeMap, - pub hosts: HostInfo, + pub service_interfaces: BTreeMap, + pub hosts: Hosts, #[ts(type = "string[]")] pub store_exposed_dependents: Vec, } diff --git a/core/startos/src/disk/mod.rs b/core/startos/src/disk/mod.rs index 68eb2b187..705a34f98 100644 --- a/core/startos/src/disk/mod.rs +++ b/core/startos/src/disk/mod.rs @@ -1,8 +1,6 @@ use std::path::{Path, PathBuf}; -use rpc_toolkit::{ - from_fn_async, CallRemoteHandler, Context, Empty, HandlerExt, ParentHandler, -}; +use rpc_toolkit::{from_fn_async, CallRemoteHandler, Context, Empty, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; use crate::context::{CliContext, RpcContext}; diff --git a/core/startos/src/net/dhcp.rs b/core/startos/src/net/dhcp.rs index 104cda961..ffcb9774b 100644 --- a/core/startos/src/net/dhcp.rs +++ b/core/startos/src/net/dhcp.rs @@ -3,7 +3,7 @@ use std::net::IpAddr; use clap::Parser; use futures::TryStreamExt; -use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; +use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; use ts_rs::TS; diff --git a/core/startos/src/net/host/binding.rs b/core/startos/src/net/host/binding.rs index 8301821f5..76dd04059 100644 --- a/core/startos/src/net/host/binding.rs +++ b/core/startos/src/net/host/binding.rs @@ -1,4 +1,3 @@ -use imbl_value::InternedString; use serde::{Deserialize, Serialize}; use ts_rs::TS; @@ -11,17 +10,31 @@ use crate::prelude::*; #[ts(export)] pub struct BindInfo { pub options: BindOptions, - pub assigned_lan_port: Option, + pub lan: LanInfo, +} +#[derive(Clone, Copy, Debug, Deserialize, Serialize, TS, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct LanInfo { + pub assigned_port: Option, + pub assigned_ssl_port: Option, } impl BindInfo { pub fn new(available_ports: &mut AvailablePorts, options: BindOptions) -> Result { - let mut assigned_lan_port = None; - if options.add_ssl.is_some() || options.secure.is_some() { - assigned_lan_port = Some(available_ports.alloc()?); + let mut assigned_port = None; + let mut assigned_ssl_port = None; + if options.secure.is_some() { + assigned_port = Some(available_ports.alloc()?); + } + if options.add_ssl.is_some() { + assigned_ssl_port = Some(available_ports.alloc()?); } Ok(Self { options, - assigned_lan_port, + lan: LanInfo { + assigned_port, + assigned_ssl_port, + }, }) } pub fn update( @@ -29,29 +42,38 @@ impl BindInfo { available_ports: &mut AvailablePorts, options: BindOptions, ) -> Result { - let Self { - mut assigned_lan_port, - .. - } = self; - if options.add_ssl.is_some() || options.secure.is_some() { - assigned_lan_port = if let Some(port) = assigned_lan_port.take() { + let Self { mut lan, .. } = self; + if options + .secure + .map_or(false, |s| !(s.ssl && options.add_ssl.is_some())) + // doesn't make sense to have 2 listening ports, both with ssl + { + lan.assigned_port = if let Some(port) = lan.assigned_port.take() { Some(port) } else { Some(available_ports.alloc()?) }; } else { - if let Some(port) = assigned_lan_port.take() { + if let Some(port) = lan.assigned_port.take() { available_ports.free([port]); } } - Ok(Self { - options, - assigned_lan_port, - }) + if options.add_ssl.is_some() { + lan.assigned_ssl_port = if let Some(port) = lan.assigned_ssl_port.take() { + Some(port) + } else { + Some(available_ports.alloc()?) + }; + } else { + if let Some(port) = lan.assigned_ssl_port.take() { + available_ports.free([port]); + } + } + Ok(Self { options, lan }) } } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] +#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, TS)] #[ts(export)] #[serde(rename_all = "camelCase")] pub struct Security { @@ -62,8 +84,6 @@ pub struct Security { #[serde(rename_all = "camelCase")] #[ts(export)] pub struct BindOptions { - #[ts(type = "string | null")] - pub scheme: Option, pub preferred_external_port: u16, pub add_ssl: Option, pub secure: Option, @@ -73,11 +93,8 @@ pub struct BindOptions { #[serde(rename_all = "camelCase")] #[ts(export)] pub struct AddSslOptions { - #[ts(type = "string | null")] - pub scheme: Option, pub preferred_external_port: u16, // #[serde(default)] // pub add_x_forwarded_headers: bool, // TODO - #[serde(default)] - pub alpn: AlpnInfo, + pub alpn: Option, } diff --git a/core/startos/src/net/host/mod.rs b/core/startos/src/net/host/mod.rs index 2d8599ba9..6cbb2dfd5 100644 --- a/core/startos/src/net/host/mod.rs +++ b/core/startos/src/net/host/mod.rs @@ -3,13 +3,13 @@ use std::collections::{BTreeMap, BTreeSet}; use imbl_value::InternedString; use models::{HostId, PackageId}; use serde::{Deserialize, Serialize}; -use torut::onion::{OnionAddressV3, TorSecretKeyV3}; use ts_rs::TS; use crate::db::model::DatabaseModel; use crate::net::forward::AvailablePorts; use crate::net::host::address::HostAddress; use crate::net::host::binding::{BindInfo, BindOptions}; +use crate::net::service_interface::HostnameInfo; use crate::prelude::*; pub mod address; @@ -23,7 +23,8 @@ pub struct Host { pub kind: HostKind, pub bindings: BTreeMap, pub addresses: BTreeSet, - pub primary: Option, + /// COMPUTED: NetService::update + pub hostname_info: BTreeMap>, // internal port -> Hostnames } impl AsRef for Host { fn as_ref(&self) -> &Host { @@ -36,7 +37,7 @@ impl Host { kind, bindings: BTreeMap::new(), addresses: BTreeSet::new(), - primary: None, + hostname_info: BTreeMap::new(), } } } @@ -53,9 +54,9 @@ pub enum HostKind { #[derive(Debug, Default, Deserialize, Serialize, HasModel, TS)] #[model = "Model"] #[ts(export)] -pub struct HostInfo(BTreeMap); +pub struct Hosts(pub BTreeMap); -impl Map for HostInfo { +impl Map for Hosts { type Key = HostId; type Value = Host; fn key_str(key: &Self::Key) -> Result, Error> { @@ -75,7 +76,7 @@ pub fn host_for<'a>( fn host_info<'a>( db: &'a mut DatabaseModel, package_id: &PackageId, - ) -> Result<&'a mut Model, Error> { + ) -> Result<&'a mut Model, Error> { Ok::<_, Error>( db.as_public_mut() .as_package_data_mut() @@ -129,9 +130,3 @@ impl Model { }) } } - -impl HostInfo { - pub fn get_host_primary(&self, host_id: &HostId) -> Option { - self.0.get(&host_id).and_then(|h| h.primary.clone()) - } -} diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index 205cc4e36..69c5c7940 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -4,7 +4,6 @@ use std::sync::{Arc, Weak}; use color_eyre::eyre::eyre; use imbl::OrdMap; -use lazy_format::lazy_format; use models::{HostId, OptionExt, PackageId}; use torut::onion::{OnionAddressV3, TorSecretKeyV3}; use tracing::instrument; @@ -15,8 +14,9 @@ use crate::hostname::Hostname; use crate::net::dns::DnsController; use crate::net::forward::LanPortForwardController; use crate::net::host::address::HostAddress; -use crate::net::host::binding::{AddSslOptions, BindOptions}; +use crate::net::host::binding::{AddSslOptions, BindOptions, LanInfo}; use crate::net::host::{host_for, Host, HostKind}; +use crate::net::service_interface::{HostnameInfo, IpHostname, OnionHostname}; use crate::net::tor::TorController; use crate::net::vhost::{AlpnInfo, VHostController}; use crate::prelude::*; @@ -164,7 +164,7 @@ impl NetController { #[derive(Default, Debug)] struct HostBinds { - lan: BTreeMap, Arc<()>)>, + lan: BTreeMap, Vec>)>, tor: BTreeMap, Vec>)>, } @@ -209,105 +209,173 @@ impl NetService { .await?; self.update(id, host).await } + pub async fn clear_bindings(&mut self) -> Result<(), Error> { + // TODO BLUJ + Ok(()) + } async fn update(&mut self, id: HostId, host: Host) -> Result<(), Error> { - dbg!(&host); - dbg!(&self.binds); let ctrl = self.net_controller()?; + let mut hostname_info = BTreeMap::new(); let binds = { if !self.binds.contains_key(&id) { self.binds.insert(id.clone(), Default::default()); } self.binds.get_mut(&id).unwrap() }; - if true - // TODO: if should listen lan - { - for (port, bind) in &host.bindings { - let old_lan_bind = binds.lan.remove(port); - let old_lan_port = old_lan_bind.as_ref().map(|(external, _, _)| *external); - let lan_bind = old_lan_bind.filter(|(external, ssl, _)| { - ssl == &bind.options.add_ssl - && bind.assigned_lan_port.as_ref() == Some(external) - }); // only keep existing binding if relevant details match - if let Some(external) = bind.assigned_lan_port { - let new_lan_bind = if let Some(b) = lan_bind { - b - } else { - if let Some(ssl) = &bind.options.add_ssl { - let rc = ctrl - .vhost + let peek = ctrl.db.peek().await; + + // LAN + let server_info = peek.as_public().as_server_info(); + let ip_info = server_info.as_ip_info().de()?; + let hostname = server_info.as_hostname().de()?; + for (port, bind) in &host.bindings { + let old_lan_bind = binds.lan.remove(port); + let old_lan_port = old_lan_bind.as_ref().map(|(external, _, _)| *external); + let lan_bind = old_lan_bind + .filter(|(external, ssl, _)| ssl == &bind.options.add_ssl && bind.lan == *external); // only keep existing binding if relevant details match + if bind.lan.assigned_port.is_some() || bind.lan.assigned_ssl_port.is_some() { + let new_lan_bind = if let Some(b) = lan_bind { + b + } else { + let mut rcs = Vec::with_capacity(2); + if let Some(ssl) = &bind.options.add_ssl { + let external = bind + .lan + .assigned_ssl_port + .or_not_found("assigned ssl port")?; + rcs.push( + ctrl.vhost .add( None, external, (self.ip, *port).into(), - if bind.options.secure.as_ref().map_or(false, |s| s.ssl) { - Ok(()) + if let Some(alpn) = ssl.alpn.clone() { + Err(alpn) } else { - Err(ssl.alpn.clone()) + if bind.options.secure.as_ref().map_or(false, |s| s.ssl) { + Ok(()) + } else { + Err(AlpnInfo::Reflect) + } }, ) - .await?; - (*port, Some(ssl.clone()), rc) + .await?, + ); + } + if let Some(security) = bind.options.secure { + if bind.options.add_ssl.is_some() && security.ssl { + // doesn't make sense to have 2 listening ports, both with ssl } else { - let rc = ctrl.forward.add(external, (self.ip, *port).into()).await?; - (*port, None, rc) + let external = + bind.lan.assigned_port.or_not_found("assigned lan port")?; + rcs.push(ctrl.forward.add(external, (self.ip, *port).into()).await?); } - }; - binds.lan.insert(*port, new_lan_bind); + } + (bind.lan, bind.options.add_ssl.clone(), rcs) + }; + let mut bind_hostname_info: Vec = + hostname_info.remove(port).unwrap_or_default(); + for (interface, ip_info) in &ip_info { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public: false, + hostname: IpHostname::Local { + value: format!("{hostname}.local"), + port: new_lan_bind.0.assigned_port, + ssl_port: new_lan_bind.0.assigned_ssl_port, + }, + }); + if let Some(ipv4) = ip_info.ipv4 { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public: false, + hostname: IpHostname::Ipv4 { + value: ipv4, + port: new_lan_bind.0.assigned_port, + ssl_port: new_lan_bind.0.assigned_ssl_port, + }, + }); + } + if let Some(ipv6) = ip_info.ipv6 { + bind_hostname_info.push(HostnameInfo::Ip { + network_interface_id: interface.clone(), + public: false, + hostname: IpHostname::Ipv6 { + value: ipv6, + port: new_lan_bind.0.assigned_port, + ssl_port: new_lan_bind.0.assigned_ssl_port, + }, + }); + } } - if let Some(external) = old_lan_port { + hostname_info.insert(*port, bind_hostname_info); + binds.lan.insert(*port, new_lan_bind); + } + if let Some(lan) = old_lan_port { + if let Some(external) = lan.assigned_ssl_port { ctrl.vhost.gc(None, external).await?; + } + if let Some(external) = lan.assigned_port { ctrl.forward.gc(external).await?; } } - let mut removed = BTreeSet::new(); - let mut removed_ssl = BTreeSet::new(); - binds.lan.retain(|internal, (external, ssl, _)| { - if host.bindings.contains_key(internal) { - true - } else { - if ssl.is_some() { - removed_ssl.insert(*external); - } else { - removed.insert(*external); - } - false - } - }); - for external in removed { - ctrl.forward.gc(external).await?; + } + let mut removed = BTreeSet::new(); + binds.lan.retain(|internal, (external, _, _)| { + if host.bindings.contains_key(internal) { + true + } else { + removed.insert(*external); + + false } - for external in removed_ssl { + }); + for lan in removed { + if let Some(external) = lan.assigned_ssl_port { ctrl.vhost.gc(None, external).await?; } + if let Some(external) = lan.assigned_port { + ctrl.forward.gc(external).await?; + } } - let tor_binds: OrdMap = host - .bindings - .iter() - .flat_map(|(internal, info)| { - let non_ssl = ( - info.options.preferred_external_port, - SocketAddr::from((self.ip, *internal)), + + struct TorHostnamePorts { + non_ssl: Option, + ssl: Option, + } + let mut tor_hostname_ports = BTreeMap::::new(); + let mut tor_binds = OrdMap::::new(); + for (internal, info) in &host.bindings { + tor_binds.insert( + info.options.preferred_external_port, + SocketAddr::from((self.ip, *internal)), + ); + if let (Some(ssl), Some(ssl_internal)) = + (&info.options.add_ssl, info.lan.assigned_ssl_port) + { + tor_binds.insert( + ssl.preferred_external_port, + SocketAddr::from(([127, 0, 0, 1], ssl_internal)), ); - if let (Some(ssl), Some(ssl_internal)) = - (&info.options.add_ssl, info.assigned_lan_port) - { - itertools::Either::Left( - [ - ( - ssl.preferred_external_port, - SocketAddr::from(([127, 0, 0, 1], ssl_internal)), - ), - non_ssl, - ] - .into_iter(), - ) - } else { - itertools::Either::Right([non_ssl].into_iter()) - } - }) - .collect(); + tor_hostname_ports.insert( + *internal, + TorHostnamePorts { + non_ssl: Some(info.options.preferred_external_port) + .filter(|p| *p != ssl.preferred_external_port), + ssl: Some(ssl.preferred_external_port), + }, + ); + } else { + tor_hostname_ports.insert( + *internal, + TorHostnamePorts { + non_ssl: Some(info.options.preferred_external_port), + ssl: None, + }, + ); + } + } let mut keep_tor_addrs = BTreeSet::new(); for addr in match host.kind { HostKind::Multi => { @@ -324,13 +392,10 @@ impl NetService { let new_tor_bind = if let Some(tor_bind) = tor_bind { tor_bind } else { - let key = ctrl - .db - .peek() - .await - .into_private() - .into_key_store() - .into_onion() + let key = peek + .as_private() + .as_key_store() + .as_onion() .get_key(address)?; let rcs = ctrl .tor @@ -338,6 +403,18 @@ impl NetService { .await?; (tor_binds.clone(), rcs) }; + for (internal, ports) in &tor_hostname_ports { + let mut bind_hostname_info = + hostname_info.remove(internal).unwrap_or_default(); + bind_hostname_info.push(HostnameInfo::Onion { + hostname: OnionHostname { + value: address.to_string(), + port: ports.non_ssl, + ssl_port: ports.ssl, + }, + }); + hostname_info.insert(*internal, bind_hostname_info); + } binds.tor.insert(address.clone(), new_tor_bind); } } @@ -347,6 +424,14 @@ impl NetService { ctrl.tor.gc(Some(addr.clone()), None).await?; } } + self.net_controller()? + .db + .mutate(|db| { + host_for(db, &self.id, &id, host.kind)? + .as_hostname_info_mut() + .ser(&hostname_info) + }) + .await?; Ok(()) } @@ -355,12 +440,13 @@ impl NetService { let mut errors = ErrorCollection::new(); if let Some(ctrl) = Weak::upgrade(&self.controller) { for (_, binds) in std::mem::take(&mut self.binds) { - for (_, (external, ssl, rc)) in binds.lan { + for (_, (lan, _, rc)) in binds.lan { drop(rc); - if ssl.is_some() { - errors.handle(ctrl.vhost.gc(None, external).await); - } else { - errors.handle(ctrl.forward.gc(external).await); + if let Some(external) = lan.assigned_ssl_port { + ctrl.vhost.gc(None, external).await?; + } + if let Some(external) = lan.assigned_port { + ctrl.forward.gc(external).await?; } } for (addr, (_, rcs)) in binds.tor { @@ -384,12 +470,12 @@ impl NetService { self.ip } - pub fn get_ext_port(&self, host_id: HostId, internal_port: u16) -> Result { + pub fn get_ext_port(&self, host_id: HostId, internal_port: u16) -> Result { let host_id_binds = self.binds.get_key_value(&host_id); match host_id_binds { Some((_, binds)) => { - if let Some(ext_port_info) = binds.lan.get(&internal_port) { - Ok(ext_port_info.0) + if let Some((lan, _, _)) = binds.lan.get(&internal_port) { + Ok(*lan) } else { Err(Error::new( eyre!( diff --git a/core/startos/src/net/service_interface.rs b/core/startos/src/net/service_interface.rs index 9a4659cfd..e905be545 100644 --- a/core/startos/src/net/service_interface.rs +++ b/core/startos/src/net/service_interface.rs @@ -1,51 +1,32 @@ use std::net::{Ipv4Addr, Ipv6Addr}; +use imbl_value::InternedString; use models::{HostId, ServiceInterfaceId}; use serde::{Deserialize, Serialize}; use ts_rs::TS; -use crate::net::host::binding::BindOptions; -use crate::net::host::HostKind; -use crate::prelude::*; - -#[derive(Clone, Debug, Deserialize, Serialize, TS)] -#[ts(export)] -#[serde(rename_all = "camelCase")] -pub struct ServiceInterfaceWithHostInfo { - #[serde(flatten)] - pub service_interface: ServiceInterface, - pub host_info: ExportedHostInfo, -} - -#[derive(Clone, Debug, Deserialize, Serialize, TS)] -#[ts(export)] -#[serde(rename_all = "camelCase")] -pub struct ExportedHostInfo { - pub id: HostId, - pub kind: HostKind, - pub hostnames: Vec, -} +use crate::net::host::address::HostAddress; #[derive(Clone, Debug, Deserialize, Serialize, TS)] #[ts(export)] #[serde(rename_all = "camelCase")] #[serde(rename_all_fields = "camelCase")] #[serde(tag = "kind")] -pub enum ExportedHostnameInfo { +pub enum HostnameInfo { Ip { network_interface_id: String, public: bool, - hostname: ExportedIpHostname, + hostname: IpHostname, }, Onion { - hostname: ExportedOnionHostname, + hostname: OnionHostname, }, } #[derive(Clone, Debug, Deserialize, Serialize, TS)] #[ts(export)] #[serde(rename_all = "camelCase")] -pub struct ExportedOnionHostname { +pub struct OnionHostname { pub value: String, pub port: Option, pub ssl_port: Option, @@ -56,7 +37,7 @@ pub struct ExportedOnionHostname { #[serde(rename_all = "camelCase")] #[serde(rename_all_fields = "camelCase")] #[serde(tag = "kind")] -pub enum ExportedIpHostname { +pub enum IpHostname { Ipv4 { value: Ipv4Addr, port: Option, @@ -110,6 +91,10 @@ pub enum ServiceInterfaceType { pub struct AddressInfo { pub username: Option, pub host_id: HostId, - pub bind_options: BindOptions, + pub internal_port: u16, + #[ts(type = "string | null")] + pub scheme: Option, + #[ts(type = "string | null")] + pub ssl_scheme: Option, pub suffix: String, } diff --git a/core/startos/src/net/vhost.rs b/core/startos/src/net/vhost.rs index 8dc1e6da3..b4f5715ae 100644 --- a/core/startos/src/net/vhost.rs +++ b/core/startos/src/net/vhost.rs @@ -205,7 +205,7 @@ impl VHostServer { .into_entries()? .into_iter() .flat_map(|(_, ips)| [ - ips.as_ipv4().de().map(|ip| ip.map(IpAddr::V4)), + ips.as_ipv4().de().map(|ip| ip.map(IpAddr::V4)), ips.as_ipv6().de().map(|ip| ip.map(IpAddr::V6)) ]) .filter_map(|a| a.transpose()) diff --git a/core/startos/src/registry/auth.rs b/core/startos/src/registry/auth.rs index 741090b4d..27655c4a8 100644 --- a/core/startos/src/registry/auth.rs +++ b/core/startos/src/registry/auth.rs @@ -138,7 +138,6 @@ impl Middleware for Auth { if request.headers().contains_key(AUTH_SIG_HEADER) { self.signer = Some( async { - let request = request; let SignatureHeader { commitment, signer, diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index e6ccdd131..da641468b 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -10,7 +10,8 @@ use persistent_container::PersistentContainer; use rpc_toolkit::{from_fn_async, CallRemoteHandler, Empty, HandlerArgs, HandlerFor}; use serde::{Deserialize, Serialize}; use start_stop::StartStop; -use tokio::{fs::File, sync::Notify}; +use tokio::fs::File; +use tokio::sync::Notify; use ts_rs::TS; use crate::context::{CliContext, RpcContext}; @@ -308,7 +309,7 @@ impl Service { .send(transition::restore::Restore { path: backup_source.path().to_path_buf(), }) - .await?; + .await??; Ok(service) } @@ -370,7 +371,7 @@ impl Service { .send(transition::backup::Backup { path: guard.path().to_path_buf(), }) - .await?; + .await??; Ok(()) } diff --git a/core/startos/src/service/persistent_container.rs b/core/startos/src/service/persistent_container.rs index 06dcdc28e..03203633a 100644 --- a/core/startos/src/service/persistent_container.rs +++ b/core/startos/src/service/persistent_container.rs @@ -10,7 +10,7 @@ use imbl_value::InternedString; use models::{ProcedureName, VolumeId}; use rpc_toolkit::{Empty, Server, ShutdownHandle}; use serde::de::DeserializeOwned; -use tokio::fs::{ File}; +use tokio::fs::File; use tokio::process::Command; use tokio::sync::{oneshot, watch, Mutex, OnceCell}; use tracing::instrument; diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index 31c5a8026..aedf4c194 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -9,7 +9,6 @@ use std::sync::{Arc, Weak}; use clap::builder::ValueParserFactory; use clap::Parser; use emver::VersionRange; -use imbl::OrdMap; use imbl_value::{json, InternedString}; use itertools::Itertools; use models::{ @@ -30,12 +29,9 @@ use crate::disk::mount::filesystem::idmapped::IdMapped; use crate::disk::mount::filesystem::loop_dev::LoopDev; use crate::disk::mount::filesystem::overlayfs::OverlayGuard; use crate::net::host::address::HostAddress; -use crate::net::host::binding::BindOptions; -use crate::net::host::{self, HostKind}; -use crate::net::service_interface::{ - AddressInfo, ExportedHostInfo, ExportedHostnameInfo, ServiceInterface, ServiceInterfaceType, - ServiceInterfaceWithHostInfo, -}; +use crate::net::host::binding::{BindOptions, LanInfo}; +use crate::net::host::{Host, HostKind}; +use crate::net::service_interface::{AddressInfo, ServiceInterface, ServiceInterfaceType}; use crate::prelude::*; use crate::s9pk::merkle_archive::source::http::HttpSource; use crate::s9pk::rpc::SKIP_ENV; @@ -193,7 +189,6 @@ pub fn service_effect_handler() -> ParentHandler { .subcommand("removeAddress", from_fn_async(remove_address).no_cli()) .subcommand("exportAction", from_fn_async(export_action).no_cli()) .subcommand("removeAction", from_fn_async(remove_action).no_cli()) - .subcommand("reverseProxy", from_fn_async(reverse_proxy).no_cli()) .subcommand("mount", from_fn_async(mount).no_cli()) // TODO Callbacks @@ -233,8 +228,6 @@ struct ExportServiceInterfaceParams { masked: bool, address_info: AddressInfo, r#type: ServiceInterfaceType, - host_kind: HostKind, - hostnames: Vec, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] #[ts(export)] @@ -242,9 +235,8 @@ struct ExportServiceInterfaceParams { struct GetPrimaryUrlParams { #[ts(type = "string | null")] package_id: Option, - service_interface_id: String, + service_interface_id: ServiceInterfaceId, callback: Callback, - host_id: HostId, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] #[ts(export)] @@ -276,37 +268,7 @@ struct RemoveActionParams { #[ts(type = "string")] id: ActionId, } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] -#[ts(export)] -#[serde(rename_all = "camelCase")] -struct ReverseProxyBind { - ip: Option, - port: u32, - ssl: bool, -} -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] -#[ts(export)] -#[serde(rename_all = "camelCase")] -struct ReverseProxyDestination { - ip: Option, - port: u32, - ssl: bool, -} -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] -#[ts(export)] -#[serde(rename_all = "camelCase")] -struct ReverseProxyHttp { - #[ts(type = "null | {[key: string]: string}")] - headers: Option>, -} -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] -#[ts(export)] -#[serde(rename_all = "camelCase")] -struct ReverseProxyParams { - bind: ReverseProxyBind, - dst: ReverseProxyDestination, - http: ReverseProxyHttp, -} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] #[ts(export)] #[serde(rename_all = "camelCase")] @@ -367,7 +329,7 @@ async fn get_container_ip(context: EffectContext, _: Empty) -> Result Result { +) -> Result { let internal_port = data.internal_port as u16; let context = context.deref()?; @@ -404,13 +366,10 @@ async fn export_service_interface( masked, address_info, r#type, - host_kind, - hostnames, }: ExportServiceInterfaceParams, ) -> Result<(), Error> { let context = context.deref()?; let package_id = context.id.clone(); - let host_id = address_info.host_id.clone(); let service_interface = ServiceInterface { id: id.clone(), @@ -422,15 +381,7 @@ async fn export_service_interface( address_info, interface_type: r#type, }; - let host_info = ExportedHostInfo { - id: host_id, - kind: host_kind, - hostnames, - }; - let svc_interface_with_host_info = ServiceInterfaceWithHostInfo { - service_interface, - host_info, - }; + let svc_interface_with_host_info = service_interface; context .ctx @@ -449,35 +400,26 @@ async fn export_service_interface( } async fn get_primary_url( context: EffectContext, - data: GetPrimaryUrlParams, -) -> Result { + GetPrimaryUrlParams { + package_id, + service_interface_id, + callback, + }: GetPrimaryUrlParams, +) -> Result, Error> { let context = context.deref()?; - let package_id = context.id.clone(); + let package_id = package_id.unwrap_or_else(|| context.id.clone()); - let db_model = context.ctx.db.peek().await; - - let pkg_data_model = db_model - .as_public() - .as_package_data() - .as_idx(&package_id) - .or_not_found(&package_id)?; - - let host = pkg_data_model.de()?.hosts.get_host_primary(&data.host_id); - - match host { - Some(host_address) => Ok(host_address), - None => Err(Error::new( - eyre!("Primary Url not found for {}", data.host_id), - crate::ErrorKind::NotFound, - )), - } + Ok(None) // TODO } async fn list_service_interfaces( context: EffectContext, - data: ListServiceInterfacesParams, -) -> Result, Error> { + ListServiceInterfacesParams { + package_id, + callback, + }: ListServiceInterfacesParams, +) -> Result, Error> { let context = context.deref()?; - let package_id = context.id.clone(); + let package_id = package_id.unwrap_or_else(|| context.id.clone()); context .ctx @@ -553,10 +495,8 @@ async fn remove_action(context: EffectContext, data: RemoveActionParams) -> Resu .await?; Ok(()) } -async fn reverse_proxy(context: EffectContext, data: ReverseProxyParams) -> Result { - todo!() -} async fn mount(context: EffectContext, data: MountParams) -> Result { + // TODO todo!() } @@ -564,49 +504,42 @@ async fn mount(context: EffectContext, data: MountParams) -> Result, - service_interface_id: String, + host_id: HostId, #[ts(type = "string | null")] package_id: Option, callback: Callback, } async fn get_host_info( ctx: EffectContext, - GetHostInfoParams { .. }: GetHostInfoParams, -) -> Result { + GetHostInfoParams { + callback, + package_id, + host_id, + }: GetHostInfoParams, +) -> Result { let ctx = ctx.deref()?; - Ok(json!({ - "id": "fakeId1", - "kind": "multi", - "hostnames": [{ - "kind": "ip", - "networkInterfaceId": "fakeNetworkInterfaceId1", - "public": true, - "hostname":{ - "kind": "domain", - "domain": format!("{}", ctx.id), - "subdomain": (), - "port": (), - "sslPort": () - } - } + let db = ctx.ctx.db.peek().await; + let package_id = package_id.unwrap_or_else(|| ctx.id.clone()); - ] - })) + db.as_public() + .as_package_data() + .as_idx(&package_id) + .or_not_found(&package_id)? + .as_hosts() + .as_idx(&host_id) + .or_not_found(&host_id)? + .de() } -async fn clear_bindings(context: EffectContext, _: Empty) -> Result { - todo!() +async fn clear_bindings(context: EffectContext, _: Empty) -> Result<(), Error> { + let ctx = context.deref()?; + let mut svc = ctx.persistent_container.net_service.lock().await; + svc.clear_bindings().await?; + Ok(()) } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)] @@ -619,15 +552,13 @@ struct BindParams { #[serde(flatten)] options: BindOptions, } -async fn bind( - context: EffectContext, - BindParams { +async fn bind(context: EffectContext, bind_params: Value) -> Result<(), Error> { + let BindParams { kind, id, internal_port, options, - }: BindParams, -) -> Result<(), Error> { + } = from_value(bind_params)?; let ctx = context.deref()?; let mut svc = ctx.persistent_container.net_service.lock().await; svc.bind(kind, id, internal_port, options).await @@ -639,39 +570,32 @@ async fn bind( struct GetServiceInterfaceParams { #[ts(type = "string | null")] package_id: Option, - service_interface_id: String, + service_interface_id: ServiceInterfaceId, callback: Callback, } + async fn get_service_interface( - _: EffectContext, + ctx: EffectContext, GetServiceInterfaceParams { callback, package_id, service_interface_id, }: GetServiceInterfaceParams, -) -> Result { - // TODO @Dr_Bonez - Ok(json!({ - "id": service_interface_id, - "name": service_interface_id, - "description": "This is a fake", - "hasPrimary": true, - "disabled": false, - "masked": false, - "addressInfo": json!({ - "username": Value::Null, - "hostId": "HostId?", - "options": json!({ - "scheme": Value::Null, - "preferredExternalPort": 80, - "addSsl":Value::Null, - "secure": false, - "ssl": false - }), - "suffix": "http" - }), - "type": "api" - })) +) -> Result { + let ctx = ctx.deref()?; + let package_id = package_id.unwrap_or_else(|| ctx.id.clone()); + let db = ctx.ctx.db.peek().await; + + let interface = db + .as_public() + .as_package_data() + .as_idx(&package_id) + .or_not_found(&package_id)? + .as_service_interfaces() + .as_idx(&service_interface_id) + .or_not_found(&service_interface_id)? + .de()?; + Ok(interface) } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Parser, TS)] @@ -764,6 +688,7 @@ async fn get_ssl_certificate( host_id, }: GetSslCertificateParams, ) -> Result { + // TODO let fake = include_str!("./fake.cert.pem"); Ok(json!([fake, fake, fake])) } @@ -785,6 +710,7 @@ async fn get_ssl_key( algorithm, }: GetSslKeyParams, ) -> Result { + // TODO let fake = include_str!("./fake.cert.key"); Ok(json!(fake)) } diff --git a/sdk/lib/StartSdk.ts b/sdk/lib/StartSdk.ts index 381914c13..f5959a084 100644 --- a/sdk/lib/StartSdk.ts +++ b/sdk/lib/StartSdk.ts @@ -59,7 +59,7 @@ import { } from "./interfaces/setupInterfaces" import { successFailure } from "./trigger/successFailure" import { HealthReceipt } from "./health/HealthReceipt" -import { MultiHost, Scheme, SingleHost, StaticHost } from "./interfaces/Host" +import { MultiHost, Scheme } from "./interfaces/Host" import { ServiceInterfaceBuilder } from "./interfaces/ServiceInterfaceBuilder" import { GetSystemSmtp } from "./util/GetSystemSmtp" import nullIfEmpty from "./util/nullIfEmpty" @@ -178,10 +178,10 @@ export class StartSdk { }, host: { - static: (effects: Effects, id: string) => - new StaticHost({ id, effects }), - single: (effects: Effects, id: string) => - new SingleHost({ id, effects }), + // static: (effects: Effects, id: string) => + // new StaticHost({ id, effects }), + // single: (effects: Effects, id: string) => + // new SingleHost({ id, effects }), multi: (effects: Effects, id: string) => new MultiHost({ id, effects }), }, nullIfEmpty, diff --git a/sdk/lib/interfaces/Host.ts b/sdk/lib/interfaces/Host.ts index 2f1353cc1..96a76628a 100644 --- a/sdk/lib/interfaces/Host.ts +++ b/sdk/lib/interfaces/Host.ts @@ -1,14 +1,14 @@ -import { object, string } from "ts-matches" +import { number, object, string } from "ts-matches" import { Effects } from "../types" import { Origin } from "./Origin" -import { AddSslOptions } from ".././osBindings" +import { AddSslOptions, BindParams } from ".././osBindings" import { Security } from ".././osBindings" import { BindOptions } from ".././osBindings" import { AlpnInfo } from ".././osBindings" export { AddSslOptions, Security, BindOptions } -const knownProtocols = { +export const knownProtocols = { http: { secure: null, defaultPort: 80, @@ -69,19 +69,17 @@ type NotProtocolsWithSslVariants = Exclude< type BindOptionsByKnownProtocol = | { protocol: ProtocolsWithSslVariants - preferredExternalPort?: number - scheme?: Scheme + preferredExternalPort: number addSsl?: Partial } | { protocol: NotProtocolsWithSslVariants - preferredExternalPort?: number - scheme?: Scheme + preferredExternalPort: number addSsl?: AddSslOptions } -type BindOptionsByProtocol = BindOptionsByKnownProtocol | BindOptions +export type BindOptionsByProtocol = BindOptionsByKnownProtocol | BindOptions -export type HostKind = "static" | "single" | "multi" +export type HostKind = BindParams["kind"] const hasStringProtocol = object({ protocol: string, @@ -110,66 +108,62 @@ export class Host { private async bindPortForUnknown( internalPort: number, options: { - scheme: Scheme preferredExternalPort: number addSsl: AddSslOptions | null secure: { ssl: boolean } | null }, ) { - await this.options.effects.bind({ + const binderOptions = { kind: this.options.kind, id: this.options.id, - internalPort: internalPort, + internalPort, ...options, - }) + } + await this.options.effects.bind(binderOptions) - return new Origin(this, options) + return new Origin(this, internalPort, null, null) } private async bindPortForKnown( options: BindOptionsByKnownProtocol, internalPort: number, ) { - const scheme = - options.scheme === undefined ? options.protocol : options.scheme const protoInfo = knownProtocols[options.protocol] const preferredExternalPort = options.preferredExternalPort || knownProtocols[options.protocol].defaultPort - const addSsl = this.getAddSsl(options, protoInfo) + const sslProto = this.getSslProto(options, protoInfo) + const addSsl = + sslProto && "alpn" in protoInfo + ? { + // addXForwardedHeaders: null, + preferredExternalPort: knownProtocols[sslProto].defaultPort, + scheme: sslProto, + alpn: protoInfo.alpn, + ...("addSsl" in options ? options.addSsl : null), + } + : null const secure: Security | null = !protoInfo.secure ? null : { ssl: false } - const newOptions = { - scheme, - preferredExternalPort, - addSsl, - secure, - } - await this.options.effects.bind({ kind: this.options.kind, id: this.options.id, internalPort, - ...newOptions, + preferredExternalPort, + addSsl, + secure, }) - return new Origin(this, newOptions) + return new Origin(this, internalPort, options.protocol, sslProto) } - private getAddSsl( + private getSslProto( options: BindOptionsByKnownProtocol, protoInfo: KnownProtocols[keyof KnownProtocols], - ): AddSslOptions | null { + ) { if (inObject("noAddSsl", options) && options.noAddSsl) return null - if ("withSsl" in protoInfo && protoInfo.withSsl) - return { - // addXForwardedHeaders: null, - preferredExternalPort: knownProtocols[protoInfo.withSsl].defaultPort, - scheme: protoInfo.withSsl, - alpn: protoInfo.alpn, - ...("addSsl" in options ? options.addSsl : null), - } + if ("withSsl" in protoInfo && protoInfo.withSsl) return protoInfo.withSsl return null } } @@ -181,17 +175,17 @@ function inObject( return key in obj } -export class StaticHost extends Host { - constructor(options: { effects: Effects; id: string }) { - super({ ...options, kind: "static" }) - } -} +// export class StaticHost extends Host { +// constructor(options: { effects: Effects; id: string }) { +// super({ ...options, kind: "static" }) +// } +// } -export class SingleHost extends Host { - constructor(options: { effects: Effects; id: string }) { - super({ ...options, kind: "single" }) - } -} +// export class SingleHost extends Host { +// constructor(options: { effects: Effects; id: string }) { +// super({ ...options, kind: "single" }) +// } +// } export class MultiHost extends Host { constructor(options: { effects: Effects; id: string }) { diff --git a/sdk/lib/interfaces/Origin.ts b/sdk/lib/interfaces/Origin.ts index aaadbea50..52afe1ed3 100644 --- a/sdk/lib/interfaces/Origin.ts +++ b/sdk/lib/interfaces/Origin.ts @@ -6,7 +6,9 @@ import { ServiceInterfaceBuilder } from "./ServiceInterfaceBuilder" export class Origin { constructor( readonly host: T, - readonly options: BindOptions, + readonly internalPort: number, + readonly scheme: string | null, + readonly sslScheme: string | null, ) {} build({ username, path, search, schemeOverride }: BuildOptions): AddressInfo { @@ -20,18 +22,9 @@ export class Origin { return { hostId: this.host.options.id, - bindOptions: { - ...this.options, - scheme: schemeOverride ? schemeOverride.noSsl : this.options.scheme, - addSsl: this.options.addSsl - ? { - ...this.options.addSsl, - scheme: schemeOverride - ? schemeOverride.ssl - : this.options.addSsl.scheme, - } - : null, - }, + internalPort: this.internalPort, + scheme: schemeOverride ? schemeOverride.noSsl : this.scheme, + sslScheme: schemeOverride ? schemeOverride.ssl : this.sslScheme, suffix: `${path}${qp}`, username, } diff --git a/sdk/lib/osBindings/AddAssetParams.ts b/sdk/lib/osBindings/AddAssetParams.ts index ce6128cf7..ffd7db675 100644 --- a/sdk/lib/osBindings/AddAssetParams.ts +++ b/sdk/lib/osBindings/AddAssetParams.ts @@ -6,7 +6,6 @@ import type { Version } from "./Version" export type AddAssetParams = { version: Version platform: string - upload: boolean url: string signature: AnySignature commitment: Blake3Commitment diff --git a/sdk/lib/osBindings/AddPackageParams.ts b/sdk/lib/osBindings/AddPackageParams.ts new file mode 100644 index 000000000..4395b9b8a --- /dev/null +++ b/sdk/lib/osBindings/AddPackageParams.ts @@ -0,0 +1,9 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { AnySignature } from "./AnySignature" +import type { MerkleArchiveCommitment } from "./MerkleArchiveCommitment" + +export type AddPackageParams = { + url: string + commitment: MerkleArchiveCommitment + signature: AnySignature +} diff --git a/sdk/lib/osBindings/AddSslOptions.ts b/sdk/lib/osBindings/AddSslOptions.ts index daa9aee0e..35071aff3 100644 --- a/sdk/lib/osBindings/AddSslOptions.ts +++ b/sdk/lib/osBindings/AddSslOptions.ts @@ -2,7 +2,6 @@ import type { AlpnInfo } from "./AlpnInfo" export type AddSslOptions = { - scheme: string | null preferredExternalPort: number - alpn: AlpnInfo + alpn: AlpnInfo | null } diff --git a/sdk/lib/osBindings/AddressInfo.ts b/sdk/lib/osBindings/AddressInfo.ts index 818b570bb..c7a1c1af1 100644 --- a/sdk/lib/osBindings/AddressInfo.ts +++ b/sdk/lib/osBindings/AddressInfo.ts @@ -1,10 +1,11 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { BindOptions } from "./BindOptions" import type { HostId } from "./HostId" export type AddressInfo = { username: string | null hostId: HostId - bindOptions: BindOptions + internalPort: number + scheme: string | null + sslScheme: string | null suffix: string } diff --git a/sdk/lib/osBindings/BindInfo.ts b/sdk/lib/osBindings/BindInfo.ts index d7b37a70f..221b1c37c 100644 --- a/sdk/lib/osBindings/BindInfo.ts +++ b/sdk/lib/osBindings/BindInfo.ts @@ -1,4 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { BindOptions } from "./BindOptions" +import type { LanInfo } from "./LanInfo" -export type BindInfo = { options: BindOptions; assignedLanPort: number | null } +export type BindInfo = { options: BindOptions; lan: LanInfo } diff --git a/sdk/lib/osBindings/BindOptions.ts b/sdk/lib/osBindings/BindOptions.ts index a462a8cfc..49d9ecbf2 100644 --- a/sdk/lib/osBindings/BindOptions.ts +++ b/sdk/lib/osBindings/BindOptions.ts @@ -3,7 +3,6 @@ import type { AddSslOptions } from "./AddSslOptions" import type { Security } from "./Security" export type BindOptions = { - scheme: string | null preferredExternalPort: number addSsl: AddSslOptions | null secure: Security | null diff --git a/sdk/lib/osBindings/BindParams.ts b/sdk/lib/osBindings/BindParams.ts index 544f65a40..fcc450476 100644 --- a/sdk/lib/osBindings/BindParams.ts +++ b/sdk/lib/osBindings/BindParams.ts @@ -8,7 +8,6 @@ export type BindParams = { kind: HostKind id: HostId internalPort: number - scheme: string | null preferredExternalPort: number addSsl: AddSslOptions | null secure: Security | null diff --git a/sdk/lib/osBindings/ExportServiceInterfaceParams.ts b/sdk/lib/osBindings/ExportServiceInterfaceParams.ts index 847a6090a..b93e83f7c 100644 --- a/sdk/lib/osBindings/ExportServiceInterfaceParams.ts +++ b/sdk/lib/osBindings/ExportServiceInterfaceParams.ts @@ -1,7 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { AddressInfo } from "./AddressInfo" -import type { ExportedHostnameInfo } from "./ExportedHostnameInfo" -import type { HostKind } from "./HostKind" import type { ServiceInterfaceId } from "./ServiceInterfaceId" import type { ServiceInterfaceType } from "./ServiceInterfaceType" @@ -14,6 +12,4 @@ export type ExportServiceInterfaceParams = { masked: boolean addressInfo: AddressInfo type: ServiceInterfaceType - hostKind: HostKind - hostnames: Array } diff --git a/sdk/lib/osBindings/ExportedHostInfo.ts b/sdk/lib/osBindings/ExportedHostInfo.ts deleted file mode 100644 index 5e8a25ad6..000000000 --- a/sdk/lib/osBindings/ExportedHostInfo.ts +++ /dev/null @@ -1,10 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { ExportedHostnameInfo } from "./ExportedHostnameInfo" -import type { HostId } from "./HostId" -import type { HostKind } from "./HostKind" - -export type ExportedHostInfo = { - id: HostId - kind: HostKind - hostnames: Array -} diff --git a/sdk/lib/osBindings/ExportedHostnameInfo.ts b/sdk/lib/osBindings/ExportedHostnameInfo.ts deleted file mode 100644 index 42f849529..000000000 --- a/sdk/lib/osBindings/ExportedHostnameInfo.ts +++ /dev/null @@ -1,12 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { ExportedIpHostname } from "./ExportedIpHostname" -import type { ExportedOnionHostname } from "./ExportedOnionHostname" - -export type ExportedHostnameInfo = - | { - kind: "ip" - networkInterfaceId: string - public: boolean - hostname: ExportedIpHostname - } - | { kind: "onion"; hostname: ExportedOnionHostname } diff --git a/sdk/lib/osBindings/GetHostInfoParams.ts b/sdk/lib/osBindings/GetHostInfoParams.ts index 1ffb95de0..120b4cfe1 100644 --- a/sdk/lib/osBindings/GetHostInfoParams.ts +++ b/sdk/lib/osBindings/GetHostInfoParams.ts @@ -1,10 +1,9 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Callback } from "./Callback" -import type { GetHostInfoParamsKind } from "./GetHostInfoParamsKind" +import type { HostId } from "./HostId" export type GetHostInfoParams = { - kind: GetHostInfoParamsKind | null - serviceInterfaceId: string + hostId: HostId packageId: string | null callback: Callback } diff --git a/sdk/lib/osBindings/GetHostInfoParamsKind.ts b/sdk/lib/osBindings/GetHostInfoParamsKind.ts deleted file mode 100644 index 482eb7177..000000000 --- a/sdk/lib/osBindings/GetHostInfoParamsKind.ts +++ /dev/null @@ -1,3 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. - -export type GetHostInfoParamsKind = "multi" diff --git a/sdk/lib/osBindings/GetPackageResponse.ts b/sdk/lib/osBindings/GetPackageResponse.ts index 5bf24bfc0..3e1dd4e9d 100644 --- a/sdk/lib/osBindings/GetPackageResponse.ts +++ b/sdk/lib/osBindings/GetPackageResponse.ts @@ -4,6 +4,7 @@ import type { PackageVersionInfo } from "./PackageVersionInfo" import type { Version } from "./Version" export type GetPackageResponse = { + categories: string[] best: { [key: Version]: PackageVersionInfo } otherVersions?: { [key: Version]: PackageInfoShort } } diff --git a/sdk/lib/osBindings/GetPackageResponseFull.ts b/sdk/lib/osBindings/GetPackageResponseFull.ts index 579924291..e375dd489 100644 --- a/sdk/lib/osBindings/GetPackageResponseFull.ts +++ b/sdk/lib/osBindings/GetPackageResponseFull.ts @@ -3,6 +3,7 @@ import type { PackageVersionInfo } from "./PackageVersionInfo" import type { Version } from "./Version" export type GetPackageResponseFull = { + categories: string[] best: { [key: Version]: PackageVersionInfo } otherVersions: { [key: Version]: PackageVersionInfo } } diff --git a/sdk/lib/osBindings/GetPrimaryUrlParams.ts b/sdk/lib/osBindings/GetPrimaryUrlParams.ts index 1a68ecc7b..dbafa4152 100644 --- a/sdk/lib/osBindings/GetPrimaryUrlParams.ts +++ b/sdk/lib/osBindings/GetPrimaryUrlParams.ts @@ -1,10 +1,9 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Callback } from "./Callback" -import type { HostId } from "./HostId" +import type { ServiceInterfaceId } from "./ServiceInterfaceId" export type GetPrimaryUrlParams = { packageId: string | null - serviceInterfaceId: string + serviceInterfaceId: ServiceInterfaceId callback: Callback - hostId: HostId } diff --git a/sdk/lib/osBindings/GetServiceInterfaceParams.ts b/sdk/lib/osBindings/GetServiceInterfaceParams.ts index ee3a0c03d..0a8bdfcb2 100644 --- a/sdk/lib/osBindings/GetServiceInterfaceParams.ts +++ b/sdk/lib/osBindings/GetServiceInterfaceParams.ts @@ -1,8 +1,9 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Callback } from "./Callback" +import type { ServiceInterfaceId } from "./ServiceInterfaceId" export type GetServiceInterfaceParams = { packageId: string | null - serviceInterfaceId: string + serviceInterfaceId: ServiceInterfaceId callback: Callback } diff --git a/sdk/lib/osBindings/HardwareRequirements.ts b/sdk/lib/osBindings/HardwareRequirements.ts index 4964bc66f..0e1da1f36 100644 --- a/sdk/lib/osBindings/HardwareRequirements.ts +++ b/sdk/lib/osBindings/HardwareRequirements.ts @@ -3,5 +3,5 @@ export type HardwareRequirements = { device: { [key: string]: string } ram: number | null - arch: Array | null + arch: string[] | null } diff --git a/sdk/lib/osBindings/Host.ts b/sdk/lib/osBindings/Host.ts index a476c0adc..7d8cf3a90 100644 --- a/sdk/lib/osBindings/Host.ts +++ b/sdk/lib/osBindings/Host.ts @@ -2,10 +2,14 @@ import type { BindInfo } from "./BindInfo" import type { HostAddress } from "./HostAddress" import type { HostKind } from "./HostKind" +import type { HostnameInfo } from "./HostnameInfo" export type Host = { kind: HostKind bindings: { [key: number]: BindInfo } addresses: Array - primary: HostAddress | null + /** + * COMPUTED: NetService::update + */ + hostnameInfo: { [key: number]: Array } } diff --git a/sdk/lib/osBindings/HostnameInfo.ts b/sdk/lib/osBindings/HostnameInfo.ts new file mode 100644 index 000000000..ef8bafac0 --- /dev/null +++ b/sdk/lib/osBindings/HostnameInfo.ts @@ -0,0 +1,12 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { IpHostname } from "./IpHostname" +import type { OnionHostname } from "./OnionHostname" + +export type HostnameInfo = + | { + kind: "ip" + networkInterfaceId: string + public: boolean + hostname: IpHostname + } + | { kind: "onion"; hostname: OnionHostname } diff --git a/sdk/lib/osBindings/HostInfo.ts b/sdk/lib/osBindings/Hosts.ts similarity index 79% rename from sdk/lib/osBindings/HostInfo.ts rename to sdk/lib/osBindings/Hosts.ts index d39b56ebe..c7aa84996 100644 --- a/sdk/lib/osBindings/HostInfo.ts +++ b/sdk/lib/osBindings/Hosts.ts @@ -2,4 +2,4 @@ import type { Host } from "./Host" import type { HostId } from "./HostId" -export type HostInfo = { [key: HostId]: Host } +export type Hosts = { [key: HostId]: Host } diff --git a/sdk/lib/osBindings/ExportedIpHostname.ts b/sdk/lib/osBindings/IpHostname.ts similarity index 94% rename from sdk/lib/osBindings/ExportedIpHostname.ts rename to sdk/lib/osBindings/IpHostname.ts index ea06fe05b..4a6b5e87c 100644 --- a/sdk/lib/osBindings/ExportedIpHostname.ts +++ b/sdk/lib/osBindings/IpHostname.ts @@ -1,6 +1,6 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type ExportedIpHostname = +export type IpHostname = | { kind: "ipv4"; value: string; port: number | null; sslPort: number | null } | { kind: "ipv6"; value: string; port: number | null; sslPort: number | null } | { diff --git a/sdk/lib/osBindings/ReverseProxyDestination.ts b/sdk/lib/osBindings/LanInfo.ts similarity index 55% rename from sdk/lib/osBindings/ReverseProxyDestination.ts rename to sdk/lib/osBindings/LanInfo.ts index 88b5dd650..59b8a5519 100644 --- a/sdk/lib/osBindings/ReverseProxyDestination.ts +++ b/sdk/lib/osBindings/LanInfo.ts @@ -1,7 +1,6 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type ReverseProxyDestination = { - ip: string | null - port: number - ssl: boolean +export type LanInfo = { + assignedPort: number | null + assignedSslPort: number | null } diff --git a/sdk/lib/osBindings/ExportedOnionHostname.ts b/sdk/lib/osBindings/OnionHostname.ts similarity index 82% rename from sdk/lib/osBindings/ExportedOnionHostname.ts rename to sdk/lib/osBindings/OnionHostname.ts index 2c7d67cec..0bea8245e 100644 --- a/sdk/lib/osBindings/ExportedOnionHostname.ts +++ b/sdk/lib/osBindings/OnionHostname.ts @@ -1,6 +1,6 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type ExportedOnionHostname = { +export type OnionHostname = { value: string port: number | null sslPort: number | null diff --git a/sdk/lib/osBindings/OsVersionInfo.ts b/sdk/lib/osBindings/OsVersionInfo.ts index 7cd0fde9e..a88115350 100644 --- a/sdk/lib/osBindings/OsVersionInfo.ts +++ b/sdk/lib/osBindings/OsVersionInfo.ts @@ -7,7 +7,7 @@ export type OsVersionInfo = { headline: string releaseNotes: string sourceVersion: string - signers: Array + authorized: Array iso: { [key: string]: RegistryAsset } squashfs: { [key: string]: RegistryAsset } img: { [key: string]: RegistryAsset } diff --git a/sdk/lib/osBindings/PackageDataEntry.ts b/sdk/lib/osBindings/PackageDataEntry.ts index f88c6d7be..ef805741b 100644 --- a/sdk/lib/osBindings/PackageDataEntry.ts +++ b/sdk/lib/osBindings/PackageDataEntry.ts @@ -3,10 +3,10 @@ import type { ActionId } from "./ActionId" import type { ActionMetadata } from "./ActionMetadata" import type { CurrentDependencies } from "./CurrentDependencies" import type { DataUrl } from "./DataUrl" -import type { HostInfo } from "./HostInfo" +import type { Hosts } from "./Hosts" import type { PackageState } from "./PackageState" +import type { ServiceInterface } from "./ServiceInterface" import type { ServiceInterfaceId } from "./ServiceInterfaceId" -import type { ServiceInterfaceWithHostInfo } from "./ServiceInterfaceWithHostInfo" import type { Status } from "./Status" export type PackageDataEntry = { @@ -18,7 +18,7 @@ export type PackageDataEntry = { lastBackup: string | null currentDependencies: CurrentDependencies actions: { [key: ActionId]: ActionMetadata } - serviceInterfaces: { [key: ServiceInterfaceId]: ServiceInterfaceWithHostInfo } - hosts: HostInfo + serviceInterfaces: { [key: ServiceInterfaceId]: ServiceInterface } + hosts: Hosts storeExposedDependents: string[] } diff --git a/sdk/lib/osBindings/PackageInfo.ts b/sdk/lib/osBindings/PackageInfo.ts index af340424f..6d07cd43e 100644 --- a/sdk/lib/osBindings/PackageInfo.ts +++ b/sdk/lib/osBindings/PackageInfo.ts @@ -4,6 +4,7 @@ import type { PackageVersionInfo } from "./PackageVersionInfo" import type { Version } from "./Version" export type PackageInfo = { - signers: Array + authorized: Array versions: { [key: Version]: PackageVersionInfo } + categories: string[] } diff --git a/sdk/lib/osBindings/PackageVersionInfo.ts b/sdk/lib/osBindings/PackageVersionInfo.ts index da82540cd..bdded46bd 100644 --- a/sdk/lib/osBindings/PackageVersionInfo.ts +++ b/sdk/lib/osBindings/PackageVersionInfo.ts @@ -17,7 +17,6 @@ export type PackageVersionInfo = { upstreamRepo: string supportSite: string marketingSite: string - categories: string[] osVersion: Version hardwareRequirements: HardwareRequirements sourceVersion: string | null diff --git a/sdk/lib/osBindings/ReverseProxyBind.ts b/sdk/lib/osBindings/ReverseProxyBind.ts deleted file mode 100644 index bd07b9489..000000000 --- a/sdk/lib/osBindings/ReverseProxyBind.ts +++ /dev/null @@ -1,3 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. - -export type ReverseProxyBind = { ip: string | null; port: number; ssl: boolean } diff --git a/sdk/lib/osBindings/ReverseProxyHttp.ts b/sdk/lib/osBindings/ReverseProxyHttp.ts deleted file mode 100644 index ba49e81dc..000000000 --- a/sdk/lib/osBindings/ReverseProxyHttp.ts +++ /dev/null @@ -1,3 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. - -export type ReverseProxyHttp = { headers: null | { [key: string]: string } } diff --git a/sdk/lib/osBindings/ReverseProxyParams.ts b/sdk/lib/osBindings/ReverseProxyParams.ts deleted file mode 100644 index 00062cbdf..000000000 --- a/sdk/lib/osBindings/ReverseProxyParams.ts +++ /dev/null @@ -1,10 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { ReverseProxyBind } from "./ReverseProxyBind" -import type { ReverseProxyDestination } from "./ReverseProxyDestination" -import type { ReverseProxyHttp } from "./ReverseProxyHttp" - -export type ReverseProxyParams = { - bind: ReverseProxyBind - dst: ReverseProxyDestination - http: ReverseProxyHttp -} diff --git a/sdk/lib/osBindings/ServiceInterfaceWithHostInfo.ts b/sdk/lib/osBindings/ServiceInterfaceWithHostInfo.ts deleted file mode 100644 index 979e6b500..000000000 --- a/sdk/lib/osBindings/ServiceInterfaceWithHostInfo.ts +++ /dev/null @@ -1,17 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { AddressInfo } from "./AddressInfo" -import type { ExportedHostInfo } from "./ExportedHostInfo" -import type { ServiceInterfaceId } from "./ServiceInterfaceId" -import type { ServiceInterfaceType } from "./ServiceInterfaceType" - -export type ServiceInterfaceWithHostInfo = { - hostInfo: ExportedHostInfo - id: ServiceInterfaceId - name: string - description: string - hasPrimary: boolean - disabled: boolean - masked: boolean - addressInfo: AddressInfo - type: ServiceInterfaceType -} diff --git a/sdk/lib/osBindings/index.ts b/sdk/lib/osBindings/index.ts index 16fc56a74..efd7a5efb 100644 --- a/sdk/lib/osBindings/index.ts +++ b/sdk/lib/osBindings/index.ts @@ -3,6 +3,7 @@ export { ActionId } from "./ActionId" export { ActionMetadata } from "./ActionMetadata" export { AddAdminParams } from "./AddAdminParams" export { AddAssetParams } from "./AddAssetParams" +export { AddPackageParams } from "./AddPackageParams" export { AddressInfo } from "./AddressInfo" export { AddSslOptions } from "./AddSslOptions" export { AddVersionParams } from "./AddVersionParams" @@ -40,15 +41,10 @@ export { Duration } from "./Duration" export { EncryptedWire } from "./EncryptedWire" export { ExecuteAction } from "./ExecuteAction" export { ExportActionParams } from "./ExportActionParams" -export { ExportedHostInfo } from "./ExportedHostInfo" -export { ExportedHostnameInfo } from "./ExportedHostnameInfo" -export { ExportedIpHostname } from "./ExportedIpHostname" -export { ExportedOnionHostname } from "./ExportedOnionHostname" export { ExportServiceInterfaceParams } from "./ExportServiceInterfaceParams" export { ExposeForDependentsParams } from "./ExposeForDependentsParams" export { FullIndex } from "./FullIndex" export { FullProgress } from "./FullProgress" -export { GetHostInfoParamsKind } from "./GetHostInfoParamsKind" export { GetHostInfoParams } from "./GetHostInfoParams" export { GetOsAssetParams } from "./GetOsAssetParams" export { GetPackageParams } from "./GetPackageParams" @@ -69,14 +65,17 @@ export { HealthCheckId } from "./HealthCheckId" export { HealthCheckResult } from "./HealthCheckResult" export { HostAddress } from "./HostAddress" export { HostId } from "./HostId" -export { HostInfo } from "./HostInfo" export { HostKind } from "./HostKind" +export { HostnameInfo } from "./HostnameInfo" +export { Hosts } from "./Hosts" export { Host } from "./Host" export { ImageId } from "./ImageId" export { InstalledState } from "./InstalledState" export { InstallingInfo } from "./InstallingInfo" export { InstallingState } from "./InstallingState" +export { IpHostname } from "./IpHostname" export { IpInfo } from "./IpInfo" +export { LanInfo } from "./LanInfo" export { ListServiceInterfacesParams } from "./ListServiceInterfacesParams" export { ListVersionSignersParams } from "./ListVersionSignersParams" export { MainStatus } from "./MainStatus" @@ -86,6 +85,7 @@ export { MerkleArchiveCommitment } from "./MerkleArchiveCommitment" export { MountParams } from "./MountParams" export { MountTarget } from "./MountTarget" export { NamedProgress } from "./NamedProgress" +export { OnionHostname } from "./OnionHostname" export { OsIndex } from "./OsIndex" export { OsVersionInfo } from "./OsVersionInfo" export { PackageDataEntry } from "./PackageDataEntry" @@ -106,10 +106,6 @@ export { RemoveActionParams } from "./RemoveActionParams" export { RemoveAddressParams } from "./RemoveAddressParams" export { RemoveVersionParams } from "./RemoveVersionParams" export { RequestCommitment } from "./RequestCommitment" -export { ReverseProxyBind } from "./ReverseProxyBind" -export { ReverseProxyDestination } from "./ReverseProxyDestination" -export { ReverseProxyHttp } from "./ReverseProxyHttp" -export { ReverseProxyParams } from "./ReverseProxyParams" export { Security } from "./Security" export { ServerInfo } from "./ServerInfo" export { ServerSpecs } from "./ServerSpecs" @@ -117,7 +113,6 @@ export { ServerStatus } from "./ServerStatus" export { ServiceInterfaceId } from "./ServiceInterfaceId" export { ServiceInterface } from "./ServiceInterface" export { ServiceInterfaceType } from "./ServiceInterfaceType" -export { ServiceInterfaceWithHostInfo } from "./ServiceInterfaceWithHostInfo" export { SessionList } from "./SessionList" export { Sessions } from "./Sessions" export { Session } from "./Session" diff --git a/sdk/lib/test/host.test.ts b/sdk/lib/test/host.test.ts index 82372f61b..a8ae317ed 100644 --- a/sdk/lib/test/host.test.ts +++ b/sdk/lib/test/host.test.ts @@ -8,6 +8,7 @@ describe("host", () => { const foo = sdk.host.multi(effects, "foo") const fooOrigin = await foo.bindPort(80, { protocol: "http" as const, + preferredExternalPort: 80, }) const fooInterface = new ServiceInterfaceBuilder({ effects, diff --git a/sdk/lib/test/startosTypeValidation.test.ts b/sdk/lib/test/startosTypeValidation.test.ts index 4f6d50f53..79ec62106 100644 --- a/sdk/lib/test/startosTypeValidation.test.ts +++ b/sdk/lib/test/startosTypeValidation.test.ts @@ -25,7 +25,6 @@ import { ListServiceInterfacesParams } from ".././osBindings" import { RemoveAddressParams } from ".././osBindings" import { ExportActionParams } from ".././osBindings" import { RemoveActionParams } from ".././osBindings" -import { ReverseProxyParams } from ".././osBindings" import { MountParams } from ".././osBindings" function typeEquality(_a: ExpectedType) {} describe("startosTypeValidation ", () => { @@ -66,7 +65,6 @@ describe("startosTypeValidation ", () => { removeAddress: {} as RemoveAddressParams, exportAction: {} as ExportActionParams, removeAction: {} as RemoveActionParams, - reverseProxy: {} as ReverseProxyParams, mount: {} as MountParams, checkDependencies: {} as CheckDependenciesParam, getDependencies: undefined, diff --git a/sdk/lib/types.ts b/sdk/lib/types.ts index 7db42e5a9..2da92cd56 100644 --- a/sdk/lib/types.ts +++ b/sdk/lib/types.ts @@ -5,6 +5,12 @@ import { SetHealth, HealthCheckResult, SetMainStatus, + ServiceInterface, + Host, + ExportServiceInterfaceParams, + GetPrimaryUrlParams, + LanInfo, + BindParams, } from "./osBindings" import { MainEffects, ServiceInterfaceType, Signals } from "./StartSdk" @@ -12,7 +18,7 @@ import { InputSpec } from "./config/configTypes" import { DependenciesReceipt } from "./config/setupConfig" import { BindOptions, Scheme } from "./interfaces/Host" import { Daemons } from "./mainFn/Daemons" -import { PathBuilder, StorePath } from "./store/PathBuilder" +import { StorePath } from "./store/PathBuilder" import { ExposedStorePaths } from "./store/setupExposeStore" import { UrlString } from "./util/getServiceInterface" export * from "./osBindings" @@ -184,14 +190,6 @@ export declare const hostName: unique symbol // asdflkjadsf.onion | 1.2.3.4 export type Hostname = string & { [hostName]: never } -/** ${scheme}://${username}@${host}:${externalPort}${suffix} */ -export type AddressInfo = { - username: string | null - hostId: string - bindOptions: BindOptions - suffix: string -} - export type HostnameInfoIp = { kind: "ip" networkInterfaceId: string @@ -219,44 +217,9 @@ export type HostnameInfoOnion = { export type HostnameInfo = HostnameInfoIp | HostnameInfoOnion -export type SingleHost = { - id: string - kind: "single" | "static" - hostname: HostnameInfo | null -} - -export type MultiHost = { - id: string - kind: "multi" - hostnames: HostnameInfo[] -} - -export type HostInfo = SingleHost | MultiHost - export type ServiceInterfaceId = string -export type ServiceInterface = { - id: ServiceInterfaceId - /** The title of this field to be displayed */ - name: string - /** Human readable description, used as tooltip usually */ - description: string - /** Whether or not one address must be the primary address */ - hasPrimary: boolean - /** Disabled interfaces do not serve, but they retain their metadata and addresses */ - disabled: boolean - /** Whether or not to mask the URIs for this interface. Useful if the URIs contain sensitive information, such as a password, macaroon, or API key */ - masked: boolean - /** URI Information */ - addressInfo: AddressInfo - /** The network interface could be several types, something like ui, p2p, or network */ - type: ServiceInterfaceType -} - -export type ServiceInterfaceWithHostInfo = ServiceInterface & { - hostInfo: HostInfo -} - +export { ServiceInterface } export type ExposeServicePaths = { /** The path to the value in the Store. [JsonPath](https://jsonpath.com/) */ paths: ExposedStorePaths @@ -326,13 +289,7 @@ export type Effects = { /** Removes all network bindings */ clearBindings(): Promise /** Creates a host connected to the specified port with the provided options */ - bind( - options: { - kind: "static" | "single" | "multi" - id: string - internalPort: number - } & BindOptions, - ): Promise + bind(options: BindParams): Promise /** Retrieves the current hostname(s) associated with a host id */ // getHostInfo(options: { // kind: "static" | "single" @@ -341,11 +298,10 @@ export type Effects = { // callback: () => void // }): Promise getHostInfo(options: { - kind: "multi" | null - serviceInterfaceId: string + hostId: string packageId: string | null callback: () => void - }): Promise + }): Promise // /** // * Run rsync between two volumes. This is used to backup data between volumes. @@ -395,14 +351,14 @@ export type Effects = { getServicePortForward(options: { internalPort: number packageId: string | null - }): Promise + }): Promise /** Removes all network interfaces */ clearServiceInterfaces(): Promise /** When we want to create a link in the front end interfaces, and example is * exposing a url to view a web service */ - exportServiceInterface(options: ServiceInterface): Promise + exportServiceInterface(options: ExportServiceInterfaceParams): Promise exposeForDependents(options: { paths: string[] }): Promise @@ -422,11 +378,7 @@ export type Effects = { * The user sets the primary url for a interface * @param options */ - getPrimaryUrl(options: { - packageId: PackageId | null - serviceInterfaceId: ServiceInterfaceId - callback: () => void - }): Promise + getPrimaryUrl(options: GetPrimaryUrlParams): Promise /** * There are times that we want to see the addresses that where exported @@ -437,7 +389,7 @@ export type Effects = { listServiceInterfaces(options: { packageId: PackageId | null callback: () => void - }): Promise + }): Promise> /** *Remove an address that was exported. Used problably during main or during setConfig. @@ -501,25 +453,6 @@ export type Effects = { /** Exists could be useful during the runtime to know if some service is running, option dep */ running(options: { packageId: PackageId }): Promise - /** Instead of creating proxies with nginx, we have a utility to create and maintain a proxy in the lifetime of this running. */ - reverseProxy(options: { - bind: { - /** Optional, default is 0.0.0.0 */ - ip: string | null - port: number - ssl: boolean - } - dst: { - /** Optional: default is 127.0.0.1 */ - ip: string | null // optional, default 127.0.0.1 - port: number - ssl: boolean - } - http: { - // optional, will do TCP layer proxy only if not present - headers: Record | null - } | null - }): Promise<{ stop(): Promise }> restart(): void shutdown(): void diff --git a/sdk/lib/util/Hostname.ts b/sdk/lib/util/Hostname.ts new file mode 100644 index 000000000..ee68abe4f --- /dev/null +++ b/sdk/lib/util/Hostname.ts @@ -0,0 +1,25 @@ +import { HostnameInfo } from "../types" + +export function hostnameInfoToAddress(hostInfo: HostnameInfo): string { + if (hostInfo.kind === "onion") { + return `${hostInfo.hostname.value}` + } + if (hostInfo.kind !== "ip") { + throw Error("Expecting that the kind is ip.") + } + const hostname = hostInfo.hostname + if (hostname.kind === "domain") { + return `${hostname.subdomain ? `${hostname.subdomain}.` : ""}${hostname.domain}` + } + const port = hostname.sslPort || hostname.port + const portString = port ? `:${port}` : "" + if ("ipv4" === hostname.kind || "ipv6" === hostname.kind) { + return `${hostname.value}${portString}` + } + if ("local" === hostname.kind) { + return `${hostname.value}${portString}` + } + throw Error( + "Expecting to have a valid hostname kind." + JSON.stringify(hostname), + ) +} diff --git a/sdk/lib/util/getServiceInterface.ts b/sdk/lib/util/getServiceInterface.ts index 3b7af1c41..737cbbc16 100644 --- a/sdk/lib/util/getServiceInterface.ts +++ b/sdk/lib/util/getServiceInterface.ts @@ -1,10 +1,15 @@ import { ServiceInterfaceType } from "../StartSdk" +import { knownProtocols } from "../interfaces/Host" import { AddressInfo, Effects, - HostInfo, + Host, + HostAddress, Hostname, HostnameInfo, + HostnameInfoIp, + HostnameInfoOnion, + IpInfo, } from "../types" export type UrlString = string @@ -20,13 +25,13 @@ export const getHostname = (url: string): Hostname | null => { } export type Filled = { - hostnames: Hostname[] - onionHostnames: Hostname[] - localHostnames: Hostname[] - ipHostnames: Hostname[] - ipv4Hostnames: Hostname[] - ipv6Hostnames: Hostname[] - nonIpHostnames: Hostname[] + hostnames: HostnameInfo[] + onionHostnames: HostnameInfo[] + localHostnames: HostnameInfo[] + ipHostnames: HostnameInfo[] + ipv4Hostnames: HostnameInfo[] + ipv6Hostnames: HostnameInfo[] + nonIpHostnames: HostnameInfo[] urls: UrlString[] onionUrls: UrlString[] @@ -50,7 +55,7 @@ export type ServiceInterfaceFilled = { /** Whether or not to mask the URIs for this interface. Useful if the URIs contain sensitive information, such as a password, macaroon, or API key */ masked: boolean /** Information about the host for this binding */ - hostInfo: HostInfo + host: Host /** URI information */ addressInfo: FilledAddressInfo /** Indicates if we are a ui/p2p/api for the kind of interface that this is representing */ @@ -69,110 +74,103 @@ const negate = (a: A) => !fn(a) const unique = (values: A[]) => Array.from(new Set(values)) -function stringifyHostname(info: HostnameInfo): Hostname { - let base: string - if ("kind" in info.hostname && info.hostname.kind === "domain") { - base = info.hostname.subdomain - ? `${info.hostname.subdomain}.${info.hostname.domain}` - : info.hostname.domain - } else { - base = info.hostname.value +export const addressHostToUrl = ( + { scheme, sslScheme, username, suffix }: AddressInfo, + host: HostnameInfo, +): UrlString[] => { + const res = [] + const fmt = (scheme: string | null, host: HostnameInfo, port: number) => { + const includePort = + scheme && + scheme in knownProtocols && + port === knownProtocols[scheme as keyof typeof knownProtocols].defaultPort + let hostname + if (host.kind === "onion") { + hostname = host.hostname.value + } else if (host.kind === "ip") { + if (host.hostname.kind === "domain") { + hostname = `${host.hostname.subdomain ? `${host.hostname.subdomain}.` : ""}${host.hostname.domain}` + } else { + hostname = host.hostname.value + } + } + return `${scheme ? `${scheme}://` : ""}${ + username ? `${username}@` : "" + }${hostname}${includePort ? `:${port}` : ""}${suffix}` } - if (info.hostname.port && info.hostname.sslPort) { - return `${base}:${info.hostname.port}` as Hostname - } else if (info.hostname.sslPort) { - return `${base}:${info.hostname.sslPort}` as Hostname - } else if (info.hostname.port) { - return `${base}:${info.hostname.port}` as Hostname + if (host.hostname.sslPort !== null) { + res.push(fmt(sslScheme, host, host.hostname.sslPort)) } - return base as Hostname -} -const addressHostToUrl = ( - { bindOptions, username, suffix }: AddressInfo, - host: Hostname, -): UrlString => { - const scheme = host.endsWith(".onion") - ? bindOptions.scheme - : bindOptions.addSsl - ? bindOptions.addSsl.scheme - : bindOptions.scheme // TODO: encode whether hostname transport is "secure"? - return `${scheme ? `${scheme}//` : ""}${ - username ? `${username}@` : "" - }${host}${suffix}` + if (host.hostname.port !== null) { + res.push(fmt(scheme, host, host.hostname.port)) + } + + return res } + export const filledAddress = ( - hostInfo: HostInfo, + host: Host, addressInfo: AddressInfo, ): FilledAddressInfo => { const toUrl = addressHostToUrl.bind(null, addressInfo) - const hostnameInfo = - hostInfo.kind == "multi" - ? hostInfo.hostnames - : hostInfo.hostname - ? [hostInfo.hostname] - : [] + const hostnames = host.hostnameInfo[addressInfo.internalPort] + return { ...addressInfo, - hostnames: hostnameInfo.flatMap((h) => stringifyHostname(h)), + hostnames, get onionHostnames() { - return hostnameInfo - .filter((h) => h.kind === "onion") - .map((h) => stringifyHostname(h)) + return hostnames.filter((h) => h.kind === "onion") }, get localHostnames() { - return hostnameInfo - .filter((h) => h.kind === "ip" && h.hostname.kind === "local") - .map((h) => stringifyHostname(h)) + return hostnames.filter( + (h) => h.kind === "ip" && h.hostname.kind === "local", + ) }, get ipHostnames() { - return hostnameInfo - .filter( - (h) => - h.kind === "ip" && - (h.hostname.kind === "ipv4" || h.hostname.kind === "ipv6"), - ) - .map((h) => stringifyHostname(h)) + return hostnames.filter( + (h) => + h.kind === "ip" && + (h.hostname.kind === "ipv4" || h.hostname.kind === "ipv6"), + ) }, get ipv4Hostnames() { - return hostnameInfo - .filter((h) => h.kind === "ip" && h.hostname.kind === "ipv4") - .map((h) => stringifyHostname(h)) + return hostnames.filter( + (h) => h.kind === "ip" && h.hostname.kind === "ipv4", + ) }, get ipv6Hostnames() { - return hostnameInfo - .filter((h) => h.kind === "ip" && h.hostname.kind === "ipv6") - .map((h) => stringifyHostname(h)) + return hostnames.filter( + (h) => h.kind === "ip" && h.hostname.kind === "ipv6", + ) }, get nonIpHostnames() { - return hostnameInfo - .filter( - (h) => - h.kind === "ip" && - h.hostname.kind !== "ipv4" && - h.hostname.kind !== "ipv6", - ) - .map((h) => stringifyHostname(h)) + return hostnames.filter( + (h) => + h.kind === "ip" && + h.hostname.kind !== "ipv4" && + h.hostname.kind !== "ipv6", + ) }, get urls() { - return this.hostnames.map(toUrl) + return this.hostnames.flatMap(toUrl) }, get onionUrls() { - return this.onionHostnames.map(toUrl) + return this.onionHostnames.flatMap(toUrl) }, get localUrls() { - return this.localHostnames.map(toUrl) + return this.localHostnames.flatMap(toUrl) }, get ipUrls() { - return this.ipHostnames.map(toUrl) + return this.ipHostnames.flatMap(toUrl) }, get ipv4Urls() { - return this.ipv4Hostnames.map(toUrl) + return this.ipv4Hostnames.flatMap(toUrl) }, get ipv6Urls() { - return this.ipv6Hostnames.map(toUrl) + return this.ipv6Hostnames.flatMap(toUrl) }, get nonIpUrls() { - return this.nonIpHostnames.map(toUrl) + return this.nonIpHostnames.flatMap(toUrl) }, } } @@ -193,23 +191,25 @@ const makeInterfaceFilled = async ({ packageId, callback, }) - const hostInfo = await effects.getHostInfo({ - packageId, - kind: null, - serviceInterfaceId: serviceInterfaceValue.id, - callback, - }) - const primaryUrl = await effects.getPrimaryUrl({ - serviceInterfaceId: id, + const hostId = serviceInterfaceValue.addressInfo.hostId + const host = await effects.getHostInfo({ packageId, + hostId, callback, }) + const primaryUrl = await effects + .getPrimaryUrl({ + serviceInterfaceId: id, + packageId, + callback, + }) + .catch((e) => null) const interfaceFilled: ServiceInterfaceFilled = { ...serviceInterfaceValue, primaryUrl: primaryUrl, - hostInfo, - addressInfo: filledAddress(hostInfo, serviceInterfaceValue.addressInfo), + host, + addressInfo: filledAddress(host, serviceInterfaceValue.addressInfo), get primaryHostname() { if (primaryUrl == null) return null return getHostname(primaryUrl) diff --git a/sdk/lib/util/getServiceInterfaces.ts b/sdk/lib/util/getServiceInterfaces.ts index a7106568b..c4cdc6b59 100644 --- a/sdk/lib/util/getServiceInterfaces.ts +++ b/sdk/lib/util/getServiceInterfaces.ts @@ -18,41 +18,27 @@ const makeManyInterfaceFilled = async ({ packageId, callback, }) - const hostIdsRecord = Object.fromEntries( - await Promise.all( - Array.from(new Set(serviceInterfaceValues.map((x) => x.id))).map( - async (id) => - [ - id, - await effects.getHostInfo({ - kind: null, - packageId, - serviceInterfaceId: id, - callback, - }), - ] as const, - ), - ), - ) const serviceInterfacesFilled: ServiceInterfaceFilled[] = await Promise.all( - serviceInterfaceValues.map(async (serviceInterfaceValue) => { - const hostInfo = await effects.getHostInfo({ - kind: null, - packageId, - serviceInterfaceId: serviceInterfaceValue.id, - callback, - }) - const primaryUrl = await effects.getPrimaryUrl({ - serviceInterfaceId: serviceInterfaceValue.id, + Object.values(serviceInterfaceValues).map(async (serviceInterfaceValue) => { + const hostId = serviceInterfaceValue.addressInfo.hostId + const host = await effects.getHostInfo({ packageId, + hostId, callback, }) + const primaryUrl = await effects + .getPrimaryUrl({ + serviceInterfaceId: serviceInterfaceValue.id, + packageId, + callback, + }) + .catch(() => null) return { ...serviceInterfaceValue, primaryUrl: primaryUrl, - hostInfo, - addressInfo: filledAddress(hostInfo, serviceInterfaceValue.addressInfo), + host, + addressInfo: filledAddress(host, serviceInterfaceValue.addressInfo), get primaryHostname() { if (primaryUrl == null) return null return getHostname(primaryUrl) diff --git a/sdk/lib/util/index.ts b/sdk/lib/util/index.ts index bd144f35a..9b19aeb38 100644 --- a/sdk/lib/util/index.ts +++ b/sdk/lib/util/index.ts @@ -10,6 +10,8 @@ import "./once" export { GetServiceInterface, getServiceInterface } from "./getServiceInterface" export { getServiceInterfaces } from "./getServiceInterfaces" +export { addressHostToUrl } from "./getServiceInterface" +export { hostnameInfoToAddress } from "./Hostname" // prettier-ignore export type FlattenIntersection = T extends ArrayLike ? T : diff --git a/web/projects/ui/src/app/pages/apps-routes/app-interfaces/app-interfaces.page.ts b/web/projects/ui/src/app/pages/apps-routes/app-interfaces/app-interfaces.page.ts index 8baa0bfde..f3fe7a0cc 100644 --- a/web/projects/ui/src/app/pages/apps-routes/app-interfaces/app-interfaces.page.ts +++ b/web/projects/ui/src/app/pages/apps-routes/app-interfaces/app-interfaces.page.ts @@ -8,6 +8,7 @@ import { PatchDB } from 'patch-db-client' import { QRComponent } from 'src/app/components/qr/qr.component' import { map } from 'rxjs' import { T } from '@start9labs/start-sdk' +import { addressHostToUrl } from '@start9labs/start-sdk/cjs/lib/util/getServiceInterface' type MappedInterface = T.ServiceInterface & { addresses: MappedAddress[] @@ -33,10 +34,14 @@ export class AppInterfacesPage { .sort(iface => iface.name.toLowerCase() > iface.name.toLowerCase() ? -1 : 1, ) - .map(iface => ({ - ...iface, - addresses: getAddresses(iface), - })) + .map(iface => { + // TODO @Matt + const host = {} as any + return { + ...iface, + addresses: getAddresses(iface, host), + } + }) return { ui: sorted.filter(val => val.type === 'ui'), @@ -99,66 +104,40 @@ export class AppInterfacesItemComponent { } function getAddresses( - serviceInterface: T.ServiceInterfaceWithHostInfo, + serviceInterface: T.ServiceInterface, + host: T.Host, ): MappedAddress[] { - const host = serviceInterface.hostInfo const addressInfo = serviceInterface.addressInfo const username = addressInfo.username ? addressInfo.username + '@' : '' const suffix = addressInfo.suffix || '' - const hostnames = host.kind === 'multi' ? host.hostnames : [] // TODO: non-multi + const hostnames = + host.kind === 'multi' ? host.hostnameInfo[addressInfo.internalPort] : [] // TODO: non-multi /* host.hostname ? [host.hostname] : [] */ - const addresses: MappedAddress[] = [] - - hostnames.forEach(h => { + return hostnames.flatMap(h => { let name = '' - let hostname = '' if (h.kind === 'onion') { name = 'Tor' - hostname = h.hostname.value } else { const hostnameKind = h.hostname.kind if (hostnameKind === 'domain') { name = 'Domain' - hostname = `${h.hostname.subdomain}.${h.hostname.domain}` } else { name = hostnameKind === 'local' ? 'Local' : `${h.networkInterfaceId} (${hostnameKind})` - hostname = h.hostname.value } } - if (h.hostname.sslPort) { - const port = h.hostname.sslPort === 443 ? '' : `:${h.hostname.sslPort}` - const scheme = addressInfo.bindOptions.addSsl?.scheme - ? `${addressInfo.bindOptions.addSsl.scheme}://` - : '' - - addresses.push({ - name: name === 'Tor' ? 'Tor (HTTPS)' : name, - url: `${scheme}${username}${hostname}${port}${suffix}`, - }) - } - - if (h.hostname.port) { - const port = h.hostname.port === 80 ? '' : `:${h.hostname.port}` - const scheme = addressInfo.bindOptions.scheme - ? `${addressInfo.bindOptions.scheme}://` - : '' - - addresses.push({ - name: name === 'Tor' ? 'Tor (HTTP)' : name, - url: `${scheme}${username}${hostname}${port}${suffix}`, - }) - } + return addressHostToUrl(addressInfo, h).map(url => ({ + name, + url, + })) }) - - return addresses } diff --git a/web/projects/ui/src/app/services/api/api.fixures.ts b/web/projects/ui/src/app/services/api/api.fixures.ts index 2b0b96d23..11d56ff15 100644 --- a/web/projects/ui/src/app/services/api/api.fixures.ts +++ b/web/projects/ui/src/app/services/api/api.fixures.ts @@ -1422,66 +1422,11 @@ export module Mock { addressInfo: { username: null, hostId: 'abcdefg', - bindOptions: { - scheme: 'http', - preferredExternalPort: 80, - addSsl: { - // addXForwardedHeaders: false, - preferredExternalPort: 443, - scheme: 'https', - alpn: { specified: ['http/1.1', 'h2'] }, - }, - secure: null, - }, + internalPort: 80, + scheme: 'http', + sslScheme: 'https', suffix: '', }, - hostInfo: { - id: 'abcdefg', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: null, - sslPort: 1234, - }, - }, - { - kind: 'onion', - hostname: { - value: 'bitcoin-ui-address.onion', - port: 80, - sslPort: 443, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: null, - sslPort: 1234, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: null, - sslPort: 1234, - }, - }, - ], - }, }, rpc: { id: 'rpc', @@ -1495,66 +1440,11 @@ export module Mock { addressInfo: { username: null, hostId: 'bcdefgh', - bindOptions: { - scheme: 'http', - preferredExternalPort: 80, - addSsl: { - // addXForwardedHeaders: false, - preferredExternalPort: 443, - scheme: 'https', - alpn: { specified: ['http/1.1'] }, - }, - secure: null, - }, + internalPort: 8332, + scheme: 'http', + sslScheme: 'https', suffix: '', }, - hostInfo: { - id: 'bcdefgh', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: null, - sslPort: 2345, - }, - }, - { - kind: 'onion', - hostname: { - value: 'bitcoin-rpc-address.onion', - port: 80, - sslPort: 443, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: null, - sslPort: 2345, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: null, - sslPort: 2345, - }, - }, - ], - }, }, p2p: { id: 'p2p', @@ -1568,63 +1458,11 @@ export module Mock { addressInfo: { username: null, hostId: 'cdefghi', - bindOptions: { - scheme: 'bitcoin', - preferredExternalPort: 8333, - addSsl: null, - secure: { - ssl: false, - }, - }, + internalPort: 8333, + scheme: 'bitcoin', + sslScheme: null, suffix: '', }, - hostInfo: { - id: 'cdefghi', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: 3456, - sslPort: null, - }, - }, - { - kind: 'onion', - hostname: { - value: 'bitcoin-p2p-address.onion', - port: 8333, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: 3456, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: 3456, - sslPort: null, - }, - }, - ], - }, }, }, currentDependencies: {}, @@ -1660,101 +1498,11 @@ export module Mock { addressInfo: { username: null, hostId: 'hijklmnop', - bindOptions: { - scheme: 'http', - preferredExternalPort: 80, - addSsl: { - // addXForwardedHeaders: false, - preferredExternalPort: 443, - scheme: 'https', - alpn: { specified: ['http/1.1', 'h2'] }, - }, - secure: { - ssl: true, - }, - }, + internalPort: 80, + scheme: 'http', + sslScheme: 'https', suffix: '', }, - hostInfo: { - id: 'hijklmnop', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: null, - sslPort: 4567, - }, - }, - { - kind: 'onion', - hostname: { - value: 'proxy-ui-address.onion', - port: 80, - sslPort: 443, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: null, - sslPort: 4567, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: null, - sslPort: 4567, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'wlan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: null, - sslPort: 4567, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'wlan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.7', - port: null, - sslPort: 4567, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'wlan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: null, - sslPort: 4567, - }, - }, - ], - }, }, }, currentDependencies: { @@ -1801,63 +1549,11 @@ export module Mock { addressInfo: { username: null, hostId: 'qrstuv', - bindOptions: { - scheme: 'grpc', - preferredExternalPort: 10009, - addSsl: null, - secure: { - ssl: true, - }, - }, + internalPort: 10009, + scheme: null, + sslScheme: 'grpc', suffix: '', }, - hostInfo: { - id: 'qrstuv', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: 5678, - sslPort: null, - }, - }, - { - kind: 'onion', - hostname: { - value: 'lnd-grpc-address.onion', - port: 10009, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: 5678, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: 5678, - sslPort: null, - }, - }, - ], - }, }, lndconnect: { id: 'lndconnect', @@ -1871,63 +1567,11 @@ export module Mock { addressInfo: { username: null, hostId: 'qrstuv', - bindOptions: { - scheme: 'lndconnect', - preferredExternalPort: 10009, - addSsl: null, - secure: { - ssl: true, - }, - }, + internalPort: 10009, + scheme: null, + sslScheme: 'lndconnect', suffix: 'cert=askjdfbjadnaskjnd&macaroon=ksjbdfnhjasbndjksand', }, - hostInfo: { - id: 'qrstuv', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: 5678, - sslPort: null, - }, - }, - { - kind: 'onion', - hostname: { - value: 'lnd-grpc-address.onion', - port: 10009, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: 5678, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: 5678, - sslPort: null, - }, - }, - ], - }, }, p2p: { id: 'p2p', @@ -1941,63 +1585,11 @@ export module Mock { addressInfo: { username: null, hostId: 'rstuvw', - bindOptions: { - scheme: null, - preferredExternalPort: 9735, - addSsl: null, - secure: { - ssl: true, - }, - }, + internalPort: 9735, + scheme: 'lightning', + sslScheme: null, suffix: '', }, - hostInfo: { - id: 'rstuvw', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: 6789, - sslPort: null, - }, - }, - { - kind: 'onion', - hostname: { - value: 'lnd-p2p-address.onion', - port: 9735, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: 6789, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: 6789, - sslPort: null, - }, - }, - ], - }, }, }, currentDependencies: { diff --git a/web/projects/ui/src/app/services/api/mock-patch.ts b/web/projects/ui/src/app/services/api/mock-patch.ts index a4ed61278..ee96f9fe2 100644 --- a/web/projects/ui/src/app/services/api/mock-patch.ts +++ b/web/projects/ui/src/app/services/api/mock-patch.ts @@ -141,66 +141,11 @@ export const mockPatchData: DataModel = { addressInfo: { username: null, hostId: 'abcdefg', - bindOptions: { - scheme: 'http', - preferredExternalPort: 80, - addSsl: { - // addXForwardedHeaders: false, - preferredExternalPort: 443, - scheme: 'https', - alpn: { specified: ['http/1.1', 'h2'] }, - }, - secure: null, - }, + internalPort: 80, + scheme: 'http', + sslScheme: 'https', suffix: '', }, - hostInfo: { - id: 'abcdefg', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: null, - sslPort: 1234, - }, - }, - { - kind: 'onion', - hostname: { - value: 'bitcoin-ui-address.onion', - port: 80, - sslPort: 443, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: null, - sslPort: 1234, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: null, - sslPort: 1234, - }, - }, - ], - }, }, rpc: { id: 'rpc', @@ -214,66 +159,11 @@ export const mockPatchData: DataModel = { addressInfo: { username: null, hostId: 'bcdefgh', - bindOptions: { - scheme: 'http', - preferredExternalPort: 80, - addSsl: { - // addXForwardedHeaders: false, - preferredExternalPort: 443, - scheme: 'https', - alpn: { specified: ['http/1.1'] }, - }, - secure: null, - }, + internalPort: 8332, + scheme: 'http', + sslScheme: 'https', suffix: '', }, - hostInfo: { - id: 'bcdefgh', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: null, - sslPort: 2345, - }, - }, - { - kind: 'onion', - hostname: { - value: 'bitcoin-rpc-address.onion', - port: 80, - sslPort: 443, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: null, - sslPort: 2345, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: null, - sslPort: 2345, - }, - }, - ], - }, }, p2p: { id: 'p2p', @@ -287,63 +177,11 @@ export const mockPatchData: DataModel = { addressInfo: { username: null, hostId: 'cdefghi', - bindOptions: { - scheme: 'bitcoin', - preferredExternalPort: 8333, - addSsl: null, - secure: { - ssl: false, - }, - }, + internalPort: 8333, + scheme: 'bitcoin', + sslScheme: null, suffix: '', }, - hostInfo: { - id: 'cdefghi', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: 3456, - sslPort: null, - }, - }, - { - kind: 'onion', - hostname: { - value: 'bitcoin-p2p-address.onion', - port: 8333, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: 3456, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: 3456, - sslPort: null, - }, - }, - ], - }, }, }, currentDependencies: {}, @@ -382,63 +220,11 @@ export const mockPatchData: DataModel = { addressInfo: { username: null, hostId: 'qrstuv', - bindOptions: { - scheme: 'grpc', - preferredExternalPort: 10009, - addSsl: null, - secure: { - ssl: true, - }, - }, + internalPort: 10009, + scheme: null, + sslScheme: 'grpc', suffix: '', }, - hostInfo: { - id: 'qrstuv', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: 5678, - sslPort: null, - }, - }, - { - kind: 'onion', - hostname: { - value: 'lnd-grpc-address.onion', - port: 10009, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: 5678, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: 5678, - sslPort: null, - }, - }, - ], - }, }, lndconnect: { id: 'lndconnect', @@ -452,63 +238,11 @@ export const mockPatchData: DataModel = { addressInfo: { username: null, hostId: 'qrstuv', - bindOptions: { - scheme: 'lndconnect', - preferredExternalPort: 10009, - addSsl: null, - secure: { - ssl: true, - }, - }, + internalPort: 10009, + scheme: null, + sslScheme: 'lndconnect', suffix: 'cert=askjdfbjadnaskjnd&macaroon=ksjbdfnhjasbndjksand', }, - hostInfo: { - id: 'qrstuv', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: 5678, - sslPort: null, - }, - }, - { - kind: 'onion', - hostname: { - value: 'lnd-grpc-address.onion', - port: 10009, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: 5678, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: 5678, - sslPort: null, - }, - }, - ], - }, }, p2p: { id: 'p2p', @@ -522,61 +256,11 @@ export const mockPatchData: DataModel = { addressInfo: { username: null, hostId: 'rstuvw', - bindOptions: { - scheme: null, - preferredExternalPort: 9735, - addSsl: null, - secure: { ssl: true }, - }, + internalPort: 8333, + scheme: 'bitcoin', + sslScheme: null, suffix: '', }, - hostInfo: { - id: 'rstuvw', - kind: 'multi', - hostnames: [ - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'local', - value: 'adjective-noun.local', - port: 6789, - sslPort: null, - }, - }, - { - kind: 'onion', - hostname: { - value: 'lnd-p2p-address.onion', - port: 9735, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv4', - value: '192.168.1.5', - port: 6789, - sslPort: null, - }, - }, - { - kind: 'ip', - networkInterfaceId: 'elan0', - public: false, - hostname: { - kind: 'ipv6', - value: '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', - port: 6789, - sslPort: null, - }, - }, - ], - }, }, }, currentDependencies: { diff --git a/web/projects/ui/src/app/services/config.service.ts b/web/projects/ui/src/app/services/config.service.ts index 3eca4f48d..42116bdd8 100644 --- a/web/projects/ui/src/app/services/config.service.ts +++ b/web/projects/ui/src/app/services/config.service.ts @@ -57,12 +57,14 @@ export class ConfigService { } /** ${scheme}://${username}@${host}:${externalPort}${suffix} */ - launchableAddress(interfaces: PackageDataEntry['serviceInterfaces']): string { + launchableAddress( + interfaces: PackageDataEntry['serviceInterfaces'], + host: T.Host, + ): string { const ui = Object.values(interfaces).find(i => i.type === 'ui') if (!ui) return '' - const host = ui.hostInfo const addressInfo = ui.addressInfo const scheme = this.isHttps() ? 'https' : 'http' const username = addressInfo.username ? addressInfo.username + '@' : '' @@ -70,20 +72,25 @@ export class ConfigService { const url = new URL(`${scheme}://${username}placeholder${suffix}`) if (host.kind === 'multi') { - const onionHostname = host.hostnames.find(h => h.kind === 'onion') - ?.hostname as T.ExportedOnionHostname + const onionHostname = host.addresses.find(h => h.kind === 'onion') + ?.address as T.OnionHostname | undefined + + if (!onionHostname) + throw new Error('Expecting that there is an onion hostname') if (this.isTor() && onionHostname) { url.hostname = onionHostname.value - } else { - const ipHostname = host.hostnames.find(h => h.kind === 'ip') - ?.hostname as T.ExportedIpHostname - - if (!ipHostname) return '' - - url.hostname = this.hostname - url.port = String(ipHostname.sslPort || ipHostname.port) } + // TODO Handle single + // else { + // const ipHostname = host.addresses.find(h => h.kind === 'ip') + // ?.hostname as T.ExportedIpHostname + + // if (!ipHostname) return '' + + // url.hostname = this.hostname + // url.port = String(ipHostname.sslPort || ipHostname.port) + // } } else { throw new Error('unimplemented') // const hostname = {} as T.ExportedHostnameInfo // host.hostname diff --git a/web/projects/ui/src/app/services/ui-launcher.service.ts b/web/projects/ui/src/app/services/ui-launcher.service.ts index badbc7a3e..9c36036f3 100644 --- a/web/projects/ui/src/app/services/ui-launcher.service.ts +++ b/web/projects/ui/src/app/services/ui-launcher.service.ts @@ -2,6 +2,7 @@ import { Inject, Injectable } from '@angular/core' import { WINDOW } from '@ng-web-apis/common' import { PackageDataEntry } from 'src/app/services/patch-db/data-model' import { ConfigService } from './config.service' +import { T } from '@start9labs/start-sdk' @Injectable({ providedIn: 'root', @@ -13,8 +14,10 @@ export class UiLauncherService { ) {} launch(interfaces: PackageDataEntry['serviceInterfaces']): void { + // TODO @Matt + const host = {} as any this.windowRef.open( - this.config.launchableAddress(interfaces), + this.config.launchableAddress(interfaces, host), '_blank', 'noreferrer', ) From 4d6cb091cca261e17f5f84ea81c190968ecbadd0 Mon Sep 17 00:00:00 2001 From: Jade <2364004+Blu-J@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:17:45 -0600 Subject: [PATCH 05/17] Feature/disk usage (#2637) * feat: Add disk usage * Fixed: let the set config work with nesting. * chore: Changes * chore: Add default route * fix: Tor only config * chore --- .../Systems/SystemForEmbassy/index.ts | 29 +++++++-- .../SystemForEmbassy/oldEmbassyTypes.ts | 4 ++ .../SystemForEmbassy/polyfillEffects.ts | 64 +++++++++++++++++-- .../src/Models/DockerProcedure.ts | 4 +- container-runtime/src/Models/Duration.ts | 26 +++++++- 5 files changed, 113 insertions(+), 14 deletions(-) diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index 8e0cb28c5..0bdb0d769 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -2,7 +2,7 @@ import { types as T, utils, EmVer } from "@start9labs/start-sdk" import * as fs from "fs/promises" import { PolyfillEffects } from "./polyfillEffects" -import { Duration, duration } from "../../../Models/Duration" +import { Duration, duration, fromDuration } from "../../../Models/Duration" import { System } from "../../../Interfaces/System" import { matchManifest, Manifest, Procedure } from "./matchManifest" import * as childProcess from "node:child_process" @@ -478,10 +478,13 @@ export class SystemForEmbassy implements System { delete this.currentRunning if (currentRunning) { await currentRunning.clean({ - timeout: this.manifest.main["sigterm-timeout"], + timeout: fromDuration(this.manifest.main["sigterm-timeout"]), }) } - const durationValue = duration(this.manifest.main["sigterm-timeout"], "s") + const durationValue = duration( + fromDuration(this.manifest.main["sigterm-timeout"]), + "s", + ) return durationValue } private async createBackup( @@ -967,7 +970,7 @@ async function updateConfig( const newConfigValue = mutConfigValue[key] if (matchSpec.test(specValue)) { - const updateObject = { spec: null } + const updateObject = { spec: newConfigValue } await updateConfig( effects, manifest, @@ -1001,6 +1004,10 @@ async function updateConfig( manifest, specInterface, ) + if (!serviceInterfaceId) { + mutConfigValue[key] = "" + return + } const filled = await utils .getServiceInterface(effects, { packageId: specValue["package-id"], @@ -1035,8 +1042,16 @@ async function updateConfig( } } function extractServiceInterfaceId(manifest: Manifest, specInterface: string) { - let serviceInterfaceId - const lanConfig = manifest.interfaces[specInterface]?.["lan-config"] || {} - serviceInterfaceId = `${specInterface}-${Object.entries(lanConfig)[0]?.[1]?.internal}` + const internalPort = + Object.entries( + manifest.interfaces[specInterface]?.["lan-config"] || {}, + )[0]?.[1]?.internal || + Object.entries( + manifest.interfaces[specInterface]?.["tor-config"]?.["port-mapping"] || + {}, + )?.[0]?.[1] + + if (!internalPort) return null + const serviceInterfaceId = `${specInterface}-${internalPort}` return serviceInterfaceId } diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts index 0d7521626..73d130c9a 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/oldEmbassyTypes.ts @@ -120,6 +120,10 @@ export type Effects = { /// Returns the body as a json json(): Promise }> + diskUsage(options?: { + volumeId: string + path: string + }): Promise<{ used: number; total: number }> runRsync(options: { srcVolume: string diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts index 12c0a58d5..3a8d5f624 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts @@ -8,7 +8,8 @@ import { HostSystemStartOs } from "../../HostSystemStartOs" import "isomorphic-fetch" import { Manifest } from "./matchManifest" import { DockerProcedureContainer } from "./DockerProcedureContainer" - +import * as cp from "child_process" +export const execFile = promisify(cp.execFile) export class PolyfillEffects implements oet.Effects { constructor( readonly effects: HostSystemStartOs, @@ -104,7 +105,9 @@ export class PolyfillEffects implements oet.Effects { stderr: x.stderr.toString(), stdout: x.stdout.toString(), })) - .then((x) => (!!x.stderr ? { error: x.stderr } : { result: x.stdout })) + .then((x: any) => + !!x.stderr ? { error: x.stderr } : { result: x.stdout }, + ) } runDaemon(input: { command: string; args?: string[] | undefined }): { wait(): Promise> @@ -163,7 +166,7 @@ export class PolyfillEffects implements oet.Effects { stderr: x.stderr.toString(), stdout: x.stdout.toString(), })) - .then((x) => { + .then((x: any) => { if (!!x.stderr) { throw new Error(x.stderr) } @@ -198,7 +201,7 @@ export class PolyfillEffects implements oet.Effects { stderr: x.stderr.toString(), stdout: x.stdout.toString(), })) - .then((x) => { + .then((x: any) => { if (!!x.stderr) { throw new Error(x.stderr) } @@ -352,7 +355,7 @@ export class PolyfillEffects implements oet.Effects { return String(pid) } const waitPromise = new Promise((resolve, reject) => { - spawned.on("exit", (code) => { + spawned.on("exit", (code: any) => { if (code === 0) { resolve(null) } else { @@ -364,4 +367,55 @@ export class PolyfillEffects implements oet.Effects { const progress = () => Promise.resolve(percentage) return { id, wait, progress } } + async diskUsage( + options?: { volumeId: string; path: string } | undefined, + ): Promise<{ used: number; total: number }> { + const output = await execFile("df", ["--block-size=1", "-P", "/"]) + .then((x: any) => ({ + stderr: x.stderr.toString(), + stdout: x.stdout.toString(), + })) + .then((x: any) => { + if (!!x.stderr) { + throw new Error(x.stderr) + } + return parseDfOutput(x.stdout) + }) + if (!!options) { + const used = await execFile("du", [ + "-s", + "--block-size=1", + "-P", + new Volume(options.volumeId, options.path).path, + ]) + .then((x: any) => ({ + stderr: x.stderr.toString(), + stdout: x.stdout.toString(), + })) + .then((x: any) => { + if (!!x.stderr) { + throw new Error(x.stderr) + } + return Number.parseInt(x.stdout.split(/\s+/)[0]) + }) + return { + ...output, + used, + } + } + return output + } +} + +function parseDfOutput(output: string): { used: number; total: number } { + const lines = output + .split("\n") + .filter((x) => x.length) + .map((x) => x.split(/\s+/)) + const index = lines.splice(0, 1)[0].map((x) => x.toLowerCase()) + const usedIndex = index.indexOf("used") + const availableIndex = index.indexOf("available") + const used = lines.map((x) => Number.parseInt(x[usedIndex]))[0] || 0 + const total = lines.map((x) => Number.parseInt(x[availableIndex]))[0] || 0 + return { used, total } } diff --git a/container-runtime/src/Models/DockerProcedure.ts b/container-runtime/src/Models/DockerProcedure.ts index 91ae73b5f..20c8145ab 100644 --- a/container-runtime/src/Models/DockerProcedure.ts +++ b/container-runtime/src/Models/DockerProcedure.ts @@ -8,7 +8,9 @@ import { literals, number, Parser, + some, } from "ts-matches" +import { matchDuration } from "./Duration" const VolumeId = string const Path = string @@ -31,7 +33,7 @@ export const matchDockerProcedure = object( "toml", "toml-pretty", ), - "sigterm-timeout": number, + "sigterm-timeout": some(number, matchDuration), inject: boolean, }, ["io-format", "sigterm-timeout", "system", "args", "inject", "mounts"], diff --git a/container-runtime/src/Models/Duration.ts b/container-runtime/src/Models/Duration.ts index 75154c782..5f61c362a 100644 --- a/container-runtime/src/Models/Duration.ts +++ b/container-runtime/src/Models/Duration.ts @@ -1,6 +1,30 @@ -export type TimeUnit = "d" | "h" | "s" | "ms" +import { string } from "ts-matches" + +export type TimeUnit = "d" | "h" | "s" | "ms" | "m" | "µs" | "ns" export type Duration = `${number}${TimeUnit}` +const durationRegex = /^([0-9]*(\.[0-9]+)?)(ns|µs|ms|s|m|d)$/ + +export const matchDuration = string.refine(isDuration) +export function isDuration(value: string): value is Duration { + return durationRegex.test(value) +} + export function duration(timeValue: number, timeUnit: TimeUnit = "s") { return `${timeValue > 0 ? timeValue : 0}${timeUnit}` as Duration } +const unitsToSeconds: Record = { + ns: 1e-9, + µs: 1e-6, + ms: 0.001, + s: 1, + m: 60, + h: 3600, + d: 86400, +} + +export function fromDuration(duration: Duration | number): number { + if (typeof duration === "number") return duration + const [, num, , unit] = duration.match(durationRegex) || [] + return Number(num) * unitsToSeconds[unit] +} From 5aefb707fa3d882098cfad2685ddf31344dd3371 Mon Sep 17 00:00:00 2001 From: Jade <2364004+Blu-J@users.noreply.github.com> Date: Mon, 10 Jun 2024 22:38:12 -0600 Subject: [PATCH 06/17] feat: Add the merge to the file. (#2643) * feat: Add the merge to the file. * chore: Fix the early escape --- sdk/lib/util/fileHelper.ts | 7 +++++++ sdk/package-lock.json | 17 +++++++++++++++-- sdk/package.json | 6 ++++-- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/sdk/lib/util/fileHelper.ts b/sdk/lib/util/fileHelper.ts index 56706f95a..07cfb2c4a 100644 --- a/sdk/lib/util/fileHelper.ts +++ b/sdk/lib/util/fileHelper.ts @@ -1,6 +1,7 @@ import * as matches from "ts-matches" import * as YAML from "yaml" import * as TOML from "@iarna/toml" +import _ from "lodash" import * as T from "../types" import * as fs from "fs" @@ -82,6 +83,12 @@ export class FileHelper { ), ) } + + async merge(data: A, effects: T.Effects) { + const fileData = (await this.read(effects).catch(() => ({}))) || {} + const mergeData = _.merge({}, fileData, data) + return await this.write(mergeData, effects) + } /** * Create a File Helper for an arbitrary file type. * diff --git a/sdk/package-lock.json b/sdk/package-lock.json index f0d61ea4d..0c457540f 100644 --- a/sdk/package-lock.json +++ b/sdk/package-lock.json @@ -1,20 +1,22 @@ { "name": "@start9labs/start-sdk", - "version": "0.4.0-rev0.lib0.rc8.beta10", + "version": "0.3.6-alpha1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@start9labs/start-sdk", - "version": "0.4.0-rev0.lib0.rc8.beta10", + "version": "0.3.6-alpha1", "license": "MIT", "dependencies": { "isomorphic-fetch": "^3.0.0", + "lodash": "4.*.*", "ts-matches": "^5.4.1" }, "devDependencies": { "@iarna/toml": "^2.2.5", "@types/jest": "^29.4.0", + "@types/lodash": "^4.17.5", "jest": "^29.4.3", "prettier": "^3.2.5", "ts-jest": "^29.0.5", @@ -1136,6 +1138,12 @@ "pretty-format": "^29.0.0" } }, + "node_modules/@types/lodash": { + "version": "4.17.5", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.5.tgz", + "integrity": "sha512-MBIOHVZqVqgfro1euRDWX7OO0fBVUUMrN6Pwm8LQsz8cWhEpihlvR70ENj3f40j58TNxZaWv2ndSkInykNBBJw==", + "dev": true + }, "node_modules/@types/node": { "version": "18.15.10", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.10.tgz", @@ -2841,6 +2849,11 @@ "node": ">=8" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", diff --git a/sdk/package.json b/sdk/package.json index db387c036..53ee291ee 100644 --- a/sdk/package.json +++ b/sdk/package.json @@ -31,6 +31,7 @@ "homepage": "https://github.com/Start9Labs/start-sdk#readme", "dependencies": { "isomorphic-fetch": "^3.0.0", + "lodash": "4.*.*", "ts-matches": "^5.4.1" }, "prettier": { @@ -41,13 +42,14 @@ }, "devDependencies": { "@iarna/toml": "^2.2.5", - "yaml": "^2.2.2", "@types/jest": "^29.4.0", + "@types/lodash": "^4.17.5", "jest": "^29.4.3", "prettier": "^3.2.5", "ts-jest": "^29.0.5", "ts-node": "^10.9.1", "tsx": "^4.7.1", - "typescript": "^5.0.4" + "typescript": "^5.0.4", + "yaml": "^2.2.2" } } From 3f380fa0da46c041db7313c9e006b1724b5af8ff Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Wed, 12 Jun 2024 11:46:59 -0600 Subject: [PATCH 07/17] feature: pack s9pk (#2642) * TODO: images * wip * pack s9pk images * include path in packsource error * debug info * add cmd as context to invoke * filehelper bugfix * fix file helper * fix exposeForDependents * misc fixes * force image removal * fix filtering * fix deadlock * fix api * chore: Up the version of the package.json * always allow concurrency within same call stack * Update core/startos/src/s9pk/merkle_archive/expected.rs Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com> --------- Co-authored-by: J H Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com> --- Makefile | 4 + container-runtime/deb-install.sh | 2 +- container-runtime/package-lock.json | 1797 +++++------------ container-runtime/package.json | 1 + .../src/Adapters/HostSystemStartOs.ts | 26 +- container-runtime/src/Adapters/RpcListener.ts | 3 + .../Systems/SystemForEmbassy/index.ts | 7 +- .../src/Adapters/Systems/SystemForStartOs.ts | 139 +- .../src/Adapters/Systems/index.ts | 22 +- container-runtime/src/Interfaces/System.ts | 1 + core/Cargo.lock | 172 +- core/startos/Cargo.toml | 4 +- core/startos/src/action.rs | 2 + core/startos/src/auth.rs | 1 + core/startos/src/backup/restore.rs | 1 - core/startos/src/config/mod.rs | 5 +- core/startos/src/context/config.rs | 21 +- core/startos/src/context/rpc.rs | 6 + core/startos/src/control.rs | 7 +- core/startos/src/dependencies.rs | 13 +- core/startos/src/disk/mount/backup.rs | 1 - .../src/disk/mount/filesystem/overlayfs.rs | 23 +- core/startos/src/disk/mount/guard.rs | 19 +- core/startos/src/init.rs | 2 +- core/startos/src/install/mod.rs | 2 - core/startos/src/lib.rs | 6 +- core/startos/src/lxc/mod.rs | 20 +- core/startos/src/os_install/gpt.rs | 2 +- core/startos/src/os_install/mod.rs | 2 +- core/startos/src/registry/device_info.rs | 2 +- core/startos/src/registry/package/add.rs | 4 +- core/startos/src/rpc_continuations.rs | 5 + .../s9pk/merkle_archive/directory_contents.rs | 5 +- .../src/s9pk/merkle_archive/expected.rs | 103 + core/startos/src/s9pk/merkle_archive/mod.rs | 4 + .../src/s9pk/merkle_archive/source/mod.rs | 5 + .../source/multi_cursor_file.rs | 2 +- core/startos/src/s9pk/rpc.rs | 130 +- core/startos/src/s9pk/v2/compat.rs | 67 +- core/startos/src/s9pk/v2/manifest.rs | 85 +- core/startos/src/s9pk/v2/mod.rs | 76 +- core/startos/src/s9pk/v2/pack.rs | 536 +++++ core/startos/src/service/action.rs | 27 +- core/startos/src/service/config.rs | 15 +- core/startos/src/service/control.rs | 13 +- core/startos/src/service/dependencies.rs | 67 +- core/startos/src/service/mod.rs | 179 +- .../src/service/persistent_container.rs | 97 +- core/startos/src/service/properties.rs | 2 + core/startos/src/service/rpc.rs | 10 +- .../src/service/service_effect_handler.rs | 326 +-- core/startos/src/service/service_map.rs | 38 +- core/startos/src/service/transition/backup.rs | 10 +- .../startos/src/service/transition/restart.rs | 9 +- .../startos/src/service/transition/restore.rs | 10 +- core/startos/src/util/actor/concurrent.rs | 39 +- core/startos/src/util/actor/mod.rs | 12 +- core/startos/src/util/actor/simple.rs | 7 +- core/startos/src/util/io.rs | 10 +- core/startos/src/util/mod.rs | 248 ++- sdk/lib/StartSdk.ts | 9 +- sdk/lib/health/HealthCheck.ts | 3 +- sdk/lib/interfaces/Host.ts | 4 +- sdk/lib/mainFn/CommandController.ts | 4 +- sdk/lib/mainFn/Daemon.ts | 4 +- sdk/lib/mainFn/Daemons.ts | 21 +- sdk/lib/manifest/ManifestTypes.ts | 6 +- sdk/lib/manifest/setupManifest.ts | 9 +- .../osBindings/CreateOverlayedImageParams.ts | 3 +- .../osBindings/DestroyOverlayedImageParams.ts | 3 +- sdk/lib/osBindings/ImageConfig.ts | 8 + sdk/lib/osBindings/ImageMetadata.ts | 3 + sdk/lib/osBindings/ImageSource.ts | 6 + sdk/lib/osBindings/LoginParams.ts | 4 + sdk/lib/osBindings/Manifest.ts | 3 +- sdk/lib/osBindings/index.ts | 4 + sdk/lib/test/configBuilder.test.ts | 2 +- sdk/lib/test/output.sdk.ts | 2 +- sdk/lib/types.ts | 24 +- sdk/lib/util/Overlay.ts | 16 +- sdk/lib/util/fileHelper.ts | 30 +- sdk/package-lock.json | 2 +- sdk/package.json | 12 +- .../ui/src/app/services/api/api.fixures.ts | 24 +- 84 files changed, 2552 insertions(+), 2108 deletions(-) create mode 100644 core/startos/src/s9pk/merkle_archive/expected.rs create mode 100644 core/startos/src/s9pk/v2/pack.rs create mode 100644 sdk/lib/osBindings/ImageConfig.ts create mode 100644 sdk/lib/osBindings/ImageMetadata.ts create mode 100644 sdk/lib/osBindings/ImageSource.ts create mode 100644 sdk/lib/osBindings/LoginParams.ts diff --git a/Makefile b/Makefile index ff5b9c4ad..34064e799 100644 --- a/Makefile +++ b/Makefile @@ -160,6 +160,10 @@ wormhole-deb: results/$(BASENAME).deb @echo "Paste the following command into the shell of your start-os server:" @wormhole send results/$(BASENAME).deb 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo /usr/lib/startos/scripts/chroot-and-upgrade '"'"'cd $$(mktemp -d) && wormhole receive --accept-file %s && apt-get install -y --reinstall ./$(BASENAME).deb'"'"'\n", $$3 }' +wormhole-cli: core/target/$(ARCH)-unknown-linux-musl/release/start-cli + @echo "Paste the following command into the shell of your start-os server:" + @wormhole send results/$(BASENAME).deb 2>&1 | awk -Winteractive '/wormhole receive/ { printf "sudo /usr/lib/startos/scripts/chroot-and-upgrade '"'"'cd $$(mktemp -d) && wormhole receive --accept-file %s && apt-get install -y --reinstall ./$(BASENAME).deb'"'"'\n", $$3 }' + update: $(ALL_TARGETS) @if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi $(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create') diff --git a/container-runtime/deb-install.sh b/container-runtime/deb-install.sh index b439c6308..697bfd10e 100644 --- a/container-runtime/deb-install.sh +++ b/container-runtime/deb-install.sh @@ -6,7 +6,7 @@ mkdir -p /run/systemd/resolve echo "nameserver 8.8.8.8" > /run/systemd/resolve/stub-resolv.conf apt-get update -apt-get install -y curl rsync +apt-get install -y curl rsync qemu-user-static curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash source ~/.bashrc diff --git a/container-runtime/package-lock.json b/container-runtime/package-lock.json index 838dcb769..9b211c077 100644 --- a/container-runtime/package-lock.json +++ b/container-runtime/package-lock.json @@ -13,6 +13,7 @@ "esbuild-plugin-resolve": "^2.0.0", "filebrowser": "^1.0.0", "isomorphic-fetch": "^3.0.0", + "lodash": "^4.17.21", "node-fetch": "^3.1.0", "ts-matches": "^5.5.1", "tslib": "^2.5.3", @@ -33,11 +34,13 @@ "license": "MIT", "dependencies": { "isomorphic-fetch": "^3.0.0", + "lodash": "^4.17.21", "ts-matches": "^5.4.1" }, "devDependencies": { "@iarna/toml": "^2.2.5", "@types/jest": "^29.4.0", + "@types/lodash": "^4.17.5", "jest": "^29.4.3", "prettier": "^3.2.5", "ts-jest": "^29.0.5", @@ -49,14 +52,12 @@ }, "node_modules/@iarna/toml": { "version": "2.2.5", - "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-2.2.5.tgz", - "integrity": "sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==" + "license": "ISC" }, "node_modules/@mole-inc/bin-wrapper": { "version": "8.0.1", - "resolved": "https://registry.npmjs.org/@mole-inc/bin-wrapper/-/bin-wrapper-8.0.1.tgz", - "integrity": "sha512-sTGoeZnjI8N4KS+sW2AN95gDBErhAguvkw/tWdCjeM8bvxpz5lqrnd0vOJABA1A+Ic3zED7PYoLP/RANLgVotA==", "dev": true, + "license": "MIT", "dependencies": { "bin-check": "^4.1.0", "bin-version-check": "^5.0.0", @@ -73,9 +74,8 @@ }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", "dev": true, + "license": "MIT", "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" @@ -86,18 +86,16 @@ }, "node_modules/@nodelib/fs.stat": { "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", "dev": true, + "license": "MIT", "engines": { "node": ">= 8" } }, "node_modules/@nodelib/fs.walk": { "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", "dev": true, + "license": "MIT", "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" @@ -108,9 +106,8 @@ }, "node_modules/@sindresorhus/is": { "version": "4.6.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", - "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -124,9 +121,8 @@ }, "node_modules/@swc/cli": { "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@swc/cli/-/cli-0.1.65.tgz", - "integrity": "sha512-4NcgsvJVHhA7trDnMmkGLLvWMHu2kSy+qHx6QwRhhJhdiYdNUrhdp+ERxen73sYtaeEOYeLJcWrQ60nzKi6rpg==", "dev": true, + "license": "MIT", "dependencies": { "@mole-inc/bin-wrapper": "^8.0.1", "commander": "^7.1.0", @@ -155,14 +151,13 @@ } }, "node_modules/@swc/core": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.4.1.tgz", - "integrity": "sha512-3y+Y8js+e7BbM16iND+6Rcs3jdiL28q3iVtYsCviYSSpP2uUVKkp5sJnCY4pg8AaVvyN7CGQHO7gLEZQ5ByozQ==", + "version": "1.5.28", "dev": true, "hasInstallScript": true, + "license": "Apache-2.0", "dependencies": { - "@swc/counter": "^0.1.2", - "@swc/types": "^0.1.5" + "@swc/counter": "^0.1.3", + "@swc/types": "^0.1.8" }, "engines": { "node": ">=10" @@ -172,19 +167,19 @@ "url": "https://opencollective.com/swc" }, "optionalDependencies": { - "@swc/core-darwin-arm64": "1.4.1", - "@swc/core-darwin-x64": "1.4.1", - "@swc/core-linux-arm-gnueabihf": "1.4.1", - "@swc/core-linux-arm64-gnu": "1.4.1", - "@swc/core-linux-arm64-musl": "1.4.1", - "@swc/core-linux-x64-gnu": "1.4.1", - "@swc/core-linux-x64-musl": "1.4.1", - "@swc/core-win32-arm64-msvc": "1.4.1", - "@swc/core-win32-ia32-msvc": "1.4.1", - "@swc/core-win32-x64-msvc": "1.4.1" + "@swc/core-darwin-arm64": "1.5.28", + "@swc/core-darwin-x64": "1.5.28", + "@swc/core-linux-arm-gnueabihf": "1.5.28", + "@swc/core-linux-arm64-gnu": "1.5.28", + "@swc/core-linux-arm64-musl": "1.5.28", + "@swc/core-linux-x64-gnu": "1.5.28", + "@swc/core-linux-x64-musl": "1.5.28", + "@swc/core-win32-arm64-msvc": "1.5.28", + "@swc/core-win32-ia32-msvc": "1.5.28", + "@swc/core-win32-x64-msvc": "1.5.28" }, "peerDependencies": { - "@swc/helpers": "^0.5.0" + "@swc/helpers": "*" }, "peerDependenciesMeta": { "@swc/helpers": { @@ -192,94 +187,13 @@ } } }, - "node_modules/@swc/core-darwin-arm64": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.4.1.tgz", - "integrity": "sha512-ePyfx0348UbR4DOAW24TedeJbafnzha8liXFGuQ4bdXtEVXhLfPngprrxKrAddCuv42F9aTxydlF6+adD3FBhA==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-darwin-x64": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-darwin-x64/-/core-darwin-x64-1.4.1.tgz", - "integrity": "sha512-eLf4JSe6VkCMdDowjM8XNC5rO+BrgfbluEzAVtKR8L2HacNYukieumN7EzpYCi0uF1BYwu1ku6tLyG2r0VcGxA==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-arm-gnueabihf": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.4.1.tgz", - "integrity": "sha512-K8VtTLWMw+rkN/jDC9o/Q9SMmzdiHwYo2CfgkwVT29NsGccwmNhCQx6XoYiPKyKGIFKt4tdQnJHKUFzxUqQVtQ==", - "cpu": [ - "arm" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-arm64-gnu": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.4.1.tgz", - "integrity": "sha512-0e8p4g0Bfkt8lkiWgcdiENH3RzkcqKtpRXIVNGOmVc0OBkvc2tpm2WTx/eoCnes2HpTT4CTtR3Zljj4knQ4Fvw==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-arm64-musl": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.4.1.tgz", - "integrity": "sha512-b/vWGQo2n7lZVUnSQ7NBq3Qrj85GrAPPiRbpqaIGwOytiFSk8VULFihbEUwDe0rXgY4LDm8z8wkgADZcLnmdUA==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, "node_modules/@swc/core-linux-x64-gnu": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.4.1.tgz", - "integrity": "sha512-AFMQlvkKEdNi1Vk2GFTxxJzbICttBsOQaXa98kFTeWTnFFIyiIj2w7Sk8XRTEJ/AjF8ia8JPKb1zddBWr9+bEQ==", + "version": "1.5.28", "cpu": [ "x64" ], "dev": true, + "license": "Apache-2.0 AND MIT", "optional": true, "os": [ "linux" @@ -289,13 +203,12 @@ } }, "node_modules/@swc/core-linux-x64-musl": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.4.1.tgz", - "integrity": "sha512-QX2MxIECX1gfvUVZY+jk528/oFkS9MAl76e3ZRvG2KC/aKlCQL0KSzcTSm13mOxkDKS30EaGRDRQWNukGpMeRg==", + "version": "1.5.28", "cpu": [ "x64" ], "dev": true, + "license": "Apache-2.0 AND MIT", "optional": true, "os": [ "linux" @@ -304,71 +217,23 @@ "node": ">=10" } }, - "node_modules/@swc/core-win32-arm64-msvc": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.4.1.tgz", - "integrity": "sha512-OklkJYXXI/tntD2zaY8i3iZldpyDw5q+NAP3k9OlQ7wXXf37djRsHLV0NW4+ZNHBjE9xp2RsXJ0jlOJhfgGoFA==", - "cpu": [ - "arm64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-win32-ia32-msvc": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.4.1.tgz", - "integrity": "sha512-MBuc3/QfKX9FnLOU7iGN+6yHRTQaPQ9WskiC8s8JFiKQ+7I2p25tay2RplR9dIEEGgVAu6L7auv96LbNTh+FaA==", - "cpu": [ - "ia32" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-win32-x64-msvc": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.4.1.tgz", - "integrity": "sha512-lu4h4wFBb/bOK6N2MuZwg7TrEpwYXgpQf5R7ObNSXL65BwZ9BG8XRzD+dLJmALu8l5N08rP/TrpoKRoGT4WSxw==", - "cpu": [ - "x64" - ], - "dev": true, - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=10" - } - }, "node_modules/@swc/counter": { "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", - "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==", - "dev": true + "dev": true, + "license": "Apache-2.0" }, "node_modules/@swc/types": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/@swc/types/-/types-0.1.5.tgz", - "integrity": "sha512-myfUej5naTBWnqOCc/MdVOLVjXUXtIA+NpDrDBKJtLLg2shUjBu3cZmB/85RyitKc55+lUUyl7oRfLOvkr2hsw==", - "dev": true + "version": "0.1.8", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@swc/counter": "^0.1.3" + } }, "node_modules/@szmarczak/http-timer": { "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.6.tgz", - "integrity": "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w==", "dev": true, + "license": "MIT", "dependencies": { "defer-to-connect": "^2.0.0" }, @@ -378,15 +243,13 @@ }, "node_modules/@tokenizer/token": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", - "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/@types/cacheable-request": { "version": "6.0.3", - "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", - "integrity": "sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==", "dev": true, + "license": "MIT", "dependencies": { "@types/http-cache-semantics": "*", "@types/keyv": "^3.1.4", @@ -396,41 +259,36 @@ }, "node_modules/@types/http-cache-semantics": { "version": "4.0.4", - "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", - "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/@types/keyv": { "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz", - "integrity": "sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==", "dev": true, + "license": "MIT", "dependencies": { "@types/node": "*" } }, "node_modules/@types/node": { - "version": "20.11.17", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.17.tgz", - "integrity": "sha512-QmgQZGWu1Yw9TDyAP9ZzpFJKynYNeOvwMJmaxABfieQoVoiVOS6MN1WSpqpRcbeA5+RW82kraAVxCCJg+780Qw==", + "version": "20.14.2", "dev": true, + "license": "MIT", "dependencies": { "undici-types": "~5.26.4" } }, "node_modules/@types/responselike": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.3.tgz", - "integrity": "sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw==", "dev": true, + "license": "MIT", "dependencies": { "@types/node": "*" } }, "node_modules/accepts": { "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", "dependencies": { "mime-types": "~2.1.34", "negotiator": "0.6.3" @@ -441,8 +299,6 @@ }, "node_modules/arch": { "version": "2.2.0", - "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz", - "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==", "dev": true, "funding": [ { @@ -457,24 +313,22 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "MIT" }, "node_modules/array-flatten": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + "license": "MIT" }, "node_modules/balanced-match": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/bin-check": { "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bin-check/-/bin-check-4.1.0.tgz", - "integrity": "sha512-b6weQyEUKsDGFlACWSIOfveEnImkJyK/FGW6FAG42loyoquvjdtOIqO6yBFzHyqyVVhNgNkQxxx09SFLK28YnA==", "dev": true, + "license": "MIT", "dependencies": { "execa": "^0.7.0", "executable": "^4.1.0" @@ -485,9 +339,8 @@ }, "node_modules/bin-version": { "version": "6.0.0", - "resolved": "https://registry.npmjs.org/bin-version/-/bin-version-6.0.0.tgz", - "integrity": "sha512-nk5wEsP4RiKjG+vF+uG8lFsEn4d7Y6FVDamzzftSunXOoOcOOkzcWdKVlGgFFwlUQCj63SgnUkLLGF8v7lufhw==", "dev": true, + "license": "MIT", "dependencies": { "execa": "^5.0.0", "find-versions": "^5.0.0" @@ -501,9 +354,8 @@ }, "node_modules/bin-version-check": { "version": "5.1.0", - "resolved": "https://registry.npmjs.org/bin-version-check/-/bin-version-check-5.1.0.tgz", - "integrity": "sha512-bYsvMqJ8yNGILLz1KP9zKLzQ6YpljV3ln1gqhuLkUtyfGi3qXKGuK2p+U4NAvjVFzDFiBBtOpCOSFNuYYEGZ5g==", "dev": true, + "license": "MIT", "dependencies": { "bin-version": "^6.0.0", "semver": "^7.5.3", @@ -518,9 +370,8 @@ }, "node_modules/bin-version/node_modules/cross-spawn": { "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "dev": true, + "license": "MIT", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -532,9 +383,8 @@ }, "node_modules/bin-version/node_modules/execa": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", - "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", "dev": true, + "license": "MIT", "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^6.0.0", @@ -555,9 +405,8 @@ }, "node_modules/bin-version/node_modules/get-stream": { "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -567,9 +416,8 @@ }, "node_modules/bin-version/node_modules/is-stream": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" }, @@ -579,9 +427,8 @@ }, "node_modules/bin-version/node_modules/npm-run-path": { "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", "dev": true, + "license": "MIT", "dependencies": { "path-key": "^3.0.0" }, @@ -591,18 +438,16 @@ }, "node_modules/bin-version/node_modules/path-key": { "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/bin-version/node_modules/shebang-command": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, + "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" }, @@ -612,18 +457,16 @@ }, "node_modules/bin-version/node_modules/shebang-regex": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/bin-version/node_modules/which": { "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, + "license": "ISC", "dependencies": { "isexe": "^2.0.0" }, @@ -635,12 +478,11 @@ } }, "node_modules/body-parser": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", - "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", + "version": "1.20.2", + "license": "MIT", "dependencies": { "bytes": "3.1.2", - "content-type": "~1.0.4", + "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", @@ -648,7 +490,7 @@ "iconv-lite": "0.4.24", "on-finished": "2.4.1", "qs": "6.11.0", - "raw-body": "2.5.1", + "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" }, @@ -659,20 +501,18 @@ }, "node_modules/brace-expansion": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", "dev": true, + "license": "MIT", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -680,26 +520,23 @@ }, "node_modules/bytes": { "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", "engines": { "node": ">= 0.8" } }, "node_modules/cacheable-lookup": { "version": "5.0.4", - "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz", - "integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==", "dev": true, + "license": "MIT", "engines": { "node": ">=10.6.0" } }, "node_modules/cacheable-request": { "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.4.tgz", - "integrity": "sha512-v+p6ongsrp0yTGbJXjgxPow2+DL93DASP4kXCDKb8/bwRtt9OEF3whggkkDkGNzgcWy2XaF4a8nZglC7uElscg==", "dev": true, + "license": "MIT", "dependencies": { "clone-response": "^1.0.2", "get-stream": "^5.1.0", @@ -715,9 +552,8 @@ }, "node_modules/cacheable-request/node_modules/get-stream": { "version": "5.2.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", - "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", "dev": true, + "license": "MIT", "dependencies": { "pump": "^3.0.0" }, @@ -730,8 +566,7 @@ }, "node_modules/call-bind": { "version": "1.0.7", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", - "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", + "license": "MIT", "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -748,9 +583,8 @@ }, "node_modules/clone-response": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz", - "integrity": "sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA==", "dev": true, + "license": "MIT", "dependencies": { "mimic-response": "^1.0.0" }, @@ -760,17 +594,15 @@ }, "node_modules/commander": { "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", "dev": true, + "license": "MIT", "engines": { "node": ">= 10" } }, "node_modules/content-disposition": { "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", "dependencies": { "safe-buffer": "5.2.1" }, @@ -780,30 +612,26 @@ }, "node_modules/content-type": { "version": "1.0.5", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", - "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "version": "0.6.0", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/cookie-signature": { "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" + "license": "MIT" }, "node_modules/cross-spawn": { "version": "5.1.0", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", - "integrity": "sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==", "dev": true, + "license": "MIT", "dependencies": { "lru-cache": "^4.0.1", "shebang-command": "^1.2.0", @@ -812,25 +640,22 @@ }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", "engines": { "node": ">= 12" } }, "node_modules/debug": { "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", "dependencies": { "ms": "2.0.0" } }, "node_modules/decompress-response": { "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", "dev": true, + "license": "MIT", "dependencies": { "mimic-response": "^3.1.0" }, @@ -843,9 +668,8 @@ }, "node_modules/decompress-response/node_modules/mimic-response": { "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -855,17 +679,15 @@ }, "node_modules/defer-to-connect": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", - "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" } }, "node_modules/define-data-property": { "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -880,16 +702,14 @@ }, "node_modules/depd": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", "engines": { "node": ">= 0.8" } }, "node_modules/destroy": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", "engines": { "node": ">= 0.8", "npm": "1.2.8000 || >= 1.4.16" @@ -897,30 +717,26 @@ }, "node_modules/ee-first": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + "license": "MIT" }, "node_modules/encodeurl": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "license": "MIT", "engines": { "node": ">= 0.8" } }, "node_modules/end-of-stream": { "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", "dev": true, + "license": "MIT", "dependencies": { "once": "^1.4.0" } }, "node_modules/es-define-property": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "license": "MIT", "dependencies": { "get-intrinsic": "^1.2.4" }, @@ -930,27 +746,23 @@ }, "node_modules/es-errors": { "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", "engines": { "node": ">= 0.4" } }, "node_modules/esbuild-plugin-resolve": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/esbuild-plugin-resolve/-/esbuild-plugin-resolve-2.0.0.tgz", - "integrity": "sha512-eJy9B8yDW5X/J48eWtR1uVmv+DKfHvYYnrrcqQoe/nUkVHVOTZlJnSevkYyGOz6hI90t036Y5QIPDrGzmppxfg==" + "license": "MIT" }, "node_modules/escape-html": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" + "license": "MIT" }, "node_modules/escape-string-regexp": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", - "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", "dev": true, + "license": "MIT", "engines": { "node": ">=12" }, @@ -960,17 +772,15 @@ }, "node_modules/etag": { "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/execa": { "version": "0.7.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", - "integrity": "sha512-RztN09XglpYI7aBBrJCPW95jEH7YF1UEPOoX9yDhUTPdp7mK+CQvnLTuD10BNXZ3byLTu2uehZ8EcKT/4CGiFw==", "dev": true, + "license": "MIT", "dependencies": { "cross-spawn": "^5.0.1", "get-stream": "^3.0.0", @@ -986,9 +796,8 @@ }, "node_modules/executable": { "version": "4.1.1", - "resolved": "https://registry.npmjs.org/executable/-/executable-4.1.1.tgz", - "integrity": "sha512-8iA79xD3uAch729dUG8xaaBBFGaEa0wdD2VkYLFHwlqosEj/jT66AzcreRDSgV7ehnNLBW2WR5jIXwGKjVdTLg==", "dev": true, + "license": "MIT", "dependencies": { "pify": "^2.2.0" }, @@ -997,16 +806,15 @@ } }, "node_modules/express": { - "version": "4.18.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", - "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "version": "4.19.2", + "license": "MIT", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.1", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -1039,9 +847,8 @@ }, "node_modules/ext-list": { "version": "2.2.2", - "resolved": "https://registry.npmjs.org/ext-list/-/ext-list-2.2.2.tgz", - "integrity": "sha512-u+SQgsubraE6zItfVA0tBuCBhfU9ogSRnsvygI7wht9TS510oLkBRXBsqopeUG/GBOIQyKZO9wjTqIu/sf5zFA==", "dev": true, + "license": "MIT", "dependencies": { "mime-db": "^1.28.0" }, @@ -1051,9 +858,8 @@ }, "node_modules/ext-name": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ext-name/-/ext-name-5.0.0.tgz", - "integrity": "sha512-yblEwXAbGv1VQDmow7s38W77hzAgJAO50ztBLMcUyUBfxv1HC+LGwtiEN+Co6LtlqT/5uwVOxsD4TNIilWhwdQ==", "dev": true, + "license": "MIT", "dependencies": { "ext-list": "^2.0.0", "sort-keys-length": "^1.0.0" @@ -1064,9 +870,8 @@ }, "node_modules/fast-glob": { "version": "3.3.2", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", - "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", "dev": true, + "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -1080,17 +885,14 @@ }, "node_modules/fastq": { "version": "1.17.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", - "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", "dev": true, + "license": "ISC", "dependencies": { "reusify": "^1.0.4" } }, "node_modules/fetch-blob": { "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", "funding": [ { "type": "github", @@ -1101,6 +903,7 @@ "url": "https://paypal.me/jimmywarting" } ], + "license": "MIT", "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" @@ -1111,9 +914,8 @@ }, "node_modules/file-type": { "version": "17.1.6", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-17.1.6.tgz", - "integrity": "sha512-hlDw5Ev+9e883s0pwUsuuYNu4tD7GgpUnOvykjv1Gya0ZIjuKumthDRua90VUn6/nlRKAjcxLUnHNTIUWwWIiw==", "dev": true, + "license": "MIT", "dependencies": { "readable-web-to-node-stream": "^3.0.2", "strtok3": "^7.0.0-alpha.9", @@ -1128,8 +930,7 @@ }, "node_modules/filebrowser": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/filebrowser/-/filebrowser-1.0.0.tgz", - "integrity": "sha512-RRONYpCDzbmWPhBX43T4dE+ptqLznJ7lKfbMaZLChB2i2ZIdFXoqT9qZTi70Dpq6fnJHuvcdeiRqMIPZKhVgTQ==", + "license": "ISC", "dependencies": { "commander": "^2.9.0", "content-disposition": "^0.5.1", @@ -1138,14 +939,12 @@ }, "node_modules/filebrowser/node_modules/commander": { "version": "2.20.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + "license": "MIT" }, "node_modules/filename-reserved-regex": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-3.0.0.tgz", - "integrity": "sha512-hn4cQfU6GOT/7cFHXBqeBg2TbrMBgdD0kcjLhvSQYYwm3s4B6cjvBfb7nBALJLAXqmU5xajSa7X2NnUud/VCdw==", "dev": true, + "license": "MIT", "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -1155,9 +954,8 @@ }, "node_modules/filenamify": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/filenamify/-/filenamify-5.1.1.tgz", - "integrity": "sha512-M45CbrJLGACfrPOkrTp3j2EcO9OBkKUYME0eiqOCa7i2poaklU0jhlIaMlr8ijLorT0uLAzrn3qXOp5684CkfA==", "dev": true, + "license": "MIT", "dependencies": { "filename-reserved-regex": "^3.0.0", "strip-outer": "^2.0.0", @@ -1171,10 +969,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", "dev": true, + "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -1184,8 +981,7 @@ }, "node_modules/finalhandler": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", - "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "license": "MIT", "dependencies": { "debug": "2.6.9", "encodeurl": "~1.0.2", @@ -1201,9 +997,8 @@ }, "node_modules/find-versions": { "version": "5.1.0", - "resolved": "https://registry.npmjs.org/find-versions/-/find-versions-5.1.0.tgz", - "integrity": "sha512-+iwzCJ7C5v5KgcBuueqVoNiHVoQpwiUK5XFLjf0affFTep+Wcw93tPvmb8tqujDNmzhBDPddnWV/qgWSXgq+Hg==", "dev": true, + "license": "MIT", "dependencies": { "semver-regex": "^4.0.5" }, @@ -1216,8 +1011,7 @@ }, "node_modules/formdata-polyfill": { "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", "dependencies": { "fetch-blob": "^3.1.2" }, @@ -1227,32 +1021,28 @@ }, "node_modules/forwarded": { "version": "0.2.0", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", - "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/fresh": { "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/function-bind": { "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/get-intrinsic": { "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "license": "MIT", "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2", @@ -1269,18 +1059,16 @@ }, "node_modules/get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", - "integrity": "sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=4" } }, "node_modules/glob-parent": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, + "license": "ISC", "dependencies": { "is-glob": "^4.0.1" }, @@ -1290,8 +1078,7 @@ }, "node_modules/gopd": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "license": "MIT", "dependencies": { "get-intrinsic": "^1.1.3" }, @@ -1301,9 +1088,8 @@ }, "node_modules/got": { "version": "11.8.6", - "resolved": "https://registry.npmjs.org/got/-/got-11.8.6.tgz", - "integrity": "sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g==", "dev": true, + "license": "MIT", "dependencies": { "@sindresorhus/is": "^4.0.0", "@szmarczak/http-timer": "^4.0.5", @@ -1326,8 +1112,7 @@ }, "node_modules/has-property-descriptors": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", "dependencies": { "es-define-property": "^1.0.0" }, @@ -1336,9 +1121,8 @@ } }, "node_modules/has-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", - "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", + "version": "1.0.3", + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -1348,8 +1132,7 @@ }, "node_modules/has-symbols": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -1358,9 +1141,8 @@ } }, "node_modules/hasown": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.1.tgz", - "integrity": "sha512-1/th4MHjnwncwXsIW6QMzlvYL9kG5e/CpVvLRZe4XPa8TOUNbCELqmvhDmnkNsAjwaG4+I8gJJL0JBvTTLO9qA==", + "version": "2.0.2", + "license": "MIT", "dependencies": { "function-bind": "^1.1.2" }, @@ -1370,14 +1152,12 @@ }, "node_modules/http-cache-semantics": { "version": "4.1.1", - "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", - "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==", - "dev": true + "dev": true, + "license": "BSD-2-Clause" }, "node_modules/http-errors": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "license": "MIT", "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", @@ -1391,9 +1171,8 @@ }, "node_modules/http2-wrapper": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.3.tgz", - "integrity": "sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==", "dev": true, + "license": "MIT", "dependencies": { "quick-lru": "^5.1.1", "resolve-alpn": "^1.0.0" @@ -1404,17 +1183,15 @@ }, "node_modules/human-signals": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", "dev": true, + "license": "Apache-2.0", "engines": { "node": ">=10.17.0" } }, "node_modules/iconv-lite": { "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", "dependencies": { "safer-buffer": ">= 2.1.2 < 3" }, @@ -1424,8 +1201,6 @@ }, "node_modules/ieee754": { "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", "dev": true, "funding": [ { @@ -1440,35 +1215,32 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "BSD-3-Clause" }, "node_modules/inherits": { "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "license": "ISC" }, "node_modules/ipaddr.js": { "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", "engines": { "node": ">= 0.10" } }, "node_modules/is-extglob": { "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/is-glob": { "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", "dev": true, + "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" }, @@ -1478,41 +1250,36 @@ }, "node_modules/is-number": { "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.12.0" } }, "node_modules/is-plain-obj": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", - "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/is-stream": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/isexe": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/isomorphic-fetch": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz", - "integrity": "sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==", + "license": "MIT", "dependencies": { "node-fetch": "^2.6.1", "whatwg-fetch": "^3.4.1" @@ -1520,8 +1287,7 @@ }, "node_modules/isomorphic-fetch/node_modules/node-fetch": { "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", "dependencies": { "whatwg-url": "^5.0.0" }, @@ -1539,33 +1305,34 @@ }, "node_modules/json-buffer": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/keyv": { "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", "dev": true, + "license": "MIT", "dependencies": { "json-buffer": "3.0.1" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "node_modules/lowercase-keys": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", - "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/lru-cache": { "version": "4.1.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", - "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", "dev": true, + "license": "ISC", "dependencies": { "pseudomap": "^1.0.2", "yallist": "^2.1.2" @@ -1573,47 +1340,41 @@ }, "node_modules/media-typer": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/merge-descriptors": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" + "license": "MIT" }, "node_modules/merge-stream": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/merge2": { "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", "dev": true, + "license": "MIT", "engines": { "node": ">= 8" } }, "node_modules/methods": { "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "version": "4.0.7", "dev": true, + "license": "MIT", "dependencies": { - "braces": "^3.0.2", + "braces": "^3.0.3", "picomatch": "^2.3.1" }, "engines": { @@ -1622,8 +1383,7 @@ }, "node_modules/mime": { "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", "bin": { "mime": "cli.js" }, @@ -1633,16 +1393,14 @@ }, "node_modules/mime-db": { "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/mime-types": { "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", "dependencies": { "mime-db": "1.52.0" }, @@ -1652,27 +1410,24 @@ }, "node_modules/mimic-fn": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/mimic-response": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=4" } }, "node_modules/minimatch": { - "version": "9.0.3", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", - "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "version": "9.0.4", "dev": true, + "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" }, @@ -1685,21 +1440,17 @@ }, "node_modules/ms": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "license": "MIT" }, "node_modules/negotiator": { "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", "engines": { "node": ">= 0.6" } }, "node_modules/node-domexception": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", "funding": [ { "type": "github", @@ -1710,14 +1461,14 @@ "url": "https://paypal.me/jimmywarting" } ], + "license": "MIT", "engines": { "node": ">=10.5.0" } }, "node_modules/node-fetch": { "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", @@ -1733,9 +1484,8 @@ }, "node_modules/normalize-url": { "version": "6.1.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", - "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -1745,9 +1495,8 @@ }, "node_modules/npm-run-path": { "version": "2.0.2", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", - "integrity": "sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==", "dev": true, + "license": "MIT", "dependencies": { "path-key": "^2.0.0" }, @@ -1757,16 +1506,14 @@ }, "node_modules/object-inspect": { "version": "1.13.1", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", - "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==", + "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/on-finished": { "version": "2.4.1", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", - "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", "dependencies": { "ee-first": "1.1.1" }, @@ -1776,18 +1523,16 @@ }, "node_modules/once": { "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dev": true, + "license": "ISC", "dependencies": { "wrappy": "1" } }, "node_modules/onetime": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", "dev": true, + "license": "MIT", "dependencies": { "mimic-fn": "^2.1.0" }, @@ -1800,9 +1545,8 @@ }, "node_modules/os-filter-obj": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/os-filter-obj/-/os-filter-obj-2.0.0.tgz", - "integrity": "sha512-uksVLsqG3pVdzzPvmAHpBK0wKxYItuzZr7SziusRPoz67tGV8rL1szZ6IdeUrbqLjGDwApBtN29eEE3IqGHOjg==", "dev": true, + "license": "MIT", "dependencies": { "arch": "^2.1.0" }, @@ -1812,49 +1556,43 @@ }, "node_modules/p-cancelable": { "version": "2.1.1", - "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", - "integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/p-finally": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", "dev": true, + "license": "MIT", "engines": { "node": ">=4" } }, "node_modules/parseurl": { "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", "engines": { "node": ">= 0.8" } }, "node_modules/path-key": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", - "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", "dev": true, + "license": "MIT", "engines": { "node": ">=4" } }, "node_modules/path-to-regexp": { "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" + "license": "MIT" }, "node_modules/peek-readable": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.0.0.tgz", - "integrity": "sha512-YtCKvLUOvwtMGmrniQPdO7MwPjgkFBtFIrmfSbYmYuq3tKDV/mcfAhBth1+C3ru7uXIZasc/pHnb+YDYNkkj4A==", "dev": true, + "license": "MIT", "engines": { "node": ">=14.16" }, @@ -1865,9 +1603,8 @@ }, "node_modules/picomatch": { "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true, + "license": "MIT", "engines": { "node": ">=8.6" }, @@ -1877,18 +1614,16 @@ }, "node_modules/pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/prettier": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz", - "integrity": "sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==", + "version": "3.3.2", "dev": true, + "license": "MIT", "bin": { "prettier": "bin/prettier.cjs" }, @@ -1901,8 +1636,7 @@ }, "node_modules/proxy-addr": { "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" @@ -1913,15 +1647,13 @@ }, "node_modules/pseudomap": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/pump": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", "dev": true, + "license": "MIT", "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -1929,8 +1661,7 @@ }, "node_modules/qs": { "version": "6.11.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", - "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "license": "BSD-3-Clause", "dependencies": { "side-channel": "^1.0.4" }, @@ -1943,120 +1674,6 @@ }, "node_modules/queue-microtask": { "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/quick-lru": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", - "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", - "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/readable-web-to-node-stream": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", - "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", - "dev": true, - "dependencies": { - "readable-stream": "^3.6.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Borewit" - } - }, - "node_modules/resolve-alpn": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", - "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==", - "dev": true - }, - "node_modules/responselike": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.1.tgz", - "integrity": "sha512-4gl03wn3hj1HP3yzgdI7d3lCkF95F21Pz4BPGvKHinyQzALR5CapwC8yIi0Rh58DEMQ/SguC03wFj2k0M/mHhw==", - "dev": true, - "dependencies": { - "lowercase-keys": "^2.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/reusify": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "dev": true, - "engines": { - "iojs": ">=1.0.0", - "node": ">=0.10.0" - } - }, - "node_modules/run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", "dev": true, "funding": [ { @@ -2072,14 +1689,95 @@ "url": "https://feross.org/support" } ], - "dependencies": { - "queue-microtask": "^1.2.2" + "license": "MIT" + }, + "node_modules/quick-lru": { + "version": "5.1.1", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "node_modules/range-parser": { + "version": "1.2.1", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.2", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readable-web-to-node-stream": { + "version": "3.0.2", + "dev": true, + "license": "MIT", + "dependencies": { + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/resolve-alpn": { + "version": "1.2.1", + "dev": true, + "license": "MIT" + }, + "node_modules/responselike": { + "version": "2.0.1", + "dev": true, + "license": "MIT", + "dependencies": { + "lowercase-keys": "^2.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "dev": true, "funding": [ { "type": "github", @@ -2093,21 +1791,38 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" }, "node_modules/safer-buffer": { "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "license": "MIT" }, "node_modules/semver": { - "version": "7.6.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", - "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", + "version": "7.6.2", "dev": true, - "dependencies": { - "lru-cache": "^6.0.0" - }, + "license": "ISC", "bin": { "semver": "bin/semver.js" }, @@ -2117,9 +1832,8 @@ }, "node_modules/semver-regex": { "version": "4.0.5", - "resolved": "https://registry.npmjs.org/semver-regex/-/semver-regex-4.0.5.tgz", - "integrity": "sha512-hunMQrEy1T6Jr2uEVjrAIqjwWcQTgOAcIM52C8MY1EZSD3DDNft04XzvYKPqjED65bNVVko0YI38nYeEHCX3yw==", "dev": true, + "license": "MIT", "engines": { "node": ">=12" }, @@ -2129,9 +1843,8 @@ }, "node_modules/semver-truncate": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/semver-truncate/-/semver-truncate-3.0.0.tgz", - "integrity": "sha512-LJWA9kSvMolR51oDE6PN3kALBNaUdkxzAGcexw8gjMA8xr5zUqK0JiR3CgARSqanYF3Z1YHvsErb1KDgh+v7Rg==", "dev": true, + "license": "MIT", "dependencies": { "semver": "^7.3.5" }, @@ -2142,28 +1855,9 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/semver/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/semver/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - }, "node_modules/send": { "version": "0.18.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", - "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "license": "MIT", "dependencies": { "debug": "2.6.9", "depd": "2.0.0", @@ -2185,13 +1879,11 @@ }, "node_modules/send/node_modules/ms": { "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + "license": "MIT" }, "node_modules/serve-static": { "version": "1.15.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", - "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "license": "MIT", "dependencies": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", @@ -2203,16 +1895,15 @@ } }, "node_modules/set-function-length": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.1.tgz", - "integrity": "sha512-j4t6ccc+VsKwYHso+kElc5neZpjtq9EnRICFZtWyBsLojhmeF/ZBd/elqm22WJh/BziDe/SBiOeAt0m2mfLD0g==", + "version": "1.2.2", + "license": "MIT", "dependencies": { - "define-data-property": "^1.1.2", + "define-data-property": "^1.1.4", "es-errors": "^1.3.0", "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.3", + "get-intrinsic": "^1.2.4", "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.1" + "has-property-descriptors": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -2220,14 +1911,12 @@ }, "node_modules/setprototypeof": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + "license": "ISC" }, "node_modules/shebang-command": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", - "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", "dev": true, + "license": "MIT", "dependencies": { "shebang-regex": "^1.0.0" }, @@ -2237,19 +1926,17 @@ }, "node_modules/shebang-regex": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", - "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/side-channel": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.5.tgz", - "integrity": "sha512-QcgiIWV4WV7qWExbN5llt6frQB/lBven9pqliLXfGPB+K9ZYXxDozp0wLkHS24kWCm+6YXH/f0HhnObZnZOBnQ==", + "version": "1.0.6", + "license": "MIT", "dependencies": { - "call-bind": "^1.0.6", + "call-bind": "^1.0.7", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.4", "object-inspect": "^1.13.1" @@ -2263,24 +1950,21 @@ }, "node_modules/signal-exit": { "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/slash": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } }, "node_modules/sort-keys": { "version": "1.1.2", - "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", - "integrity": "sha512-vzn8aSqKgytVik0iwdBEi+zevbTYZogewTUM6dtpmGwEcdzbub/TX4bCzRhebDCRC3QzXgJsLRKB2V/Oof7HXg==", "dev": true, + "license": "MIT", "dependencies": { "is-plain-obj": "^1.0.0" }, @@ -2290,9 +1974,8 @@ }, "node_modules/sort-keys-length": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/sort-keys-length/-/sort-keys-length-1.0.1.tgz", - "integrity": "sha512-GRbEOUqCxemTAk/b32F2xa8wDTs+Z1QHOkbhJDQTvv/6G3ZkbJ+frYWsTcc7cBB3Fu4wy4XlLCuNtJuMn7Gsvw==", "dev": true, + "license": "MIT", "dependencies": { "sort-keys": "^1.0.0" }, @@ -2302,53 +1985,47 @@ }, "node_modules/source-map": { "version": "0.7.4", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", - "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", "dev": true, + "license": "BSD-3-Clause", "engines": { "node": ">= 8" } }, "node_modules/statuses": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "license": "MIT", "engines": { "node": ">= 0.8" } }, "node_modules/string_decoder": { "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "dev": true, + "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" } }, "node_modules/strip-eof": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", - "integrity": "sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.10.0" } }, "node_modules/strip-final-newline": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } }, "node_modules/strip-outer": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-outer/-/strip-outer-2.0.0.tgz", - "integrity": "sha512-A21Xsm1XzUkK0qK1ZrytDUvqsQWict2Cykhvi0fBQntGG5JSprESasEyV1EZ/4CiR5WB5KjzLTrP/bO37B0wPg==", "dev": true, + "license": "MIT", "engines": { "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, @@ -2358,9 +2035,8 @@ }, "node_modules/strtok3": { "version": "7.0.0", - "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-7.0.0.tgz", - "integrity": "sha512-pQ+V+nYQdC5H3Q7qBZAz/MO6lwGhoC2gOAjuouGf/VO0m7vQRh8QNMl2Uf6SwAtzZ9bOw3UIeBukEGNJl5dtXQ==", "dev": true, + "license": "MIT", "dependencies": { "@tokenizer/token": "^0.3.0", "peek-readable": "^5.0.0" @@ -2375,9 +2051,8 @@ }, "node_modules/to-regex-range": { "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, + "license": "MIT", "dependencies": { "is-number": "^7.0.0" }, @@ -2387,17 +2062,15 @@ }, "node_modules/toidentifier": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", "engines": { "node": ">=0.6" } }, "node_modules/token-types": { "version": "5.0.1", - "resolved": "https://registry.npmjs.org/token-types/-/token-types-5.0.1.tgz", - "integrity": "sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg==", "dev": true, + "license": "MIT", "dependencies": { "@tokenizer/token": "^0.3.0", "ieee754": "^1.2.1" @@ -2412,14 +2085,12 @@ }, "node_modules/tr46": { "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + "license": "MIT" }, "node_modules/trim-repeated": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/trim-repeated/-/trim-repeated-2.0.0.tgz", - "integrity": "sha512-QUHBFTJGdOwmp0tbOG505xAgOp/YliZP/6UgafFXYZ26WT1bvQmSMJUvkeVSASuJJHbqsFbynTvkd5W8RBTipg==", "dev": true, + "license": "MIT", "dependencies": { "escape-string-regexp": "^5.0.0" }, @@ -2429,18 +2100,15 @@ }, "node_modules/ts-matches": { "version": "5.5.1", - "resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-5.5.1.tgz", - "integrity": "sha512-UFYaKgfqlg9FROK7bdpYqFwG1CJvP4kOJdjXuWoqxo9jCmANoDw1GxkSCpJgoTeIiSTaTH5Qr1klSspb8c+ydg==" + "license": "MIT" }, "node_modules/tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + "version": "2.6.3", + "license": "0BSD" }, "node_modules/type-is": { "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", "dependencies": { "media-typer": "0.3.0", "mime-types": "~2.1.24" @@ -2450,10 +2118,9 @@ } }, "node_modules/typescript": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", - "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "version": "5.4.5", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -2464,62 +2131,53 @@ }, "node_modules/undici-types": { "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/unpipe": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", "engines": { "node": ">= 0.8" } }, "node_modules/util-deprecate": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/utils-merge": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", "engines": { "node": ">= 0.4.0" } }, "node_modules/vary": { "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", "engines": { "node": ">= 0.8" } }, "node_modules/web-streams-polyfill": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.2.tgz", - "integrity": "sha512-3pRGuxRF5gpuZc0W+EpwQRmCD7gRqcDOMt688KmdlDAgAyaB1XlN0zq2njfDNm44XVdIouE7pZ6GzbdyH47uIQ==", + "version": "3.3.3", + "license": "MIT", "engines": { "node": ">= 8" } }, "node_modules/webidl-conversions": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + "license": "BSD-2-Clause" }, "node_modules/whatwg-fetch": { "version": "3.6.20", - "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", - "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==" + "license": "MIT" }, "node_modules/whatwg-url": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -2527,9 +2185,8 @@ }, "node_modules/which": { "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", "dev": true, + "license": "ISC", "dependencies": { "isexe": "^2.0.0" }, @@ -2539,20 +2196,20 @@ }, "node_modules/wrappy": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/yallist": { "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/yaml": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.4.tgz", - "integrity": "sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==", + "version": "2.4.5", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, "engines": { "node": ">= 14" } @@ -2560,14 +2217,10 @@ }, "dependencies": { "@iarna/toml": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-2.2.5.tgz", - "integrity": "sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==" + "version": "2.2.5" }, "@mole-inc/bin-wrapper": { "version": "8.0.1", - "resolved": "https://registry.npmjs.org/@mole-inc/bin-wrapper/-/bin-wrapper-8.0.1.tgz", - "integrity": "sha512-sTGoeZnjI8N4KS+sW2AN95gDBErhAguvkw/tWdCjeM8bvxpz5lqrnd0vOJABA1A+Ic3zED7PYoLP/RANLgVotA==", "dev": true, "requires": { "bin-check": "^4.1.0", @@ -2582,8 +2235,6 @@ }, "@nodelib/fs.scandir": { "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", "dev": true, "requires": { "@nodelib/fs.stat": "2.0.5", @@ -2592,14 +2243,10 @@ }, "@nodelib/fs.stat": { "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", "dev": true }, "@nodelib/fs.walk": { "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", "dev": true, "requires": { "@nodelib/fs.scandir": "2.1.5", @@ -2608,8 +2255,6 @@ }, "@sindresorhus/is": { "version": "4.6.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", - "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==", "dev": true }, "@start9labs/start-sdk": { @@ -2617,8 +2262,10 @@ "requires": { "@iarna/toml": "^2.2.5", "@types/jest": "^29.4.0", + "@types/lodash": "^4.17.5", "isomorphic-fetch": "^3.0.0", "jest": "^29.4.3", + "lodash": "^4.17.21", "prettier": "^3.2.5", "ts-jest": "^29.0.5", "ts-matches": "^5.4.1", @@ -2630,8 +2277,6 @@ }, "@swc/cli": { "version": "0.1.65", - "resolved": "https://registry.npmjs.org/@swc/cli/-/cli-0.1.65.tgz", - "integrity": "sha512-4NcgsvJVHhA7trDnMmkGLLvWMHu2kSy+qHx6QwRhhJhdiYdNUrhdp+ERxen73sYtaeEOYeLJcWrQ60nzKi6rpg==", "dev": true, "requires": { "@mole-inc/bin-wrapper": "^8.0.1", @@ -2644,111 +2289,46 @@ } }, "@swc/core": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.4.1.tgz", - "integrity": "sha512-3y+Y8js+e7BbM16iND+6Rcs3jdiL28q3iVtYsCviYSSpP2uUVKkp5sJnCY4pg8AaVvyN7CGQHO7gLEZQ5ByozQ==", + "version": "1.5.28", "dev": true, "requires": { - "@swc/core-darwin-arm64": "1.4.1", - "@swc/core-darwin-x64": "1.4.1", - "@swc/core-linux-arm-gnueabihf": "1.4.1", - "@swc/core-linux-arm64-gnu": "1.4.1", - "@swc/core-linux-arm64-musl": "1.4.1", - "@swc/core-linux-x64-gnu": "1.4.1", - "@swc/core-linux-x64-musl": "1.4.1", - "@swc/core-win32-arm64-msvc": "1.4.1", - "@swc/core-win32-ia32-msvc": "1.4.1", - "@swc/core-win32-x64-msvc": "1.4.1", - "@swc/counter": "^0.1.2", - "@swc/types": "^0.1.5" + "@swc/core-darwin-arm64": "1.5.28", + "@swc/core-darwin-x64": "1.5.28", + "@swc/core-linux-arm-gnueabihf": "1.5.28", + "@swc/core-linux-arm64-gnu": "1.5.28", + "@swc/core-linux-arm64-musl": "1.5.28", + "@swc/core-linux-x64-gnu": "1.5.28", + "@swc/core-linux-x64-musl": "1.5.28", + "@swc/core-win32-arm64-msvc": "1.5.28", + "@swc/core-win32-ia32-msvc": "1.5.28", + "@swc/core-win32-x64-msvc": "1.5.28", + "@swc/counter": "^0.1.3", + "@swc/types": "^0.1.8" } }, - "@swc/core-darwin-arm64": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.4.1.tgz", - "integrity": "sha512-ePyfx0348UbR4DOAW24TedeJbafnzha8liXFGuQ4bdXtEVXhLfPngprrxKrAddCuv42F9aTxydlF6+adD3FBhA==", - "dev": true, - "optional": true - }, - "@swc/core-darwin-x64": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-darwin-x64/-/core-darwin-x64-1.4.1.tgz", - "integrity": "sha512-eLf4JSe6VkCMdDowjM8XNC5rO+BrgfbluEzAVtKR8L2HacNYukieumN7EzpYCi0uF1BYwu1ku6tLyG2r0VcGxA==", - "dev": true, - "optional": true - }, - "@swc/core-linux-arm-gnueabihf": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.4.1.tgz", - "integrity": "sha512-K8VtTLWMw+rkN/jDC9o/Q9SMmzdiHwYo2CfgkwVT29NsGccwmNhCQx6XoYiPKyKGIFKt4tdQnJHKUFzxUqQVtQ==", - "dev": true, - "optional": true - }, - "@swc/core-linux-arm64-gnu": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.4.1.tgz", - "integrity": "sha512-0e8p4g0Bfkt8lkiWgcdiENH3RzkcqKtpRXIVNGOmVc0OBkvc2tpm2WTx/eoCnes2HpTT4CTtR3Zljj4knQ4Fvw==", - "dev": true, - "optional": true - }, - "@swc/core-linux-arm64-musl": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.4.1.tgz", - "integrity": "sha512-b/vWGQo2n7lZVUnSQ7NBq3Qrj85GrAPPiRbpqaIGwOytiFSk8VULFihbEUwDe0rXgY4LDm8z8wkgADZcLnmdUA==", - "dev": true, - "optional": true - }, "@swc/core-linux-x64-gnu": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.4.1.tgz", - "integrity": "sha512-AFMQlvkKEdNi1Vk2GFTxxJzbICttBsOQaXa98kFTeWTnFFIyiIj2w7Sk8XRTEJ/AjF8ia8JPKb1zddBWr9+bEQ==", + "version": "1.5.28", "dev": true, "optional": true }, "@swc/core-linux-x64-musl": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.4.1.tgz", - "integrity": "sha512-QX2MxIECX1gfvUVZY+jk528/oFkS9MAl76e3ZRvG2KC/aKlCQL0KSzcTSm13mOxkDKS30EaGRDRQWNukGpMeRg==", - "dev": true, - "optional": true - }, - "@swc/core-win32-arm64-msvc": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.4.1.tgz", - "integrity": "sha512-OklkJYXXI/tntD2zaY8i3iZldpyDw5q+NAP3k9OlQ7wXXf37djRsHLV0NW4+ZNHBjE9xp2RsXJ0jlOJhfgGoFA==", - "dev": true, - "optional": true - }, - "@swc/core-win32-ia32-msvc": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.4.1.tgz", - "integrity": "sha512-MBuc3/QfKX9FnLOU7iGN+6yHRTQaPQ9WskiC8s8JFiKQ+7I2p25tay2RplR9dIEEGgVAu6L7auv96LbNTh+FaA==", - "dev": true, - "optional": true - }, - "@swc/core-win32-x64-msvc": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.4.1.tgz", - "integrity": "sha512-lu4h4wFBb/bOK6N2MuZwg7TrEpwYXgpQf5R7ObNSXL65BwZ9BG8XRzD+dLJmALu8l5N08rP/TrpoKRoGT4WSxw==", + "version": "1.5.28", "dev": true, "optional": true }, "@swc/counter": { "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", - "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==", "dev": true }, "@swc/types": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/@swc/types/-/types-0.1.5.tgz", - "integrity": "sha512-myfUej5naTBWnqOCc/MdVOLVjXUXtIA+NpDrDBKJtLLg2shUjBu3cZmB/85RyitKc55+lUUyl7oRfLOvkr2hsw==", - "dev": true + "version": "0.1.8", + "dev": true, + "requires": { + "@swc/counter": "^0.1.3" + } }, "@szmarczak/http-timer": { "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.6.tgz", - "integrity": "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w==", "dev": true, "requires": { "defer-to-connect": "^2.0.0" @@ -2756,14 +2336,10 @@ }, "@tokenizer/token": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", - "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", "dev": true }, "@types/cacheable-request": { "version": "6.0.3", - "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", - "integrity": "sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==", "dev": true, "requires": { "@types/http-cache-semantics": "*", @@ -2774,23 +2350,17 @@ }, "@types/http-cache-semantics": { "version": "4.0.4", - "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", - "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==", "dev": true }, "@types/keyv": { "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz", - "integrity": "sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==", "dev": true, "requires": { "@types/node": "*" } }, "@types/node": { - "version": "20.11.17", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.17.tgz", - "integrity": "sha512-QmgQZGWu1Yw9TDyAP9ZzpFJKynYNeOvwMJmaxABfieQoVoiVOS6MN1WSpqpRcbeA5+RW82kraAVxCCJg+780Qw==", + "version": "20.14.2", "dev": true, "requires": { "undici-types": "~5.26.4" @@ -2798,8 +2368,6 @@ }, "@types/responselike": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.3.tgz", - "integrity": "sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw==", "dev": true, "requires": { "@types/node": "*" @@ -2807,8 +2375,6 @@ }, "accepts": { "version": "1.3.8", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", - "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", "requires": { "mime-types": "~2.1.34", "negotiator": "0.6.3" @@ -2816,25 +2382,17 @@ }, "arch": { "version": "2.2.0", - "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz", - "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==", "dev": true }, "array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + "version": "1.1.1" }, "balanced-match": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, "bin-check": { "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bin-check/-/bin-check-4.1.0.tgz", - "integrity": "sha512-b6weQyEUKsDGFlACWSIOfveEnImkJyK/FGW6FAG42loyoquvjdtOIqO6yBFzHyqyVVhNgNkQxxx09SFLK28YnA==", "dev": true, "requires": { "execa": "^0.7.0", @@ -2843,8 +2401,6 @@ }, "bin-version": { "version": "6.0.0", - "resolved": "https://registry.npmjs.org/bin-version/-/bin-version-6.0.0.tgz", - "integrity": "sha512-nk5wEsP4RiKjG+vF+uG8lFsEn4d7Y6FVDamzzftSunXOoOcOOkzcWdKVlGgFFwlUQCj63SgnUkLLGF8v7lufhw==", "dev": true, "requires": { "execa": "^5.0.0", @@ -2853,8 +2409,6 @@ "dependencies": { "cross-spawn": { "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "dev": true, "requires": { "path-key": "^3.1.0", @@ -2864,8 +2418,6 @@ }, "execa": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", - "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", "dev": true, "requires": { "cross-spawn": "^7.0.3", @@ -2881,20 +2433,14 @@ }, "get-stream": { "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", "dev": true }, "is-stream": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", "dev": true }, "npm-run-path": { "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", "dev": true, "requires": { "path-key": "^3.0.0" @@ -2902,14 +2448,10 @@ }, "path-key": { "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true }, "shebang-command": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, "requires": { "shebang-regex": "^3.0.0" @@ -2917,14 +2459,10 @@ }, "shebang-regex": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, "which": { "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, "requires": { "isexe": "^2.0.0" @@ -2934,8 +2472,6 @@ }, "bin-version-check": { "version": "5.1.0", - "resolved": "https://registry.npmjs.org/bin-version-check/-/bin-version-check-5.1.0.tgz", - "integrity": "sha512-bYsvMqJ8yNGILLz1KP9zKLzQ6YpljV3ln1gqhuLkUtyfGi3qXKGuK2p+U4NAvjVFzDFiBBtOpCOSFNuYYEGZ5g==", "dev": true, "requires": { "bin-version": "^6.0.0", @@ -2944,12 +2480,10 @@ } }, "body-parser": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", - "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", + "version": "1.20.2", "requires": { "bytes": "3.1.2", - "content-type": "~1.0.4", + "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", @@ -2957,44 +2491,34 @@ "iconv-lite": "0.4.24", "on-finished": "2.4.1", "qs": "6.11.0", - "raw-body": "2.5.1", + "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" } }, "brace-expansion": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dev": true, "requires": { "balanced-match": "^1.0.0" } }, "braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", "dev": true, "requires": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" } }, "bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + "version": "3.1.2" }, "cacheable-lookup": { "version": "5.0.4", - "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz", - "integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==", "dev": true }, "cacheable-request": { "version": "7.0.4", - "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.4.tgz", - "integrity": "sha512-v+p6ongsrp0yTGbJXjgxPow2+DL93DASP4kXCDKb8/bwRtt9OEF3whggkkDkGNzgcWy2XaF4a8nZglC7uElscg==", "dev": true, "requires": { "clone-response": "^1.0.2", @@ -3008,8 +2532,6 @@ "dependencies": { "get-stream": { "version": "5.2.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", - "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", "dev": true, "requires": { "pump": "^3.0.0" @@ -3019,8 +2541,6 @@ }, "call-bind": { "version": "1.0.7", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", - "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", "requires": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -3031,8 +2551,6 @@ }, "clone-response": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz", - "integrity": "sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA==", "dev": true, "requires": { "mimic-response": "^1.0.0" @@ -3040,37 +2558,25 @@ }, "commander": { "version": "7.2.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", - "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", "dev": true }, "content-disposition": { "version": "0.5.4", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", - "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", "requires": { "safe-buffer": "5.2.1" } }, "content-type": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", - "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" + "version": "1.0.5" }, "cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==" + "version": "0.6.0" }, "cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" + "version": "1.0.6" }, "cross-spawn": { "version": "5.1.0", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", - "integrity": "sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==", "dev": true, "requires": { "lru-cache": "^4.0.1", @@ -3079,22 +2585,16 @@ } }, "data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==" + "version": "4.0.1" }, "debug": { "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", "requires": { "ms": "2.0.0" } }, "decompress-response": { "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", "dev": true, "requires": { "mimic-response": "^3.1.0" @@ -3102,22 +2602,16 @@ "dependencies": { "mimic-response": { "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", "dev": true } } }, "defer-to-connect": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", - "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==", "dev": true }, "define-data-property": { "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", "requires": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", @@ -3125,29 +2619,19 @@ } }, "depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" + "version": "2.0.0" }, "destroy": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", - "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" + "version": "1.2.0" }, "ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + "version": "1.1.1" }, "encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==" + "version": "1.0.2" }, "end-of-stream": { "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", "dev": true, "requires": { "once": "^1.4.0" @@ -3155,42 +2639,28 @@ }, "es-define-property": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", "requires": { "get-intrinsic": "^1.2.4" } }, "es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==" + "version": "1.3.0" }, "esbuild-plugin-resolve": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/esbuild-plugin-resolve/-/esbuild-plugin-resolve-2.0.0.tgz", - "integrity": "sha512-eJy9B8yDW5X/J48eWtR1uVmv+DKfHvYYnrrcqQoe/nUkVHVOTZlJnSevkYyGOz6hI90t036Y5QIPDrGzmppxfg==" + "version": "2.0.0" }, "escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" + "version": "1.0.3" }, "escape-string-regexp": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", - "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", "dev": true }, "etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" + "version": "1.8.1" }, "execa": { "version": "0.7.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", - "integrity": "sha512-RztN09XglpYI7aBBrJCPW95jEH7YF1UEPOoX9yDhUTPdp7mK+CQvnLTuD10BNXZ3byLTu2uehZ8EcKT/4CGiFw==", "dev": true, "requires": { "cross-spawn": "^5.0.1", @@ -3204,24 +2674,20 @@ }, "executable": { "version": "4.1.1", - "resolved": "https://registry.npmjs.org/executable/-/executable-4.1.1.tgz", - "integrity": "sha512-8iA79xD3uAch729dUG8xaaBBFGaEa0wdD2VkYLFHwlqosEj/jT66AzcreRDSgV7ehnNLBW2WR5jIXwGKjVdTLg==", "dev": true, "requires": { "pify": "^2.2.0" } }, "express": { - "version": "4.18.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", - "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "version": "4.19.2", "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.1", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -3251,8 +2717,6 @@ }, "ext-list": { "version": "2.2.2", - "resolved": "https://registry.npmjs.org/ext-list/-/ext-list-2.2.2.tgz", - "integrity": "sha512-u+SQgsubraE6zItfVA0tBuCBhfU9ogSRnsvygI7wht9TS510oLkBRXBsqopeUG/GBOIQyKZO9wjTqIu/sf5zFA==", "dev": true, "requires": { "mime-db": "^1.28.0" @@ -3260,8 +2724,6 @@ }, "ext-name": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ext-name/-/ext-name-5.0.0.tgz", - "integrity": "sha512-yblEwXAbGv1VQDmow7s38W77hzAgJAO50ztBLMcUyUBfxv1HC+LGwtiEN+Co6LtlqT/5uwVOxsD4TNIilWhwdQ==", "dev": true, "requires": { "ext-list": "^2.0.0", @@ -3270,8 +2732,6 @@ }, "fast-glob": { "version": "3.3.2", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", - "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", "dev": true, "requires": { "@nodelib/fs.stat": "^2.0.2", @@ -3283,8 +2743,6 @@ }, "fastq": { "version": "1.17.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", - "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", "dev": true, "requires": { "reusify": "^1.0.4" @@ -3292,8 +2750,6 @@ }, "fetch-blob": { "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", "requires": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" @@ -3301,8 +2757,6 @@ }, "file-type": { "version": "17.1.6", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-17.1.6.tgz", - "integrity": "sha512-hlDw5Ev+9e883s0pwUsuuYNu4tD7GgpUnOvykjv1Gya0ZIjuKumthDRua90VUn6/nlRKAjcxLUnHNTIUWwWIiw==", "dev": true, "requires": { "readable-web-to-node-stream": "^3.0.2", @@ -3312,8 +2766,6 @@ }, "filebrowser": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/filebrowser/-/filebrowser-1.0.0.tgz", - "integrity": "sha512-RRONYpCDzbmWPhBX43T4dE+ptqLznJ7lKfbMaZLChB2i2ZIdFXoqT9qZTi70Dpq6fnJHuvcdeiRqMIPZKhVgTQ==", "requires": { "commander": "^2.9.0", "content-disposition": "^0.5.1", @@ -3321,22 +2773,16 @@ }, "dependencies": { "commander": { - "version": "2.20.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + "version": "2.20.3" } } }, "filename-reserved-regex": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-3.0.0.tgz", - "integrity": "sha512-hn4cQfU6GOT/7cFHXBqeBg2TbrMBgdD0kcjLhvSQYYwm3s4B6cjvBfb7nBALJLAXqmU5xajSa7X2NnUud/VCdw==", "dev": true }, "filenamify": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/filenamify/-/filenamify-5.1.1.tgz", - "integrity": "sha512-M45CbrJLGACfrPOkrTp3j2EcO9OBkKUYME0eiqOCa7i2poaklU0jhlIaMlr8ijLorT0uLAzrn3qXOp5684CkfA==", "dev": true, "requires": { "filename-reserved-regex": "^3.0.0", @@ -3345,9 +2791,7 @@ } }, "fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", "dev": true, "requires": { "to-regex-range": "^5.0.1" @@ -3355,8 +2799,6 @@ }, "finalhandler": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", - "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", "requires": { "debug": "2.6.9", "encodeurl": "~1.0.2", @@ -3369,8 +2811,6 @@ }, "find-versions": { "version": "5.1.0", - "resolved": "https://registry.npmjs.org/find-versions/-/find-versions-5.1.0.tgz", - "integrity": "sha512-+iwzCJ7C5v5KgcBuueqVoNiHVoQpwiUK5XFLjf0affFTep+Wcw93tPvmb8tqujDNmzhBDPddnWV/qgWSXgq+Hg==", "dev": true, "requires": { "semver-regex": "^4.0.5" @@ -3378,31 +2818,21 @@ }, "formdata-polyfill": { "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", "requires": { "fetch-blob": "^3.1.2" } }, "forwarded": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", - "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" + "version": "0.2.0" }, "fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==" + "version": "0.5.2" }, "function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" + "version": "1.1.2" }, "get-intrinsic": { "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", "requires": { "es-errors": "^1.3.0", "function-bind": "^1.1.2", @@ -3413,14 +2843,10 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", - "integrity": "sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==", "dev": true }, "glob-parent": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, "requires": { "is-glob": "^4.0.1" @@ -3428,16 +2854,12 @@ }, "gopd": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", "requires": { "get-intrinsic": "^1.1.3" } }, "got": { "version": "11.8.6", - "resolved": "https://registry.npmjs.org/got/-/got-11.8.6.tgz", - "integrity": "sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g==", "dev": true, "requires": { "@sindresorhus/is": "^4.0.0", @@ -3455,40 +2877,28 @@ }, "has-property-descriptors": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", "requires": { "es-define-property": "^1.0.0" } }, "has-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", - "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==" + "version": "1.0.3" }, "has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + "version": "1.0.3" }, "hasown": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.1.tgz", - "integrity": "sha512-1/th4MHjnwncwXsIW6QMzlvYL9kG5e/CpVvLRZe4XPa8TOUNbCELqmvhDmnkNsAjwaG4+I8gJJL0JBvTTLO9qA==", + "version": "2.0.2", "requires": { "function-bind": "^1.1.2" } }, "http-cache-semantics": { "version": "4.1.1", - "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", - "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==", "dev": true }, "http-errors": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", - "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", "requires": { "depd": "2.0.0", "inherits": "2.0.4", @@ -3499,8 +2909,6 @@ }, "http2-wrapper": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.3.tgz", - "integrity": "sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==", "dev": true, "requires": { "quick-lru": "^5.1.1", @@ -3509,44 +2917,30 @@ }, "human-signals": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", "dev": true }, "iconv-lite": { "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "requires": { "safer-buffer": ">= 2.1.2 < 3" } }, "ieee754": { "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", "dev": true }, "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "version": "2.0.4" }, "ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" + "version": "1.9.1" }, "is-extglob": { "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", "dev": true }, "is-glob": { "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", "dev": true, "requires": { "is-extglob": "^2.1.1" @@ -3554,32 +2948,22 @@ }, "is-number": { "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true }, "is-plain-obj": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", - "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", "dev": true }, "is-stream": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", "dev": true }, "isexe": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, "isomorphic-fetch": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz", - "integrity": "sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==", "requires": { "node-fetch": "^2.6.1", "whatwg-fetch": "^3.4.1" @@ -3587,8 +2971,6 @@ "dependencies": { "node-fetch": { "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "requires": { "whatwg-url": "^5.0.0" } @@ -3597,29 +2979,26 @@ }, "json-buffer": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", "dev": true }, "keyv": { "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", "dev": true, "requires": { "json-buffer": "3.0.1" } }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, "lowercase-keys": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", - "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", "dev": true }, "lru-cache": { "version": "4.1.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", - "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", "dev": true, "requires": { "pseudomap": "^1.0.2", @@ -3627,100 +3006,68 @@ } }, "media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" + "version": "0.3.0" }, "merge-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" + "version": "1.0.1" }, "merge-stream": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true }, "merge2": { "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", "dev": true }, "methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==" + "version": "1.1.2" }, "micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "version": "4.0.7", "dev": true, "requires": { - "braces": "^3.0.2", + "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==" + "version": "1.6.0" }, "mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + "version": "1.52.0" }, "mime-types": { "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "requires": { "mime-db": "1.52.0" } }, "mimic-fn": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, "mimic-response": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", "dev": true }, "minimatch": { - "version": "9.0.3", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", - "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "version": "9.0.4", "dev": true, "requires": { "brace-expansion": "^2.0.1" } }, "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "version": "2.0.0" }, "negotiator": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", - "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==" + "version": "0.6.3" }, "node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==" + "version": "1.0.0" }, "node-fetch": { "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", "requires": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", @@ -3729,36 +3076,26 @@ }, "normalize-url": { "version": "6.1.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", - "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==", "dev": true }, "npm-run-path": { "version": "2.0.2", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", - "integrity": "sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==", "dev": true, "requires": { "path-key": "^2.0.0" } }, "object-inspect": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", - "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==" + "version": "1.13.1" }, "on-finished": { "version": "2.4.1", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", - "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", "requires": { "ee-first": "1.1.1" } }, "once": { "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dev": true, "requires": { "wrappy": "1" @@ -3766,8 +3103,6 @@ }, "onetime": { "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", "dev": true, "requires": { "mimic-fn": "^2.1.0" @@ -3775,8 +3110,6 @@ }, "os-filter-obj": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/os-filter-obj/-/os-filter-obj-2.0.0.tgz", - "integrity": "sha512-uksVLsqG3pVdzzPvmAHpBK0wKxYItuzZr7SziusRPoz67tGV8rL1szZ6IdeUrbqLjGDwApBtN29eEE3IqGHOjg==", "dev": true, "requires": { "arch": "^2.1.0" @@ -3784,60 +3117,40 @@ }, "p-cancelable": { "version": "2.1.1", - "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", - "integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==", "dev": true }, "p-finally": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", "dev": true }, "parseurl": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==" + "version": "1.3.3" }, "path-key": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", - "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", "dev": true }, "path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" + "version": "0.1.7" }, "peek-readable": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.0.0.tgz", - "integrity": "sha512-YtCKvLUOvwtMGmrniQPdO7MwPjgkFBtFIrmfSbYmYuq3tKDV/mcfAhBth1+C3ru7uXIZasc/pHnb+YDYNkkj4A==", "dev": true }, "picomatch": { "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true }, "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", "dev": true }, "prettier": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz", - "integrity": "sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==", + "version": "3.3.2", "dev": true }, "proxy-addr": { "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "requires": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" @@ -3845,14 +3158,10 @@ }, "pseudomap": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", - "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==", "dev": true }, "pump": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", "dev": true, "requires": { "end-of-stream": "^1.1.0", @@ -3861,33 +3170,23 @@ }, "qs": { "version": "6.11.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", - "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", "requires": { "side-channel": "^1.0.4" } }, "queue-microtask": { "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "dev": true }, "quick-lru": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", - "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", "dev": true }, "range-parser": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", - "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" + "version": "1.2.1" }, "raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "version": "2.5.2", "requires": { "bytes": "3.1.2", "http-errors": "2.0.0", @@ -3897,8 +3196,6 @@ }, "readable-stream": { "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", "dev": true, "requires": { "inherits": "^2.0.3", @@ -3908,8 +3205,6 @@ }, "readable-web-to-node-stream": { "version": "3.0.2", - "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", - "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", "dev": true, "requires": { "readable-stream": "^3.6.0" @@ -3917,14 +3212,10 @@ }, "resolve-alpn": { "version": "1.2.1", - "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", - "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==", "dev": true }, "responselike": { "version": "2.0.1", - "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.1.tgz", - "integrity": "sha512-4gl03wn3hj1HP3yzgdI7d3lCkF95F21Pz4BPGvKHinyQzALR5CapwC8yIi0Rh58DEMQ/SguC03wFj2k0M/mHhw==", "dev": true, "requires": { "lowercase-keys": "^2.0.0" @@ -3932,65 +3223,31 @@ }, "reusify": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", "dev": true }, "run-parallel": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", "dev": true, "requires": { "queue-microtask": "^1.2.2" } }, "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + "version": "5.2.1" }, "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "version": "2.1.2" }, "semver": { - "version": "7.6.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", - "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", - "dev": true, - "requires": { - "lru-cache": "^6.0.0" - }, - "dependencies": { - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "requires": { - "yallist": "^4.0.0" - } - }, - "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - } - } + "version": "7.6.2", + "dev": true }, "semver-regex": { "version": "4.0.5", - "resolved": "https://registry.npmjs.org/semver-regex/-/semver-regex-4.0.5.tgz", - "integrity": "sha512-hunMQrEy1T6Jr2uEVjrAIqjwWcQTgOAcIM52C8MY1EZSD3DDNft04XzvYKPqjED65bNVVko0YI38nYeEHCX3yw==", "dev": true }, "semver-truncate": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/semver-truncate/-/semver-truncate-3.0.0.tgz", - "integrity": "sha512-LJWA9kSvMolR51oDE6PN3kALBNaUdkxzAGcexw8gjMA8xr5zUqK0JiR3CgARSqanYF3Z1YHvsErb1KDgh+v7Rg==", "dev": true, "requires": { "semver": "^7.3.5" @@ -3998,8 +3255,6 @@ }, "send": { "version": "0.18.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", - "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", "requires": { "debug": "2.6.9", "depd": "2.0.0", @@ -4017,16 +3272,12 @@ }, "dependencies": { "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + "version": "2.1.3" } } }, "serve-static": { "version": "1.15.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", - "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", "requires": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", @@ -4035,27 +3286,21 @@ } }, "set-function-length": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.1.tgz", - "integrity": "sha512-j4t6ccc+VsKwYHso+kElc5neZpjtq9EnRICFZtWyBsLojhmeF/ZBd/elqm22WJh/BziDe/SBiOeAt0m2mfLD0g==", + "version": "1.2.2", "requires": { - "define-data-property": "^1.1.2", + "define-data-property": "^1.1.4", "es-errors": "^1.3.0", "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.3", + "get-intrinsic": "^1.2.4", "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.1" + "has-property-descriptors": "^1.0.2" } }, "setprototypeof": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + "version": "1.2.0" }, "shebang-command": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", - "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", "dev": true, "requires": { "shebang-regex": "^1.0.0" @@ -4063,16 +3308,12 @@ }, "shebang-regex": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", - "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==", "dev": true }, "side-channel": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.5.tgz", - "integrity": "sha512-QcgiIWV4WV7qWExbN5llt6frQB/lBven9pqliLXfGPB+K9ZYXxDozp0wLkHS24kWCm+6YXH/f0HhnObZnZOBnQ==", + "version": "1.0.6", "requires": { - "call-bind": "^1.0.6", + "call-bind": "^1.0.7", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.4", "object-inspect": "^1.13.1" @@ -4080,20 +3321,14 @@ }, "signal-exit": { "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true }, "slash": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", "dev": true }, "sort-keys": { "version": "1.1.2", - "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", - "integrity": "sha512-vzn8aSqKgytVik0iwdBEi+zevbTYZogewTUM6dtpmGwEcdzbub/TX4bCzRhebDCRC3QzXgJsLRKB2V/Oof7HXg==", "dev": true, "requires": { "is-plain-obj": "^1.0.0" @@ -4101,8 +3336,6 @@ }, "sort-keys-length": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/sort-keys-length/-/sort-keys-length-1.0.1.tgz", - "integrity": "sha512-GRbEOUqCxemTAk/b32F2xa8wDTs+Z1QHOkbhJDQTvv/6G3ZkbJ+frYWsTcc7cBB3Fu4wy4XlLCuNtJuMn7Gsvw==", "dev": true, "requires": { "sort-keys": "^1.0.0" @@ -4110,19 +3343,13 @@ }, "source-map": { "version": "0.7.4", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", - "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", "dev": true }, "statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" + "version": "2.0.1" }, "string_decoder": { "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", "dev": true, "requires": { "safe-buffer": "~5.2.0" @@ -4130,26 +3357,18 @@ }, "strip-eof": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", - "integrity": "sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==", "dev": true }, "strip-final-newline": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", "dev": true }, "strip-outer": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-outer/-/strip-outer-2.0.0.tgz", - "integrity": "sha512-A21Xsm1XzUkK0qK1ZrytDUvqsQWict2Cykhvi0fBQntGG5JSprESasEyV1EZ/4CiR5WB5KjzLTrP/bO37B0wPg==", "dev": true }, "strtok3": { "version": "7.0.0", - "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-7.0.0.tgz", - "integrity": "sha512-pQ+V+nYQdC5H3Q7qBZAz/MO6lwGhoC2gOAjuouGf/VO0m7vQRh8QNMl2Uf6SwAtzZ9bOw3UIeBukEGNJl5dtXQ==", "dev": true, "requires": { "@tokenizer/token": "^0.3.0", @@ -4158,22 +3377,16 @@ }, "to-regex-range": { "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "dev": true, "requires": { "is-number": "^7.0.0" } }, "toidentifier": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" + "version": "1.0.1" }, "token-types": { "version": "5.0.1", - "resolved": "https://registry.npmjs.org/token-types/-/token-types-5.0.1.tgz", - "integrity": "sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg==", "dev": true, "requires": { "@tokenizer/token": "^0.3.0", @@ -4181,90 +3394,60 @@ } }, "tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + "version": "0.0.3" }, "trim-repeated": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/trim-repeated/-/trim-repeated-2.0.0.tgz", - "integrity": "sha512-QUHBFTJGdOwmp0tbOG505xAgOp/YliZP/6UgafFXYZ26WT1bvQmSMJUvkeVSASuJJHbqsFbynTvkd5W8RBTipg==", "dev": true, "requires": { "escape-string-regexp": "^5.0.0" } }, "ts-matches": { - "version": "5.5.1", - "resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-5.5.1.tgz", - "integrity": "sha512-UFYaKgfqlg9FROK7bdpYqFwG1CJvP4kOJdjXuWoqxo9jCmANoDw1GxkSCpJgoTeIiSTaTH5Qr1klSspb8c+ydg==" + "version": "5.5.1" }, "tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + "version": "2.6.3" }, "type-is": { "version": "1.6.18", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", - "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", "requires": { "media-typer": "0.3.0", "mime-types": "~2.1.24" } }, "typescript": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", - "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "version": "5.4.5", "dev": true }, "undici-types": { "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "dev": true }, "unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" + "version": "1.0.0" }, "util-deprecate": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, "utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" + "version": "1.0.1" }, "vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" + "version": "1.1.2" }, "web-streams-polyfill": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.2.tgz", - "integrity": "sha512-3pRGuxRF5gpuZc0W+EpwQRmCD7gRqcDOMt688KmdlDAgAyaB1XlN0zq2njfDNm44XVdIouE7pZ6GzbdyH47uIQ==" + "version": "3.3.3" }, "webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + "version": "3.0.1" }, "whatwg-fetch": { - "version": "3.6.20", - "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", - "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==" + "version": "3.6.20" }, "whatwg-url": { "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", "requires": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -4272,8 +3455,6 @@ }, "which": { "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", "dev": true, "requires": { "isexe": "^2.0.0" @@ -4281,20 +3462,14 @@ }, "wrappy": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "dev": true }, "yallist": { "version": "2.1.2", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", - "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==", "dev": true }, "yaml": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.4.tgz", - "integrity": "sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==" + "version": "2.4.5" } } } diff --git a/container-runtime/package.json b/container-runtime/package.json index e2c56afff..357c606fc 100644 --- a/container-runtime/package.json +++ b/container-runtime/package.json @@ -21,6 +21,7 @@ "esbuild-plugin-resolve": "^2.0.0", "filebrowser": "^1.0.0", "isomorphic-fetch": "^3.0.0", + "lodash": "^4.17.21", "node-fetch": "^3.1.0", "ts-matches": "^5.5.1", "tslib": "^2.5.3", diff --git a/container-runtime/src/Adapters/HostSystemStartOs.ts b/container-runtime/src/Adapters/HostSystemStartOs.ts index ba2076f52..93905825a 100644 --- a/container-runtime/src/Adapters/HostSystemStartOs.ts +++ b/container-runtime/src/Adapters/HostSystemStartOs.ts @@ -32,6 +32,8 @@ type RpcError = typeof matchRpcError._TYPE const SOCKET_PATH = "/media/startos/rpc/host.sock" const MAIN = "/main" as const export class HostSystemStartOs implements Effects { + procedureId: string | null = null + static of(callbackHolder: CallbackHolder) { return new HostSystemStartOs(callbackHolder) } @@ -40,7 +42,7 @@ export class HostSystemStartOs implements Effects { id = 0 rpcRound( method: K, - params: unknown, + params: Record, ) { const id = this.id++ const client = net.createConnection({ path: SOCKET_PATH }, () => { @@ -48,7 +50,7 @@ export class HostSystemStartOs implements Effects { JSON.stringify({ id, method, - params, + params: { ...params, procedureId: this.procedureId }, }) + "\n", ) }) @@ -102,14 +104,14 @@ export class HostSystemStartOs implements Effects { }) as ReturnType } clearBindings(...[]: Parameters) { - return this.rpcRound("clearBindings", null) as ReturnType< + return this.rpcRound("clearBindings", {}) as ReturnType< T.Effects["clearBindings"] > } clearServiceInterfaces( ...[]: Parameters ) { - return this.rpcRound("clearServiceInterfaces", null) as ReturnType< + return this.rpcRound("clearServiceInterfaces", {}) as ReturnType< T.Effects["clearServiceInterfaces"] > } @@ -145,18 +147,20 @@ export class HostSystemStartOs implements Effects { T.Effects["exportServiceInterface"] > } - exposeForDependents(...[options]: any) { - return this.rpcRound("exposeForDependents", null) as ReturnType< + exposeForDependents( + ...[options]: Parameters + ) { + return this.rpcRound("exposeForDependents", options) as ReturnType< T.Effects["exposeForDependents"] > } getConfigured(...[]: Parameters) { - return this.rpcRound("getConfigured", null) as ReturnType< + return this.rpcRound("getConfigured", {}) as ReturnType< T.Effects["getConfigured"] > } getContainerIp(...[]: Parameters) { - return this.rpcRound("getContainerIp", null) as ReturnType< + return this.rpcRound("getContainerIp", {}) as ReturnType< T.Effects["getContainerIp"] > } @@ -229,7 +233,7 @@ export class HostSystemStartOs implements Effects { > } restart(...[]: Parameters) { - return this.rpcRound("restart", null) + return this.rpcRound("restart", {}) as ReturnType } running(...[packageId]: Parameters) { return this.rpcRound("running", { packageId }) as ReturnType< @@ -262,7 +266,7 @@ export class HostSystemStartOs implements Effects { > } getDependencies(): ReturnType { - return this.rpcRound("getDependencies", null) as ReturnType< + return this.rpcRound("getDependencies", {}) as ReturnType< T.Effects["getDependencies"] > } @@ -279,7 +283,7 @@ export class HostSystemStartOs implements Effects { } shutdown(...[]: Parameters) { - return this.rpcRound("shutdown", null) + return this.rpcRound("shutdown", {}) as ReturnType } stopped(...[packageId]: Parameters) { return this.rpcRound("stopped", { packageId }) as ReturnType< diff --git a/container-runtime/src/Adapters/RpcListener.ts b/container-runtime/src/Adapters/RpcListener.ts index faff253fe..5391d943e 100644 --- a/container-runtime/src/Adapters/RpcListener.ts +++ b/container-runtime/src/Adapters/RpcListener.ts @@ -58,6 +58,7 @@ const runType = object({ method: literal("execute"), params: object( { + id: string, procedure: string, input: any, timeout: number, @@ -70,6 +71,7 @@ const sandboxRunType = object({ method: literal("sandbox"), params: object( { + id: string, procedure: string, input: any, timeout: number, @@ -195,6 +197,7 @@ export class RpcListener { const procedure = jsonPath.unsafeCast(params.procedure) return system .execute(this.effects, { + id: params.id, procedure, input: params.input, timeout: params.timeout, diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index 0bdb0d769..731b38903 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -49,7 +49,7 @@ function todo(): never { const execFile = promisify(childProcess.execFile) const MANIFEST_LOCATION = "/usr/lib/startos/package/embassyManifest.json" -const EMBASSY_JS_LOCATION = "/usr/lib/startos/package/embassy.js" +export const EMBASSY_JS_LOCATION = "/usr/lib/startos/package/embassy.js" const EMBASSY_POINTER_PATH_PREFIX = "/embassyConfig" const matchSetResult = object( @@ -199,11 +199,14 @@ export class SystemForEmbassy implements System { async execute( effects: HostSystemStartOs, options: { + id: string procedure: JsonPath input: unknown timeout?: number | undefined }, ): Promise { + effects = Object.create(effects) + effects.procedureId = options.id return this._execute(effects, options) .then((x) => matches(x) @@ -724,7 +727,7 @@ export class SystemForEmbassy implements System { private async properties( effects: HostSystemStartOs, timeoutMs: number | null, - ): Promise> { + ): Promise> { // TODO BLU-J set the properties ever so often const setConfigValue = this.manifest.properties if (!setConfigValue) throw new Error("There is no properties") diff --git a/container-runtime/src/Adapters/Systems/SystemForStartOs.ts b/container-runtime/src/Adapters/Systems/SystemForStartOs.ts index 7549bf0f2..bf27f222d 100644 --- a/container-runtime/src/Adapters/Systems/SystemForStartOs.ts +++ b/container-runtime/src/Adapters/Systems/SystemForStartOs.ts @@ -1,20 +1,23 @@ import { ExecuteResult, System } from "../../Interfaces/System" import { unNestPath } from "../../Models/JsonPath" -import { string } from "ts-matches" +import matches, { any, number, object, string, tuple } from "ts-matches" import { HostSystemStartOs } from "../HostSystemStartOs" import { Effects } from "../../Models/Effects" -import { RpcResult } from "../RpcListener" +import { RpcResult, matchRpcResult } from "../RpcListener" import { duration } from "../../Models/Duration" -const LOCATION = "/usr/lib/startos/package/startos" +import { T } from "@start9labs/start-sdk" +import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk" +export const STARTOS_JS_LOCATION = "/usr/lib/startos/package/index.js" export class SystemForStartOs implements System { private onTerm: (() => Promise) | undefined static of() { - return new SystemForStartOs() + return new SystemForStartOs(require(STARTOS_JS_LOCATION)) } - constructor() {} + constructor(readonly abi: T.ABI) {} async execute( effects: HostSystemStartOs, options: { + id: string procedure: | "/init" | "/uninit" @@ -33,7 +36,61 @@ export class SystemForStartOs implements System { timeout?: number | undefined }, ): Promise { - return { result: await this._execute(effects, options) } + effects = Object.create(effects) + effects.procedureId = options.id + return this._execute(effects, options) + .then((x) => + matches(x) + .when( + object({ + result: any, + }), + (x) => x, + ) + .when( + object({ + error: string, + }), + (x) => ({ + error: { + code: 0, + message: x.error, + }, + }), + ) + .when( + object({ + "error-code": tuple(number, string), + }), + ({ "error-code": [code, message] }) => ({ + error: { + code, + message, + }, + }), + ) + .defaultTo({ result: x }), + ) + .catch((error: unknown) => { + if (error instanceof Error) + return { + error: { + code: 0, + message: error.name, + data: { + details: error.message, + debug: `${error?.cause ?? "[noCause]"}:${error?.stack ?? "[noStack]"}`, + }, + }, + } + if (matchRpcResult.test(error)) return error + return { + error: { + code: 0, + message: String(error), + }, + } + }) } async _execute( effects: Effects, @@ -58,26 +115,27 @@ export class SystemForStartOs implements System { ): Promise { switch (options.procedure) { case "/init": { - const path = `${LOCATION}/procedures/init` - const procedure: any = await import(path).catch(() => require(path)) - const previousVersion = string.optional().unsafeCast(options) - return procedure.init({ effects, previousVersion }) + const previousVersion = + string.optional().unsafeCast(options.input) || null + return this.abi.init({ effects, previousVersion }) } case "/uninit": { - const path = `${LOCATION}/procedures/init` - const procedure: any = await import(path).catch(() => require(path)) - const nextVersion = string.optional().unsafeCast(options) - return procedure.uninit({ effects, nextVersion }) + const nextVersion = string.optional().unsafeCast(options.input) || null + return this.abi.uninit({ effects, nextVersion }) } case "/main/start": { - const path = `${LOCATION}/procedures/main` - const procedure: any = await import(path).catch(() => require(path)) const started = async (onTerm: () => Promise) => { await effects.setMainStatus({ status: "running" }) if (this.onTerm) await this.onTerm() this.onTerm = onTerm } - return procedure.main({ effects, started }) + const daemons = await ( + await this.abi.main({ + effects: { ...effects, _type: "main" }, + started, + }) + ).build() + this.onTerm = daemons.term } case "/main/stop": { await effects.setMainStatus({ status: "stopped" }) @@ -86,67 +144,50 @@ export class SystemForStartOs implements System { return duration(30, "s") } case "/config/set": { - const path = `${LOCATION}/procedures/config` - const procedure: any = await import(path).catch(() => require(path)) - const input = options.input - return procedure.setConfig({ effects, input }) + const input = options.input as any // TODO + return this.abi.setConfig({ effects, input }) } case "/config/get": { - const path = `${LOCATION}/procedures/config` - const procedure: any = await import(path).catch(() => require(path)) - return procedure.getConfig({ effects }) + return this.abi.getConfig({ effects }) } case "/backup/create": case "/backup/restore": throw new Error("this should be called with the init/unit") case "/actions/metadata": { - const path = `${LOCATION}/procedures/actions` - const procedure: any = await import(path).catch(() => require(path)) - return procedure.actionsMetadata({ effects }) + return this.abi.actionsMetadata({ effects }) } default: const procedures = unNestPath(options.procedure) const id = procedures[2] switch (true) { case procedures[1] === "actions" && procedures[3] === "get": { - const path = `${LOCATION}/procedures/actions` - const action: any = (await import(path).catch(() => require(path))) - .actions[id] + const action = (await this.abi.actions({ effects }))[id] if (!action) throw new Error(`Action ${id} not found`) - return action.get({ effects }) + return action.getConfig({ effects }) } case procedures[1] === "actions" && procedures[3] === "run": { - const path = `${LOCATION}/procedures/actions` - const action: any = (await import(path).catch(() => require(path))) - .actions[id] + const action = (await this.abi.actions({ effects }))[id] if (!action) throw new Error(`Action ${id} not found`) - const input = options.input - return action.run({ effects, input }) + return action.run({ effects, input: options.input as any }) // TODO } case procedures[1] === "dependencies" && procedures[3] === "query": { - const path = `${LOCATION}/procedures/dependencies` - const dependencyConfig: any = ( - await import(path).catch(() => require(path)) - ).dependencyConfig[id] + const dependencyConfig = this.abi.dependencyConfig[id] if (!dependencyConfig) throw new Error(`dependencyConfig ${id} not found`) const localConfig = options.input - return dependencyConfig.query({ effects, localConfig }) + return dependencyConfig.query({ effects }) } case procedures[1] === "dependencies" && procedures[3] === "update": { - const path = `${LOCATION}/procedures/dependencies` - const dependencyConfig: any = ( - await import(path).catch(() => require(path)) - ).dependencyConfig[id] + const dependencyConfig = this.abi.dependencyConfig[id] if (!dependencyConfig) throw new Error(`dependencyConfig ${id} not found`) - return dependencyConfig.update(options.input) + return dependencyConfig.update(options.input as any) // TODO } } } - throw new Error("Method not implemented.") + throw new Error(`Method ${options.procedure} not implemented.`) } - exit(effects: Effects): Promise { - throw new Error("Method not implemented.") + async exit(effects: Effects): Promise { + return void null } } diff --git a/container-runtime/src/Adapters/Systems/index.ts b/container-runtime/src/Adapters/Systems/index.ts index eadc67318..a44ad533e 100644 --- a/container-runtime/src/Adapters/Systems/index.ts +++ b/container-runtime/src/Adapters/Systems/index.ts @@ -1,6 +1,22 @@ +import * as fs from "node:fs/promises" import { System } from "../../Interfaces/System" -import { SystemForEmbassy } from "./SystemForEmbassy" -import { SystemForStartOs } from "./SystemForStartOs" +import { EMBASSY_JS_LOCATION, SystemForEmbassy } from "./SystemForEmbassy" +import { STARTOS_JS_LOCATION, SystemForStartOs } from "./SystemForStartOs" export async function getSystem(): Promise { - return SystemForEmbassy.of() + if ( + await fs.access(STARTOS_JS_LOCATION).then( + () => true, + () => false, + ) + ) { + return SystemForStartOs.of() + } else if ( + await fs.access(EMBASSY_JS_LOCATION).then( + () => true, + () => false, + ) + ) { + return SystemForEmbassy.of() + } + throw new Error(`${STARTOS_JS_LOCATION} not found`) } diff --git a/container-runtime/src/Interfaces/System.ts b/container-runtime/src/Interfaces/System.ts index 86b2aa492..85ba0fb0f 100644 --- a/container-runtime/src/Interfaces/System.ts +++ b/container-runtime/src/Interfaces/System.ts @@ -14,6 +14,7 @@ export interface System { execute( effects: T.Effects, options: { + id: string procedure: JsonPath input: unknown timeout?: number diff --git a/core/Cargo.lock b/core/Cargo.lock index 4ae02e98e..cf10152c2 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -387,6 +387,22 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backhand" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f2fc1bc7bb7fd449e02000cc1592cc63dcdcd61710f8b9efe32bab2d1784603" +dependencies = [ + "deku", + "flate2", + "rustc-hash", + "thiserror", + "tracing", + "xz2", + "zstd", + "zstd-safe", +] + [[package]] name = "backtrace" version = "0.3.71" @@ -588,6 +604,11 @@ name = "cc" version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "065a29261d53ba54260972629f9ca6bffa69bac13cd1fed61420f7fa68b9f8bd" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] [[package]] name = "cfg-if" @@ -1058,12 +1079,6 @@ dependencies = [ "cipher 0.3.0", ] -[[package]] -name = "current_platform" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a74858bcfe44b22016cb49337d7b6f04618c58e5dbfdef61b06b8c434324a0bc" - [[package]] name = "curve25519-dalek" version = "3.2.0" @@ -1146,6 +1161,31 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +[[package]] +name = "deku" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709ade444d53896e60f6265660eb50480dd08b77bfc822e5dcc233b88b0b2fba" +dependencies = [ + "bitvec", + "deku_derive", + "no_std_io", + "rustversion", +] + +[[package]] +name = "deku_derive" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7534973f93f9de83203e41c8ddd32d230599fa73fa889f3deb1580ccd186913" +dependencies = [ + "darling", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.60", +] + [[package]] name = "der" version = "0.7.9" @@ -2405,6 +2445,15 @@ dependencies = [ "jaq-parse", ] +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + [[package]] name = "josekit" version = "0.8.6" @@ -2561,6 +2610,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libyml" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e281a65eeba3d4503a2839252f86374528f9ceafe6fed97c1d3b52e1fb625c1" + [[package]] name = "linux-raw-sys" version = "0.4.13" @@ -2583,6 +2638,17 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "matchers" version = "0.1.0" @@ -2788,6 +2854,15 @@ dependencies = [ "libc", ] +[[package]] +name = "no_std_io" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fa5f306a6f2c01b4fd172f29bb46195b1764061bf926c75e96ff55df3178208" +dependencies = [ + "memchr", +] + [[package]] name = "nom" version = "7.1.3" @@ -3800,6 +3875,12 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.0" @@ -3895,9 +3976,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "rusty-fork" @@ -3929,9 +4010,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "same-file" @@ -4015,9 +4096,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.200" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] @@ -4041,9 +4122,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.200" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", @@ -4052,9 +4133,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "indexmap 2.2.6", "itoa", @@ -4124,16 +4205,20 @@ dependencies = [ ] [[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" +name = "serde_yml" +version = "0.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +checksum = "78ce6afeda22f0b55dde2c34897bce76a629587348480384231205c14b59a01f" dependencies = [ "indexmap 2.2.6", "itoa", + "libyml", + "log", + "memchr", "ryu", "serde", - "unsafe-libyaml", + "serde_json", + "tempfile", ] [[package]] @@ -4600,6 +4685,7 @@ dependencies = [ "async-trait", "axum 0.7.5", "axum-server", + "backhand", "base32", "base64 0.21.7", "base64ct", @@ -4614,7 +4700,6 @@ dependencies = [ "console-subscriber", "cookie 0.18.1", "cookie_store", - "current_platform", "der", "digest 0.10.7", "divrem", @@ -4681,7 +4766,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "serde_with", - "serde_yaml", + "serde_yml", "sha2 0.10.8", "shell-words", "simple-logging", @@ -5537,12 +5622,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "untrusted" version = "0.9.0" @@ -5993,6 +6072,15 @@ dependencies = [ "rustix", ] +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yajrc" version = "0.1.3" @@ -6056,3 +6144,31 @@ dependencies = [ "quote", "syn 2.0.60", ] + +[[package]] +name = "zstd" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.10+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index bd064a167..3ad4f9eeb 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -59,6 +59,7 @@ async-stream = "0.3.5" async-trait = "0.1.74" axum = { version = "0.7.3", features = ["ws"] } axum-server = "0.6.0" +backhand = "0.18.0" base32 = "0.4.0" base64 = "0.21.4" base64ct = "1.6.0" @@ -72,7 +73,6 @@ console = "0.15.7" console-subscriber = { version = "0.2", optional = true } cookie = "0.18.0" cookie_store = "0.20.0" -current_platform = "0.2.0" der = { version = "0.7.9", features = ["derive", "pem"] } digest = "0.10.7" divrem = "1.0.0" @@ -154,7 +154,7 @@ serde_json = "1.0" serde_toml = { package = "toml", version = "0.8.2" } serde_urlencoded = "0.7" serde_with = { version = "3.4.0", features = ["macros", "json"] } -serde_yaml = "0.9.25" +serde_yaml = { package = "serde_yml", version = "0.0.10" } sha2 = "0.10.2" shell-words = "1" simple-logging = "2.0.2" diff --git a/core/startos/src/action.rs b/core/startos/src/action.rs index 87ff317f8..e93af4a4d 100644 --- a/core/startos/src/action.rs +++ b/core/startos/src/action.rs @@ -8,6 +8,7 @@ use ts_rs::TS; use crate::config::Config; use crate::context::RpcContext; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::util::serde::{display_serializable, StdinDeserializable, WithIoFormat}; #[derive(Debug, Serialize, Deserialize)] @@ -77,6 +78,7 @@ pub async fn action( .as_ref() .or_not_found(lazy_format!("Manager for {}", package_id))? .action( + Guid::new(), action_id, input.map(|c| to_value(&c)).transpose()?.unwrap_or_default(), ) diff --git a/core/startos/src/auth.rs b/core/startos/src/auth.rs index 4838f2ea2..d33320b78 100644 --- a/core/startos/src/auth.rs +++ b/core/startos/src/auth.rs @@ -178,6 +178,7 @@ pub fn check_password_against_db(db: &DatabaseModel, password: &str) -> Result<( #[derive(Deserialize, Serialize, Parser, TS)] #[serde(rename_all = "camelCase")] #[command(rename_all = "kebab-case")] +#[ts(export)] pub struct LoginParams { password: Option, #[ts(skip)] diff --git a/core/startos/src/backup/restore.rs b/core/startos/src/backup/restore.rs index 774b1f1cf..4753a4290 100644 --- a/core/startos/src/backup/restore.rs +++ b/core/startos/src/backup/restore.rs @@ -149,7 +149,6 @@ async fn restore_packages( S9pk::open( backup_dir.path().join(&id).with_extension("s9pk"), Some(&id), - true, ) .await?, Some(backup_dir), diff --git a/core/startos/src/config/mod.rs b/core/startos/src/config/mod.rs index e61517794..01309a16f 100644 --- a/core/startos/src/config/mod.rs +++ b/core/startos/src/config/mod.rs @@ -16,6 +16,7 @@ use ts_rs::TS; use crate::context::{CliContext, RpcContext}; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::util::serde::{HandlerExtSerde, StdinDeserializable}; #[derive(Clone, Debug, Default, Serialize, Deserialize)] @@ -156,7 +157,7 @@ pub async fn get(ctx: RpcContext, _: Empty, id: PackageId) -> Result, - #[arg(long = "ethernet-interface")] + #[arg(long)] pub ethernet_interface: Option, #[arg(skip)] pub os_partitions: Option, - #[arg(long = "bind-rpc")] + #[arg(long)] pub bind_rpc: Option, - #[arg(long = "tor-control")] + #[arg(long)] pub tor_control: Option, - #[arg(long = "tor-socks")] + #[arg(long)] pub tor_socks: Option, - #[arg(long = "dns-bind")] + #[arg(long)] pub dns_bind: Option>, - #[arg(long = "revision-cache-size")] + #[arg(long)] pub revision_cache_size: Option, - #[arg(short = 'd', long = "datadir")] + #[arg(short, long)] pub datadir: Option, - #[arg(long = "disable-encryption")] + #[arg(long)] pub disable_encryption: Option, + #[arg(long)] + pub multi_arch_s9pks: Option, } impl ContextConfig for ServerConfig { fn next(&mut self) -> Option { @@ -131,6 +133,7 @@ impl ContextConfig for ServerConfig { .or(other.revision_cache_size); self.datadir = self.datadir.take().or(other.datadir); self.disable_encryption = self.disable_encryption.take().or(other.disable_encryption); + self.multi_arch_s9pks = self.multi_arch_s9pks.take().or(other.multi_arch_s9pks); } } diff --git a/core/startos/src/context/rpc.rs b/core/startos/src/context/rpc.rs index cf2d28085..a3e77a62c 100644 --- a/core/startos/src/context/rpc.rs +++ b/core/startos/src/context/rpc.rs @@ -43,6 +43,7 @@ pub struct RpcContextSeed { pub db: TypedPatchDb, pub account: RwLock, pub net_controller: Arc, + pub s9pk_arch: Option<&'static str>, pub services: ServiceMap, pub metrics_cache: RwLock>, pub shutdown: broadcast::Sender>, @@ -152,6 +153,11 @@ impl RpcContext { db, account: RwLock::new(account), net_controller, + s9pk_arch: if config.multi_arch_s9pks.unwrap_or(false) { + None + } else { + Some(crate::ARCH) + }, services, metrics_cache, shutdown, diff --git a/core/startos/src/control.rs b/core/startos/src/control.rs index d4a595a61..e831e07d6 100644 --- a/core/startos/src/control.rs +++ b/core/startos/src/control.rs @@ -7,6 +7,7 @@ use ts_rs::TS; use crate::context::RpcContext; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::Error; #[derive(Deserialize, Serialize, Parser, TS)] @@ -23,7 +24,7 @@ pub async fn start(ctx: RpcContext, ControlParams { id }: ControlParams) -> Resu .await .as_ref() .or_not_found(lazy_format!("Manager for {id}"))? - .start() + .start(Guid::new()) .await?; Ok(()) @@ -36,7 +37,7 @@ pub async fn stop(ctx: RpcContext, ControlParams { id }: ControlParams) -> Resul .await .as_ref() .ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))? - .stop() + .stop(Guid::new()) .await?; Ok(()) @@ -48,7 +49,7 @@ pub async fn restart(ctx: RpcContext, ControlParams { id }: ControlParams) -> Re .await .as_ref() .ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))? - .restart() + .restart(Guid::new()) .await?; Ok(()) diff --git a/core/startos/src/dependencies.rs b/core/startos/src/dependencies.rs index a746a42c9..f6ccc53ad 100644 --- a/core/startos/src/dependencies.rs +++ b/core/startos/src/dependencies.rs @@ -13,6 +13,7 @@ use crate::config::{Config, ConfigSpec, ConfigureContext}; use crate::context::RpcContext; use crate::db::model::package::CurrentDependencies; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::Error; pub fn dependency() -> ParentHandler { @@ -86,7 +87,7 @@ pub async fn configure_impl( ErrorKind::Unknown, ) })? - .configure(configure_context) + .configure(Guid::new(), configure_context) .await?; Ok(()) } @@ -103,14 +104,15 @@ pub async fn configure_logic( ctx: RpcContext, (dependent_id, dependency_id): (PackageId, PackageId), ) -> Result { + let procedure_id = Guid::new(); let dependency_guard = ctx.services.get(&dependency_id).await; let dependency = dependency_guard.as_ref().or_not_found(&dependency_id)?; let dependent_guard = ctx.services.get(&dependent_id).await; let dependent = dependent_guard.as_ref().or_not_found(&dependent_id)?; - let config_res = dependency.get_config().await?; + let config_res = dependency.get_config(procedure_id.clone()).await?; let diff = Value::Object( dependent - .dependency_config(dependency_id, config_res.config.clone()) + .dependency_config(procedure_id, dependency_id, config_res.config.clone()) .await? .unwrap_or_default(), ); @@ -129,6 +131,7 @@ pub async fn compute_dependency_config_errs( id: &PackageId, current_dependencies: &mut CurrentDependencies, ) -> Result<(), Error> { + let procedure_id = Guid::new(); let service_guard = ctx.services.get(id).await; let service = service_guard.as_ref().or_not_found(id)?; for (dep_id, dep_info) in current_dependencies.0.iter_mut() { @@ -137,10 +140,10 @@ pub async fn compute_dependency_config_errs( continue; }; - let dep_config = dependency.get_config().await?.config; + let dep_config = dependency.get_config(procedure_id.clone()).await?.config; dep_info.config_satisfied = service - .dependency_config(dep_id.clone(), dep_config) + .dependency_config(procedure_id.clone(), dep_id.clone(), dep_config) .await? .is_none(); } diff --git a/core/startos/src/disk/mount/backup.rs b/core/startos/src/disk/mount/backup.rs index 5dbd80db3..ad9f5090b 100644 --- a/core/startos/src/disk/mount/backup.rs +++ b/core/startos/src/disk/mount/backup.rs @@ -178,7 +178,6 @@ impl BackupMountGuard { Ok(()) } } -#[async_trait::async_trait] impl GenericMountGuard for BackupMountGuard { fn path(&self) -> &Path { if let Some(guard) = &self.encrypted_guard { diff --git a/core/startos/src/disk/mount/filesystem/overlayfs.rs b/core/startos/src/disk/mount/filesystem/overlayfs.rs index f96de7b11..5e40a21a1 100644 --- a/core/startos/src/disk/mount/filesystem/overlayfs.rs +++ b/core/startos/src/disk/mount/filesystem/overlayfs.rs @@ -6,8 +6,8 @@ use digest::generic_array::GenericArray; use digest::{Digest, OutputSizeUser}; use sha2::Sha256; -use crate::disk::mount::filesystem::{FileSystem, ReadOnly, ReadWrite}; -use crate::disk::mount::guard::{GenericMountGuard, MountGuard, TmpMountGuard}; +use crate::disk::mount::filesystem::{FileSystem, ReadWrite}; +use crate::disk::mount::guard::{GenericMountGuard, MountGuard}; use crate::prelude::*; use crate::util::io::TmpDir; @@ -94,17 +94,13 @@ impl< } #[derive(Debug)] -pub struct OverlayGuard { - lower: Option, +pub struct OverlayGuard { + lower: Option, upper: Option, inner_guard: MountGuard, } -impl OverlayGuard { - pub async fn mount( - base: &impl FileSystem, - mountpoint: impl AsRef, - ) -> Result { - let lower = TmpMountGuard::mount(base, ReadOnly).await?; +impl OverlayGuard { + pub async fn mount(lower: G, mountpoint: impl AsRef) -> Result { let upper = TmpDir::new().await?; let inner_guard = MountGuard::mount( &OverlayFs::new( @@ -140,16 +136,15 @@ impl OverlayGuard { } } } -#[async_trait::async_trait] -impl GenericMountGuard for OverlayGuard { +impl GenericMountGuard for OverlayGuard { fn path(&self) -> &Path { self.inner_guard.path() } - async fn unmount(mut self) -> Result<(), Error> { + async fn unmount(self) -> Result<(), Error> { self.unmount(false).await } } -impl Drop for OverlayGuard { +impl Drop for OverlayGuard { fn drop(&mut self) { let lower = self.lower.take(); let upper = self.upper.take(); diff --git a/core/startos/src/disk/mount/guard.rs b/core/startos/src/disk/mount/guard.rs index d08b04881..4686a10b8 100644 --- a/core/startos/src/disk/mount/guard.rs +++ b/core/startos/src/disk/mount/guard.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::sync::{Arc, Weak}; +use futures::Future; use lazy_static::lazy_static; use models::ResultExt; use tokio::sync::Mutex; @@ -14,23 +15,20 @@ use crate::Error; pub const TMP_MOUNTPOINT: &'static str = "/media/startos/tmp"; -#[async_trait::async_trait] pub trait GenericMountGuard: std::fmt::Debug + Send + Sync + 'static { fn path(&self) -> &Path; - async fn unmount(mut self) -> Result<(), Error>; + fn unmount(self) -> impl Future> + Send; } -#[async_trait::async_trait] impl GenericMountGuard for Never { fn path(&self) -> &Path { match *self {} } - async fn unmount(mut self) -> Result<(), Error> { + async fn unmount(self) -> Result<(), Error> { match self {} } } -#[async_trait::async_trait] impl GenericMountGuard for Arc where T: GenericMountGuard, @@ -38,7 +36,7 @@ where fn path(&self) -> &Path { (&**self).path() } - async fn unmount(mut self) -> Result<(), Error> { + async fn unmount(self) -> Result<(), Error> { if let Ok(guard) = Arc::try_unwrap(self) { guard.unmount().await?; } @@ -102,12 +100,11 @@ impl Drop for MountGuard { } } } -#[async_trait::async_trait] impl GenericMountGuard for MountGuard { fn path(&self) -> &Path { &self.mountpoint } - async fn unmount(mut self) -> Result<(), Error> { + async fn unmount(self) -> Result<(), Error> { MountGuard::unmount(self, false).await } } @@ -165,12 +162,11 @@ impl TmpMountGuard { std::mem::replace(self, unmounted) } } -#[async_trait::async_trait] impl GenericMountGuard for TmpMountGuard { fn path(&self) -> &Path { self.guard.path() } - async fn unmount(mut self) -> Result<(), Error> { + async fn unmount(self) -> Result<(), Error> { self.guard.unmount().await } } @@ -187,12 +183,11 @@ impl SubPath { Self { guard, path } } } -#[async_trait::async_trait] impl GenericMountGuard for SubPath { fn path(&self) -> &Path { self.path.as_path() } - async fn unmount(mut self) -> Result<(), Error> { + async fn unmount(self) -> Result<(), Error> { self.guard.unmount().await } } diff --git a/core/startos/src/init.rs b/core/startos/src/init.rs index 694d4e3a3..97c674ac5 100644 --- a/core/startos/src/init.rs +++ b/core/startos/src/init.rs @@ -242,7 +242,7 @@ pub async fn init(cfg: &ServerConfig) -> Result { let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok() || &*server_info.version < &emver::Version::new(0, 3, 2, 0) - || (*ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0)); + || (ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0)); let log_dir = cfg.datadir().join("main/logs"); if tokio::fs::metadata(&log_dir).await.is_err() { diff --git a/core/startos/src/install/mod.rs b/core/startos/src/install/mod.rs index c4e4452b7..bb7d1a02e 100644 --- a/core/startos/src/install/mod.rs +++ b/core/startos/src/install/mod.rs @@ -152,7 +152,6 @@ pub async fn install( .await?, ), None, // TODO - true, ) .await?; @@ -262,7 +261,6 @@ pub async fn sideload(ctx: RpcContext) -> Result { if let Err(e) = async { let s9pk = S9pk::deserialize( &file, None, // TODO - true, ) .await?; let _ = id_send.send(s9pk.as_manifest().id.clone()); diff --git a/core/startos/src/lib.rs b/core/startos/src/lib.rs index d60a2db24..0e125af98 100644 --- a/core/startos/src/lib.rs +++ b/core/startos/src/lib.rs @@ -4,12 +4,8 @@ pub const CAP_1_KiB: usize = 1024; pub const CAP_1_MiB: usize = CAP_1_KiB * CAP_1_KiB; pub const CAP_10_MiB: usize = 10 * CAP_1_MiB; pub const HOST_IP: [u8; 4] = [172, 18, 0, 1]; -pub const TARGET: &str = current_platform::CURRENT_PLATFORM; +pub use std::env::consts::ARCH; lazy_static::lazy_static! { - pub static ref ARCH: &'static str = { - let (arch, _) = TARGET.split_once("-").unwrap(); - arch - }; pub static ref PLATFORM: String = { if let Ok(platform) = std::fs::read_to_string("/usr/lib/startos/PLATFORM.txt") { platform diff --git a/core/startos/src/lxc/mod.rs b/core/startos/src/lxc/mod.rs index f64ecebe7..8d37120ba 100644 --- a/core/startos/src/lxc/mod.rs +++ b/core/startos/src/lxc/mod.rs @@ -29,7 +29,7 @@ use crate::disk::mount::filesystem::bind::Bind; use crate::disk::mount::filesystem::block_dev::BlockDev; use crate::disk::mount::filesystem::idmapped::IdMapped; use crate::disk::mount::filesystem::overlayfs::OverlayGuard; -use crate::disk::mount::filesystem::{MountType, ReadWrite}; +use crate::disk::mount::filesystem::{MountType, ReadOnly, ReadWrite}; use crate::disk::mount::guard::{GenericMountGuard, MountGuard, TmpMountGuard}; use crate::disk::mount::util::unmount; use crate::prelude::*; @@ -153,7 +153,7 @@ impl LxcManager { pub struct LxcContainer { manager: Weak, - rootfs: OverlayGuard, + rootfs: OverlayGuard, pub guid: Arc, rpc_bind: TmpMountGuard, log_mount: Option, @@ -184,12 +184,16 @@ impl LxcContainer { .invoke(ErrorKind::Filesystem) .await?; let rootfs = OverlayGuard::mount( - &IdMapped::new( - BlockDev::new("/usr/lib/startos/container-runtime/rootfs.squashfs"), - 0, - 100000, - 65536, - ), + TmpMountGuard::mount( + &IdMapped::new( + BlockDev::new("/usr/lib/startos/container-runtime/rootfs.squashfs"), + 0, + 100000, + 65536, + ), + ReadOnly, + ) + .await?, &rootfs_dir, ) .await?; diff --git a/core/startos/src/os_install/gpt.rs b/core/startos/src/os_install/gpt.rs index 4139b4cf2..01703083b 100644 --- a/core/startos/src/os_install/gpt.rs +++ b/core/startos/src/os_install/gpt.rs @@ -87,7 +87,7 @@ pub async fn partition(disk: &DiskInfo, overwrite: bool) -> Result gpt::partition_types::LINUX_ROOT_X64, "aarch64" => gpt::partition_types::LINUX_ROOT_ARM_64, _ => gpt::partition_types::LINUX_FS, diff --git a/core/startos/src/os_install/mod.rs b/core/startos/src/os_install/mod.rs index c3931b236..4e5c7ed15 100644 --- a/core/startos/src/os_install/mod.rs +++ b/core/startos/src/os_install/mod.rs @@ -366,7 +366,7 @@ pub async fn execute( if tokio::fs::metadata("/sys/firmware/efi").await.is_err() { install.arg("--target=i386-pc"); } else { - match *ARCH { + match ARCH { "x86_64" => install.arg("--target=x86_64-efi"), "aarch64" => install.arg("--target=arm64-efi"), _ => &mut install, diff --git a/core/startos/src/registry/device_info.rs b/core/startos/src/registry/device_info.rs index 7da5bd8b9..51d6ac46b 100644 --- a/core/startos/src/registry/device_info.rs +++ b/core/startos/src/registry/device_info.rs @@ -134,7 +134,7 @@ pub struct HardwareInfo { impl From<&RpcContext> for HardwareInfo { fn from(value: &RpcContext) -> Self { Self { - arch: InternedString::intern(&**crate::ARCH), + arch: InternedString::intern(crate::ARCH), ram: value.hardware.ram, devices: value .hardware diff --git a/core/startos/src/registry/package/add.rs b/core/startos/src/registry/package/add.rs index 6a1050b99..d28aeaaa4 100644 --- a/core/startos/src/registry/package/add.rs +++ b/core/startos/src/registry/package/add.rs @@ -53,7 +53,6 @@ pub async fn add_package( let s9pk = S9pk::deserialize( &Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?), Some(&commitment), - false, ) .await?; @@ -109,7 +108,7 @@ pub async fn cli_add_package( .. }: HandlerArgs, ) -> Result<(), Error> { - let s9pk = S9pk::open(&file, None, false).await?; + let s9pk = S9pk::open(&file, None).await?; let mut progress = FullProgressTracker::new(); let progress_handle = progress.handle(); @@ -143,7 +142,6 @@ pub async fn cli_add_package( let mut src = S9pk::deserialize( &Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?), Some(&commitment), - false, ) .await?; src.serialize(&mut TrackingIO::new(0, tokio::io::sink()), true) diff --git a/core/startos/src/rpc_continuations.rs b/core/startos/src/rpc_continuations.rs index ce8bf43fd..e6b823ef9 100644 --- a/core/startos/src/rpc_continuations.rs +++ b/core/startos/src/rpc_continuations.rs @@ -39,6 +39,11 @@ impl Guid { Some(Guid(InternedString::intern(r))) } } +impl Default for Guid { + fn default() -> Self { + Self::new() + } +} impl AsRef for Guid { fn as_ref(&self) -> &str { self.0.as_ref() diff --git a/core/startos/src/s9pk/merkle_archive/directory_contents.rs b/core/startos/src/s9pk/merkle_archive/directory_contents.rs index 77c3582d9..c5e5c4a7d 100644 --- a/core/startos/src/s9pk/merkle_archive/directory_contents.rs +++ b/core/startos/src/s9pk/merkle_archive/directory_contents.rs @@ -211,7 +211,10 @@ impl DirectoryContents { if !filter(path) { if v.hash.is_none() { return Err(Error::new( - eyre!("cannot filter out unhashed file, run `update_hashes` first"), + eyre!( + "cannot filter out unhashed file {}, run `update_hashes` first", + path.display() + ), ErrorKind::InvalidRequest, )); } diff --git a/core/startos/src/s9pk/merkle_archive/expected.rs b/core/startos/src/s9pk/merkle_archive/expected.rs new file mode 100644 index 000000000..a0f095b7a --- /dev/null +++ b/core/startos/src/s9pk/merkle_archive/expected.rs @@ -0,0 +1,103 @@ + +use std::ffi::OsStr; +use std::path::Path; + +use crate::prelude::*; +use crate::s9pk::merkle_archive::directory_contents::DirectoryContents; +use crate::s9pk::merkle_archive::source::FileSource; +use crate::s9pk::merkle_archive::Entry; + +/// An object for tracking the files expected to be in an s9pk +pub struct Expected<'a, T> { + keep: DirectoryContents<()>, + dir: &'a DirectoryContents, +} +impl<'a, T> Expected<'a, T> { + pub fn new(dir: &'a DirectoryContents,) -> Self { + Self { + keep: DirectoryContents::new(), + dir + } + } +} +impl<'a, T: Clone> Expected<'a, T> { + pub fn check_file(&mut self, path: impl AsRef) -> Result<(), Error> { + if self + .dir + .get_path(path.as_ref()) + .and_then(|e| e.as_file()) + .is_some() + { + self.keep.insert_path(path, Entry::file(()))?; + Ok(()) + } else { + Err(Error::new( + eyre!("file {} missing from archive", path.as_ref().display()), + ErrorKind::ParseS9pk, + )) + } + } + pub fn check_stem( + &mut self, + path: impl AsRef, + mut valid_extension: impl FnMut(Option<&OsStr>) -> bool, + ) -> Result<(), Error> { + let (dir, stem) = if let Some(parent) = path.as_ref().parent().filter(|p| *p != Path::new("")) { + ( + self.dir + .get_path(parent) + .and_then(|e| e.as_directory()) + .ok_or_else(|| { + Error::new( + eyre!("directory {} missing from archive", parent.display()), + ErrorKind::ParseS9pk, + ) + })?, + path.as_ref().strip_prefix(parent).unwrap(), + ) + } else { + (self.dir, path.as_ref()) + }; + let name = dir + .with_stem(&stem.as_os_str().to_string_lossy()) + .filter(|(_, e)| e.as_file().is_some()) + .try_fold( + Err(Error::new( + eyre!( + "file {} with valid extension missing from archive", + path.as_ref().display() + ), + ErrorKind::ParseS9pk, + )), + |acc, (name, _)| + if valid_extension(Path::new(&*name).extension()) { + match acc { + Ok(_) => Err(Error::new( + eyre!( + "more than one file matching {} with valid extension in archive", + path.as_ref().display() + ), + ErrorKind::ParseS9pk, + )), + Err(_) => Ok(Ok(name)) + } + } else { + Ok(acc) + } + )??; + self.keep + .insert_path(path.as_ref().with_file_name(name), Entry::file(()))?; + Ok(()) + } + pub fn into_filter(self) -> Filter { + Filter(self.keep) + } +} + +pub struct Filter(DirectoryContents<()>); +impl Filter { + pub fn keep_checked(&self, dir: &mut DirectoryContents) -> Result<(), Error> { + dir.filter(|path| self.0.get_path(path).is_some()) + } +} + diff --git a/core/startos/src/s9pk/merkle_archive/mod.rs b/core/startos/src/s9pk/merkle_archive/mod.rs index 1c0d6b786..00ead65c5 100644 --- a/core/startos/src/s9pk/merkle_archive/mod.rs +++ b/core/startos/src/s9pk/merkle_archive/mod.rs @@ -19,6 +19,7 @@ use crate::util::serde::Base64; use crate::CAP_1_MiB; pub mod directory_contents; +pub mod expected; pub mod file_contents; pub mod hash; pub mod sink; @@ -217,6 +218,9 @@ impl Entry { pub fn file(source: S) -> Self { Self::new(EntryContents::File(FileContents::new(source))) } + pub fn directory(directory: DirectoryContents) -> Self { + Self::new(EntryContents::Directory(directory)) + } pub fn hash(&self) -> Option<(Hash, u64)> { self.hash } diff --git a/core/startos/src/s9pk/merkle_archive/source/mod.rs b/core/startos/src/s9pk/merkle_archive/source/mod.rs index f6922d109..0a00b18dd 100644 --- a/core/startos/src/s9pk/merkle_archive/source/mod.rs +++ b/core/startos/src/s9pk/merkle_archive/source/mod.rs @@ -280,3 +280,8 @@ impl FileSource for Section { self.source.copy_to(self.position, self.size, w).await } } + +pub type DynRead = Box; +pub fn into_dyn_read(r: R) -> DynRead { + Box::new(r) +} diff --git a/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs b/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs index 9cc162f0e..92eb40f9d 100644 --- a/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs +++ b/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs @@ -97,7 +97,7 @@ impl ArchiveSource for MultiCursorFile { .ok() .map(|m| m.len()) } - async fn fetch_all(&self) -> Result { + async fn fetch_all(&self) -> Result { use tokio::io::AsyncSeekExt; let mut file = self.cursor().await?; diff --git a/core/startos/src/s9pk/rpc.rs b/core/startos/src/s9pk/rpc.rs index 89cfc9b5a..fac9e6724 100644 --- a/core/startos/src/s9pk/rpc.rs +++ b/core/startos/src/s9pk/rpc.rs @@ -1,32 +1,26 @@ -use std::collections::BTreeSet; -use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::path::PathBuf; use clap::Parser; -use itertools::Itertools; use models::ImageId; use rpc_toolkit::{from_fn_async, Empty, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; use tokio::fs::File; -use tokio::process::Command; use ts_rs::TS; use crate::context::CliContext; use crate::prelude::*; use crate::s9pk::manifest::Manifest; -use crate::s9pk::merkle_archive::source::DynFileSource; -use crate::s9pk::merkle_archive::Entry; -use crate::s9pk::v2::compat::CONTAINER_TOOL; +use crate::s9pk::v2::pack::ImageConfig; use crate::s9pk::v2::SIG_CONTEXT; use crate::s9pk::S9pk; use crate::util::io::TmpDir; use crate::util::serde::{apply_expr, HandlerExtSerde}; -use crate::util::Invoke; pub const SKIP_ENV: &[&str] = &["TERM", "container", "HOME", "HOSTNAME"]; pub fn s9pk() -> ParentHandler { ParentHandler::new() + .subcommand("pack", from_fn_async(super::v2::pack::pack).no_display()) .subcommand("edit", edit()) .subcommand("inspect", inspect()) } @@ -77,117 +71,21 @@ fn inspect() -> ParentHandler { #[derive(Deserialize, Serialize, Parser, TS)] struct AddImageParams { id: ImageId, - image: String, - arches: Option>, + #[command(flatten)] + config: ImageConfig, } async fn add_image( ctx: CliContext, - AddImageParams { id, image, arches }: AddImageParams, + AddImageParams { id, config }: AddImageParams, S9pkPath { s9pk: s9pk_path }: S9pkPath, ) -> Result<(), Error> { - let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?, false) + let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?) .await? .into_dyn(); - let arches: BTreeSet<_> = arches - .unwrap_or_else(|| vec!["x86_64".to_owned(), "aarch64".to_owned()]) - .into_iter() - .collect(); + s9pk.as_manifest_mut().images.insert(id, config); let tmpdir = TmpDir::new().await?; - for arch in arches { - let sqfs_path = tmpdir.join(format!("image.{arch}.squashfs")); - let docker_platform = if arch == "x86_64" { - "--platform=linux/amd64".to_owned() - } else if arch == "aarch64" { - "--platform=linux/arm64".to_owned() - } else { - format!("--platform=linux/{arch}") - }; - let env = String::from_utf8( - Command::new(CONTAINER_TOOL) - .arg("run") - .arg("--rm") - .arg(&docker_platform) - .arg("--entrypoint") - .arg("env") - .arg(&image) - .invoke(ErrorKind::Docker) - .await?, - )? - .lines() - .filter(|l| { - l.trim() - .split_once("=") - .map_or(false, |(v, _)| !SKIP_ENV.contains(&v)) - }) - .join("\n") - + "\n"; - let workdir = Path::new( - String::from_utf8( - Command::new(CONTAINER_TOOL) - .arg("run") - .arg(&docker_platform) - .arg("--rm") - .arg("--entrypoint") - .arg("pwd") - .arg(&image) - .invoke(ErrorKind::Docker) - .await?, - )? - .trim(), - ) - .to_owned(); - let container_id = String::from_utf8( - Command::new(CONTAINER_TOOL) - .arg("create") - .arg(&docker_platform) - .arg(&image) - .invoke(ErrorKind::Docker) - .await?, - )?; - Command::new("bash") - .arg("-c") - .arg(format!( - "{CONTAINER_TOOL} export {container_id} | mksquashfs - {sqfs} -tar", - container_id = container_id.trim(), - sqfs = sqfs_path.display() - )) - .invoke(ErrorKind::Docker) - .await?; - Command::new(CONTAINER_TOOL) - .arg("rm") - .arg(container_id.trim()) - .invoke(ErrorKind::Docker) - .await?; - let archive = s9pk.as_archive_mut(); - archive.set_signer(ctx.developer_key()?.clone(), SIG_CONTEXT); - archive.contents_mut().insert_path( - Path::new("images") - .join(&arch) - .join(&id) - .with_extension("squashfs"), - Entry::file(DynFileSource::new(sqfs_path)), - )?; - archive.contents_mut().insert_path( - Path::new("images") - .join(&arch) - .join(&id) - .with_extension("env"), - Entry::file(DynFileSource::new(Arc::<[u8]>::from(Vec::from(env)))), - )?; - archive.contents_mut().insert_path( - Path::new("images") - .join(&arch) - .join(&id) - .with_extension("json"), - Entry::file(DynFileSource::new(Arc::<[u8]>::from( - serde_json::to_vec(&serde_json::json!({ - "workdir": workdir - })) - .with_kind(ErrorKind::Serialization)?, - ))), - )?; - } - s9pk.as_manifest_mut().images.insert(id); + s9pk.load_images(&tmpdir).await?; + s9pk.validate_and_filter(None)?; let tmp_path = s9pk_path.with_extension("s9pk.tmp"); let mut tmp_file = File::create(&tmp_path).await?; s9pk.serialize(&mut tmp_file, true).await?; @@ -206,7 +104,7 @@ async fn edit_manifest( EditManifestParams { expression }: EditManifestParams, S9pkPath { s9pk: s9pk_path }: S9pkPath, ) -> Result { - let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?, false).await?; + let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?).await?; let old = serde_json::to_value(s9pk.as_manifest()).with_kind(ErrorKind::Serialization)?; *s9pk.as_manifest_mut() = serde_json::from_value(apply_expr(old.into(), &expression)?.into()) .with_kind(ErrorKind::Serialization)?; @@ -227,7 +125,7 @@ async fn file_tree( _: Empty, S9pkPath { s9pk }: S9pkPath, ) -> Result, Error> { - let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?, false).await?; + let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?).await?; Ok(s9pk.as_archive().contents().file_paths("")) } @@ -244,7 +142,7 @@ async fn cat( ) -> Result<(), Error> { use crate::s9pk::merkle_archive::source::FileSource; - let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?, false).await?; + let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?).await?; tokio::io::copy( &mut s9pk .as_archive() @@ -266,6 +164,6 @@ async fn inspect_manifest( _: Empty, S9pkPath { s9pk }: S9pkPath, ) -> Result { - let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?, false).await?; + let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?).await?; Ok(s9pk.as_manifest().clone()) } diff --git a/core/startos/src/s9pk/v2/compat.rs b/core/startos/src/s9pk/v2/compat.rs index 5d4ad2f44..835c86b87 100644 --- a/core/startos/src/s9pk/v2/compat.rs +++ b/core/startos/src/s9pk/v2/compat.rs @@ -1,6 +1,5 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::io::Cursor; -use std::path::{Path, PathBuf}; +use std::collections::BTreeMap; +use std::path::Path; use std::sync::Arc; use itertools::Itertools; @@ -14,49 +13,18 @@ use crate::prelude::*; use crate::s9pk::manifest::Manifest; use crate::s9pk::merkle_archive::directory_contents::DirectoryContents; use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; -use crate::s9pk::merkle_archive::source::{FileSource, Section}; +use crate::s9pk::merkle_archive::source::Section; use crate::s9pk::merkle_archive::{Entry, MerkleArchive}; use crate::s9pk::rpc::SKIP_ENV; use crate::s9pk::v1::manifest::{Manifest as ManifestV1, PackageProcedure}; use crate::s9pk::v1::reader::S9pkReader; +use crate::s9pk::v2::pack::{PackSource, CONTAINER_TOOL}; use crate::s9pk::v2::{S9pk, SIG_CONTEXT}; use crate::util::io::TmpDir; use crate::util::Invoke; pub const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x01]; -#[cfg(not(feature = "docker"))] -pub const CONTAINER_TOOL: &str = "podman"; - -#[cfg(feature = "docker")] -pub const CONTAINER_TOOL: &str = "docker"; - -type DynRead = Box; -fn into_dyn_read(r: R) -> DynRead { - Box::new(r) -} - -#[derive(Clone)] -enum CompatSource { - Buffered(Arc<[u8]>), - File(PathBuf), -} -impl FileSource for CompatSource { - type Reader = Box; - async fn size(&self) -> Result { - match self { - Self::Buffered(a) => Ok(a.len() as u64), - Self::File(f) => Ok(tokio::fs::metadata(f).await?.len()), - } - } - async fn reader(&self) -> Result { - match self { - Self::Buffered(a) => Ok(into_dyn_read(Cursor::new(a.clone()))), - Self::File(f) => Ok(into_dyn_read(File::open(f).await?)), - } - } -} - impl S9pk> { #[instrument(skip_all)] pub async fn from_v1( @@ -66,7 +34,7 @@ impl S9pk> { ) -> Result { let scratch_dir = TmpDir::new().await?; - let mut archive = DirectoryContents::::new(); + let mut archive = DirectoryContents::::new(); // manifest.json let manifest_raw = reader.manifest().await?; @@ -88,21 +56,21 @@ impl S9pk> { let license: Arc<[u8]> = reader.license().await?.to_vec().await?.into(); archive.insert_path( "LICENSE.md", - Entry::file(CompatSource::Buffered(license.into())), + Entry::file(PackSource::Buffered(license.into())), )?; // instructions.md let instructions: Arc<[u8]> = reader.instructions().await?.to_vec().await?.into(); archive.insert_path( "instructions.md", - Entry::file(CompatSource::Buffered(instructions.into())), + Entry::file(PackSource::Buffered(instructions.into())), )?; // icon.md let icon: Arc<[u8]> = reader.icon().await?.to_vec().await?.into(); archive.insert_path( format!("icon.{}", manifest.assets.icon_type()), - Entry::file(CompatSource::Buffered(icon.into())), + Entry::file(PackSource::Buffered(icon.into())), )?; // images @@ -122,7 +90,9 @@ impl S9pk> { .invoke(ErrorKind::Docker) .await?; for (image, system) in &images { - new_manifest.images.insert(image.clone()); + let mut image_config = new_manifest.images.remove(image).unwrap_or_default(); + image_config.arch.insert(arch.as_str().into()); + new_manifest.images.insert(image.clone(), image_config); let sqfs_path = images_dir.join(image).with_extension("squashfs"); let image_name = if *system { format!("start9/{}:latest", image) @@ -190,21 +160,21 @@ impl S9pk> { .join(&arch) .join(&image) .with_extension("squashfs"), - Entry::file(CompatSource::File(sqfs_path)), + Entry::file(PackSource::File(sqfs_path)), )?; archive.insert_path( Path::new("images") .join(&arch) .join(&image) .with_extension("env"), - Entry::file(CompatSource::Buffered(Vec::from(env).into())), + Entry::file(PackSource::Buffered(Vec::from(env).into())), )?; archive.insert_path( Path::new("images") .join(&arch) .join(&image) .with_extension("json"), - Entry::file(CompatSource::Buffered( + Entry::file(PackSource::Buffered( serde_json::to_vec(&serde_json::json!({ "workdir": workdir })) @@ -240,7 +210,7 @@ impl S9pk> { .await?; archive.insert_path( Path::new("assets").join(&asset_id), - Entry::file(CompatSource::File(sqfs_path)), + Entry::file(PackSource::File(sqfs_path)), )?; } @@ -267,12 +237,12 @@ impl S9pk> { .await?; archive.insert_path( Path::new("javascript.squashfs"), - Entry::file(CompatSource::File(sqfs_path)), + Entry::file(PackSource::File(sqfs_path)), )?; archive.insert_path( "manifest.json", - Entry::file(CompatSource::Buffered( + Entry::file(PackSource::Buffered( serde_json::to_vec::(&new_manifest) .with_kind(ErrorKind::Serialization)? .into(), @@ -289,7 +259,6 @@ impl S9pk> { Ok(S9pk::deserialize( &MultiCursorFile::from(File::open(destination.as_ref()).await?), None, - false, ) .await?) } @@ -310,7 +279,7 @@ impl From for Manifest { marketing_site: value.marketing_site.unwrap_or_else(|| default_url.clone()), donation_url: value.donation_url, description: value.description, - images: BTreeSet::new(), + images: BTreeMap::new(), assets: value .volumes .iter() diff --git a/core/startos/src/s9pk/v2/manifest.rs b/core/startos/src/s9pk/v2/manifest.rs index b5ada621b..77e48c126 100644 --- a/core/startos/src/s9pk/v2/manifest.rs +++ b/core/startos/src/s9pk/v2/manifest.rs @@ -1,10 +1,11 @@ use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; use color_eyre::eyre::eyre; use helpers::const_true; use imbl_value::InternedString; pub use models::PackageId; -use models::{ImageId, VolumeId}; +use models::{mime, ImageId, VolumeId}; use serde::{Deserialize, Serialize}; use ts_rs::TS; use url::Url; @@ -12,6 +13,9 @@ use url::Url; use crate::dependencies::Dependencies; use crate::prelude::*; use crate::s9pk::git_hash::GitHash; +use crate::s9pk::merkle_archive::directory_contents::DirectoryContents; +use crate::s9pk::merkle_archive::expected::{Expected, Filter}; +use crate::s9pk::v2::pack::ImageConfig; use crate::util::serde::Regex; use crate::util::VersionString; use crate::version::{Current, VersionT}; @@ -42,7 +46,7 @@ pub struct Manifest { #[ts(type = "string | null")] pub donation_url: Option, pub description: Description, - pub images: BTreeSet, + pub images: BTreeMap, pub assets: BTreeSet, // TODO: AssetsId pub volumes: BTreeSet, #[serde(default)] @@ -59,6 +63,83 @@ pub struct Manifest { #[serde(default = "const_true")] pub has_config: bool, } +impl Manifest { + pub fn validate_for<'a, T: Clone>( + &self, + arch: Option<&str>, + archive: &'a DirectoryContents, + ) -> Result { + let mut expected = Expected::new(archive); + expected.check_file("manifest.json")?; + expected.check_stem("icon", |ext| { + ext.and_then(|e| e.to_str()) + .and_then(mime) + .map_or(false, |mime| mime.starts_with("image/")) + })?; + expected.check_file("LICENSE.md")?; + expected.check_file("instructions.md")?; + expected.check_file("javascript.squashfs")?; + for assets in &self.assets { + expected.check_file(Path::new("assets").join(assets).with_extension("squashfs"))?; + } + for (image_id, config) in &self.images { + let mut check_arch = |arch: &str| { + let mut arch = arch; + if let Err(e) = expected.check_file( + Path::new("images") + .join(arch) + .join(image_id) + .with_extension("squashfs"), + ) { + if let Some(emulate_as) = &config.emulate_missing_as { + expected.check_file( + Path::new("images") + .join(arch) + .join(image_id) + .with_extension("squashfs"), + )?; + arch = &**emulate_as; + } else { + return Err(e); + } + } + expected.check_file( + Path::new("images") + .join(arch) + .join(image_id) + .with_extension("json"), + )?; + expected.check_file( + Path::new("images") + .join(arch) + .join(image_id) + .with_extension("env"), + )?; + Ok(()) + }; + if let Some(arch) = arch { + check_arch(arch)?; + } else if let Some(arches) = &self.hardware_requirements.arch { + for arch in arches { + check_arch(arch)?; + } + } else if let Some(arch) = config.emulate_missing_as.as_deref() { + if !config.arch.contains(arch) { + return Err(Error::new( + eyre!("`emulateMissingAs` must match an included `arch`"), + ErrorKind::ParseS9pk, + )); + } + for arch in &config.arch { + check_arch(&arch)?; + } + } else { + return Err(Error::new(eyre!("`emulateMissingAs` required for all images if no `arch` specified in `hardwareRequirements`"), ErrorKind::ParseS9pk)); + } + } + Ok(expected.into_filter()) + } +} #[derive(Clone, Debug, Default, Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] diff --git a/core/startos/src/s9pk/v2/mod.rs b/core/startos/src/s9pk/v2/mod.rs index e206c8553..a1183efa8 100644 --- a/core/startos/src/s9pk/v2/mod.rs +++ b/core/startos/src/s9pk/v2/mod.rs @@ -14,7 +14,8 @@ use crate::s9pk::merkle_archive::sink::Sink; use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section}; use crate::s9pk::merkle_archive::{Entry, MerkleArchive}; -use crate::ARCH; +use crate::s9pk::v2::pack::{ImageSource, PackSource}; +use crate::util::io::TmpDir; const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x02]; @@ -22,6 +23,7 @@ pub const SIG_CONTEXT: &str = "s9pk"; pub mod compat; pub mod manifest; +pub mod pack; /** / @@ -34,10 +36,14 @@ pub mod manifest; │ └── .squashfs (xN) └── images └── + ├── .json (xN) ├── .env (xN) └── .squashfs (xN) */ +// this sorts the s9pk to optimize such that the parts that are used first appear earlier in the s9pk +// this is useful for manipulating an s9pk while partially downloaded on a source that does not support +// random access fn priority(s: &str) -> Option { match s { "manifest.json" => Some(0), @@ -51,26 +57,6 @@ fn priority(s: &str) -> Option { } } -fn filter(p: &Path) -> bool { - match p.iter().count() { - 1 if p.file_name() == Some(OsStr::new("manifest.json")) => true, - 1 if p.file_stem() == Some(OsStr::new("icon")) => true, - 1 if p.file_name() == Some(OsStr::new("LICENSE.md")) => true, - 1 if p.file_name() == Some(OsStr::new("instructions.md")) => true, - 1 if p.file_name() == Some(OsStr::new("javascript.squashfs")) => true, - 1 if p.file_name() == Some(OsStr::new("assets")) => true, - 1 if p.file_name() == Some(OsStr::new("images")) => true, - 2 if p.parent() == Some(Path::new("assets")) => { - p.extension().map_or(false, |ext| ext == "squashfs") - } - 2 if p.parent() == Some(Path::new("images")) => p.file_name() == Some(OsStr::new(&*ARCH)), - 3 if p.parent() == Some(&*Path::new("images").join(&*ARCH)) => p - .extension() - .map_or(false, |ext| ext == "squashfs" || ext == "env"), - _ => false, - } -} - #[derive(Clone)] pub struct S9pk> { pub manifest: Manifest, @@ -108,6 +94,11 @@ impl S9pk { }) } + pub fn validate_and_filter(&mut self, arch: Option<&str>) -> Result<(), Error> { + let filter = self.manifest.validate_for(arch, self.archive.contents())?; + filter.keep_checked(self.archive.contents_mut()) + } + pub async fn icon(&self) -> Result<(InternedString, FileContents), Error> { let mut best_icon = None; for (path, icon) in self @@ -174,12 +165,37 @@ impl S9pk { } } +impl + FileSource + Clone> S9pk { + pub async fn load_images(&mut self, tmpdir: &TmpDir) -> Result<(), Error> { + let id = &self.manifest.id; + let version = &self.manifest.version; + for (image_id, image_config) in &mut self.manifest.images { + self.manifest_dirty = true; + for arch in &image_config.arch { + image_config + .source + .load( + tmpdir, + id, + version, + image_id, + arch, + self.archive.contents_mut(), + ) + .await?; + } + image_config.source = ImageSource::Packed; + } + + Ok(()) + } +} + impl S9pk> { #[instrument(skip_all)] pub async fn deserialize( source: &S, commitment: Option<&MerkleArchiveCommitment>, - apply_filter: bool, ) -> Result { use tokio::io::AsyncReadExt; @@ -201,10 +217,6 @@ impl S9pk> { let mut archive = MerkleArchive::deserialize(source, SIG_CONTEXT, &mut header, commitment).await?; - if apply_filter { - archive.filter(filter)?; - } - archive.sort_by(|a, b| match (priority(a), priority(b)) { (Some(a), Some(b)) => a.cmp(&b), (Some(_), None) => std::cmp::Ordering::Less, @@ -216,15 +228,11 @@ impl S9pk> { } } impl S9pk { - pub async fn from_file(file: File, apply_filter: bool) -> Result { - Self::deserialize(&MultiCursorFile::from(file), None, apply_filter).await + pub async fn from_file(file: File) -> Result { + Self::deserialize(&MultiCursorFile::from(file), None).await } - pub async fn open( - path: impl AsRef, - id: Option<&PackageId>, - apply_filter: bool, - ) -> Result { - let res = Self::from_file(tokio::fs::File::open(path).await?, apply_filter).await?; + pub async fn open(path: impl AsRef, id: Option<&PackageId>) -> Result { + let res = Self::from_file(tokio::fs::File::open(path).await?).await?; if let Some(id) = id { ensure_code!( &res.as_manifest().id == id, diff --git a/core/startos/src/s9pk/v2/pack.rs b/core/startos/src/s9pk/v2/pack.rs new file mode 100644 index 000000000..18fe0472b --- /dev/null +++ b/core/startos/src/s9pk/v2/pack.rs @@ -0,0 +1,536 @@ +use std::collections::BTreeSet; +use std::ffi::OsStr; +use std::io::Cursor; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use clap::Parser; +use futures::future::{ready, BoxFuture}; +use futures::{FutureExt, TryStreamExt}; +use imbl_value::InternedString; +use models::{ImageId, PackageId, VersionString}; +use serde::{Deserialize, Serialize}; +use tokio::fs::File; +use tokio::io::AsyncRead; +use tokio::process::Command; +use tokio::sync::OnceCell; +use tokio_stream::wrappers::ReadDirStream; +use ts_rs::TS; + +use crate::context::CliContext; +use crate::prelude::*; +use crate::rpc_continuations::Guid; +use crate::s9pk::merkle_archive::directory_contents::DirectoryContents; +use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; +use crate::s9pk::merkle_archive::source::{ + into_dyn_read, ArchiveSource, DynFileSource, FileSource, +}; +use crate::s9pk::merkle_archive::{Entry, MerkleArchive}; +use crate::s9pk::v2::SIG_CONTEXT; +use crate::s9pk::S9pk; +use crate::util::io::TmpDir; +use crate::util::Invoke; + +#[cfg(not(feature = "docker"))] +pub const CONTAINER_TOOL: &str = "podman"; + +#[cfg(feature = "docker")] +pub const CONTAINER_TOOL: &str = "docker"; + +pub struct SqfsDir { + path: PathBuf, + tmpdir: Arc, + sqfs: OnceCell, +} +impl SqfsDir { + pub fn new(path: PathBuf, tmpdir: Arc) -> Self { + Self { + path, + tmpdir, + sqfs: OnceCell::new(), + } + } + async fn file(&self) -> Result<&MultiCursorFile, Error> { + self.sqfs + .get_or_try_init(|| async move { + let guid = Guid::new(); + let path = self.tmpdir.join(guid.as_ref()).with_extension("squashfs"); + let mut cmd = Command::new("mksquashfs"); + if self.path.extension().and_then(|s| s.to_str()) == Some("tar") { + cmd.arg("-tar"); + } + cmd.arg(&self.path) + .arg(&path) + .invoke(ErrorKind::Filesystem) + .await?; + Ok(MultiCursorFile::from( + File::open(&path) + .await + .with_ctx(|_| (ErrorKind::Filesystem, path.display()))?, + )) + }) + .await + } +} + +#[derive(Clone)] +pub enum PackSource { + Buffered(Arc<[u8]>), + File(PathBuf), + Squashfs(Arc), +} +impl FileSource for PackSource { + type Reader = Box; + async fn size(&self) -> Result { + match self { + Self::Buffered(a) => Ok(a.len() as u64), + Self::File(f) => Ok(tokio::fs::metadata(f) + .await + .with_ctx(|_| (ErrorKind::Filesystem, f.display()))? + .len()), + Self::Squashfs(dir) => dir + .file() + .await + .with_ctx(|_| (ErrorKind::Filesystem, dir.path.display()))? + .size() + .await + .or_not_found("file metadata"), + } + } + async fn reader(&self) -> Result { + match self { + Self::Buffered(a) => Ok(into_dyn_read(Cursor::new(a.clone()))), + Self::File(f) => Ok(into_dyn_read( + File::open(f) + .await + .with_ctx(|_| (ErrorKind::Filesystem, f.display()))?, + )), + Self::Squashfs(dir) => dir.file().await?.fetch_all().await.map(into_dyn_read), + } + } +} +impl From for DynFileSource { + fn from(value: PackSource) -> Self { + DynFileSource::new(value) + } +} + +#[derive(Deserialize, Serialize, Parser)] +pub struct PackParams { + pub path: Option, + #[arg(short = 'o', long = "output")] + pub output: Option, + #[arg(long = "javascript")] + pub javascript: Option, + #[arg(long = "icon")] + pub icon: Option, + #[arg(long = "license")] + pub license: Option, + #[arg(long = "instructions")] + pub instructions: Option, + #[arg(long = "assets")] + pub assets: Option, +} +impl PackParams { + fn path(&self) -> &Path { + self.path.as_deref().unwrap_or(Path::new(".")) + } + fn output(&self, id: &PackageId) -> PathBuf { + self.output + .as_ref() + .cloned() + .unwrap_or_else(|| self.path().join(id).with_extension("s9pk")) + } + fn javascript(&self) -> PathBuf { + self.javascript + .as_ref() + .cloned() + .unwrap_or_else(|| self.path().join("javascript")) + } + async fn icon(&self) -> Result { + if let Some(icon) = &self.icon { + Ok(icon.clone()) + } else { + ReadDirStream::new(tokio::fs::read_dir(self.path()).await?).try_filter(|x| ready(x.path().file_stem() == Some(OsStr::new("icon")))).map_err(Error::from).try_fold(Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)), |acc, x| async move { match acc { + Ok(_) => Err(Error::new(eyre!("multiple icons found in working directory, please specify which to use with `--icon`"), ErrorKind::InvalidRequest)), + Err(e) => Ok({ + let path = x.path(); + if path.file_stem().and_then(|s| s.to_str()) == Some("icon") { + Ok(path) + } else { + Err(e) + } + }) + }}).await? + } + } + fn license(&self) -> PathBuf { + self.license + .as_ref() + .cloned() + .unwrap_or_else(|| self.path().join("LICENSE.md")) + } + fn instructions(&self) -> PathBuf { + self.instructions + .as_ref() + .cloned() + .unwrap_or_else(|| self.path().join("instructions.md")) + } + fn assets(&self) -> PathBuf { + self.assets + .as_ref() + .cloned() + .unwrap_or_else(|| self.path().join("assets")) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct ImageConfig { + pub source: ImageSource, + #[ts(type = "string[]")] + pub arch: BTreeSet, + #[ts(type = "string | null")] + pub emulate_missing_as: Option, +} +impl Default for ImageConfig { + fn default() -> Self { + Self { + source: ImageSource::Packed, + arch: BTreeSet::new(), + emulate_missing_as: None, + } + } +} + +#[derive(Parser)] +struct CliImageConfig { + #[arg(long, conflicts_with("docker-tag"))] + docker_build: bool, + #[arg(long, requires("docker-build"))] + dockerfile: Option, + #[arg(long, requires("docker-build"))] + workdir: Option, + #[arg(long, conflicts_with_all(["dockerfile", "workdir"]))] + docker_tag: Option, + #[arg(long)] + arch: Vec, + #[arg(long)] + emulate_missing_as: Option, +} +impl TryFrom for ImageConfig { + type Error = clap::Error; + fn try_from(value: CliImageConfig) -> Result { + let res = Self { + source: if value.docker_build { + ImageSource::DockerBuild { + dockerfile: value.dockerfile, + workdir: value.workdir, + } + } else if let Some(tag) = value.docker_tag { + ImageSource::DockerTag(tag) + } else { + ImageSource::Packed + }, + arch: value.arch.into_iter().collect(), + emulate_missing_as: value.emulate_missing_as, + }; + res.emulate_missing_as + .as_ref() + .map(|a| { + if !res.arch.contains(a) { + Err(clap::Error::raw( + clap::error::ErrorKind::InvalidValue, + "`emulate-missing-as` must match one of the provided `arch`es", + )) + } else { + Ok(()) + } + }) + .transpose()?; + Ok(res) + } +} +impl clap::Args for ImageConfig { + fn augment_args(cmd: clap::Command) -> clap::Command { + CliImageConfig::augment_args(cmd) + } + fn augment_args_for_update(cmd: clap::Command) -> clap::Command { + CliImageConfig::augment_args_for_update(cmd) + } +} +impl clap::FromArgMatches for ImageConfig { + fn from_arg_matches(matches: &clap::ArgMatches) -> Result { + Self::try_from(CliImageConfig::from_arg_matches(matches)?) + } + fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> { + *self = Self::try_from(CliImageConfig::from_arg_matches(matches)?)?; + Ok(()) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub enum ImageSource { + Packed, + #[serde(rename_all = "camelCase")] + DockerBuild { + workdir: Option, + dockerfile: Option, + }, + DockerTag(String), +} +impl ImageSource { + #[instrument(skip_all)] + pub fn load<'a, S: From + FileSource + Clone>( + &'a self, + tmpdir: &'a TmpDir, + id: &'a PackageId, + version: &'a VersionString, + image_id: &'a ImageId, + arch: &'a str, + into: &'a mut DirectoryContents, + ) -> BoxFuture<'a, Result<(), Error>> { + #[derive(Deserialize)] + #[serde(rename_all = "PascalCase")] + struct DockerImageConfig { + env: Vec, + #[serde(default)] + working_dir: PathBuf, + #[serde(default)] + user: String, + } + async move { + match self { + ImageSource::Packed => Ok(()), + ImageSource::DockerBuild { + workdir, + dockerfile, + } => { + let workdir = workdir.as_deref().unwrap_or(Path::new(".")); + let dockerfile = dockerfile + .clone() + .unwrap_or_else(|| workdir.join("Dockerfile")); + let docker_platform = if arch == "x86_64" { + "--platform=linux/amd64".to_owned() + } else if arch == "aarch64" { + "--platform=linux/arm64".to_owned() + } else { + format!("--platform=linux/{arch}") + }; + // docker buildx build ${path} -o type=image,name=start9/${id} + let tag = format!("start9/{id}/{image_id}:{version}"); + Command::new(CONTAINER_TOOL) + .arg("build") + .arg(workdir) + .arg("-f") + .arg(dockerfile) + .arg("-t") + .arg(&tag) + .arg(&docker_platform) + .arg("-o") + .arg("type=image") + .capture(false) + .invoke(ErrorKind::Docker) + .await?; + ImageSource::DockerTag(tag.clone()) + .load(tmpdir, id, version, image_id, arch, into) + .await?; + Command::new(CONTAINER_TOOL) + .arg("rmi") + .arg("-f") + .arg(&tag) + .invoke(ErrorKind::Docker) + .await?; + Ok(()) + } + ImageSource::DockerTag(tag) => { + let docker_platform = if arch == "x86_64" { + "--platform=linux/amd64".to_owned() + } else if arch == "aarch64" { + "--platform=linux/arm64".to_owned() + } else { + format!("--platform=linux/{arch}") + }; + let mut inspect_cmd = Command::new(CONTAINER_TOOL); + inspect_cmd + .arg("image") + .arg("inspect") + .arg("--format") + .arg("{{json .Config}}") + .arg(&tag); + let inspect_res = match inspect_cmd.invoke(ErrorKind::Docker).await { + Ok(a) => a, + Err(e) + if { + let msg = e.source.to_string(); + #[cfg(feature = "docker")] + let matches = msg.contains("No such image:"); + #[cfg(not(feature = "docker"))] + let matches = msg.contains(": image not known"); + matches + } => + { + Command::new(CONTAINER_TOOL) + .arg("pull") + .arg(&docker_platform) + .arg(tag) + .capture(false) + .invoke(ErrorKind::Docker) + .await?; + inspect_cmd.invoke(ErrorKind::Docker).await? + } + Err(e) => return Err(e), + }; + let config = serde_json::from_slice::(&inspect_res) + .with_kind(ErrorKind::Deserialization)?; + let base_path = Path::new("images").join(arch).join(image_id); + into.insert_path( + base_path.with_extension("json"), + Entry::file( + PackSource::Buffered( + serde_json::to_vec(&ImageMetadata { + workdir: if config.working_dir == Path::new("") { + "/".into() + } else { + config.working_dir + }, + user: if config.user.is_empty() { + "root".into() + } else { + config.user.into() + }, + }) + .with_kind(ErrorKind::Serialization)? + .into(), + ) + .into(), + ), + )?; + into.insert_path( + base_path.with_extension("env"), + Entry::file( + PackSource::Buffered(config.env.join("\n").into_bytes().into()).into(), + ), + )?; + let dest = tmpdir.join(Guid::new().as_ref()).with_extension("squashfs"); + let container = String::from_utf8( + Command::new(CONTAINER_TOOL) + .arg("create") + .arg(&docker_platform) + .arg(&tag) + .invoke(ErrorKind::Docker) + .await?, + )?; + Command::new(CONTAINER_TOOL) + .arg("export") + .arg(container.trim()) + .pipe(Command::new("mksquashfs").arg("-").arg(&dest).arg("-tar")) + .capture(false) + .invoke(ErrorKind::Docker) + .await?; + Command::new(CONTAINER_TOOL) + .arg("rm") + .arg(container.trim()) + .invoke(ErrorKind::Docker) + .await?; + into.insert_path( + base_path.with_extension("squashfs"), + Entry::file(PackSource::File(dest).into()), + )?; + + Ok(()) + } + } + } + .boxed() + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct ImageMetadata { + pub workdir: PathBuf, + #[ts(type = "string")] + pub user: InternedString, +} + +#[instrument(skip_all)] +pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> { + let tmpdir = Arc::new(TmpDir::new().await?); + let mut files = DirectoryContents::::new(); + let js_dir = params.javascript(); + let manifest: Arc<[u8]> = Command::new("node") + .arg("-e") + .arg(format!( + "console.log(JSON.stringify(require('{}/index.js').manifest))", + js_dir.display() + )) + .invoke(ErrorKind::Javascript) + .await? + .into(); + files.insert( + "manifest.json".into(), + Entry::file(PackSource::Buffered(manifest.clone())), + ); + let icon = params.icon().await?; + let icon_ext = icon + .extension() + .or_not_found("icon file extension")? + .to_string_lossy(); + files.insert( + InternedString::from_display(&lazy_format!("icon.{}", icon_ext)), + Entry::file(PackSource::File(icon)), + ); + files.insert( + "LICENSE.md".into(), + Entry::file(PackSource::File(params.license())), + ); + files.insert( + "instructions.md".into(), + Entry::file(PackSource::File(params.instructions())), + ); + files.insert( + "javascript.squashfs".into(), + Entry::file(PackSource::Squashfs(Arc::new(SqfsDir::new( + js_dir, + tmpdir.clone(), + )))), + ); + + let mut s9pk = S9pk::new( + MerkleArchive::new(files, ctx.developer_key()?.clone(), SIG_CONTEXT), + None, + ) + .await?; + + let assets_dir = params.assets(); + for assets in s9pk.as_manifest().assets.clone() { + s9pk.as_archive_mut().contents_mut().insert_path( + Path::new("assets").join(&assets).with_extension("squashfs"), + Entry::file(PackSource::Squashfs(Arc::new(SqfsDir::new( + assets_dir.join(&assets), + tmpdir.clone(), + )))), + )?; + } + + s9pk.load_images(&*tmpdir).await?; + + s9pk.validate_and_filter(None)?; + + s9pk.serialize( + &mut File::create(params.output(&s9pk.as_manifest().id)).await?, + false, + ) + .await?; + + drop(s9pk); + + tmpdir.gc().await?; + + Ok(()) +} diff --git a/core/startos/src/service/action.rs b/core/startos/src/service/action.rs index 5f97685ae..6c5ac4eab 100644 --- a/core/startos/src/service/action.rs +++ b/core/startos/src/service/action.rs @@ -4,6 +4,7 @@ use models::{ActionId, ProcedureName}; use crate::action::ActionResult; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::service::config::GetConfig; use crate::service::dependencies::DependencyConfig; use crate::service::{Service, ServiceActor}; @@ -23,13 +24,18 @@ impl Handler for ServiceActor { } async fn handle( &mut self, - Action { id, input }: Action, + id: Guid, + Action { + id: action_id, + input, + }: Action, _: &BackgroundJobQueue, ) -> Self::Response { let container = &self.0.persistent_container; container .execute::( - ProcedureName::RunAction(id), + id, + ProcedureName::RunAction(action_id), input, Some(Duration::from_secs(30)), ) @@ -39,7 +45,20 @@ impl Handler for ServiceActor { } impl Service { - pub async fn action(&self, id: ActionId, input: Value) -> Result { - self.actor.send(Action { id, input }).await? + pub async fn action( + &self, + id: Guid, + action_id: ActionId, + input: Value, + ) -> Result { + self.actor + .send( + id, + Action { + id: action_id, + input, + }, + ) + .await? } } diff --git a/core/startos/src/service/config.rs b/core/startos/src/service/config.rs index 0f166eedb..faa70fc41 100644 --- a/core/startos/src/service/config.rs +++ b/core/startos/src/service/config.rs @@ -5,6 +5,7 @@ use models::ProcedureName; use crate::config::action::ConfigRes; use crate::config::ConfigureContext; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::service::dependencies::DependencyConfig; use crate::service::{Service, ServiceActor}; use crate::util::actor::background::BackgroundJobQueue; @@ -19,6 +20,7 @@ impl Handler for ServiceActor { } async fn handle( &mut self, + id: Guid, Configure(ConfigureContext { timeout, config }): Configure, _: &BackgroundJobQueue, ) -> Self::Response { @@ -26,7 +28,7 @@ impl Handler for ServiceActor { let package_id = &self.0.id; container - .execute::(ProcedureName::SetConfig, to_value(&config)?, timeout) + .execute::(id, ProcedureName::SetConfig, to_value(&config)?, timeout) .await .with_kind(ErrorKind::ConfigRulesViolation)?; self.0 @@ -52,10 +54,11 @@ impl Handler for ServiceActor { fn conflicts_with(_: &GetConfig) -> ConflictBuilder { ConflictBuilder::nothing().except::() } - async fn handle(&mut self, _: GetConfig, _: &BackgroundJobQueue) -> Self::Response { + async fn handle(&mut self, id: Guid, _: GetConfig, _: &BackgroundJobQueue) -> Self::Response { let container = &self.0.persistent_container; container .execute::( + id, ProcedureName::GetConfig, Value::Null, Some(Duration::from_secs(30)), // TODO timeout @@ -66,10 +69,10 @@ impl Handler for ServiceActor { } impl Service { - pub async fn configure(&self, ctx: ConfigureContext) -> Result<(), Error> { - self.actor.send(Configure(ctx)).await? + pub async fn configure(&self, id: Guid, ctx: ConfigureContext) -> Result<(), Error> { + self.actor.send(id, Configure(ctx)).await? } - pub async fn get_config(&self) -> Result { - self.actor.send(GetConfig).await? + pub async fn get_config(&self, id: Guid) -> Result { + self.actor.send(id, GetConfig).await? } } diff --git a/core/startos/src/service/control.rs b/core/startos/src/service/control.rs index 0443b80b1..7c4bdf815 100644 --- a/core/startos/src/service/control.rs +++ b/core/startos/src/service/control.rs @@ -1,4 +1,5 @@ use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::service::config::GetConfig; use crate::service::dependencies::DependencyConfig; use crate::service::start_stop::StartStop; @@ -15,7 +16,7 @@ impl Handler for ServiceActor { .except::() .except::() } - async fn handle(&mut self, _: Start, _: &BackgroundJobQueue) -> Self::Response { + async fn handle(&mut self, _: Guid, _: Start, _: &BackgroundJobQueue) -> Self::Response { self.0.persistent_container.state.send_modify(|x| { x.desired_state = StartStop::Start; }); @@ -23,8 +24,8 @@ impl Handler for ServiceActor { } } impl Service { - pub async fn start(&self) -> Result<(), Error> { - self.actor.send(Start).await + pub async fn start(&self, id: Guid) -> Result<(), Error> { + self.actor.send(id, Start).await } } @@ -36,7 +37,7 @@ impl Handler for ServiceActor { .except::() .except::() } - async fn handle(&mut self, _: Stop, _: &BackgroundJobQueue) -> Self::Response { + async fn handle(&mut self, _: Guid, _: Stop, _: &BackgroundJobQueue) -> Self::Response { let mut transition_state = None; self.0.persistent_container.state.send_modify(|x| { x.desired_state = StartStop::Stop; @@ -51,7 +52,7 @@ impl Handler for ServiceActor { } } impl Service { - pub async fn stop(&self) -> Result<(), Error> { - self.actor.send(Stop).await + pub async fn stop(&self, id: Guid) -> Result<(), Error> { + self.actor.send(id, Stop).await } } diff --git a/core/startos/src/service/dependencies.rs b/core/startos/src/service/dependencies.rs index e71b62917..60fd76ad6 100644 --- a/core/startos/src/service/dependencies.rs +++ b/core/startos/src/service/dependencies.rs @@ -4,35 +4,28 @@ use imbl_value::json; use models::{PackageId, ProcedureName}; use crate::prelude::*; -use crate::service::{Service, ServiceActor}; +use crate::rpc_continuations::Guid; +use crate::service::{Service, ServiceActor, ServiceActorSeed}; use crate::util::actor::background::BackgroundJobQueue; use crate::util::actor::{ConflictBuilder, Handler}; use crate::Config; -pub(super) struct DependencyConfig { - dependency_id: PackageId, - remote_config: Option, -} -impl Handler for ServiceActor { - type Response = Result, Error>; - fn conflicts_with(_: &DependencyConfig) -> ConflictBuilder { - ConflictBuilder::nothing() - } - async fn handle( - &mut self, - DependencyConfig { - dependency_id, - remote_config, - }: DependencyConfig, - _: &BackgroundJobQueue, - ) -> Self::Response { - let container = &self.0.persistent_container; +impl ServiceActorSeed { + async fn dependency_config( + &self, + id: Guid, + dependency_id: PackageId, + remote_config: Option, + ) -> Result, Error> { + let container = &self.persistent_container; container .sanboxed::>( + id.clone(), ProcedureName::UpdateDependency(dependency_id.clone()), json!({ "queryResults": container .execute::( + id, ProcedureName::QueryDependency(dependency_id), Value::Null, Some(Duration::from_secs(30)), @@ -49,17 +42,45 @@ impl Handler for ServiceActor { } } +pub(super) struct DependencyConfig { + dependency_id: PackageId, + remote_config: Option, +} +impl Handler for ServiceActor { + type Response = Result, Error>; + fn conflicts_with(_: &DependencyConfig) -> ConflictBuilder { + ConflictBuilder::nothing() + } + async fn handle( + &mut self, + id: Guid, + DependencyConfig { + dependency_id, + remote_config, + }: DependencyConfig, + _: &BackgroundJobQueue, + ) -> Self::Response { + self.0 + .dependency_config(id, dependency_id, remote_config) + .await + } +} + impl Service { pub async fn dependency_config( &self, + id: Guid, dependency_id: PackageId, remote_config: Option, ) -> Result, Error> { self.actor - .send(DependencyConfig { - dependency_id, - remote_config, - }) + .send( + id, + DependencyConfig { + dependency_id, + remote_config, + }, + ) .await? } } diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index da641468b..6588de836 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -1,4 +1,5 @@ -use std::sync::Arc; +use std::ops::Deref; +use std::sync::{Arc, Weak}; use std::time::Duration; use chrono::{DateTime, Utc}; @@ -68,13 +69,87 @@ pub enum LoadDisposition { Undo, } +pub struct ServiceRef(Arc); +impl ServiceRef { + pub fn weak(&self) -> Weak { + Arc::downgrade(&self.0) + } + pub async fn uninstall( + self, + target_version: Option, + ) -> Result<(), Error> { + self.seed + .persistent_container + .execute( + Guid::new(), + ProcedureName::Uninit, + to_value(&target_version)?, + None, + ) // TODO timeout + .await?; + let id = self.seed.persistent_container.s9pk.as_manifest().id.clone(); + let ctx = self.seed.ctx.clone(); + self.shutdown().await?; + if target_version.is_none() { + ctx.db + .mutate(|d| d.as_public_mut().as_package_data_mut().remove(&id)) + .await?; + } + Ok(()) + } + pub async fn shutdown(self) -> Result<(), Error> { + if let Some((hdl, shutdown)) = self.seed.persistent_container.rpc_server.send_replace(None) + { + self.seed + .persistent_container + .rpc_client + .request(rpc::Exit, Empty {}) + .await?; + shutdown.shutdown(); + hdl.await.with_kind(ErrorKind::Cancelled)?; + } + let service = Arc::try_unwrap(self.0).map_err(|_| { + Error::new( + eyre!("ServiceActor held somewhere after actor shutdown"), + ErrorKind::Unknown, + ) + })?; + service + .actor + .shutdown(crate::util::actor::PendingMessageStrategy::FinishAll { timeout: None }) // TODO timeout + .await; + Arc::try_unwrap(service.seed) + .map_err(|_| { + Error::new( + eyre!("ServiceActorSeed held somewhere after actor shutdown"), + ErrorKind::Unknown, + ) + })? + .persistent_container + .exit() + .await?; + Ok(()) + } +} +impl Deref for ServiceRef { + type Target = Service; + fn deref(&self) -> &Self::Target { + &*self.0 + } +} +impl From for ServiceRef { + fn from(value: Service) -> Self { + Self(Arc::new(value)) + } +} + pub struct Service { actor: ConcurrentActor, seed: Arc, } impl Service { #[instrument(skip_all)] - async fn new(ctx: RpcContext, s9pk: S9pk, start: StartStop) -> Result { + async fn new(ctx: RpcContext, s9pk: S9pk, start: StartStop) -> Result { let id = s9pk.as_manifest().id.clone(); let persistent_container = PersistentContainer::new( &ctx, s9pk, @@ -89,13 +164,17 @@ impl Service { ctx, synchronized: Arc::new(Notify::new()), }); - seed.persistent_container - .init(Arc::downgrade(&seed)) - .await?; - Ok(Self { + let service: ServiceRef = Self { actor: ConcurrentActor::new(ServiceActor(seed.clone())), seed, - }) + } + .into(); + service + .seed + .persistent_container + .init(service.weak()) + .await?; + Ok(service) } #[instrument(skip_all)] @@ -103,7 +182,7 @@ impl Service { ctx: &RpcContext, id: &PackageId, disposition: LoadDisposition, - ) -> Result, Error> { + ) -> Result, Error> { let handle_installed = { let ctx = ctx.clone(); move |s9pk: S9pk, i: Model| async move { @@ -137,7 +216,7 @@ impl Service { match entry.as_state_info().as_match() { PackageStateMatchModelRef::Installing(_) => { if disposition == LoadDisposition::Retry { - if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id), true).await.map_err(|e| { + if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id)).await.map_err(|e| { tracing::error!("Error opening s9pk for install: {e}"); tracing::debug!("{e:?}") }) { @@ -170,7 +249,7 @@ impl Service { && progress == &Progress::Complete(true) }) { - if let Ok(s9pk) = S9pk::open(&s9pk_path, Some(id), true).await.map_err(|e| { + if let Ok(s9pk) = S9pk::open(&s9pk_path, Some(id)).await.map_err(|e| { tracing::error!("Error opening s9pk for update: {e}"); tracing::debug!("{e:?}") }) { @@ -189,7 +268,7 @@ impl Service { } } } - let s9pk = S9pk::open(s9pk_path, Some(id), true).await?; + let s9pk = S9pk::open(s9pk_path, Some(id)).await?; ctx.db .mutate({ |db| { @@ -214,7 +293,7 @@ impl Service { handle_installed(s9pk, entry).await } PackageStateMatchModelRef::Removing(_) | PackageStateMatchModelRef::Restoring(_) => { - if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id), true).await.map_err(|e| { + if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id)).await.map_err(|e| { tracing::error!("Error opening s9pk for removal: {e}"); tracing::debug!("{e:?}") }) { @@ -225,7 +304,7 @@ impl Service { tracing::debug!("{e:?}") }) { - match service.uninstall(None).await { + match ServiceRef::from(service).uninstall(None).await { Err(e) => { tracing::error!("Error uninstalling service: {e}"); tracing::debug!("{e:?}") @@ -242,7 +321,7 @@ impl Service { Ok(None) } PackageStateMatchModelRef::Installed(_) => { - handle_installed(S9pk::open(s9pk_path, Some(id), true).await?, entry).await + handle_installed(S9pk::open(s9pk_path, Some(id)).await?, entry).await } PackageStateMatchModelRef::Error(e) => Err(Error::new( eyre!("Failed to parse PackageDataEntry, found {e:?}"), @@ -257,7 +336,7 @@ impl Service { s9pk: S9pk, src_version: Option, progress: Option, - ) -> Result { + ) -> Result { let manifest = s9pk.as_manifest().clone(); let developer_key = s9pk.as_archive().signer(); let icon = s9pk.icon_data_url().await?; @@ -265,7 +344,12 @@ impl Service { service .seed .persistent_container - .execute(ProcedureName::Init, to_value(&src_version)?, None) // TODO timeout + .execute( + Guid::new(), + ProcedureName::Init, + to_value(&src_version)?, + None, + ) // TODO timeout .await .with_kind(ErrorKind::MigrationFailed)?; // TODO: handle cancellation if let Some(mut progress) = progress { @@ -301,61 +385,21 @@ impl Service { s9pk: S9pk, backup_source: impl GenericMountGuard, progress: Option, - ) -> Result { + ) -> Result { let service = Service::install(ctx.clone(), s9pk, None, progress).await?; service .actor - .send(transition::restore::Restore { - path: backup_source.path().to_path_buf(), - }) + .send( + Guid::new(), + transition::restore::Restore { + path: backup_source.path().to_path_buf(), + }, + ) .await??; Ok(service) } - pub async fn shutdown(self) -> Result<(), Error> { - self.actor - .shutdown(crate::util::actor::PendingMessageStrategy::FinishAll { timeout: None }) // TODO timeout - .await; - if let Some((hdl, shutdown)) = self.seed.persistent_container.rpc_server.send_replace(None) - { - self.seed - .persistent_container - .rpc_client - .request(rpc::Exit, Empty {}) - .await?; - shutdown.shutdown(); - hdl.await.with_kind(ErrorKind::Cancelled)?; - } - Arc::try_unwrap(self.seed) - .map_err(|_| { - Error::new( - eyre!("ServiceActorSeed held somewhere after actor shutdown"), - ErrorKind::Unknown, - ) - })? - .persistent_container - .exit() - .await?; - Ok(()) - } - - pub async fn uninstall(self, target_version: Option) -> Result<(), Error> { - self.seed - .persistent_container - .execute(ProcedureName::Uninit, to_value(&target_version)?, None) // TODO timeout - .await?; - let id = self.seed.persistent_container.s9pk.as_manifest().id.clone(); - let ctx = self.seed.ctx.clone(); - self.shutdown().await?; - if target_version.is_none() { - ctx.db - .mutate(|d| d.as_public_mut().as_package_data_mut().remove(&id)) - .await?; - } - Ok(()) - } - #[instrument(skip_all)] pub async fn backup(&self, guard: impl GenericMountGuard) -> Result<(), Error> { let id = &self.seed.id; @@ -368,9 +412,12 @@ impl Service { .await?; drop(file); self.actor - .send(transition::backup::Backup { - path: guard.path().to_path_buf(), - }) + .send( + Guid::new(), + transition::backup::Backup { + path: guard.path().to_path_buf(), + }, + ) .await??; Ok(()) } diff --git a/core/startos/src/service/persistent_container.rs b/core/startos/src/service/persistent_container.rs index 03203633a..d9d08e5d3 100644 --- a/core/startos/src/service/persistent_container.rs +++ b/core/startos/src/service/persistent_container.rs @@ -6,8 +6,7 @@ use std::time::Duration; use futures::future::ready; use futures::{Future, FutureExt}; use helpers::NonDetachingJoinHandle; -use imbl_value::InternedString; -use models::{ProcedureName, VolumeId}; +use models::{ImageId, ProcedureName, VolumeId}; use rpc_toolkit::{Empty, Server, ShutdownHandle}; use serde::de::DeserializeOwned; use tokio::fs::File; @@ -24,14 +23,15 @@ use crate::disk::mount::filesystem::idmapped::IdMapped; use crate::disk::mount::filesystem::loop_dev::LoopDev; use crate::disk::mount::filesystem::overlayfs::OverlayGuard; use crate::disk::mount::filesystem::{MountType, ReadOnly}; -use crate::disk::mount::guard::MountGuard; +use crate::disk::mount::guard::{GenericMountGuard, MountGuard}; use crate::lxc::{LxcConfig, LxcContainer, HOST_RPC_SERVER_SOCKET}; use crate::net::net_controller::NetService; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::s9pk::merkle_archive::source::FileSource; use crate::s9pk::S9pk; use crate::service::start_stop::StartStop; -use crate::service::{rpc, RunningStatus}; +use crate::service::{rpc, RunningStatus, Service}; use crate::util::rpc_client::UnixRpcClient; use crate::util::Invoke; use crate::volume::{asset_dir, data_dir}; @@ -89,7 +89,8 @@ pub struct PersistentContainer { js_mount: MountGuard, volumes: BTreeMap, assets: BTreeMap, - pub(super) overlays: Arc>>, + pub(super) images: BTreeMap>, + pub(super) overlays: Arc>>>>, pub(super) state: Arc>, pub(super) net_service: Mutex, destroyed: bool, @@ -178,14 +179,62 @@ impl PersistentContainer { .await?, ); } + + let mut images = BTreeMap::new(); let image_path = lxc_container.rootfs_dir().join("media/startos/images"); tokio::fs::create_dir_all(&image_path).await?; - for image in &s9pk.as_manifest().images { + for (image, config) in &s9pk.as_manifest().images { + let mut arch = ARCH; + let mut sqfs_path = Path::new("images") + .join(arch) + .join(image) + .with_extension("squashfs"); + if !s9pk + .as_archive() + .contents() + .get_path(&sqfs_path) + .and_then(|e| e.as_file()) + .is_some() + { + arch = if let Some(arch) = config.emulate_missing_as.as_deref() { + arch + } else { + continue; + }; + sqfs_path = Path::new("images") + .join(arch) + .join(image) + .with_extension("squashfs"); + } + let sqfs = s9pk + .as_archive() + .contents() + .get_path(&sqfs_path) + .and_then(|e| e.as_file()) + .or_not_found(sqfs_path.display())?; + let mountpoint = image_path.join(image); + tokio::fs::create_dir_all(&mountpoint).await?; + Command::new("chown") + .arg("100000:100000") + .arg(&mountpoint) + .invoke(ErrorKind::Filesystem) + .await?; + images.insert( + image.clone(), + Arc::new( + MountGuard::mount( + &IdMapped::new(LoopDev::from(&**sqfs), 0, 100000, 65536), + &mountpoint, + ReadOnly, + ) + .await?, + ), + ); let env_filename = Path::new(image.as_ref()).with_extension("env"); if let Some(env) = s9pk .as_archive() .contents() - .get_path(Path::new("images").join(*ARCH).join(&env_filename)) + .get_path(Path::new("images").join(arch).join(&env_filename)) .and_then(|e| e.as_file()) { env.copy(&mut File::create(image_path.join(&env_filename)).await?) @@ -195,7 +244,7 @@ impl PersistentContainer { if let Some(json) = s9pk .as_archive() .contents() - .get_path(Path::new("images").join(*ARCH).join(&json_filename)) + .get_path(Path::new("images").join(arch).join(&json_filename)) .and_then(|e| e.as_file()) { json.copy(&mut File::create(image_path.join(&json_filename)).await?) @@ -215,6 +264,7 @@ impl PersistentContainer { js_mount, volumes, assets, + images, overlays: Arc::new(Mutex::new(BTreeMap::new())), state: Arc::new(watch::channel(ServiceState::new(start)).0), net_service: Mutex::new(net_service), @@ -257,7 +307,7 @@ impl PersistentContainer { } #[instrument(skip_all)] - pub async fn init(&self, seed: Weak) -> Result<(), Error> { + pub async fn init(&self, seed: Weak) -> Result<(), Error> { let socket_server_context = EffectContext::new(seed); let server = Server::new( move || ready(Ok(socket_server_context.clone())), @@ -330,6 +380,7 @@ impl PersistentContainer { let js_mount = self.js_mount.take(); let volumes = std::mem::take(&mut self.volumes); let assets = std::mem::take(&mut self.assets); + let images = std::mem::take(&mut self.images); let overlays = self.overlays.clone(); let lxc_container = self.lxc_container.take(); self.destroyed = true; @@ -352,6 +403,9 @@ impl PersistentContainer { for (_, overlay) in std::mem::take(&mut *overlays.lock().await) { errs.handle(overlay.unmount(true).await); } + for (_, images) in images { + errs.handle(images.unmount().await); + } errs.handle(js_mount.unmount(true).await); if let Some(lxc_container) = lxc_container { errs.handle(lxc_container.exit().await); @@ -378,6 +432,7 @@ impl PersistentContainer { #[instrument(skip_all)] pub async fn start(&self) -> Result<(), Error> { self.execute( + Guid::new(), ProcedureName::StartMain, Value::Null, Some(Duration::from_secs(5)), // TODO @@ -389,7 +444,7 @@ impl PersistentContainer { #[instrument(skip_all)] pub async fn stop(&self) -> Result { let timeout: Option = self - .execute(ProcedureName::StopMain, Value::Null, None) + .execute(Guid::new(), ProcedureName::StopMain, Value::Null, None) .await?; Ok(timeout.map(|a| *a).unwrap_or(Duration::from_secs(30))) } @@ -397,6 +452,7 @@ impl PersistentContainer { #[instrument(skip_all)] pub async fn execute( &self, + id: Guid, name: ProcedureName, input: Value, timeout: Option, @@ -404,7 +460,7 @@ impl PersistentContainer { where O: DeserializeOwned, { - self._execute(name, input, timeout) + self._execute(id, name, input, timeout) .await .and_then(from_value) } @@ -412,6 +468,7 @@ impl PersistentContainer { #[instrument(skip_all)] pub async fn sanboxed( &self, + id: Guid, name: ProcedureName, input: Value, timeout: Option, @@ -419,7 +476,7 @@ impl PersistentContainer { where O: DeserializeOwned, { - self._sandboxed(name, input, timeout) + self._sandboxed(id, name, input, timeout) .await .and_then(from_value) } @@ -427,13 +484,15 @@ impl PersistentContainer { #[instrument(skip_all)] async fn _execute( &self, + id: Guid, name: ProcedureName, input: Value, timeout: Option, ) -> Result { - let fut = self - .rpc_client - .request(rpc::Execute, rpc::ExecuteParams::new(name, input, timeout)); + let fut = self.rpc_client.request( + rpc::Execute, + rpc::ExecuteParams::new(id, name, input, timeout), + ); Ok(if let Some(timeout) = timeout { tokio::time::timeout(timeout, fut) @@ -447,13 +506,15 @@ impl PersistentContainer { #[instrument(skip_all)] async fn _sandboxed( &self, + id: Guid, name: ProcedureName, input: Value, timeout: Option, ) -> Result { - let fut = self - .rpc_client - .request(rpc::Sandbox, rpc::ExecuteParams::new(name, input, timeout)); + let fut = self.rpc_client.request( + rpc::Sandbox, + rpc::ExecuteParams::new(id, name, input, timeout), + ); Ok(if let Some(timeout) = timeout { tokio::time::timeout(timeout, fut) diff --git a/core/startos/src/service/properties.rs b/core/startos/src/service/properties.rs index 3e795207a..3f5201f1d 100644 --- a/core/startos/src/service/properties.rs +++ b/core/startos/src/service/properties.rs @@ -3,6 +3,7 @@ use std::time::Duration; use models::ProcedureName; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::service::Service; impl Service { @@ -11,6 +12,7 @@ impl Service { let container = &self.seed.persistent_container; container .execute::( + Guid::new(), ProcedureName::Properties, Value::Null, Some(Duration::from_secs(30)), diff --git a/core/startos/src/service/rpc.rs b/core/startos/src/service/rpc.rs index 65c8b98fe..eff44b2cf 100644 --- a/core/startos/src/service/rpc.rs +++ b/core/startos/src/service/rpc.rs @@ -7,6 +7,7 @@ use rpc_toolkit::Empty; use ts_rs::TS; use crate::prelude::*; +use crate::rpc_continuations::Guid; #[derive(Clone)] pub struct Init; @@ -46,14 +47,21 @@ impl serde::Serialize for Exit { #[derive(Clone, serde::Deserialize, serde::Serialize, TS)] pub struct ExecuteParams { + id: Guid, procedure: String, #[ts(type = "any")] input: Value, timeout: Option, } impl ExecuteParams { - pub fn new(procedure: ProcedureName, input: Value, timeout: Option) -> Self { + pub fn new( + id: Guid, + procedure: ProcedureName, + input: Value, + timeout: Option, + ) -> Self { Self { + id, procedure: procedure.js_function_name(), input, timeout: timeout.map(|d| d.as_millis()), diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs index aedf4c194..28a611854 100644 --- a/core/startos/src/service/service_effect_handler.rs +++ b/core/startos/src/service/service_effect_handler.rs @@ -7,9 +7,9 @@ use std::str::FromStr; use std::sync::{Arc, Weak}; use clap::builder::ValueParserFactory; -use clap::Parser; +use clap::{CommandFactory, FromArgMatches, Parser}; use emver::VersionRange; -use imbl_value::{json, InternedString}; +use imbl_value::json; use itertools::Itertools; use models::{ ActionId, DataUrl, HealthCheckId, HostId, ImageId, PackageId, ServiceInterfaceId, VolumeId, @@ -25,35 +25,34 @@ use crate::db::model::package::{ ActionMetadata, CurrentDependencies, CurrentDependencyInfo, CurrentDependencyKind, ManifestPreference, }; -use crate::disk::mount::filesystem::idmapped::IdMapped; -use crate::disk::mount::filesystem::loop_dev::LoopDev; use crate::disk::mount::filesystem::overlayfs::OverlayGuard; +use crate::echo; use crate::net::host::address::HostAddress; use crate::net::host::binding::{BindOptions, LanInfo}; use crate::net::host::{Host, HostKind}; use crate::net::service_interface::{AddressInfo, ServiceInterface, ServiceInterfaceType}; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::s9pk::merkle_archive::source::http::HttpSource; use crate::s9pk::rpc::SKIP_ENV; use crate::s9pk::S9pk; use crate::service::cli::ContainerCliContext; -use crate::service::ServiceActorSeed; +use crate::service::Service; use crate::status::health_check::HealthCheckResult; use crate::status::MainStatus; use crate::util::clap::FromStrParser; -use crate::util::{new_guid, Invoke}; -use crate::{echo, ARCH}; +use crate::util::Invoke; #[derive(Clone)] -pub(super) struct EffectContext(Weak); +pub(super) struct EffectContext(Weak); impl EffectContext { - pub fn new(seed: Weak) -> Self { - Self(seed) + pub fn new(service: Weak) -> Self { + Self(service) } } impl Context for EffectContext {} impl EffectContext { - fn deref(&self) -> Result, Error> { + fn deref(&self) -> Result, Error> { if let Some(seed) = Weak::upgrade(&self.0) { Ok(seed) } else { @@ -66,11 +65,55 @@ impl EffectContext { } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -struct RpcData { - id: i64, - method: String, - params: Value, +#[serde(rename_all = "camelCase")] +pub struct WithProcedureId { + #[serde(default)] + procedure_id: Guid, + #[serde(flatten)] + rest: T, } +impl FromArgMatches for WithProcedureId { + fn from_arg_matches(matches: &clap::ArgMatches) -> Result { + let rest = T::from_arg_matches(matches)?; + Ok(Self { + procedure_id: matches.get_one("procedure-id").cloned().unwrap_or_default(), + rest, + }) + } + fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result { + let rest = T::from_arg_matches_mut(matches)?; + Ok(Self { + procedure_id: matches.get_one("procedure-id").cloned().unwrap_or_default(), + rest, + }) + } + fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> { + self.rest.update_from_arg_matches(matches)?; + self.procedure_id = matches.get_one("procedure-id").cloned().unwrap_or_default(); + Ok(()) + } + fn update_from_arg_matches_mut( + &mut self, + matches: &mut clap::ArgMatches, + ) -> Result<(), clap::Error> { + self.rest.update_from_arg_matches_mut(matches)?; + self.procedure_id = matches.get_one("procedure-id").cloned().unwrap_or_default(); + Ok(()) + } +} +impl CommandFactory for WithProcedureId { + fn command() -> clap::Command { + T::command_for_update().arg( + clap::Arg::new("procedure-id") + .action(clap::ArgAction::Set) + .value_parser(clap::value_parser!(Guid)), + ) + } + fn command_for_update() -> clap::Command { + Self::command() + } +} + pub fn service_effect_handler() -> ParentHandler { ParentHandler::new() .subcommand("gitInfo", from_fn(|_: C| crate::version::git_info())) @@ -290,6 +333,7 @@ struct MountParams { async fn set_system_smtp(context: EffectContext, data: SetSystemSmtpParams) -> Result<(), Error> { let context = context.deref()?; context + .seed .ctx .db .mutate(|db| { @@ -304,6 +348,7 @@ async fn get_system_smtp( ) -> Result { let context = context.deref()?; let res = context + .seed .ctx .db .peek() @@ -323,7 +368,7 @@ async fn get_system_smtp( } async fn get_container_ip(context: EffectContext, _: Empty) -> Result { let context = context.deref()?; - let net_service = context.persistent_container.net_service.lock().await; + let net_service = context.seed.persistent_container.net_service.lock().await; Ok(net_service.get_ip()) } async fn get_service_port_forward( @@ -333,14 +378,15 @@ async fn get_service_port_forward( let internal_port = data.internal_port as u16; let context = context.deref()?; - let net_service = context.persistent_container.net_service.lock().await; + let net_service = context.seed.persistent_container.net_service.lock().await; net_service.get_ext_port(data.host_id, internal_port) } async fn clear_network_interfaces(context: EffectContext, _: Empty) -> Result<(), Error> { let context = context.deref()?; - let package_id = context.id.clone(); + let package_id = context.seed.id.clone(); context + .seed .ctx .db .mutate(|db| { @@ -369,7 +415,7 @@ async fn export_service_interface( }: ExportServiceInterfaceParams, ) -> Result<(), Error> { let context = context.deref()?; - let package_id = context.id.clone(); + let package_id = context.seed.id.clone(); let service_interface = ServiceInterface { id: id.clone(), @@ -384,6 +430,7 @@ async fn export_service_interface( let svc_interface_with_host_info = service_interface; context + .seed .ctx .db .mutate(|db| { @@ -407,7 +454,7 @@ async fn get_primary_url( }: GetPrimaryUrlParams, ) -> Result, Error> { let context = context.deref()?; - let package_id = package_id.unwrap_or_else(|| context.id.clone()); + let package_id = package_id.unwrap_or_else(|| context.seed.id.clone()); Ok(None) // TODO } @@ -419,9 +466,10 @@ async fn list_service_interfaces( }: ListServiceInterfacesParams, ) -> Result, Error> { let context = context.deref()?; - let package_id = package_id.unwrap_or_else(|| context.id.clone()); + let package_id = package_id.unwrap_or_else(|| context.seed.id.clone()); context + .seed .ctx .db .peek() @@ -435,9 +483,10 @@ async fn list_service_interfaces( } async fn remove_address(context: EffectContext, data: RemoveAddressParams) -> Result<(), Error> { let context = context.deref()?; - let package_id = context.id.clone(); + let package_id = context.seed.id.clone(); context + .seed .ctx .db .mutate(|db| { @@ -454,8 +503,9 @@ async fn remove_address(context: EffectContext, data: RemoveAddressParams) -> Re } async fn export_action(context: EffectContext, data: ExportActionParams) -> Result<(), Error> { let context = context.deref()?; - let package_id = context.id.clone(); + let package_id = context.seed.id.clone(); context + .seed .ctx .db .mutate(|db| { @@ -477,8 +527,9 @@ async fn export_action(context: EffectContext, data: ExportActionParams) -> Resu } async fn remove_action(context: EffectContext, data: RemoveActionParams) -> Result<(), Error> { let context = context.deref()?; - let package_id = context.id.clone(); + let package_id = context.seed.id.clone(); context + .seed .ctx .db .mutate(|db| { @@ -514,16 +565,16 @@ struct GetHostInfoParams { callback: Callback, } async fn get_host_info( - ctx: EffectContext, + context: EffectContext, GetHostInfoParams { callback, package_id, host_id, }: GetHostInfoParams, ) -> Result { - let ctx = ctx.deref()?; - let db = ctx.ctx.db.peek().await; - let package_id = package_id.unwrap_or_else(|| ctx.id.clone()); + let context = context.deref()?; + let db = context.seed.ctx.db.peek().await; + let package_id = package_id.unwrap_or_else(|| context.seed.id.clone()); db.as_public() .as_package_data() @@ -536,8 +587,8 @@ async fn get_host_info( } async fn clear_bindings(context: EffectContext, _: Empty) -> Result<(), Error> { - let ctx = context.deref()?; - let mut svc = ctx.persistent_container.net_service.lock().await; + let context = context.deref()?; + let mut svc = context.seed.persistent_container.net_service.lock().await; svc.clear_bindings().await?; Ok(()) } @@ -559,8 +610,8 @@ async fn bind(context: EffectContext, bind_params: Value) -> Result<(), Error> { internal_port, options, } = from_value(bind_params)?; - let ctx = context.deref()?; - let mut svc = ctx.persistent_container.net_service.lock().await; + let context = context.deref()?; + let mut svc = context.seed.persistent_container.net_service.lock().await; svc.bind(kind, id, internal_port, options).await } @@ -575,16 +626,16 @@ struct GetServiceInterfaceParams { } async fn get_service_interface( - ctx: EffectContext, + context: EffectContext, GetServiceInterfaceParams { callback, package_id, service_interface_id, }: GetServiceInterfaceParams, ) -> Result { - let ctx = ctx.deref()?; - let package_id = package_id.unwrap_or_else(|| ctx.id.clone()); - let db = ctx.ctx.db.peek().await; + let context = context.deref()?; + let package_id = package_id.unwrap_or_else(|| context.seed.id.clone()); + let db = context.seed.ctx.db.peek().await; let interface = db .as_public() @@ -729,8 +780,8 @@ async fn get_store( GetStoreParams { package_id, path }: GetStoreParams, ) -> Result { let context = context.deref()?; - let peeked = context.ctx.db.peek().await; - let package_id = package_id.unwrap_or(context.id.clone()); + let peeked = context.seed.ctx.db.peek().await; + let package_id = package_id.unwrap_or(context.seed.id.clone()); let value = peeked .as_private() .as_package_stores() @@ -758,8 +809,9 @@ async fn set_store( SetStoreParams { value, path }: SetStoreParams, ) -> Result<(), Error> { let context = context.deref()?; - let package_id = context.id.clone(); + let package_id = context.seed.id.clone(); context + .seed .ctx .db .mutate(|db| { @@ -812,7 +864,7 @@ struct ParamsMaybePackageId { async fn exists(context: EffectContext, params: ParamsPackageId) -> Result { let context = context.deref()?; - let peeked = context.ctx.db.peek().await; + let peeked = context.seed.ctx.db.peek().await; let package = peeked .as_public() .as_package_data() @@ -834,31 +886,30 @@ struct ExecuteAction { } async fn execute_action( context: EffectContext, - ExecuteAction { - action_id, - input, - service_id, - }: ExecuteAction, + WithProcedureId { + procedure_id, + rest: + ExecuteAction { + service_id, + action_id, + input, + }, + }: WithProcedureId, ) -> Result { let context = context.deref()?; - let package_id = service_id.clone().unwrap_or_else(|| context.id.clone()); - let service = context.ctx.services.get(&package_id).await; - let service = service.as_ref().ok_or_else(|| { - Error::new( - eyre!("Could not find package {package_id}"), - ErrorKind::Unknown, - ) - })?; + let package_id = service_id + .clone() + .unwrap_or_else(|| context.seed.id.clone()); - Ok(json!(service.action(action_id, input).await?)) + Ok(json!(context.action(procedure_id, action_id, input).await?)) } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] struct FromService {} async fn get_configured(context: EffectContext, _: Empty) -> Result { let context = context.deref()?; - let peeked = context.ctx.db.peek().await; - let package_id = &context.id; + let peeked = context.seed.ctx.db.peek().await; + let package_id = &context.seed.id; let package = peeked .as_public() .as_package_data() @@ -872,8 +923,8 @@ async fn get_configured(context: EffectContext, _: Empty) -> Result Result { let context = context.deref()?; - let peeked = context.ctx.db.peek().await; - let package_id = params.package_id.unwrap_or_else(|| context.id.clone()); + let peeked = context.seed.ctx.db.peek().await; + let package_id = params.package_id.unwrap_or_else(|| context.seed.id.clone()); let package = peeked .as_public() .as_package_data() @@ -887,7 +938,7 @@ async fn stopped(context: EffectContext, params: ParamsMaybePackageId) -> Result async fn running(context: EffectContext, params: ParamsPackageId) -> Result { dbg!("Starting the running {params:?}"); let context = context.deref()?; - let peeked = context.ctx.db.peek().await; + let peeked = context.seed.ctx.db.peek().await; let package_id = params.package_id; let package = peeked .as_public() @@ -900,30 +951,24 @@ async fn running(context: EffectContext, params: ParamsPackageId) -> Result Result { +async fn restart( + context: EffectContext, + WithProcedureId { procedure_id, .. }: WithProcedureId, +) -> Result<(), Error> { let context = context.deref()?; - let service = context.ctx.services.get(&context.id).await; - let service = service.as_ref().ok_or_else(|| { - Error::new( - eyre!("Could not find package {}", context.id), - ErrorKind::Unknown, - ) - })?; - service.restart().await?; - Ok(json!(())) + dbg!("here"); + context.restart(procedure_id).await?; + dbg!("here"); + Ok(()) } -async fn shutdown(context: EffectContext, _: Empty) -> Result { +async fn shutdown( + context: EffectContext, + WithProcedureId { procedure_id, .. }: WithProcedureId, +) -> Result<(), Error> { let context = context.deref()?; - let service = context.ctx.services.get(&context.id).await; - let service = service.as_ref().ok_or_else(|| { - Error::new( - eyre!("Could not find package {}", context.id), - ErrorKind::Unknown, - ) - })?; - service.stop().await?; - Ok(json!(())) + context.stop(procedure_id).await?; + Ok(()) } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Parser, TS)] @@ -935,8 +980,9 @@ struct SetConfigured { } async fn set_configured(context: EffectContext, params: SetConfigured) -> Result { let context = context.deref()?; - let package_id = &context.id; + let package_id = &context.seed.id; context + .seed .ctx .db .mutate(|db| { @@ -989,9 +1035,9 @@ async fn set_main_status(context: EffectContext, params: SetMainStatus) -> Resul dbg!(format!("Status for main will be is {params:?}")); let context = context.deref()?; match params.status { - SetMainStatusStatus::Running => context.started(), - SetMainStatusStatus::Stopped => context.stopped(), - SetMainStatusStatus::Starting => context.stopped(), + SetMainStatusStatus::Running => context.seed.started(), + SetMainStatusStatus::Stopped => context.seed.stopped(), + SetMainStatusStatus::Starting => context.seed.stopped(), } Ok(Value::Null) } @@ -1011,8 +1057,9 @@ async fn set_health( ) -> Result { let context = context.deref()?; - let package_id = &context.id; + let package_id = &context.seed.id; context + .seed .ctx .db .mutate(move |db| { @@ -1041,17 +1088,17 @@ async fn set_health( #[command(rename_all = "camelCase")] #[ts(export)] pub struct DestroyOverlayedImageParams { - #[ts(type = "string")] - guid: InternedString, + guid: Guid, } #[instrument(skip_all)] pub async fn destroy_overlayed_image( - ctx: EffectContext, + context: EffectContext, DestroyOverlayedImageParams { guid }: DestroyOverlayedImageParams, ) -> Result<(), Error> { - let ctx = ctx.deref()?; - if ctx + let context = context.deref()?; + if context + .seed .persistent_container .overlays .lock() @@ -1068,30 +1115,25 @@ pub async fn destroy_overlayed_image( #[command(rename_all = "camelCase")] #[ts(export)] pub struct CreateOverlayedImageParams { - #[ts(type = "string")] image_id: ImageId, } #[instrument(skip_all)] pub async fn create_overlayed_image( - ctx: EffectContext, + context: EffectContext, CreateOverlayedImageParams { image_id }: CreateOverlayedImageParams, -) -> Result<(PathBuf, InternedString), Error> { - let ctx = ctx.deref()?; - let path = Path::new("images") - .join(*ARCH) - .join(&image_id) - .with_extension("squashfs"); - if let Some(image) = ctx +) -> Result<(PathBuf, Guid), Error> { + let context = context.deref()?; + if let Some(image) = context + .seed .persistent_container - .s9pk - .as_archive() - .contents() - .get_path(&path) - .and_then(|e| e.as_file()) + .images + .get(&image_id) + .cloned() { - let guid = new_guid(); - let rootfs_dir = ctx + let guid = Guid::new(); + let rootfs_dir = context + .seed .persistent_container .lxc_container .get() @@ -1102,7 +1144,9 @@ pub async fn create_overlayed_image( ) })? .rootfs_dir(); - let mountpoint = rootfs_dir.join("media/startos/overlays").join(&*guid); + let mountpoint = rootfs_dir + .join("media/startos/overlays") + .join(guid.as_ref()); tokio::fs::create_dir_all(&mountpoint).await?; let container_mountpoint = Path::new("/").join( mountpoint @@ -1110,18 +1154,16 @@ pub async fn create_overlayed_image( .with_kind(ErrorKind::Incoherent)?, ); tracing::info!("Mounting overlay {guid} for {image_id}"); - let guard = OverlayGuard::mount( - &IdMapped::new(LoopDev::from(&**image), 0, 100000, 65536), - &mountpoint, - ) - .await?; + let guard = OverlayGuard::mount(image, &mountpoint).await?; Command::new("chown") .arg("100000:100000") .arg(&mountpoint) .invoke(ErrorKind::Filesystem) .await?; tracing::info!("Mounted overlay {guid} for {image_id}"); - ctx.persistent_container + context + .seed + .persistent_container .overlays .lock() .await @@ -1228,13 +1270,15 @@ struct SetDependenciesParams { } async fn set_dependencies( - ctx: EffectContext, - SetDependenciesParams { dependencies }: SetDependenciesParams, + context: EffectContext, + WithProcedureId { + procedure_id, + rest: SetDependenciesParams { dependencies }, + }: WithProcedureId, ) -> Result<(), Error> { - let ctx = ctx.deref()?; - let id = &ctx.id; - let service_guard = ctx.ctx.services.get(id).await; - let service = service_guard.as_ref().or_not_found(id)?; + let context = context.deref()?; + let id = &context.seed.id; + let mut deps = BTreeMap::new(); for dependency in dependencies { let (dep_id, kind, registry_url, version_spec) = match dependency { @@ -1264,14 +1308,13 @@ async fn set_dependencies( let remote_s9pk = S9pk::deserialize( &Arc::new( HttpSource::new( - ctx.ctx.client.clone(), + context.seed.ctx.client.clone(), registry_url .join(&format!("package/v2/{}.s9pk?spec={}", dep_id, version_spec))?, ) .await?, ), None, // TODO - true, ) .await?; @@ -1291,14 +1334,19 @@ async fn set_dependencies( ) } }; - let config_satisfied = if let Some(dep_service) = &*ctx.ctx.services.get(&dep_id).await { - service - .dependency_config(dep_id.clone(), dep_service.get_config().await?.config) - .await? - .is_none() - } else { - true - }; + let config_satisfied = + if let Some(dep_service) = &*context.seed.ctx.services.get(&dep_id).await { + context + .dependency_config( + procedure_id.clone(), + dep_id.clone(), + dep_service.get_config(procedure_id.clone()).await?.config, + ) + .await? + .is_none() + } else { + true + }; deps.insert( dep_id, CurrentDependencyInfo { @@ -1311,7 +1359,9 @@ async fn set_dependencies( }, ); } - ctx.ctx + context + .seed + .ctx .db .mutate(|db| { db.as_public_mut() @@ -1324,10 +1374,10 @@ async fn set_dependencies( .await } -async fn get_dependencies(ctx: EffectContext) -> Result, Error> { - let ctx = ctx.deref()?; - let id = &ctx.id; - let db = ctx.ctx.db.peek().await; +async fn get_dependencies(context: EffectContext) -> Result, Error> { + let context = context.deref()?; + let id = &context.seed.id; + let db = context.seed.ctx.db.peek().await; let data = db .as_public() .as_package_data() @@ -1384,16 +1434,16 @@ struct CheckDependenciesResult { } async fn check_dependencies( - ctx: EffectContext, + context: EffectContext, CheckDependenciesParam { package_ids }: CheckDependenciesParam, ) -> Result, Error> { - let ctx = ctx.deref()?; - let db = ctx.ctx.db.peek().await; + let context = context.deref()?; + let db = context.seed.ctx.db.peek().await; let current_dependencies = db .as_public() .as_package_data() - .as_idx(&ctx.id) - .or_not_found(&ctx.id)? + .as_idx(&context.seed.id) + .or_not_found(&context.seed.id)? .as_current_dependencies() .de()?; let package_ids: Vec<_> = package_ids diff --git a/core/startos/src/service/service_map.rs b/core/startos/src/service/service_map.rs index ba9188f32..1474ea35e 100644 --- a/core/startos/src/service/service_map.rs +++ b/core/startos/src/service/service_map.rs @@ -25,7 +25,7 @@ use crate::progress::{ use crate::s9pk::manifest::PackageId; use crate::s9pk::merkle_archive::source::FileSource; use crate::s9pk::S9pk; -use crate::service::{LoadDisposition, Service}; +use crate::service::{LoadDisposition, Service, ServiceRef}; use crate::status::{MainStatus, Status}; use crate::util::serde::Pem; @@ -39,23 +39,22 @@ pub struct InstallProgressHandles { /// This is the structure to contain all the services #[derive(Default)] -pub struct ServiceMap(Mutex>>>>); +pub struct ServiceMap(Mutex>>>>); impl ServiceMap { - async fn entry(&self, id: &PackageId) -> Arc>> { + async fn entry(&self, id: &PackageId) -> Arc>> { let mut lock = self.0.lock().await; - dbg!(lock.keys().collect::>()); lock.entry(id.clone()) .or_insert_with(|| Arc::new(RwLock::new(None))) .clone() } #[instrument(skip_all)] - pub async fn get(&self, id: &PackageId) -> OwnedRwLockReadGuard> { + pub async fn get(&self, id: &PackageId) -> OwnedRwLockReadGuard> { self.entry(id).await.read_owned().await } #[instrument(skip_all)] - pub async fn get_mut(&self, id: &PackageId) -> OwnedRwLockWriteGuard> { + pub async fn get_mut(&self, id: &PackageId) -> OwnedRwLockWriteGuard> { self.entry(id).await.write_owned().await } @@ -83,7 +82,7 @@ impl ServiceMap { shutdown_err = service.shutdown().await; } // TODO: retry on error? - *service = Service::load(ctx, id, disposition).await?; + *service = Service::load(ctx, id, disposition).await?.map(From::from); shutdown_err?; Ok(()) } @@ -95,6 +94,7 @@ impl ServiceMap { mut s9pk: S9pk, recovery_source: Option, ) -> Result { + s9pk.validate_and_filter(ctx.s9pk_arch)?; let manifest = s9pk.as_manifest().clone(); let id = manifest.id.clone(); let icon = s9pk.icon_data_url().await?; @@ -128,7 +128,7 @@ impl ServiceMap { ); let restoring = recovery_source.is_some(); - let mut reload_guard = ServiceReloadGuard::new(ctx.clone(), id.clone(), op_name); + let mut reload_guard = ServiceRefReloadGuard::new(ctx.clone(), id.clone(), op_name); reload_guard .handle(ctx.db.mutate({ @@ -231,7 +231,7 @@ impl ServiceMap { Ok(reload_guard .handle_last(async move { finalization_progress.start(); - let s9pk = S9pk::open(&installed_path, Some(&id), true).await?; + let s9pk = S9pk::open(&installed_path, Some(&id)).await?; let prev = if let Some(service) = service.take() { ensure_code!( recovery_source.is_none(), @@ -264,7 +264,8 @@ impl ServiceMap { progress_handle, }), ) - .await?, + .await? + .into(), ); } else { *service = Some( @@ -277,7 +278,8 @@ impl ServiceMap { progress_handle, }), ) - .await?, + .await? + .into(), ); } sync_progress_task.await.map_err(|_| { @@ -295,7 +297,7 @@ impl ServiceMap { pub async fn uninstall(&self, ctx: &RpcContext, id: &PackageId) -> Result<(), Error> { let mut guard = self.get_mut(id).await; if let Some(service) = guard.take() { - ServiceReloadGuard::new(ctx.clone(), id.clone(), "Uninstall") + ServiceRefReloadGuard::new(ctx.clone(), id.clone(), "Uninstall") .handle_last(async move { let res = service.uninstall(None).await; drop(guard); @@ -326,17 +328,17 @@ impl ServiceMap { } } -pub struct ServiceReloadGuard(Option); -impl Drop for ServiceReloadGuard { +pub struct ServiceRefReloadGuard(Option); +impl Drop for ServiceRefReloadGuard { fn drop(&mut self) { if let Some(info) = self.0.take() { tokio::spawn(info.reload(None)); } } } -impl ServiceReloadGuard { +impl ServiceRefReloadGuard { pub fn new(ctx: RpcContext, id: PackageId, operation: &'static str) -> Self { - Self(Some(ServiceReloadInfo { ctx, id, operation })) + Self(Some(ServiceRefReloadInfo { ctx, id, operation })) } pub async fn handle( @@ -365,12 +367,12 @@ impl ServiceReloadGuard { } } -struct ServiceReloadInfo { +struct ServiceRefReloadInfo { ctx: RpcContext, id: PackageId, operation: &'static str, } -impl ServiceReloadInfo { +impl ServiceRefReloadInfo { async fn reload(self, error: Option) -> Result<(), Error> { self.ctx .services diff --git a/core/startos/src/service/transition/backup.rs b/core/startos/src/service/transition/backup.rs index e6a5cb817..f7591f0d9 100644 --- a/core/startos/src/service/transition/backup.rs +++ b/core/startos/src/service/transition/backup.rs @@ -6,6 +6,7 @@ use models::ProcedureName; use super::TempDesiredRestore; use crate::disk::mount::filesystem::ReadWrite; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::service::config::GetConfig; use crate::service::dependencies::DependencyConfig; use crate::service::transition::{TransitionKind, TransitionState}; @@ -24,7 +25,12 @@ impl Handler for ServiceActor { .except::() .except::() } - async fn handle(&mut self, backup: Backup, jobs: &BackgroundJobQueue) -> Self::Response { + async fn handle( + &mut self, + id: Guid, + backup: Backup, + jobs: &BackgroundJobQueue, + ) -> Self::Response { // So Need a handle to just a single field in the state let temp: TempDesiredRestore = TempDesiredRestore::new(&self.0.persistent_container.state); let mut current = self.0.persistent_container.state.subscribe(); @@ -45,7 +51,7 @@ impl Handler for ServiceActor { .mount_backup(path, ReadWrite) .await?; seed.persistent_container - .execute(ProcedureName::CreateBackup, Value::Null, None) + .execute(id, ProcedureName::CreateBackup, Value::Null, None) .await?; backup_guard.unmount(true).await?; diff --git a/core/startos/src/service/transition/restart.rs b/core/startos/src/service/transition/restart.rs index 07466def1..a39291621 100644 --- a/core/startos/src/service/transition/restart.rs +++ b/core/startos/src/service/transition/restart.rs @@ -2,6 +2,7 @@ use futures::FutureExt; use super::TempDesiredRestore; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::service::config::GetConfig; use crate::service::dependencies::DependencyConfig; use crate::service::transition::{TransitionKind, TransitionState}; @@ -18,7 +19,8 @@ impl Handler for ServiceActor { .except::() .except::() } - async fn handle(&mut self, _: Restart, jobs: &BackgroundJobQueue) -> Self::Response { + async fn handle(&mut self, _: Guid, _: Restart, jobs: &BackgroundJobQueue) -> Self::Response { + dbg!("here"); // So Need a handle to just a single field in the state let temp = TempDesiredRestore::new(&self.0.persistent_container.state); let mut current = self.0.persistent_container.state.subscribe(); @@ -74,7 +76,8 @@ impl Handler for ServiceActor { } impl Service { #[instrument(skip_all)] - pub async fn restart(&self) -> Result<(), Error> { - self.actor.send(Restart).await + pub async fn restart(&self, id: Guid) -> Result<(), Error> { + dbg!("here"); + self.actor.send(id, Restart).await } } diff --git a/core/startos/src/service/transition/restore.rs b/core/startos/src/service/transition/restore.rs index b32ade4f5..1c4020ea4 100644 --- a/core/startos/src/service/transition/restore.rs +++ b/core/startos/src/service/transition/restore.rs @@ -5,6 +5,7 @@ use models::ProcedureName; use crate::disk::mount::filesystem::ReadOnly; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::service::transition::{TransitionKind, TransitionState}; use crate::service::ServiceActor; use crate::util::actor::background::BackgroundJobQueue; @@ -19,7 +20,12 @@ impl Handler for ServiceActor { fn conflicts_with(_: &Restore) -> ConflictBuilder { ConflictBuilder::everything() } - async fn handle(&mut self, restore: Restore, jobs: &BackgroundJobQueue) -> Self::Response { + async fn handle( + &mut self, + id: Guid, + restore: Restore, + jobs: &BackgroundJobQueue, + ) -> Self::Response { // So Need a handle to just a single field in the state let path = restore.path.clone(); let seed = self.0.clone(); @@ -32,7 +38,7 @@ impl Handler for ServiceActor { .mount_backup(path, ReadOnly) .await?; seed.persistent_container - .execute(ProcedureName::RestoreBackup, Value::Null, None) + .execute(id, ProcedureName::RestoreBackup, Value::Null, None) .await?; backup_guard.unmount(true).await?; diff --git a/core/startos/src/util/actor/concurrent.rs b/core/startos/src/util/actor/concurrent.rs index e32470d80..7b26fc4c7 100644 --- a/core/startos/src/util/actor/concurrent.rs +++ b/core/startos/src/util/actor/concurrent.rs @@ -8,6 +8,7 @@ use helpers::NonDetachingJoinHandle; use tokio::sync::{mpsc, oneshot}; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::util::actor::background::{BackgroundJobQueue, BackgroundJobRunner}; use crate::util::actor::{Actor, ConflictFn, Handler, PendingMessageStrategy, Request}; @@ -18,6 +19,7 @@ struct ConcurrentRunner { waiting: Vec>, recv: mpsc::UnboundedReceiver>, handlers: Vec<( + Guid, Arc>, oneshot::Sender>, BoxFuture<'static, Box>, @@ -41,16 +43,21 @@ impl Future for ConcurrentRunner { } }); if this.shutdown.is_some() { - while let std::task::Poll::Ready(Some((msg, reply))) = this.recv.poll_recv(cx) { - if this.handlers.iter().any(|(f, _, _)| f(&*msg)) { - this.waiting.push((msg, reply)); + while let std::task::Poll::Ready(Some((id, msg, reply))) = this.recv.poll_recv(cx) { + if this + .handlers + .iter() + .any(|(hid, f, _, _)| &id != hid && f(&*msg)) + { + this.waiting.push((id, msg, reply)); } else { let mut actor = this.actor.clone(); let queue = this.queue.clone(); this.handlers.push(( + id.clone(), msg.conflicts_with(), reply, - async move { msg.handle_with(&mut actor, &queue).await }.boxed(), + async move { msg.handle_with(id, &mut actor, &queue).await }.boxed(), )) } } @@ -62,29 +69,34 @@ impl Future for ConcurrentRunner { .handlers .iter_mut() .enumerate() - .filter_map(|(i, (_, _, f))| match f.poll_unpin(cx) { + .filter_map(|(i, (_, _, _, f))| match f.poll_unpin(cx) { std::task::Poll::Pending => None, std::task::Poll::Ready(res) => Some((i, res)), }) .collect::>(); for (idx, res) in complete.into_iter().rev() { #[allow(clippy::let_underscore_future)] - let (f, reply, _) = this.handlers.swap_remove(idx); + let (_, f, reply, _) = this.handlers.swap_remove(idx); let _ = reply.send(res); // TODO: replace with Vec::extract_if once stable if this.shutdown.is_some() { let mut i = 0; while i < this.waiting.len() { - if f(&*this.waiting[i].0) - && !this.handlers.iter().any(|(f, _, _)| f(&*this.waiting[i].0)) + if f(&*this.waiting[i].1) + && !this + .handlers + .iter() + .any(|(_, f, _, _)| f(&*this.waiting[i].1)) { - let (msg, reply) = this.waiting.remove(i); + let (id, msg, reply) = this.waiting.remove(i); let mut actor = this.actor.clone(); let queue = this.queue.clone(); this.handlers.push(( + id.clone(), msg.conflicts_with(), reply, - async move { msg.handle_with(&mut actor, &queue).await }.boxed(), + async move { msg.handle_with(id, &mut actor, &queue).await } + .boxed(), )); cont = true; } else { @@ -137,6 +149,7 @@ impl ConcurrentActor { /// Message is guaranteed to be queued immediately pub fn queue( &self, + id: Guid, message: M, ) -> impl Future> where @@ -150,7 +163,7 @@ impl ConcurrentActor { } let (reply_send, reply_recv) = oneshot::channel(); self.messenger - .send((Box::new(message), reply_send)) + .send((id, Box::new(message), reply_send)) .unwrap(); futures::future::Either::Right( reply_recv @@ -170,11 +183,11 @@ impl ConcurrentActor { ) } - pub async fn send(&self, message: M) -> Result + pub async fn send(&self, id: Guid, message: M) -> Result where A: Handler, { - self.queue(message).await + self.queue(id, message).await } pub async fn shutdown(self, strategy: PendingMessageStrategy) { diff --git a/core/startos/src/util/actor/mod.rs b/core/startos/src/util/actor/mod.rs index 5cef7c22e..d85e53757 100644 --- a/core/startos/src/util/actor/mod.rs +++ b/core/startos/src/util/actor/mod.rs @@ -9,6 +9,7 @@ use tokio::sync::oneshot; #[allow(unused_imports)] use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::util::actor::background::BackgroundJobQueue; pub mod background; @@ -28,6 +29,7 @@ pub trait Handler: Actor { } fn handle( &mut self, + id: Guid, msg: M, jobs: &BackgroundJobQueue, ) -> impl Future + Send; @@ -39,6 +41,7 @@ trait Message: Send + Any { fn conflicts_with(&self) -> Arc>; fn handle_with<'a>( self: Box, + id: Guid, actor: &'a mut A, jobs: &'a BackgroundJobQueue, ) -> BoxFuture<'a, Box>; @@ -52,10 +55,11 @@ where } fn handle_with<'a>( self: Box, + id: Guid, actor: &'a mut A, jobs: &'a BackgroundJobQueue, ) -> BoxFuture<'a, Box> { - async move { Box::new(actor.handle(*self, jobs).await) as Box }.boxed() + async move { Box::new(actor.handle(id, *self, jobs).await) as Box }.boxed() } } impl dyn Message { @@ -80,7 +84,11 @@ impl dyn Message { } } -type Request = (Box>, oneshot::Sender>); +type Request = ( + Guid, + Box>, + oneshot::Sender>, +); pub enum PendingMessageStrategy { CancelAll, diff --git a/core/startos/src/util/actor/simple.rs b/core/startos/src/util/actor/simple.rs index 6f880a57a..7f2ad388e 100644 --- a/core/startos/src/util/actor/simple.rs +++ b/core/startos/src/util/actor/simple.rs @@ -7,6 +7,7 @@ use tokio::sync::oneshot::error::TryRecvError; use tokio::sync::{mpsc, oneshot}; use crate::prelude::*; +use crate::rpc_continuations::Guid; use crate::util::actor::background::BackgroundJobQueue; use crate::util::actor::{Actor, Handler, PendingMessageStrategy, Request}; @@ -26,9 +27,9 @@ impl SimpleActor { tokio::select! { _ = &mut runner => (), msg = messenger_recv.recv() => match msg { - Some((msg, reply)) if shutdown_recv.try_recv() == Err(TryRecvError::Empty) => { + Some((id, msg, reply)) if shutdown_recv.try_recv() == Err(TryRecvError::Empty) => { tokio::select! { - res = msg.handle_with(&mut actor, &queue) => { let _ = reply.send(res); }, + res = msg.handle_with(id, &mut actor, &queue) => { let _ = reply.send(res); }, _ = &mut runner => (), } } @@ -60,7 +61,7 @@ impl SimpleActor { } let (reply_send, reply_recv) = oneshot::channel(); self.messenger - .send((Box::new(message), reply_send)) + .send((Guid::new(), Box::new(message), reply_send)) .unwrap(); futures::future::Either::Right( reply_recv diff --git a/core/startos/src/util/io.rs b/core/startos/src/util/io.rs index 16bafd8f6..f4476ee2b 100644 --- a/core/startos/src/util/io.rs +++ b/core/startos/src/util/io.rs @@ -681,8 +681,6 @@ impl AsyncWrite for TimeoutStream { } } -pub struct TmpFile {} - #[derive(Debug)] pub struct TmpDir { path: PathBuf, @@ -707,6 +705,14 @@ impl TmpDir { tokio::fs::remove_dir_all(&self.path).await?; Ok(()) } + + pub async fn gc(self: Arc) -> Result<(), Error> { + if let Ok(dir) = Arc::try_unwrap(self) { + dir.delete().await + } else { + Ok(()) + } + } } impl std::ops::Deref for TmpDir { type Target = Path; diff --git a/core/startos/src/util/mod.rs b/core/startos/src/util/mod.rs index aa17b6d7a..4346a0b1e 100644 --- a/core/startos/src/util/mod.rs +++ b/core/startos/src/util/mod.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, VecDeque}; use std::future::Future; use std::marker::PhantomData; use std::path::{Path, PathBuf}; @@ -11,6 +11,8 @@ use std::time::Duration; use async_trait::async_trait; use color_eyre::eyre::{self, eyre}; use fd_lock_rs::FdLock; +use futures::future::BoxFuture; +use futures::FutureExt; use helpers::canonicalize; pub use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; @@ -19,7 +21,8 @@ pub use models::VersionString; use pin_project::pin_project; use sha2::Digest; use tokio::fs::File; -use tokio::sync::{Mutex, OwnedMutexGuard, RwLock}; +use tokio::io::{AsyncRead, AsyncReadExt, BufReader}; +use tokio::sync::{oneshot, Mutex, OwnedMutexGuard, RwLock}; use tracing::instrument; use crate::shutdown::Shutdown; @@ -62,11 +65,16 @@ pub trait Invoke<'a> { where Self: 'ext, 'ext: 'a; + fn pipe<'ext: 'a>( + &'ext mut self, + next: &'ext mut tokio::process::Command, + ) -> Self::Extended<'ext>; fn timeout<'ext: 'a>(&'ext mut self, timeout: Option) -> Self::Extended<'ext>; fn input<'ext: 'a, Input: tokio::io::AsyncRead + Unpin + Send>( &'ext mut self, input: Option<&'ext mut Input>, ) -> Self::Extended<'ext>; + fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext>; fn invoke( &mut self, error_kind: crate::ErrorKind, @@ -76,7 +84,20 @@ pub trait Invoke<'a> { pub struct ExtendedCommand<'a> { cmd: &'a mut tokio::process::Command, timeout: Option, - input: Option<&'a mut (dyn tokio::io::AsyncRead + Unpin + Send)>, + input: Option<&'a mut (dyn AsyncRead + Unpin + Send)>, + pipe: VecDeque<&'a mut tokio::process::Command>, + capture: bool, +} +impl<'a> From<&'a mut tokio::process::Command> for ExtendedCommand<'a> { + fn from(value: &'a mut tokio::process::Command) -> Self { + ExtendedCommand { + cmd: value, + timeout: None, + input: None, + pipe: VecDeque::new(), + capture: true, + } + } } impl<'a> std::ops::Deref for ExtendedCommand<'a> { type Target = tokio::process::Command; @@ -95,35 +116,38 @@ impl<'a> Invoke<'a> for tokio::process::Command { where Self: 'ext, 'ext: 'a; - fn timeout<'ext: 'a>(&'ext mut self, timeout: Option) -> Self::Extended<'ext> { - ExtendedCommand { - cmd: self, - timeout, - input: None, - } + fn pipe<'ext: 'a>( + &'ext mut self, + next: &'ext mut tokio::process::Command, + ) -> Self::Extended<'ext> { + let mut cmd = ExtendedCommand::from(self); + cmd.pipe.push_back(next); + cmd } - fn input<'ext: 'a, Input: tokio::io::AsyncRead + Unpin + Send>( + fn timeout<'ext: 'a>(&'ext mut self, timeout: Option) -> Self::Extended<'ext> { + let mut cmd = ExtendedCommand::from(self); + cmd.timeout = timeout; + cmd + } + fn input<'ext: 'a, Input: AsyncRead + Unpin + Send>( &'ext mut self, input: Option<&'ext mut Input>, ) -> Self::Extended<'ext> { - ExtendedCommand { - cmd: self, - timeout: None, - input: if let Some(input) = input { - Some(&mut *input) - } else { - None - }, - } + let mut cmd = ExtendedCommand::from(self); + cmd.input = if let Some(input) = input { + Some(&mut *input) + } else { + None + }; + cmd + } + fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext> { + let mut cmd = ExtendedCommand::from(self); + cmd.capture = capture; + cmd } async fn invoke(&mut self, error_kind: crate::ErrorKind) -> Result, Error> { - ExtendedCommand { - cmd: self, - timeout: None, - input: None, - } - .invoke(error_kind) - .await + ExtendedCommand::from(self).invoke(error_kind).await } } @@ -132,6 +156,13 @@ impl<'a> Invoke<'a> for ExtendedCommand<'a> { where Self: 'ext, 'ext: 'a; + fn pipe<'ext: 'a>( + &'ext mut self, + next: &'ext mut tokio::process::Command, + ) -> Self::Extended<'ext> { + self.pipe.push_back(next.kill_on_drop(true)); + self + } fn timeout<'ext: 'a>(&'ext mut self, timeout: Option) -> Self::Extended<'ext> { self.timeout = timeout; self @@ -147,39 +178,150 @@ impl<'a> Invoke<'a> for ExtendedCommand<'a> { }; self } + fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext> { + self.capture = capture; + self + } + #[instrument(skip_all)] async fn invoke(&mut self, error_kind: crate::ErrorKind) -> Result, Error> { + let cmd_str = self + .cmd + .as_std() + .get_program() + .to_string_lossy() + .into_owned(); self.cmd.kill_on_drop(true); if self.input.is_some() { self.cmd.stdin(Stdio::piped()); } - self.cmd.stdout(Stdio::piped()); - self.cmd.stderr(Stdio::piped()); - let mut child = self.cmd.spawn().with_kind(error_kind)?; - if let (Some(mut stdin), Some(input)) = (child.stdin.take(), self.input.take()) { - use tokio::io::AsyncWriteExt; - tokio::io::copy(input, &mut stdin).await?; - stdin.flush().await?; - stdin.shutdown().await?; - drop(stdin); + if self.pipe.is_empty() { + if self.capture { + self.cmd.stdout(Stdio::piped()); + self.cmd.stderr(Stdio::piped()); + } + let mut child = self.cmd.spawn().with_ctx(|_| (error_kind, &cmd_str))?; + if let (Some(mut stdin), Some(input)) = (child.stdin.take(), self.input.take()) { + use tokio::io::AsyncWriteExt; + tokio::io::copy(input, &mut stdin).await?; + stdin.flush().await?; + stdin.shutdown().await?; + drop(stdin); + } + let res = match self.timeout { + None => child + .wait_with_output() + .await + .with_ctx(|_| (error_kind, &cmd_str))?, + Some(t) => tokio::time::timeout(t, child.wait_with_output()) + .await + .with_kind(ErrorKind::Timeout)? + .with_ctx(|_| (error_kind, &cmd_str))?, + }; + crate::ensure_code!( + res.status.success(), + error_kind, + "{}", + Some(&res.stderr) + .filter(|a| !a.is_empty()) + .or(Some(&res.stdout)) + .filter(|a| !a.is_empty()) + .and_then(|a| std::str::from_utf8(a).ok()) + .unwrap_or(&format!( + "{} exited with code {}", + self.cmd.as_std().get_program().to_string_lossy(), + res.status + )) + ); + Ok(res.stdout) + } else { + let mut futures = Vec::>>::new(); // todo: predict capacity + + let mut cmds = std::mem::take(&mut self.pipe); + cmds.push_front(&mut *self.cmd); + let len = cmds.len(); + + let timeout = self.timeout; + + let mut prev = self + .input + .take() + .map(|i| Box::new(i) as Box); + for (idx, cmd) in IntoIterator::into_iter(cmds).enumerate() { + let last = idx == len - 1; + if self.capture || !last { + cmd.stdout(Stdio::piped()); + } + if self.capture { + cmd.stderr(Stdio::piped()); + } + if prev.is_some() { + cmd.stdin(Stdio::piped()); + } + let mut child = cmd.spawn().with_kind(error_kind)?; + let input = std::mem::replace( + &mut prev, + child + .stdout + .take() + .map(|i| Box::new(BufReader::new(i)) as Box), + ); + futures.push( + async move { + if let (Some(mut stdin), Some(mut input)) = (child.stdin.take(), input) { + use tokio::io::AsyncWriteExt; + tokio::io::copy(&mut input, &mut stdin).await?; + stdin.flush().await?; + stdin.shutdown().await?; + drop(stdin); + } + let res = match timeout { + None => child.wait_with_output().await?, + Some(t) => tokio::time::timeout(t, child.wait_with_output()) + .await + .with_kind(ErrorKind::Timeout)??, + }; + crate::ensure_code!( + res.status.success(), + error_kind, + "{}", + Some(&res.stderr) + .filter(|a| !a.is_empty()) + .or(Some(&res.stdout)) + .filter(|a| !a.is_empty()) + .and_then(|a| std::str::from_utf8(a).ok()) + .unwrap_or(&format!( + "{} exited with code {}", + cmd.as_std().get_program().to_string_lossy(), + res.status + )) + ); + + Ok(()) + } + .boxed(), + ); + } + + let (send, recv) = oneshot::channel(); + futures.push( + async move { + if let Some(mut prev) = prev { + let mut res = Vec::new(); + prev.read_to_end(&mut res).await?; + send.send(res).unwrap(); + } else { + send.send(Vec::new()).unwrap(); + } + + Ok(()) + } + .boxed(), + ); + + futures::future::try_join_all(futures).await?; + + Ok(recv.await.unwrap()) } - let res = match self.timeout { - None => child.wait_with_output().await?, - Some(t) => tokio::time::timeout(t, child.wait_with_output()) - .await - .with_kind(ErrorKind::Timeout)??, - }; - crate::ensure_code!( - res.status.success(), - error_kind, - "{}", - Some(&res.stderr) - .filter(|a| !a.is_empty()) - .or(Some(&res.stdout)) - .filter(|a| !a.is_empty()) - .and_then(|a| std::str::from_utf8(a).ok()) - .unwrap_or(&format!("Unknown Error ({})", res.status)) - ); - Ok(res.stdout) } } diff --git a/sdk/lib/StartSdk.ts b/sdk/lib/StartSdk.ts index f5959a084..4208928c9 100644 --- a/sdk/lib/StartSdk.ts +++ b/sdk/lib/StartSdk.ts @@ -187,7 +187,10 @@ export class StartSdk { nullIfEmpty, runCommand: async ( effects: Effects, - image: { id: Manifest["images"][number]; sharedRun?: boolean }, + image: { + id: keyof Manifest["images"] & T.ImageId + sharedRun?: boolean + }, command: ValidIfNoStupidEscape | [string, ...string[]], options: CommandOptions & { mounts?: { path: string; options: MountOptions }[] @@ -396,7 +399,7 @@ export class StartSdk { setupProperties: ( fn: (options: { effects: Effects }) => Promise, - ): T.ExpectedExports.Properties => + ): T.ExpectedExports.properties => (options) => fn(options).then(nullifyProperties), setupUninstall: (fn: UninstallFn) => @@ -743,7 +746,7 @@ export class StartSdk { export async function runCommand( effects: Effects, - image: { id: Manifest["images"][number]; sharedRun?: boolean }, + image: { id: keyof Manifest["images"] & T.ImageId; sharedRun?: boolean }, command: string | [string, ...string[]], options: CommandOptions & { mounts?: { path: string; options: MountOptions }[] diff --git a/sdk/lib/health/HealthCheck.ts b/sdk/lib/health/HealthCheck.ts index ca0e3ebb9..1ed8652bf 100644 --- a/sdk/lib/health/HealthCheck.ts +++ b/sdk/lib/health/HealthCheck.ts @@ -8,12 +8,13 @@ import { defaultTrigger } from "../trigger/defaultTrigger" import { once } from "../util/once" import { Overlay } from "../util/Overlay" import { object, unknown } from "ts-matches" +import { T } from ".." export type HealthCheckParams = { effects: Effects name: string image: { - id: Manifest["images"][number] + id: keyof Manifest["images"] & T.ImageId sharedRun?: boolean } trigger?: Trigger diff --git a/sdk/lib/interfaces/Host.ts b/sdk/lib/interfaces/Host.ts index 96a76628a..aa27a289c 100644 --- a/sdk/lib/interfaces/Host.ts +++ b/sdk/lib/interfaces/Host.ts @@ -69,12 +69,12 @@ type NotProtocolsWithSslVariants = Exclude< type BindOptionsByKnownProtocol = | { protocol: ProtocolsWithSslVariants - preferredExternalPort: number + preferredExternalPort?: number addSsl?: Partial } | { protocol: NotProtocolsWithSslVariants - preferredExternalPort: number + preferredExternalPort?: number addSsl?: AddSslOptions } export type BindOptionsByProtocol = BindOptionsByKnownProtocol | BindOptions diff --git a/sdk/lib/mainFn/CommandController.ts b/sdk/lib/mainFn/CommandController.ts index a68e1e426..264574f7c 100644 --- a/sdk/lib/mainFn/CommandController.ts +++ b/sdk/lib/mainFn/CommandController.ts @@ -1,6 +1,6 @@ import { NO_TIMEOUT, SIGKILL, SIGTERM } from "../StartSdk" import { SDKManifest } from "../manifest/ManifestTypes" -import { Effects, ValidIfNoStupidEscape } from "../types" +import { Effects, ImageId, ValidIfNoStupidEscape } from "../types" import { MountOptions, Overlay } from "../util/Overlay" import { splitCommand } from "../util/splitCommand" import { cpExecFile, cpExec } from "./Daemons" @@ -15,7 +15,7 @@ export class CommandController { return async ( effects: Effects, imageId: { - id: Manifest["images"][number] + id: keyof Manifest["images"] & ImageId sharedRun?: boolean }, command: ValidIfNoStupidEscape | [string, ...string[]], diff --git a/sdk/lib/mainFn/Daemon.ts b/sdk/lib/mainFn/Daemon.ts index 2ec438801..c48865f94 100644 --- a/sdk/lib/mainFn/Daemon.ts +++ b/sdk/lib/mainFn/Daemon.ts @@ -1,5 +1,5 @@ import { SDKManifest } from "../manifest/ManifestTypes" -import { Effects, ValidIfNoStupidEscape } from "../types" +import { Effects, ImageId, ValidIfNoStupidEscape } from "../types" import { MountOptions, Overlay } from "../util/Overlay" import { CommandController } from "./CommandController" @@ -18,7 +18,7 @@ export class Daemon { return async ( effects: Effects, imageId: { - id: Manifest["images"][number] + id: keyof Manifest["images"] & ImageId sharedRun?: boolean }, command: ValidIfNoStupidEscape | [string, ...string[]], diff --git a/sdk/lib/mainFn/Daemons.ts b/sdk/lib/mainFn/Daemons.ts index 8962061f5..d51d444a8 100644 --- a/sdk/lib/mainFn/Daemons.ts +++ b/sdk/lib/mainFn/Daemons.ts @@ -5,7 +5,12 @@ import { SDKManifest } from "../manifest/ManifestTypes" import { Trigger } from "../trigger" import { TriggerInput } from "../trigger/TriggerInput" import { defaultTrigger } from "../trigger/defaultTrigger" -import { DaemonReturned, Effects, ValidIfNoStupidEscape } from "../types" +import { + DaemonReturned, + Effects, + ImageId, + ValidIfNoStupidEscape, +} from "../types" import { Mounts } from "./Mounts" import { CommandOptions, MountOptions, Overlay } from "../util/Overlay" import { splitCommand } from "../util/splitCommand" @@ -34,8 +39,8 @@ type DaemonsParams< Id extends string, > = { command: ValidIfNoStupidEscape | [string, ...string[]] - image: { id: Manifest["images"][number]; sharedRun?: boolean } - mounts: { path: string; options: MountOptions }[] + image: { id: keyof Manifest["images"] & ImageId; sharedRun?: boolean } + mounts: Mounts env?: Record ready: Ready requires: Exclude[] @@ -116,12 +121,10 @@ export class Daemons { options: DaemonsParams, ) { const daemonIndex = this.daemons.length - const daemon = Daemon.of()( - this.effects, - options.image, - options.command, - options, - ) + const daemon = Daemon.of()(this.effects, options.image, options.command, { + ...options, + mounts: options.mounts.build(), + }) const healthDaemon = new HealthDaemon( daemon, daemonIndex, diff --git a/sdk/lib/manifest/ManifestTypes.ts b/sdk/lib/manifest/ManifestTypes.ts index ed8703bf2..c820930c8 100644 --- a/sdk/lib/manifest/ManifestTypes.ts +++ b/sdk/lib/manifest/ManifestTypes.ts @@ -1,5 +1,5 @@ import { ValidEmVer } from "../emverLite/mod" -import { ActionMetadata } from "../types" +import { ActionMetadata, ImageConfig, ImageId } from "../types" export interface Container { /** This should be pointing to a docker container name */ @@ -28,8 +28,6 @@ export type SDKManifest = { readonly releaseNotes: string /** The type of license for the project. Include the LICENSE in the root of the project directory. A license is required for a Start9 package.*/ readonly license: string // name of license - /** A list of normie (hosted, SaaS, custodial, etc) services this services intends to replace */ - readonly replaces: Readonly /** The Start9 wrapper repository URL for the package. This repo contains the manifest file (this), * any scripts necessary for configuration, backups, actions, or health checks (more below). This key * must exist. But could be embedded into the source repository @@ -52,7 +50,7 @@ export type SDKManifest = { } /** Defines the os images needed to run the container processes */ - readonly images: string[] + readonly images: Record /** This denotes readonly asset directories that should be available to mount to the container. * Assuming that there will be three files with names along the lines: * icon.* : the icon that will be this packages icon on the ui diff --git a/sdk/lib/manifest/setupManifest.ts b/sdk/lib/manifest/setupManifest.ts index 41c74baa0..8bd39a7aa 100644 --- a/sdk/lib/manifest/setupManifest.ts +++ b/sdk/lib/manifest/setupManifest.ts @@ -1,18 +1,19 @@ +import { ImageConfig, ImageId, VolumeId } from "../osBindings" import { SDKManifest, ManifestVersion } from "./ManifestTypes" export function setupManifest< Id extends string, Version extends ManifestVersion, Dependencies extends Record, - VolumesTypes extends string, - AssetTypes extends string, - ImagesTypes extends string, + VolumesTypes extends VolumeId, + AssetTypes extends VolumeId, + ImagesTypes extends ImageId, Manifest extends SDKManifest & { dependencies: Dependencies id: Id version: Version assets: AssetTypes[] - images: ImagesTypes[] + images: Record volumes: VolumesTypes[] }, >(manifest: Manifest): Manifest { diff --git a/sdk/lib/osBindings/CreateOverlayedImageParams.ts b/sdk/lib/osBindings/CreateOverlayedImageParams.ts index b8579ac7d..aad94f01f 100644 --- a/sdk/lib/osBindings/CreateOverlayedImageParams.ts +++ b/sdk/lib/osBindings/CreateOverlayedImageParams.ts @@ -1,3 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ImageId } from "./ImageId" -export type CreateOverlayedImageParams = { imageId: string } +export type CreateOverlayedImageParams = { imageId: ImageId } diff --git a/sdk/lib/osBindings/DestroyOverlayedImageParams.ts b/sdk/lib/osBindings/DestroyOverlayedImageParams.ts index 82fc1cc67..b5b7484a2 100644 --- a/sdk/lib/osBindings/DestroyOverlayedImageParams.ts +++ b/sdk/lib/osBindings/DestroyOverlayedImageParams.ts @@ -1,3 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Guid } from "./Guid" -export type DestroyOverlayedImageParams = { guid: string } +export type DestroyOverlayedImageParams = { guid: Guid } diff --git a/sdk/lib/osBindings/ImageConfig.ts b/sdk/lib/osBindings/ImageConfig.ts new file mode 100644 index 000000000..2b1033b83 --- /dev/null +++ b/sdk/lib/osBindings/ImageConfig.ts @@ -0,0 +1,8 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ImageSource } from "./ImageSource" + +export type ImageConfig = { + source: ImageSource + arch: string[] + emulateMissingAs: string | null +} diff --git a/sdk/lib/osBindings/ImageMetadata.ts b/sdk/lib/osBindings/ImageMetadata.ts new file mode 100644 index 000000000..b50f7a084 --- /dev/null +++ b/sdk/lib/osBindings/ImageMetadata.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ImageMetadata = { workdir: string; user: string } diff --git a/sdk/lib/osBindings/ImageSource.ts b/sdk/lib/osBindings/ImageSource.ts new file mode 100644 index 000000000..a71684e46 --- /dev/null +++ b/sdk/lib/osBindings/ImageSource.ts @@ -0,0 +1,6 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ImageSource = + | "packed" + | { dockerBuild: { workdir: string | null; dockerfile: string | null } } + | { dockerTag: string } diff --git a/sdk/lib/osBindings/LoginParams.ts b/sdk/lib/osBindings/LoginParams.ts new file mode 100644 index 000000000..272f7ed07 --- /dev/null +++ b/sdk/lib/osBindings/LoginParams.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { PasswordType } from "./PasswordType" + +export type LoginParams = { password: PasswordType | null; metadata: any } diff --git a/sdk/lib/osBindings/Manifest.ts b/sdk/lib/osBindings/Manifest.ts index 7f06be25e..15b96bd13 100644 --- a/sdk/lib/osBindings/Manifest.ts +++ b/sdk/lib/osBindings/Manifest.ts @@ -3,6 +3,7 @@ import type { Alerts } from "./Alerts" import type { Dependencies } from "./Dependencies" import type { Description } from "./Description" import type { HardwareRequirements } from "./HardwareRequirements" +import type { ImageConfig } from "./ImageConfig" import type { ImageId } from "./ImageId" import type { PackageId } from "./PackageId" import type { Version } from "./Version" @@ -20,7 +21,7 @@ export type Manifest = { marketingSite: string donationUrl: string | null description: Description - images: Array + images: { [key: ImageId]: ImageConfig } assets: Array volumes: Array alerts: Alerts diff --git a/sdk/lib/osBindings/index.ts b/sdk/lib/osBindings/index.ts index efd7a5efb..06a4bed7e 100644 --- a/sdk/lib/osBindings/index.ts +++ b/sdk/lib/osBindings/index.ts @@ -69,7 +69,10 @@ export { HostKind } from "./HostKind" export { HostnameInfo } from "./HostnameInfo" export { Hosts } from "./Hosts" export { Host } from "./Host" +export { ImageConfig } from "./ImageConfig" export { ImageId } from "./ImageId" +export { ImageMetadata } from "./ImageMetadata" +export { ImageSource } from "./ImageSource" export { InstalledState } from "./InstalledState" export { InstallingInfo } from "./InstallingInfo" export { InstallingState } from "./InstallingState" @@ -78,6 +81,7 @@ export { IpInfo } from "./IpInfo" export { LanInfo } from "./LanInfo" export { ListServiceInterfacesParams } from "./ListServiceInterfacesParams" export { ListVersionSignersParams } from "./ListVersionSignersParams" +export { LoginParams } from "./LoginParams" export { MainStatus } from "./MainStatus" export { Manifest } from "./Manifest" export { MaybeUtf8String } from "./MaybeUtf8String" diff --git a/sdk/lib/test/configBuilder.test.ts b/sdk/lib/test/configBuilder.test.ts index 2df40b95c..9738475a6 100644 --- a/sdk/lib/test/configBuilder.test.ts +++ b/sdk/lib/test/configBuilder.test.ts @@ -400,7 +400,7 @@ describe("values", () => { long: "", }, containers: {}, - images: [], + images: {}, volumes: [], assets: [], alerts: { diff --git a/sdk/lib/test/output.sdk.ts b/sdk/lib/test/output.sdk.ts index a0bab1f6e..189491be5 100644 --- a/sdk/lib/test/output.sdk.ts +++ b/sdk/lib/test/output.sdk.ts @@ -21,7 +21,7 @@ export const sdk = StartSdk.of() long: "", }, containers: {}, - images: [], + images: {}, volumes: [], assets: [], alerts: { diff --git a/sdk/lib/types.ts b/sdk/lib/types.ts index 2da92cd56..1f1245adc 100644 --- a/sdk/lib/types.ts +++ b/sdk/lib/types.ts @@ -11,6 +11,7 @@ import { GetPrimaryUrlParams, LanInfo, BindParams, + Manifest, } from "./osBindings" import { MainEffects, ServiceInterfaceType, Signals } from "./StartSdk" @@ -110,9 +111,26 @@ export namespace ExpectedExports { */ export type dependencyConfig = Record - export type Properties = (options: { + export type properties = (options: { effects: Effects }) => Promise + + export type manifest = Manifest +} +export type ABI = { + setConfig: ExpectedExports.setConfig + getConfig: ExpectedExports.getConfig + createBackup: ExpectedExports.createBackup + restoreBackup: ExpectedExports.restoreBackup + actions: ExpectedExports.actions + actionsMetadata: ExpectedExports.actionsMetadata + main: ExpectedExports.main + afterShutdown: ExpectedExports.afterShutdown + init: ExpectedExports.init + uninit: ExpectedExports.uninit + dependencyConfig: ExpectedExports.dependencyConfig + properties: ExpectedExports.properties + manifest: ExpectedExports.manifest } export type TimeMs = number export type VersionString = string @@ -453,8 +471,8 @@ export type Effects = { /** Exists could be useful during the runtime to know if some service is running, option dep */ running(options: { packageId: PackageId }): Promise - restart(): void - shutdown(): void + restart(): Promise + shutdown(): Promise mount(options: { location: string diff --git a/sdk/lib/util/Overlay.ts b/sdk/lib/util/Overlay.ts index 794b78732..fbc854c57 100644 --- a/sdk/lib/util/Overlay.ts +++ b/sdk/lib/util/Overlay.ts @@ -8,16 +8,18 @@ const WORKDIR = (imageId: string) => `/media/startos/images/${imageId}/` export class Overlay { private constructor( readonly effects: T.Effects, - readonly imageId: string, + readonly imageId: T.ImageId, readonly rootfs: string, - readonly guid: string, + readonly guid: T.Guid, ) {} static async of( effects: T.Effects, - image: { id: string; sharedRun?: boolean }, + image: { id: T.ImageId; sharedRun?: boolean }, ) { - const { id: imageId, sharedRun } = image - const [rootfs, guid] = await effects.createOverlayedImage({ imageId }) + const { id, sharedRun } = image + const [rootfs, guid] = await effects.createOverlayedImage({ + imageId: id as string, + }) const shared = ["dev", "sys", "proc"] if (!!sharedRun) { @@ -33,7 +35,7 @@ export class Overlay { ]) } - return new Overlay(effects, imageId, rootfs, guid) + return new Overlay(effects, id, rootfs, guid) } async mount(options: MountOptions, path: string): Promise { @@ -97,7 +99,7 @@ export class Overlay { stdout: string | Buffer stderr: string | Buffer }> { - const imageMeta: any = await fs + const imageMeta: T.ImageMetadata = await fs .readFile(`/media/startos/images/${this.imageId}.json`, { encoding: "utf8", }) diff --git a/sdk/lib/util/fileHelper.ts b/sdk/lib/util/fileHelper.ts index 07cfb2c4a..54f1eca06 100644 --- a/sdk/lib/util/fileHelper.ts +++ b/sdk/lib/util/fileHelper.ts @@ -3,7 +3,7 @@ import * as YAML from "yaml" import * as TOML from "@iarna/toml" import _ from "lodash" import * as T from "../types" -import * as fs from "fs" +import * as fs from "node:fs/promises" const previousPath = /(.+?)\/([^/]*)$/ @@ -59,28 +59,24 @@ export class FileHelper { readonly readData: (stringValue: string) => A, ) {} async write(data: A, effects: T.Effects) { - if (previousPath.exec(this.path)) { - await new Promise((resolve, reject) => - fs.mkdir(this.path, (err: any) => (!err ? resolve(null) : reject(err))), - ) + const parent = previousPath.exec(this.path) + if (parent) { + await fs.mkdir(parent[1], { recursive: true }) } - await new Promise((resolve, reject) => - fs.writeFile(this.path, this.writeData(data), (err: any) => - !err ? resolve(null) : reject(err), - ), - ) + await fs.writeFile(this.path, this.writeData(data)) } async read(effects: T.Effects) { - if (!fs.existsSync(this.path)) { + if ( + !(await fs.access(this.path).then( + () => true, + () => false, + )) + ) { return null } return this.readData( - await new Promise((resolve, reject) => - fs.readFile(this.path, (err: any, data: any) => - !err ? resolve(data.toString("utf-8")) : reject(err), - ), - ), + await fs.readFile(this.path).then((data) => data.toString("utf-8")), ) } @@ -142,7 +138,7 @@ export class FileHelper { return new FileHelper( path, (inData) => { - return JSON.stringify(inData, null, 2) + return YAML.stringify(inData, null, 2) }, (inString) => { return shape.unsafeCast(YAML.parse(inString)) diff --git a/sdk/package-lock.json b/sdk/package-lock.json index 0c457540f..a50066b34 100644 --- a/sdk/package-lock.json +++ b/sdk/package-lock.json @@ -10,7 +10,7 @@ "license": "MIT", "dependencies": { "isomorphic-fetch": "^3.0.0", - "lodash": "4.*.*", + "lodash": "^4.17.21", "ts-matches": "^5.4.1" }, "devDependencies": { diff --git a/sdk/package.json b/sdk/package.json index 53ee291ee..d6c3a1ca5 100644 --- a/sdk/package.json +++ b/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@start9labs/start-sdk", - "version": "0.3.6-alpha1", + "version": "0.3.6-alpha5", "description": "Software development kit to facilitate packaging services for StartOS", "main": "./cjs/lib/index.js", "types": "./cjs/lib/index.d.ts", @@ -31,8 +31,10 @@ "homepage": "https://github.com/Start9Labs/start-sdk#readme", "dependencies": { "isomorphic-fetch": "^3.0.0", - "lodash": "4.*.*", - "ts-matches": "^5.4.1" + "lodash": "^4.17.21", + "ts-matches": "^5.4.1", + "yaml": "^2.2.2", + "@iarna/toml": "^2.2.5" }, "prettier": { "trailingComma": "all", @@ -41,7 +43,6 @@ "singleQuote": false }, "devDependencies": { - "@iarna/toml": "^2.2.5", "@types/jest": "^29.4.0", "@types/lodash": "^4.17.5", "jest": "^29.4.3", @@ -49,7 +50,6 @@ "ts-jest": "^29.0.5", "ts-node": "^10.9.1", "tsx": "^4.7.1", - "typescript": "^5.0.4", - "yaml": "^2.2.2" + "typescript": "^5.0.4" } } diff --git a/web/projects/ui/src/app/services/api/api.fixures.ts b/web/projects/ui/src/app/services/api/api.fixures.ts index 11d56ff15..5642eece5 100644 --- a/web/projects/ui/src/app/services/api/api.fixures.ts +++ b/web/projects/ui/src/app/services/api/api.fixures.ts @@ -69,7 +69,13 @@ export module Mock { osVersion: '0.2.12', dependencies: {}, hasConfig: true, - images: ['main'], + images: { + main: { + source: 'packed', + arch: ['x86_64', 'aarch64'], + emulateMissingAs: 'aarch64', + }, + }, assets: [], volumes: ['main'], hardwareRequirements: { @@ -116,7 +122,13 @@ export module Mock { }, }, hasConfig: true, - images: ['main'], + images: { + main: { + source: 'packed', + arch: ['x86_64', 'aarch64'], + emulateMissingAs: 'aarch64', + }, + }, assets: [], volumes: ['main'], hardwareRequirements: { @@ -157,7 +169,13 @@ export module Mock { }, }, hasConfig: false, - images: ['main'], + images: { + main: { + source: 'packed', + arch: ['x86_64', 'aarch64'], + emulateMissingAs: 'aarch64', + }, + }, assets: [], volumes: ['main'], hardwareRequirements: { From bb514d621657333284c622a7bb66673328535a3c Mon Sep 17 00:00:00 2001 From: Jade <2364004+Blu-J@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:16:12 -0600 Subject: [PATCH 08/17] Chore/refactoring effects (#2644) * fix mac build * wip * chore: Update the effects to get rid of bad pattern * chore: Some small changes --------- Co-authored-by: Aiden McClelland --- .../src/Adapters/HostSystemStartOs.ts | 519 ++++++------ container-runtime/src/Adapters/RpcListener.ts | 2 +- .../Systems/SystemForEmbassy/MainLoop.ts | 11 +- .../Systems/SystemForEmbassy/index.ts | 72 +- .../SystemForEmbassy/polyfillEffects.ts | 770 +++++++++--------- .../src/Adapters/Systems/SystemForStartOs.ts | 7 +- .../src/Interfaces/HostSystem.ts | 5 +- container-runtime/src/Interfaces/System.ts | 4 +- container-runtime/src/index.ts | 4 +- download-firmware.sh | 3 +- 10 files changed, 704 insertions(+), 693 deletions(-) diff --git a/container-runtime/src/Adapters/HostSystemStartOs.ts b/container-runtime/src/Adapters/HostSystemStartOs.ts index 93905825a..1996af0fd 100644 --- a/container-runtime/src/Adapters/HostSystemStartOs.ts +++ b/container-runtime/src/Adapters/HostSystemStartOs.ts @@ -31,274 +31,273 @@ type RpcError = typeof matchRpcError._TYPE const SOCKET_PATH = "/media/startos/rpc/host.sock" const MAIN = "/main" as const -export class HostSystemStartOs implements Effects { - procedureId: string | null = null - - static of(callbackHolder: CallbackHolder) { - return new HostSystemStartOs(callbackHolder) - } - - constructor(readonly callbackHolder: CallbackHolder) {} - id = 0 - rpcRound( - method: K, - params: Record, - ) { - const id = this.id++ - const client = net.createConnection({ path: SOCKET_PATH }, () => { - client.write( - JSON.stringify({ - id, - method, - params: { ...params, procedureId: this.procedureId }, - }) + "\n", - ) - }) - let bufs: Buffer[] = [] - return new Promise((resolve, reject) => { - client.on("data", (data) => { - try { - bufs.push(data) - if (data.reduce((acc, x) => acc || x == 10, false)) { - const res: unknown = JSON.parse( - Buffer.concat(bufs).toString().split("\n")[0], - ) - if (testRpcError(res)) { - let message = res.error.message - console.error({ method, params, hostSystemStartOs: true }) - if (string.test(res.error.data)) { - message += ": " + res.error.data - console.error(res.error.data) +let hostSystemId = 0 +export const hostSystemStartOs = + (callbackHolder: CallbackHolder) => + (procedureId: null | string): Effects => { + const rpcRound = ( + method: K, + params: Record, + ) => { + const id = hostSystemId++ + const client = net.createConnection({ path: SOCKET_PATH }, () => { + client.write( + JSON.stringify({ + id, + method, + params: { ...params, procedureId: procedureId }, + }) + "\n", + ) + }) + let bufs: Buffer[] = [] + return new Promise((resolve, reject) => { + client.on("data", (data) => { + try { + bufs.push(data) + if (data.reduce((acc, x) => acc || x == 10, false)) { + const res: unknown = JSON.parse( + Buffer.concat(bufs).toString().split("\n")[0], + ) + if (testRpcError(res)) { + let message = res.error.message + console.error({ method, params, hostSystemStartOs: true }) + if (string.test(res.error.data)) { + message += ": " + res.error.data + console.error(res.error.data) + } else { + if (res.error.data?.details) { + message += ": " + res.error.data.details + console.error(res.error.data.details) + } + if (res.error.data?.debug) { + message += "\n" + res.error.data.debug + console.error("Debug: " + res.error.data.debug) + } + } + reject(new Error(`${message}@${method}`)) + } else if (testRpcResult(res)) { + resolve(res.result) } else { - if (res.error.data?.details) { - message += ": " + res.error.data.details - console.error(res.error.data.details) - } - if (res.error.data?.debug) { - message += "\n" + res.error.data.debug - console.error("Debug: " + res.error.data.debug) - } + reject(new Error(`malformed response ${JSON.stringify(res)}`)) } - reject(new Error(`${message}@${method}`)) - } else if (testRpcResult(res)) { - resolve(res.result) - } else { - reject(new Error(`malformed response ${JSON.stringify(res)}`)) } + } catch (error) { + reject(error) } - } catch (error) { + client.end() + }) + client.on("error", (error) => { reject(error) - } - client.end() + }) }) - client.on("error", (error) => { - reject(error) - }) - }) - } - - bind(...[options]: Parameters) { - return this.rpcRound("bind", { - ...options, - stack: new Error().stack, - }) as ReturnType - } - clearBindings(...[]: Parameters) { - return this.rpcRound("clearBindings", {}) as ReturnType< - T.Effects["clearBindings"] - > - } - clearServiceInterfaces( - ...[]: Parameters - ) { - return this.rpcRound("clearServiceInterfaces", {}) as ReturnType< - T.Effects["clearServiceInterfaces"] - > - } - createOverlayedImage(options: { - imageId: string - }): Promise<[string, string]> { - return this.rpcRound("createOverlayedImage", options) as ReturnType< - T.Effects["createOverlayedImage"] - > - } - destroyOverlayedImage(options: { guid: string }): Promise { - return this.rpcRound("destroyOverlayedImage", options) as ReturnType< - T.Effects["destroyOverlayedImage"] - > - } - executeAction(...[options]: Parameters) { - return this.rpcRound("executeAction", options) as ReturnType< - T.Effects["executeAction"] - > - } - exists(...[packageId]: Parameters) { - return this.rpcRound("exists", packageId) as ReturnType - } - exportAction(...[options]: Parameters) { - return this.rpcRound("exportAction", options) as ReturnType< - T.Effects["exportAction"] - > - } - exportServiceInterface: Effects["exportServiceInterface"] = ( - ...[options]: Parameters - ) => { - return this.rpcRound("exportServiceInterface", options) as ReturnType< - T.Effects["exportServiceInterface"] - > - } - exposeForDependents( - ...[options]: Parameters - ) { - return this.rpcRound("exposeForDependents", options) as ReturnType< - T.Effects["exposeForDependents"] - > - } - getConfigured(...[]: Parameters) { - return this.rpcRound("getConfigured", {}) as ReturnType< - T.Effects["getConfigured"] - > - } - getContainerIp(...[]: Parameters) { - return this.rpcRound("getContainerIp", {}) as ReturnType< - T.Effects["getContainerIp"] - > - } - getHostInfo: Effects["getHostInfo"] = (...[allOptions]: any[]) => { - const options = { - ...allOptions, - callback: this.callbackHolder.addCallback(allOptions.callback), } - return this.rpcRound("getHostInfo", options) as ReturnType< - T.Effects["getHostInfo"] - > as any - } - getServiceInterface( - ...[options]: Parameters - ) { - return this.rpcRound("getServiceInterface", { - ...options, - callback: this.callbackHolder.addCallback(options.callback), - }) as ReturnType - } + const self: Effects = { + bind(...[options]: Parameters) { + return rpcRound("bind", { + ...options, + stack: new Error().stack, + }) as ReturnType + }, + clearBindings(...[]: Parameters) { + return rpcRound("clearBindings", {}) as ReturnType< + T.Effects["clearBindings"] + > + }, + clearServiceInterfaces( + ...[]: Parameters + ) { + return rpcRound("clearServiceInterfaces", {}) as ReturnType< + T.Effects["clearServiceInterfaces"] + > + }, + createOverlayedImage(options: { + imageId: string + }): Promise<[string, string]> { + return rpcRound("createOverlayedImage", options) as ReturnType< + T.Effects["createOverlayedImage"] + > + }, + destroyOverlayedImage(options: { guid: string }): Promise { + return rpcRound("destroyOverlayedImage", options) as ReturnType< + T.Effects["destroyOverlayedImage"] + > + }, + executeAction(...[options]: Parameters) { + return rpcRound("executeAction", options) as ReturnType< + T.Effects["executeAction"] + > + }, + exists(...[packageId]: Parameters) { + return rpcRound("exists", packageId) as ReturnType + }, + exportAction(...[options]: Parameters) { + return rpcRound("exportAction", options) as ReturnType< + T.Effects["exportAction"] + > + }, + exportServiceInterface: (( + ...[options]: Parameters + ) => { + return rpcRound("exportServiceInterface", options) as ReturnType< + T.Effects["exportServiceInterface"] + > + }) as Effects["exportServiceInterface"], + exposeForDependents( + ...[options]: Parameters + ) { + return rpcRound("exposeForDependents", options) as ReturnType< + T.Effects["exposeForDependents"] + > + }, + getConfigured(...[]: Parameters) { + return rpcRound("getConfigured", {}) as ReturnType< + T.Effects["getConfigured"] + > + }, + getContainerIp(...[]: Parameters) { + return rpcRound("getContainerIp", {}) as ReturnType< + T.Effects["getContainerIp"] + > + }, + getHostInfo: ((...[allOptions]: any[]) => { + const options = { + ...allOptions, + callback: callbackHolder.addCallback(allOptions.callback), + } + return rpcRound("getHostInfo", options) as ReturnType< + T.Effects["getHostInfo"] + > as any + }) as Effects["getHostInfo"], + getServiceInterface( + ...[options]: Parameters + ) { + return rpcRound("getServiceInterface", { + ...options, + callback: callbackHolder.addCallback(options.callback), + }) as ReturnType + }, - getPrimaryUrl(...[options]: Parameters) { - return this.rpcRound("getPrimaryUrl", { - ...options, - callback: this.callbackHolder.addCallback(options.callback), - }) as ReturnType - } - getServicePortForward( - ...[options]: Parameters - ) { - return this.rpcRound("getServicePortForward", options) as ReturnType< - T.Effects["getServicePortForward"] - > - } - getSslCertificate(options: Parameters[0]) { - return this.rpcRound("getSslCertificate", options) as ReturnType< - T.Effects["getSslCertificate"] - > - } - getSslKey(options: Parameters[0]) { - return this.rpcRound("getSslKey", options) as ReturnType< - T.Effects["getSslKey"] - > - } - getSystemSmtp(...[options]: Parameters) { - return this.rpcRound("getSystemSmtp", { - ...options, - callback: this.callbackHolder.addCallback(options.callback), - }) as ReturnType - } - listServiceInterfaces( - ...[options]: Parameters - ) { - return this.rpcRound("listServiceInterfaces", { - ...options, - callback: this.callbackHolder.addCallback(options.callback), - }) as ReturnType - } - mount(...[options]: Parameters) { - return this.rpcRound("mount", options) as ReturnType - } - removeAction(...[options]: Parameters) { - return this.rpcRound("removeAction", options) as ReturnType< - T.Effects["removeAction"] - > - } - removeAddress(...[options]: Parameters) { - return this.rpcRound("removeAddress", options) as ReturnType< - T.Effects["removeAddress"] - > - } - restart(...[]: Parameters) { - return this.rpcRound("restart", {}) as ReturnType - } - running(...[packageId]: Parameters) { - return this.rpcRound("running", { packageId }) as ReturnType< - T.Effects["running"] - > - } - // runRsync(...[options]: Parameters) { - // - // return this.rpcRound('executeAction', options) as ReturnType - // - // return this.rpcRound('executeAction', options) as ReturnType - // } - setConfigured(...[configured]: Parameters) { - return this.rpcRound("setConfigured", { configured }) as ReturnType< - T.Effects["setConfigured"] - > - } - setDependencies( - dependencies: Parameters[0], - ): ReturnType { - return this.rpcRound("setDependencies", dependencies) as ReturnType< - T.Effects["setDependencies"] - > - } - checkDependencies( - options: Parameters[0], - ): ReturnType { - return this.rpcRound("checkDependencies", options) as ReturnType< - T.Effects["checkDependencies"] - > - } - getDependencies(): ReturnType { - return this.rpcRound("getDependencies", {}) as ReturnType< - T.Effects["getDependencies"] - > - } - setHealth(...[options]: Parameters) { - return this.rpcRound("setHealth", options) as ReturnType< - T.Effects["setHealth"] - > - } + getPrimaryUrl(...[options]: Parameters) { + return rpcRound("getPrimaryUrl", { + ...options, + callback: callbackHolder.addCallback(options.callback), + }) as ReturnType + }, + getServicePortForward( + ...[options]: Parameters + ) { + return rpcRound("getServicePortForward", options) as ReturnType< + T.Effects["getServicePortForward"] + > + }, + getSslCertificate( + options: Parameters[0], + ) { + return rpcRound("getSslCertificate", options) as ReturnType< + T.Effects["getSslCertificate"] + > + }, + getSslKey(options: Parameters[0]) { + return rpcRound("getSslKey", options) as ReturnType< + T.Effects["getSslKey"] + > + }, + getSystemSmtp(...[options]: Parameters) { + return rpcRound("getSystemSmtp", { + ...options, + callback: callbackHolder.addCallback(options.callback), + }) as ReturnType + }, + listServiceInterfaces( + ...[options]: Parameters + ) { + return rpcRound("listServiceInterfaces", { + ...options, + callback: callbackHolder.addCallback(options.callback), + }) as ReturnType + }, + mount(...[options]: Parameters) { + return rpcRound("mount", options) as ReturnType + }, + removeAction(...[options]: Parameters) { + return rpcRound("removeAction", options) as ReturnType< + T.Effects["removeAction"] + > + }, + removeAddress(...[options]: Parameters) { + return rpcRound("removeAddress", options) as ReturnType< + T.Effects["removeAddress"] + > + }, + restart(...[]: Parameters) { + return rpcRound("restart", {}) as ReturnType + }, + running(...[packageId]: Parameters) { + return rpcRound("running", { packageId }) as ReturnType< + T.Effects["running"] + > + }, + // runRsync(...[options]: Parameters) { + // + // return rpcRound('executeAction', options) as ReturnType + // + // return rpcRound('executeAction', options) as ReturnType + // } + setConfigured(...[configured]: Parameters) { + return rpcRound("setConfigured", { configured }) as ReturnType< + T.Effects["setConfigured"] + > + }, + setDependencies( + dependencies: Parameters[0], + ): ReturnType { + return rpcRound("setDependencies", dependencies) as ReturnType< + T.Effects["setDependencies"] + > + }, + checkDependencies( + options: Parameters[0], + ): ReturnType { + return rpcRound("checkDependencies", options) as ReturnType< + T.Effects["checkDependencies"] + > + }, + getDependencies(): ReturnType { + return rpcRound("getDependencies", {}) as ReturnType< + T.Effects["getDependencies"] + > + }, + setHealth(...[options]: Parameters) { + return rpcRound("setHealth", options) as ReturnType< + T.Effects["setHealth"] + > + }, - setMainStatus(o: { status: "running" | "stopped" }): Promise { - return this.rpcRound("setMainStatus", o) as ReturnType< - T.Effects["setHealth"] - > - } + setMainStatus(o: { status: "running" | "stopped" }): Promise { + return rpcRound("setMainStatus", o) as ReturnType< + T.Effects["setHealth"] + > + }, - shutdown(...[]: Parameters) { - return this.rpcRound("shutdown", {}) as ReturnType + shutdown(...[]: Parameters) { + return rpcRound("shutdown", {}) as ReturnType + }, + stopped(...[packageId]: Parameters) { + return rpcRound("stopped", { packageId }) as ReturnType< + T.Effects["stopped"] + > + }, + store: { + get: async (options: any) => + rpcRound("getStore", { + ...options, + callback: callbackHolder.addCallback(options.callback), + }) as any, + set: async (options: any) => + rpcRound("setStore", options) as ReturnType< + T.Effects["store"]["set"] + >, + } as T.Effects["store"], + } + return self } - stopped(...[packageId]: Parameters) { - return this.rpcRound("stopped", { packageId }) as ReturnType< - T.Effects["stopped"] - > - } - store: T.Effects["store"] = { - get: async (options: any) => - this.rpcRound("getStore", { - ...options, - callback: this.callbackHolder.addCallback(options.callback), - }) as any, - set: async (options: any) => - this.rpcRound("setStore", options) as ReturnType< - T.Effects["store"]["set"] - >, - } -} diff --git a/container-runtime/src/Adapters/RpcListener.ts b/container-runtime/src/Adapters/RpcListener.ts index 5391d943e..04e9bc40f 100644 --- a/container-runtime/src/Adapters/RpcListener.ts +++ b/container-runtime/src/Adapters/RpcListener.ts @@ -247,7 +247,7 @@ export class RpcListener { })), ) .when(exitType, async ({ id }) => { - if (this._system) await this._system.exit(this.effects) + if (this._system) await this._system.exit(this.effects(null)) delete this._system delete this._effects diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts index 08bf944ed..975bb52ca 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/MainLoop.ts @@ -1,9 +1,10 @@ -import { PolyfillEffects } from "./polyfillEffects" +import { polyfillEffects } from "./polyfillEffects" import { DockerProcedureContainer } from "./DockerProcedureContainer" import { SystemForEmbassy } from "." -import { HostSystemStartOs } from "../../HostSystemStartOs" +import { hostSystemStartOs } from "../../HostSystemStartOs" import { Daemons, T, daemons } from "@start9labs/start-sdk" import { Daemon } from "@start9labs/start-sdk/cjs/lib/mainFn/Daemon" +import { Effects } from "../../../Models/Effects" const EMBASSY_HEALTH_INTERVAL = 15 * 1000 const EMBASSY_PROPERTIES_LOOP = 30 * 1000 @@ -27,7 +28,7 @@ export class MainLoop { | undefined constructor( readonly system: SystemForEmbassy, - readonly effects: HostSystemStartOs, + readonly effects: Effects, ) { this.healthLoops = this.constructHealthLoops() this.mainEvent = this.constructMainEvent() @@ -65,7 +66,7 @@ export class MainLoop { } } - private async setupInterfaces(effects: HostSystemStartOs) { + private async setupInterfaces(effects: T.Effects) { for (const interfaceId in this.system.manifest.interfaces) { const iface = this.system.manifest.interfaces[interfaceId] const internalPorts = new Set() @@ -211,7 +212,7 @@ export class MainLoop { } const result = await method( - new PolyfillEffects(effects, this.system.manifest), + polyfillEffects(effects, this.system.manifest), timeChanged, ) diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index 731b38903..127fb0e09 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -1,7 +1,7 @@ import { types as T, utils, EmVer } from "@start9labs/start-sdk" import * as fs from "fs/promises" -import { PolyfillEffects } from "./polyfillEffects" +import { polyfillEffects } from "./polyfillEffects" import { Duration, duration, fromDuration } from "../../../Models/Duration" import { System } from "../../../Interfaces/System" import { matchManifest, Manifest, Procedure } from "./matchManifest" @@ -27,7 +27,7 @@ import { Parser, array, } from "ts-matches" -import { HostSystemStartOs } from "../../HostSystemStartOs" +import { hostSystemStartOs } from "../../HostSystemStartOs" import { JsonPath, unNestPath } from "../../../Models/JsonPath" import { RpcResult, matchRpcResult } from "../../RpcListener" import { CT } from "@start9labs/start-sdk" @@ -41,6 +41,7 @@ import { MultiHost, } from "@start9labs/start-sdk/cjs/lib/interfaces/Host" import { ServiceInterfaceBuilder } from "@start9labs/start-sdk/cjs/lib/interfaces/ServiceInterfaceBuilder" +import { Effects } from "../../../Models/Effects" type Optional = A | undefined | null function todo(): never { @@ -197,7 +198,7 @@ export class SystemForEmbassy implements System { readonly moduleCode: Partial, ) {} async execute( - effects: HostSystemStartOs, + effectCreator: ReturnType, options: { id: string procedure: JsonPath @@ -205,8 +206,7 @@ export class SystemForEmbassy implements System { timeout?: number | undefined }, ): Promise { - effects = Object.create(effects) - effects.procedureId = options.id + const effects = effectCreator(options.id) return this._execute(effects, options) .then((x) => matches(x) @@ -261,12 +261,12 @@ export class SystemForEmbassy implements System { } }) } - async exit(effects: HostSystemStartOs): Promise { + async exit(): Promise { if (this.currentRunning) await this.currentRunning.clean() delete this.currentRunning } async _execute( - effects: HostSystemStartOs, + effects: Effects, options: { procedure: JsonPath input: unknown @@ -340,7 +340,7 @@ export class SystemForEmbassy implements System { throw new Error(`Could not find the path for ${options.procedure}`) } private async init( - effects: HostSystemStartOs, + effects: Effects, previousVersion: Optional, timeoutMs: number | null, ): Promise { @@ -350,7 +350,7 @@ export class SystemForEmbassy implements System { await this.exportActions(effects) await this.exportNetwork(effects) } - async exportNetwork(effects: HostSystemStartOs) { + async exportNetwork(effects: Effects) { for (const [id, interfaceValue] of Object.entries( this.manifest.interfaces, )) { @@ -428,7 +428,7 @@ export class SystemForEmbassy implements System { ) } } - async exportActions(effects: HostSystemStartOs) { + async exportActions(effects: Effects) { const manifest = this.manifest if (!manifest.actions) return for (const [actionId, action] of Object.entries(manifest.actions)) { @@ -457,7 +457,7 @@ export class SystemForEmbassy implements System { } } private async uninit( - effects: HostSystemStartOs, + effects: Effects, nextVersion: Optional, timeoutMs: number | null, ): Promise { @@ -465,7 +465,7 @@ export class SystemForEmbassy implements System { await effects.setMainStatus({ status: "stopped" }) } private async mainStart( - effects: HostSystemStartOs, + effects: Effects, timeoutMs: number | null, ): Promise { if (!!this.currentRunning) return @@ -473,7 +473,7 @@ export class SystemForEmbassy implements System { this.currentRunning = new MainLoop(this, effects) } private async mainStop( - effects: HostSystemStartOs, + effects: Effects, timeoutMs: number | null, ): Promise { const { currentRunning } = this @@ -491,7 +491,7 @@ export class SystemForEmbassy implements System { return durationValue } private async createBackup( - effects: HostSystemStartOs, + effects: Effects, timeoutMs: number | null, ): Promise { const backup = this.manifest.backup.create @@ -503,13 +503,11 @@ export class SystemForEmbassy implements System { await container.execFail([backup.entrypoint, ...backup.args], timeoutMs) } else { const moduleCode = await this.moduleCode - await moduleCode.createBackup?.( - new PolyfillEffects(effects, this.manifest), - ) + await moduleCode.createBackup?.(polyfillEffects(effects, this.manifest)) } } private async restoreBackup( - effects: HostSystemStartOs, + effects: Effects, timeoutMs: number | null, ): Promise { const restoreBackup = this.manifest.backup.restore @@ -528,19 +526,17 @@ export class SystemForEmbassy implements System { ) } else { const moduleCode = await this.moduleCode - await moduleCode.restoreBackup?.( - new PolyfillEffects(effects, this.manifest), - ) + await moduleCode.restoreBackup?.(polyfillEffects(effects, this.manifest)) } } private async getConfig( - effects: HostSystemStartOs, + effects: Effects, timeoutMs: number | null, ): Promise { return this.getConfigUncleaned(effects, timeoutMs).then(removePointers) } private async getConfigUncleaned( - effects: HostSystemStartOs, + effects: Effects, timeoutMs: number | null, ): Promise { const config = this.manifest.config?.get @@ -564,7 +560,7 @@ export class SystemForEmbassy implements System { const moduleCode = await this.moduleCode const method = moduleCode.getConfig if (!method) throw new Error("Expecting that the method getConfig exists") - return (await method(new PolyfillEffects(effects, this.manifest)).then( + return (await method(polyfillEffects(effects, this.manifest)).then( (x) => { if ("result" in x) return x.result if ("error" in x) throw new Error("Error getting config: " + x.error) @@ -574,7 +570,7 @@ export class SystemForEmbassy implements System { } } private async setConfig( - effects: HostSystemStartOs, + effects: Effects, newConfigWithoutPointers: unknown, timeoutMs: number | null, ): Promise { @@ -617,7 +613,7 @@ export class SystemForEmbassy implements System { const answer = matchSetResult.unsafeCast( await method( - new PolyfillEffects(effects, this.manifest), + polyfillEffects(effects, this.manifest), newConfig as U.Config, ).then((x): T.SetResult => { if ("result" in x) @@ -636,7 +632,7 @@ export class SystemForEmbassy implements System { } } private async setConfigSetConfig( - effects: HostSystemStartOs, + effects: Effects, dependsOn: { [x: string]: readonly string[] }, ) { await effects.setDependencies({ @@ -660,7 +656,7 @@ export class SystemForEmbassy implements System { } private async migration( - effects: HostSystemStartOs, + effects: Effects, fromVersion: string, timeoutMs: number | null, ): Promise { @@ -713,7 +709,7 @@ export class SystemForEmbassy implements System { if (!method) throw new Error("Expecting that the method migration exists") return (await method( - new PolyfillEffects(effects, this.manifest), + polyfillEffects(effects, this.manifest), fromVersion as string, ).then((x) => { if ("result" in x) return x.result @@ -725,7 +721,7 @@ export class SystemForEmbassy implements System { return { configured: true } } private async properties( - effects: HostSystemStartOs, + effects: Effects, timeoutMs: number | null, ): Promise> { // TODO BLU-J set the properties ever so often @@ -754,7 +750,7 @@ export class SystemForEmbassy implements System { if (!method) throw new Error("Expecting that the method properties exists") const properties = matchProperties.unsafeCast( - await method(new PolyfillEffects(effects, this.manifest)).then((x) => { + await method(polyfillEffects(effects, this.manifest)).then((x) => { if ("result" in x) return x.result if ("error" in x) throw new Error("Error getting config: " + x.error) throw new Error("Error getting config: " + x["error-code"][1]) @@ -765,7 +761,7 @@ export class SystemForEmbassy implements System { throw new Error(`Unknown type in the fetch properties: ${setConfigValue}`) } private async action( - effects: HostSystemStartOs, + effects: Effects, actionId: string, formData: unknown, timeoutMs: number | null, @@ -795,7 +791,7 @@ export class SystemForEmbassy implements System { const method = moduleCode.action?.[actionId] if (!method) throw new Error("Expecting that the method action exists") return (await method( - new PolyfillEffects(effects, this.manifest), + polyfillEffects(effects, this.manifest), formData as any, ).then((x) => { if ("result" in x) return x.result @@ -805,7 +801,7 @@ export class SystemForEmbassy implements System { } } private async dependenciesCheck( - effects: HostSystemStartOs, + effects: Effects, id: string, oldConfig: unknown, timeoutMs: number | null, @@ -838,7 +834,7 @@ export class SystemForEmbassy implements System { `Expecting that the method dependency check ${id} exists`, ) return (await method( - new PolyfillEffects(effects, this.manifest), + polyfillEffects(effects, this.manifest), oldConfig as any, ).then((x) => { if ("result" in x) return x.result @@ -850,7 +846,7 @@ export class SystemForEmbassy implements System { } } private async dependenciesAutoconfig( - effects: HostSystemStartOs, + effects: Effects, id: string, oldConfig: unknown, timeoutMs: number | null, @@ -863,7 +859,7 @@ export class SystemForEmbassy implements System { `Expecting that the method dependency autoConfigure ${id} exists`, ) return (await method( - new PolyfillEffects(effects, this.manifest), + polyfillEffects(effects, this.manifest), oldConfig as any, ).then((x) => { if ("result" in x) return x.result @@ -961,7 +957,7 @@ function cleanConfigFromPointers( } async function updateConfig( - effects: HostSystemStartOs, + effects: Effects, manifest: Manifest, spec: unknown, mutConfigValue: unknown, diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts index 3a8d5f624..2b7363cbf 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts @@ -4,390 +4,175 @@ import { Volume } from "../../../Models/Volume" import * as child_process from "child_process" import { promisify } from "util" import { daemons, startSdk, T } from "@start9labs/start-sdk" -import { HostSystemStartOs } from "../../HostSystemStartOs" import "isomorphic-fetch" import { Manifest } from "./matchManifest" import { DockerProcedureContainer } from "./DockerProcedureContainer" import * as cp from "child_process" +import { Effects } from "../../../Models/Effects" export const execFile = promisify(cp.execFile) -export class PolyfillEffects implements oet.Effects { - constructor( - readonly effects: HostSystemStartOs, - private manifest: Manifest, - ) {} - async writeFile(input: { - path: string - volumeId: string - toWrite: string - }): Promise { - await fs.writeFile( - new Volume(input.volumeId, input.path).path, - input.toWrite, - ) - } - async readFile(input: { volumeId: string; path: string }): Promise { - return ( - await fs.readFile(new Volume(input.volumeId, input.path).path) - ).toString() - } - async metadata(input: { - volumeId: string - path: string - }): Promise { - const stats = await fs.stat(new Volume(input.volumeId, input.path).path) - return { - fileType: stats.isFile() ? "file" : "directory", - gid: stats.gid, - uid: stats.uid, - mode: stats.mode, - isDir: stats.isDirectory(), - isFile: stats.isFile(), - isSymlink: stats.isSymbolicLink(), - len: stats.size, - readonly: (stats.mode & 0o200) > 0, - } - } - async createDir(input: { volumeId: string; path: string }): Promise { - const path = new Volume(input.volumeId, input.path).path - await fs.mkdir(path, { recursive: true }) - return path - } - async readDir(input: { volumeId: string; path: string }): Promise { - return fs.readdir(new Volume(input.volumeId, input.path).path) - } - async removeDir(input: { volumeId: string; path: string }): Promise { - const path = new Volume(input.volumeId, input.path).path - await fs.rmdir(new Volume(input.volumeId, input.path).path, { - recursive: true, - }) - return path - } - removeFile(input: { volumeId: string; path: string }): Promise { - return fs.rm(new Volume(input.volumeId, input.path).path) - } - async writeJsonFile(input: { - volumeId: string - path: string - toWrite: Record - }): Promise { - await fs.writeFile( - new Volume(input.volumeId, input.path).path, - JSON.stringify(input.toWrite), - ) - } - async readJsonFile(input: { - volumeId: string - path: string - }): Promise> { - return JSON.parse( - ( +export const polyfillEffects = ( + effects: Effects, + manifest: Manifest, +): oet.Effects => { + const self = { + effects, + manifest, + async writeFile(input: { + path: string + volumeId: string + toWrite: string + }): Promise { + await fs.writeFile( + new Volume(input.volumeId, input.path).path, + input.toWrite, + ) + }, + async readFile(input: { volumeId: string; path: string }): Promise { + return ( await fs.readFile(new Volume(input.volumeId, input.path).path) - ).toString(), - ) - } - runCommand({ - command, - args, - timeoutMillis, - }: { - command: string - args?: string[] | undefined - timeoutMillis?: number | undefined - }): Promise> { - return startSdk - .runCommand( - this.effects, - { id: this.manifest.main.image }, - [command, ...(args || [])], - {}, + ).toString() + }, + async metadata(input: { + volumeId: string + path: string + }): Promise { + const stats = await fs.stat(new Volume(input.volumeId, input.path).path) + return { + fileType: stats.isFile() ? "file" : "directory", + gid: stats.gid, + uid: stats.uid, + mode: stats.mode, + isDir: stats.isDirectory(), + isFile: stats.isFile(), + isSymlink: stats.isSymbolicLink(), + len: stats.size, + readonly: (stats.mode & 0o200) > 0, + } + }, + async createDir(input: { + volumeId: string + path: string + }): Promise { + const path = new Volume(input.volumeId, input.path).path + await fs.mkdir(path, { recursive: true }) + return path + }, + async readDir(input: { + volumeId: string + path: string + }): Promise { + return fs.readdir(new Volume(input.volumeId, input.path).path) + }, + async removeDir(input: { + volumeId: string + path: string + }): Promise { + const path = new Volume(input.volumeId, input.path).path + await fs.rmdir(new Volume(input.volumeId, input.path).path, { + recursive: true, + }) + return path + }, + removeFile(input: { volumeId: string; path: string }): Promise { + return fs.rm(new Volume(input.volumeId, input.path).path) + }, + async writeJsonFile(input: { + volumeId: string + path: string + toWrite: Record + }): Promise { + await fs.writeFile( + new Volume(input.volumeId, input.path).path, + JSON.stringify(input.toWrite), ) - .then((x: any) => ({ - stderr: x.stderr.toString(), - stdout: x.stdout.toString(), - })) - .then((x: any) => - !!x.stderr ? { error: x.stderr } : { result: x.stdout }, + }, + async readJsonFile(input: { + volumeId: string + path: string + }): Promise> { + return JSON.parse( + ( + await fs.readFile(new Volume(input.volumeId, input.path).path) + ).toString(), ) - } - runDaemon(input: { command: string; args?: string[] | undefined }): { - wait(): Promise> - term(): Promise - } { - const dockerProcedureContainer = DockerProcedureContainer.of( - this.effects, - this.manifest.main, - this.manifest.volumes, - ) - const daemon = dockerProcedureContainer.then((dockerProcedureContainer) => - daemons.runCommand()( - this.effects, - { id: this.manifest.main.image }, - [input.command, ...(input.args || [])], - { - overlay: dockerProcedureContainer.overlay, - }, - ), - ) - return { - wait: () => - daemon.then((daemon) => - daemon.wait().then(() => { - return { result: "" } - }), + }, + runCommand({ + command, + args, + timeoutMillis, + }: { + command: string + args?: string[] | undefined + timeoutMillis?: number | undefined + }): Promise> { + return startSdk + .runCommand( + effects, + { id: manifest.main.image }, + [command, ...(args || [])], + {}, + ) + .then((x: any) => ({ + stderr: x.stderr.toString(), + stdout: x.stdout.toString(), + })) + .then((x: any) => + !!x.stderr ? { error: x.stderr } : { result: x.stdout }, + ) + }, + runDaemon(input: { command: string; args?: string[] | undefined }): { + wait(): Promise> + term(): Promise + } { + const dockerProcedureContainer = DockerProcedureContainer.of( + effects, + manifest.main, + manifest.volumes, + ) + const daemon = dockerProcedureContainer.then((dockerProcedureContainer) => + daemons.runCommand()( + effects, + { id: manifest.main.image }, + [input.command, ...(input.args || [])], + { + overlay: dockerProcedureContainer.overlay, + }, ), - term: () => daemon.then((daemon) => daemon.term()), - } - } - async chown(input: { - volumeId: string - path: string - uid: string - }): Promise { - await startSdk - .runCommand( - this.effects, - { id: this.manifest.main.image }, - ["chown", "--recursive", input.uid, `/drive/${input.path}`], - { - mounts: [ - { - path: "/drive", - options: { - type: "volume", - id: input.volumeId, - subpath: null, - readonly: false, - }, - }, - ], - }, ) - .then((x: any) => ({ - stderr: x.stderr.toString(), - stdout: x.stdout.toString(), - })) - .then((x: any) => { - if (!!x.stderr) { - throw new Error(x.stderr) - } - }) - return null - } - async chmod(input: { - volumeId: string - path: string - mode: string - }): Promise { - await startSdk - .runCommand( - this.effects, - { id: this.manifest.main.image }, - ["chmod", "--recursive", input.mode, `/drive/${input.path}`], - { - mounts: [ - { - path: "/drive", - options: { - type: "volume", - id: input.volumeId, - subpath: null, - readonly: false, + return { + wait: () => + daemon.then((daemon) => + daemon.wait().then(() => { + return { result: "" } + }), + ), + term: () => daemon.then((daemon) => daemon.term()), + } + }, + async chown(input: { + volumeId: string + path: string + uid: string + }): Promise { + await startSdk + .runCommand( + effects, + { id: manifest.main.image }, + ["chown", "--recursive", input.uid, `/drive/${input.path}`], + { + mounts: [ + { + path: "/drive", + options: { + type: "volume", + id: input.volumeId, + subpath: null, + readonly: false, + }, }, - }, - ], - }, - ) - .then((x: any) => ({ - stderr: x.stderr.toString(), - stdout: x.stdout.toString(), - })) - .then((x: any) => { - if (!!x.stderr) { - throw new Error(x.stderr) - } - }) - return null - } - sleep(timeMs: number): Promise { - return new Promise((resolve) => setTimeout(resolve, timeMs)) - } - trace(whatToPrint: string): void { - console.trace(whatToPrint) - } - warn(whatToPrint: string): void { - console.warn(whatToPrint) - } - error(whatToPrint: string): void { - console.error(whatToPrint) - } - debug(whatToPrint: string): void { - console.debug(whatToPrint) - } - info(whatToPrint: string): void { - console.log(false) - } - is_sandboxed(): boolean { - return false - } - exists(input: { volumeId: string; path: string }): Promise { - return this.metadata(input) - .then(() => true) - .catch(() => false) - } - async fetch( - url: string, - options?: - | { - method?: - | "GET" - | "POST" - | "PUT" - | "DELETE" - | "HEAD" - | "PATCH" - | undefined - headers?: Record | undefined - body?: string | undefined - } - | undefined, - ): Promise<{ - method: string - ok: boolean - status: number - headers: Record - body?: string | null | undefined - text(): Promise - json(): Promise - }> { - const fetched = await fetch(url, options) - return { - method: fetched.type, - ok: fetched.ok, - status: fetched.status, - headers: Object.fromEntries(fetched.headers.entries()), - body: await fetched.text(), - text: () => fetched.text(), - json: () => fetched.json(), - } - } - - runRsync(rsyncOptions: { - srcVolume: string - dstVolume: string - srcPath: string - dstPath: string - options: oet.BackupOptions - }): { - id: () => Promise - wait: () => Promise - progress: () => Promise - } { - let secondRun: ReturnType | undefined - let firstRun = this._runRsync(rsyncOptions) - let waitValue = firstRun.wait().then((x) => { - secondRun = this._runRsync(rsyncOptions) - return secondRun.wait() - }) - const id = async () => { - return secondRun?.id?.() ?? firstRun.id() - } - const wait = () => waitValue - const progress = async () => { - const secondProgress = secondRun?.progress?.() - if (secondProgress) { - return (await secondProgress) / 2.0 + 0.5 - } - return (await firstRun.progress()) / 2.0 - } - return { id, wait, progress } - } - _runRsync(rsyncOptions: { - srcVolume: string - dstVolume: string - srcPath: string - dstPath: string - options: oet.BackupOptions - }): { - id: () => Promise - wait: () => Promise - progress: () => Promise - } { - const { srcVolume, dstVolume, srcPath, dstPath, options } = rsyncOptions - const command = "rsync" - const args: string[] = [] - if (options.delete) { - args.push("--delete") - } - if (options.force) { - args.push("--force") - } - if (options.ignoreExisting) { - args.push("--ignore-existing") - } - for (const exclude of options.exclude) { - args.push(`--exclude=${exclude}`) - } - args.push("-actAXH") - args.push("--info=progress2") - args.push("--no-inc-recursive") - args.push(new Volume(srcVolume, srcPath).path) - args.push(new Volume(dstVolume, dstPath).path) - const spawned = child_process.spawn(command, args, { detached: true }) - let percentage = 0.0 - spawned.stdout.on("data", (data: unknown) => { - const lines = String(data).replace("\r", "\n").split("\n") - for (const line of lines) { - const parsed = /$([0-9.]+)%/.exec(line)?.[1] - if (!parsed) continue - percentage = Number.parseFloat(parsed) - } - }) - - spawned.stderr.on("data", (data: unknown) => { - console.error(String(data)) - }) - - const id = async () => { - const pid = spawned.pid - if (pid === undefined) { - throw new Error("rsync process has no pid") - } - return String(pid) - } - const waitPromise = new Promise((resolve, reject) => { - spawned.on("exit", (code: any) => { - if (code === 0) { - resolve(null) - } else { - reject(new Error(`rsync exited with code ${code}`)) - } - }) - }) - const wait = () => waitPromise - const progress = () => Promise.resolve(percentage) - return { id, wait, progress } - } - async diskUsage( - options?: { volumeId: string; path: string } | undefined, - ): Promise<{ used: number; total: number }> { - const output = await execFile("df", ["--block-size=1", "-P", "/"]) - .then((x: any) => ({ - stderr: x.stderr.toString(), - stdout: x.stdout.toString(), - })) - .then((x: any) => { - if (!!x.stderr) { - throw new Error(x.stderr) - } - return parseDfOutput(x.stdout) - }) - if (!!options) { - const used = await execFile("du", [ - "-s", - "--block-size=1", - "-P", - new Volume(options.volumeId, options.path).path, - ]) + ], + }, + ) .then((x: any) => ({ stderr: x.stderr.toString(), stdout: x.stdout.toString(), @@ -396,15 +181,244 @@ export class PolyfillEffects implements oet.Effects { if (!!x.stderr) { throw new Error(x.stderr) } - return Number.parseInt(x.stdout.split(/\s+/)[0]) }) + return null + }, + async chmod(input: { + volumeId: string + path: string + mode: string + }): Promise { + await startSdk + .runCommand( + effects, + { id: manifest.main.image }, + ["chmod", "--recursive", input.mode, `/drive/${input.path}`], + { + mounts: [ + { + path: "/drive", + options: { + type: "volume", + id: input.volumeId, + subpath: null, + readonly: false, + }, + }, + ], + }, + ) + .then((x: any) => ({ + stderr: x.stderr.toString(), + stdout: x.stdout.toString(), + })) + .then((x: any) => { + if (!!x.stderr) { + throw new Error(x.stderr) + } + }) + return null + }, + sleep(timeMs: number): Promise { + return new Promise((resolve) => setTimeout(resolve, timeMs)) + }, + trace(whatToPrint: string): void { + console.trace(whatToPrint) + }, + warn(whatToPrint: string): void { + console.warn(whatToPrint) + }, + error(whatToPrint: string): void { + console.error(whatToPrint) + }, + debug(whatToPrint: string): void { + console.debug(whatToPrint) + }, + info(whatToPrint: string): void { + console.log(false) + }, + is_sandboxed(): boolean { + return false + }, + exists(input: { volumeId: string; path: string }): Promise { + return self + .metadata(input) + .then(() => true) + .catch(() => false) + }, + async fetch( + url: string, + options?: + | { + method?: + | "GET" + | "POST" + | "PUT" + | "DELETE" + | "HEAD" + | "PATCH" + | undefined + headers?: Record | undefined + body?: string | undefined + } + | undefined, + ): Promise<{ + method: string + ok: boolean + status: number + headers: Record + body?: string | null | undefined + text(): Promise + json(): Promise + }> { + const fetched = await fetch(url, options) return { - ...output, - used, + method: fetched.type, + ok: fetched.ok, + status: fetched.status, + headers: Object.fromEntries(fetched.headers.entries()), + body: await fetched.text(), + text: () => fetched.text(), + json: () => fetched.json(), } - } - return output + }, + + runRsync(rsyncOptions: { + srcVolume: string + dstVolume: string + srcPath: string + dstPath: string + options: oet.BackupOptions + }): { + id: () => Promise + wait: () => Promise + progress: () => Promise + } { + let secondRun: ReturnType | undefined + let firstRun = self._runRsync(rsyncOptions) + let waitValue = firstRun.wait().then((x) => { + secondRun = self._runRsync(rsyncOptions) + return secondRun.wait() + }) + const id = async () => { + return secondRun?.id?.() ?? firstRun.id() + } + const wait = () => waitValue + const progress = async () => { + const secondProgress = secondRun?.progress?.() + if (secondProgress) { + return (await secondProgress) / 2.0 + 0.5 + } + return (await firstRun.progress()) / 2.0 + } + return { id, wait, progress } + }, + _runRsync(rsyncOptions: { + srcVolume: string + dstVolume: string + srcPath: string + dstPath: string + options: oet.BackupOptions + }): { + id: () => Promise + wait: () => Promise + progress: () => Promise + } { + const { srcVolume, dstVolume, srcPath, dstPath, options } = rsyncOptions + const command = "rsync" + const args: string[] = [] + if (options.delete) { + args.push("--delete") + } + if (options.force) { + args.push("--force") + } + if (options.ignoreExisting) { + args.push("--ignore-existing") + } + for (const exclude of options.exclude) { + args.push(`--exclude=${exclude}`) + } + args.push("-actAXH") + args.push("--info=progress2") + args.push("--no-inc-recursive") + args.push(new Volume(srcVolume, srcPath).path) + args.push(new Volume(dstVolume, dstPath).path) + const spawned = child_process.spawn(command, args, { detached: true }) + let percentage = 0.0 + spawned.stdout.on("data", (data: unknown) => { + const lines = String(data).replace("\r", "\n").split("\n") + for (const line of lines) { + const parsed = /$([0-9.]+)%/.exec(line)?.[1] + if (!parsed) continue + percentage = Number.parseFloat(parsed) + } + }) + + spawned.stderr.on("data", (data: unknown) => { + console.error(String(data)) + }) + + const id = async () => { + const pid = spawned.pid + if (pid === undefined) { + throw new Error("rsync process has no pid") + } + return String(pid) + } + const waitPromise = new Promise((resolve, reject) => { + spawned.on("exit", (code: any) => { + if (code === 0) { + resolve(null) + } else { + reject(new Error(`rsync exited with code ${code}`)) + } + }) + }) + const wait = () => waitPromise + const progress = () => Promise.resolve(percentage) + return { id, wait, progress } + }, + async diskUsage( + options?: { volumeId: string; path: string } | undefined, + ): Promise<{ used: number; total: number }> { + const output = await execFile("df", ["--block-size=1", "-P", "/"]) + .then((x: any) => ({ + stderr: x.stderr.toString(), + stdout: x.stdout.toString(), + })) + .then((x: any) => { + if (!!x.stderr) { + throw new Error(x.stderr) + } + return parseDfOutput(x.stdout) + }) + if (!!options) { + const used = await execFile("du", [ + "-s", + "--block-size=1", + "-P", + new Volume(options.volumeId, options.path).path, + ]) + .then((x: any) => ({ + stderr: x.stderr.toString(), + stdout: x.stdout.toString(), + })) + .then((x: any) => { + if (!!x.stderr) { + throw new Error(x.stderr) + } + return Number.parseInt(x.stdout.split(/\s+/)[0]) + }) + return { + ...output, + used, + } + } + return output + }, } + return self } function parseDfOutput(output: string): { used: number; total: number } { diff --git a/container-runtime/src/Adapters/Systems/SystemForStartOs.ts b/container-runtime/src/Adapters/Systems/SystemForStartOs.ts index bf27f222d..e10434032 100644 --- a/container-runtime/src/Adapters/Systems/SystemForStartOs.ts +++ b/container-runtime/src/Adapters/Systems/SystemForStartOs.ts @@ -1,7 +1,7 @@ import { ExecuteResult, System } from "../../Interfaces/System" import { unNestPath } from "../../Models/JsonPath" import matches, { any, number, object, string, tuple } from "ts-matches" -import { HostSystemStartOs } from "../HostSystemStartOs" +import { hostSystemStartOs } from "../HostSystemStartOs" import { Effects } from "../../Models/Effects" import { RpcResult, matchRpcResult } from "../RpcListener" import { duration } from "../../Models/Duration" @@ -15,7 +15,7 @@ export class SystemForStartOs implements System { } constructor(readonly abi: T.ABI) {} async execute( - effects: HostSystemStartOs, + effectCreator: ReturnType, options: { id: string procedure: @@ -36,8 +36,7 @@ export class SystemForStartOs implements System { timeout?: number | undefined }, ): Promise { - effects = Object.create(effects) - effects.procedureId = options.id + const effects = effectCreator(options.id) return this._execute(effects, options) .then((x) => matches(x) diff --git a/container-runtime/src/Interfaces/HostSystem.ts b/container-runtime/src/Interfaces/HostSystem.ts index 4e04bbcc8..4ba986e3b 100644 --- a/container-runtime/src/Interfaces/HostSystem.ts +++ b/container-runtime/src/Interfaces/HostSystem.ts @@ -2,6 +2,7 @@ import { types as T } from "@start9labs/start-sdk" import { CallbackHolder } from "../Models/CallbackHolder" import { Effects } from "../Models/Effects" - export type HostSystem = Effects -export type GetHostSystem = (callbackHolder: CallbackHolder) => HostSystem +export type GetHostSystem = ( + callbackHolder: CallbackHolder, +) => (procedureId: null | string) => Effects diff --git a/container-runtime/src/Interfaces/System.ts b/container-runtime/src/Interfaces/System.ts index 85ba0fb0f..986288411 100644 --- a/container-runtime/src/Interfaces/System.ts +++ b/container-runtime/src/Interfaces/System.ts @@ -1,7 +1,7 @@ import { types as T } from "@start9labs/start-sdk" import { JsonPath } from "../Models/JsonPath" -import { HostSystemStartOs } from "../Adapters/HostSystemStartOs" import { RpcResult } from "../Adapters/RpcListener" +import { hostSystemStartOs } from "../Adapters/HostSystemStartOs" export type ExecuteResult = | { ok: unknown } | { err: { code: number; message: string } } @@ -12,7 +12,7 @@ export interface System { // stop(effects: Effects, options: { timeout: number, signal?: number }): Promise execute( - effects: T.Effects, + effectCreator: ReturnType, options: { id: string procedure: JsonPath diff --git a/container-runtime/src/index.ts b/container-runtime/src/index.ts index d86111ecb..74be5b73a 100644 --- a/container-runtime/src/index.ts +++ b/container-runtime/src/index.ts @@ -1,12 +1,12 @@ import { RpcListener } from "./Adapters/RpcListener" import { SystemForEmbassy } from "./Adapters/Systems/SystemForEmbassy" -import { HostSystemStartOs } from "./Adapters/HostSystemStartOs" +import { hostSystemStartOs } from "./Adapters/HostSystemStartOs" import { AllGetDependencies } from "./Interfaces/AllGetDependencies" import { getSystem } from "./Adapters/Systems" const getDependencies: AllGetDependencies = { system: getSystem, - hostSystem: () => HostSystemStartOs.of, + hostSystem: () => hostSystemStartOs, } new RpcListener(getDependencies) diff --git a/download-firmware.sh b/download-firmware.sh index 2457b3062..be72e6a6d 100755 --- a/download-firmware.sh +++ b/download-firmware.sh @@ -16,7 +16,8 @@ mkdir -p ./firmware/$PLATFORM cd ./firmware/$PLATFORM -mapfile -t firmwares <<< "$(jq -c ".[] | select(.platform[] | contains(\"$PLATFORM\"))" ../../build/lib/firmware.json)" +firmwares=() +while IFS= read -r line; do firmwares+=("$line"); done < <(jq -c ".[] | select(.platform[] | contains(\"$PLATFORM\"))" ../../build/lib/firmware.json) for firmware in "${firmwares[@]}"; do if [ -n "$firmware" ]; then id=$(echo "$firmware" | jq --raw-output '.id') From e92d4ff1471ec5c3fea8edb7962b2427a1c1fb5e Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Mon, 17 Jun 2024 10:37:57 -0600 Subject: [PATCH 09/17] fix compat assets (#2645) * fix compat assets * return error on s9pk parse fail in sideload * return parse error over websocket --- core/startos/src/install/mod.rs | 40 ++++++++++++++++++++++-------- core/startos/src/s9pk/v2/compat.rs | 4 ++- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/core/startos/src/install/mod.rs b/core/startos/src/install/mod.rs index bb7d1a02e..707503615 100644 --- a/core/startos/src/install/mod.rs +++ b/core/startos/src/install/mod.rs @@ -198,12 +198,22 @@ pub async fn sideload(ctx: RpcContext) -> Result { use axum::extract::ws::Message; async move { if let Err(e) = async { - let id = id_recv.await.map_err(|_| { + let id = match id_recv.await.map_err(|_| { Error::new( eyre!("Could not get id to watch progress"), ErrorKind::Cancelled, ) - })?; + }).and_then(|a|a) { + Ok(a) => a, + Err(e) =>{ ws.send(Message::Text( + serde_json::to_string(&Err::<(), _>(RpcError::from(e.clone_output()))) + .with_kind(ErrorKind::Serialization)?, + )) + .await + .with_kind(ErrorKind::Network)?; + return Err(e); + } + }; tokio::select! { res = async { while let Some(_) = sub.recv().await { @@ -259,17 +269,25 @@ pub async fn sideload(ctx: RpcContext) -> Result { .await; tokio::spawn(async move { if let Err(e) = async { - let s9pk = S9pk::deserialize( + match S9pk::deserialize( &file, None, // TODO ) - .await?; - let _ = id_send.send(s9pk.as_manifest().id.clone()); - ctx.services - .install(ctx.clone(), s9pk, None::) - .await? - .await? - .await?; - file.delete().await + .await + { + Ok(s9pk) => { + let _ = id_send.send(Ok(s9pk.as_manifest().id.clone())); + ctx.services + .install(ctx.clone(), s9pk, None::) + .await? + .await? + .await?; + file.delete().await + } + Err(e) => { + let _ = id_send.send(Err(e.clone_output())); + return Err(e); + } + } } .await { diff --git a/core/startos/src/s9pk/v2/compat.rs b/core/startos/src/s9pk/v2/compat.rs index 835c86b87..ec5b586ea 100644 --- a/core/startos/src/s9pk/v2/compat.rs +++ b/core/startos/src/s9pk/v2/compat.rs @@ -209,7 +209,9 @@ impl S9pk> { .invoke(ErrorKind::Filesystem) .await?; archive.insert_path( - Path::new("assets").join(&asset_id), + Path::new("assets") + .join(&asset_id) + .with_extension("squashfs"), Entry::file(PackSource::File(sqfs_path)), )?; } From da3720c7a9e158d64a6d9825f883f972597e05ae Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Wed, 19 Jun 2024 13:51:44 -0600 Subject: [PATCH 10/17] Feat/combine uis (#2633) * wip * restructure backend for new ui structure * new patchdb bootstrap, single websocket api, local storage migration, more * update db websocket * init apis * update patch-db * setup progress * feat: implement state service, alert and routing Signed-off-by: waterplea * update setup wizard for new types * feat: add init page Signed-off-by: waterplea * chore: refactor message, patch-db source stream and connection service Signed-off-by: waterplea * fix method not found on state * fix backend bugs * fix compat assets * address comments * remove unneeded styling * cleaner progress * bugfixes * fix init logs * fix progress reporting * fix navigation by getting state after init * remove patch dependency from live api * fix caching * re-add patchDB to live api * fix metrics values * send close frame * add bootId and fix polling --------- Signed-off-by: waterplea Co-authored-by: Aiden McClelland Co-authored-by: waterplea --- Makefile | 7 +- core/Cargo.lock | 718 ++++++++++++------ core/startos/Cargo.toml | 4 +- core/startos/src/backup/restore.rs | 60 +- core/startos/src/bins/registry.rs | 3 +- core/startos/src/bins/start_init.rs | 211 +++-- core/startos/src/bins/startd.rs | 76 +- core/startos/src/context/cli.rs | 7 +- core/startos/src/context/diagnostic.rs | 2 +- core/startos/src/context/init.rs | 47 ++ core/startos/src/context/install.rs | 9 + core/startos/src/context/mod.rs | 2 + core/startos/src/context/rpc.rs | 114 ++- core/startos/src/context/setup.rs | 147 +++- core/startos/src/db/mod.rs | 244 ++---- core/startos/src/diagnostic.rs | 13 +- core/startos/src/disk/main.rs | 2 +- core/startos/src/firmware.rs | 83 +- core/startos/src/init.rs | 323 +++++++- core/startos/src/install/mod.rs | 30 +- core/startos/src/lib.rs | 87 ++- core/startos/src/lxc/mod.rs | 80 +- core/startos/src/middleware/auth.rs | 18 +- core/startos/src/middleware/diagnostic.rs | 42 - core/startos/src/middleware/mod.rs | 1 - core/startos/src/net/net_controller.rs | 53 +- core/startos/src/net/refresher.html | 11 + core/startos/src/net/static_server.rs | 359 ++++----- core/startos/src/net/web_server.rs | 103 ++- core/startos/src/progress.rs | 163 ++-- core/startos/src/registry/context.rs | 3 +- core/startos/src/registry/mod.rs | 6 +- core/startos/src/registry/os/asset/add.rs | 27 +- core/startos/src/registry/os/asset/get.rs | 26 +- core/startos/src/registry/os/asset/sign.rs | 27 +- core/startos/src/registry/package/add.rs | 29 +- .../src/registry/signer/commitment/request.rs | 2 +- core/startos/src/rpc_continuations.rs | 131 +++- .../source/multi_cursor_file.rs | 1 + core/startos/src/service/mod.rs | 2 +- core/startos/src/service/service_map.rs | 35 +- core/startos/src/setup.rs | 361 ++++----- core/startos/src/update/mod.rs | 103 ++- core/startos/src/upload.rs | 101 +-- core/startos/src/util/io.rs | 77 +- core/startos/src/util/mod.rs | 1 + core/startos/src/util/net.rs | 24 + core/startos/src/util/serde.rs | 12 +- core/startos/src/version/mod.rs | 34 +- core/startos/src/volume.rs | 6 +- patch-db | 2 +- sdk/lib/osBindings/AttachParams.ts | 7 + sdk/lib/osBindings/BackupTargetFS.ts | 7 + sdk/lib/osBindings/BlockDev.ts | 3 + sdk/lib/osBindings/Cifs.ts | 8 + sdk/lib/osBindings/InitProgressRes.ts | 5 + sdk/lib/osBindings/RecoverySource.ts | 6 + sdk/lib/osBindings/SetupExecuteParams.ts | 10 + sdk/lib/osBindings/SetupProgress.ts | 5 + sdk/lib/osBindings/SetupResult.ts | 7 + sdk/lib/osBindings/SetupStatusRes.ts | 7 + sdk/lib/osBindings/VerifyCifsParams.ts | 9 + sdk/lib/osBindings/index.ts | 11 + web/package-lock.json | 19 +- web/package.json | 3 +- .../src/app/app-routing.module.ts | 27 - .../diagnostic-ui/src/app/app.component.html | 5 - .../diagnostic-ui/src/app/app.component.scss | 8 - .../diagnostic-ui/src/app/app.component.ts | 10 - .../diagnostic-ui/src/app/app.module.ts | 43 -- .../src/app/pages/home/home-routing.module.ts | 16 - .../src/app/services/api/api.service.ts | 16 - .../src/app/services/api/live-api.service.ts | 68 -- .../src/app/services/api/mock-api.service.ts | 67 -- .../src/environments/environment.prod.ts | 3 - .../src/environments/environment.ts | 16 - web/projects/diagnostic-ui/src/index.html | 23 - web/projects/diagnostic-ui/src/main.ts | 12 - web/projects/diagnostic-ui/src/polyfills.ts | 64 -- web/projects/diagnostic-ui/src/styles.scss | 41 - web/projects/diagnostic-ui/src/zone-flags.ts | 6 - web/projects/diagnostic-ui/tsconfig.json | 9 - .../setup-wizard/src/app/app.component.ts | 2 +- .../src/app/pages/embassy/embassy.page.ts | 24 +- .../src/app/pages/loading/loading.module.ts | 4 +- .../src/app/pages/loading/loading.page.html | 54 +- .../src/app/pages/loading/loading.page.scss | 3 - .../src/app/pages/loading/loading.page.ts | 162 ++-- .../src/app/services/api/api.service.ts | 63 +- .../src/app/services/api/live-api.service.ts | 60 +- .../src/app/services/api/mock-api.service.ts | 186 ++++- .../src/app/services/state.service.ts | 6 +- web/projects/shared/src/types/api.ts | 1 + web/projects/ui/src/app/app-routing.module.ts | 27 +- web/projects/ui/src/app/app.component.html | 1 + web/projects/ui/src/app/app.component.scss | 6 +- web/projects/ui/src/app/app.component.ts | 20 +- web/projects/ui/src/app/app.module.ts | 2 + web/projects/ui/src/app/app.providers.ts | 7 +- .../ui/src/app/app/menu/menu.component.ts | 4 +- .../connection-bar.component.ts | 12 +- .../src/app/components/logs/logs.component.ts | 68 +- .../components/status/status.component.html | 4 +- .../app/components/status/status.component.ts | 4 +- .../app/modals/os-update/os-update.page.ts | 3 +- .../app-list-icon.component.html | 2 +- .../app-list-icon/app-list-icon.component.ts | 4 +- .../app-show-health-checks.component.html | 2 +- .../app-show-health-checks.component.ts | 4 +- .../app-show-status.component.html | 2 +- .../app-show-status.component.ts | 4 +- .../diagnostic-routing.module.ts | 21 + .../diagnostic-routes}/home/home.module.ts | 12 +- .../diagnostic-routes}/home/home.page.html | 6 - .../diagnostic-routes}/home/home.page.scss | 0 .../diagnostic-routes}/home/home.page.ts | 59 +- .../diagnostic-routes}/logs/logs.module.ts | 0 .../diagnostic-routes}/logs/logs.page.html | 0 .../diagnostic-routes}/logs/logs.page.scss | 0 .../diagnostic-routes}/logs/logs.page.ts | 4 +- .../ui/src/app/pages/init/init.module.ts | 24 + .../ui/src/app/pages/init/init.page.html | 18 + .../ui/src/app/pages/init/init.page.scss | 23 + .../ui/src/app/pages/init/init.page.ts | 11 + .../ui/src/app/pages/init/init.service.ts | 91 +++ .../src/app/pages/init/logs/logs.component.ts | 33 + .../ui/src/app/pages/init/logs/logs.module.ts | 20 + .../src/app/pages/init/logs/logs.service.ts | 49 ++ .../app/pages/init/logs/logs.template.html | 9 + .../login/ca-wizard/ca-wizard.component.ts | 2 +- .../server-metrics/server-metrics.page.html | 4 +- .../server-show/server-show.page.ts | 49 -- .../ui/src/app/services/api/api.fixures.ts | 21 +- .../ui/src/app/services/api/api.types.ts | 47 +- .../app/services/api/embassy-api.service.ts | 53 +- .../services/api/embassy-live-api.service.ts | 133 ++-- .../services/api/embassy-mock-api.service.ts | 246 ++++-- .../ui/src/app/services/auth.service.ts | 2 +- .../ui/src/app/services/connection.service.ts | 30 +- .../ui/src/app/services/eos.service.ts | 11 +- .../ui/src/app/services/network.service.ts | 22 + .../ui/src/app/services/patch-data.service.ts | 27 +- .../patch-db/local-storage-bootstrap.ts | 16 +- .../app/services/patch-db/patch-db.factory.ts | 56 +- .../src/app/services/patch-monitor.service.ts | 9 +- .../ui/src/app/services/state.service.ts | 136 ++++ .../ui/src/app/services/storage.service.ts | 21 +- 147 files changed, 3939 insertions(+), 2637 deletions(-) create mode 100644 core/startos/src/context/init.rs delete mode 100644 core/startos/src/middleware/diagnostic.rs create mode 100644 core/startos/src/net/refresher.html create mode 100644 core/startos/src/util/net.rs create mode 100644 sdk/lib/osBindings/AttachParams.ts create mode 100644 sdk/lib/osBindings/BackupTargetFS.ts create mode 100644 sdk/lib/osBindings/BlockDev.ts create mode 100644 sdk/lib/osBindings/Cifs.ts create mode 100644 sdk/lib/osBindings/InitProgressRes.ts create mode 100644 sdk/lib/osBindings/RecoverySource.ts create mode 100644 sdk/lib/osBindings/SetupExecuteParams.ts create mode 100644 sdk/lib/osBindings/SetupProgress.ts create mode 100644 sdk/lib/osBindings/SetupResult.ts create mode 100644 sdk/lib/osBindings/SetupStatusRes.ts create mode 100644 sdk/lib/osBindings/VerifyCifsParams.ts delete mode 100644 web/projects/diagnostic-ui/src/app/app-routing.module.ts delete mode 100644 web/projects/diagnostic-ui/src/app/app.component.html delete mode 100644 web/projects/diagnostic-ui/src/app/app.component.scss delete mode 100644 web/projects/diagnostic-ui/src/app/app.component.ts delete mode 100644 web/projects/diagnostic-ui/src/app/app.module.ts delete mode 100644 web/projects/diagnostic-ui/src/app/pages/home/home-routing.module.ts delete mode 100644 web/projects/diagnostic-ui/src/app/services/api/api.service.ts delete mode 100644 web/projects/diagnostic-ui/src/app/services/api/live-api.service.ts delete mode 100644 web/projects/diagnostic-ui/src/app/services/api/mock-api.service.ts delete mode 100644 web/projects/diagnostic-ui/src/environments/environment.prod.ts delete mode 100644 web/projects/diagnostic-ui/src/environments/environment.ts delete mode 100644 web/projects/diagnostic-ui/src/index.html delete mode 100644 web/projects/diagnostic-ui/src/main.ts delete mode 100644 web/projects/diagnostic-ui/src/polyfills.ts delete mode 100644 web/projects/diagnostic-ui/src/styles.scss delete mode 100644 web/projects/diagnostic-ui/src/zone-flags.ts delete mode 100644 web/projects/diagnostic-ui/tsconfig.json create mode 100644 web/projects/ui/src/app/pages/diagnostic-routes/diagnostic-routing.module.ts rename web/projects/{diagnostic-ui/src/app/pages => ui/src/app/pages/diagnostic-routes}/home/home.module.ts (55%) rename web/projects/{diagnostic-ui/src/app/pages => ui/src/app/pages/diagnostic-routes}/home/home.page.html (92%) rename web/projects/{diagnostic-ui/src/app/pages => ui/src/app/pages/diagnostic-routes}/home/home.page.scss (100%) rename web/projects/{diagnostic-ui/src/app/pages => ui/src/app/pages/diagnostic-routes}/home/home.page.ts (74%) rename web/projects/{diagnostic-ui/src/app/pages => ui/src/app/pages/diagnostic-routes}/logs/logs.module.ts (100%) rename web/projects/{diagnostic-ui/src/app/pages => ui/src/app/pages/diagnostic-routes}/logs/logs.page.html (100%) rename web/projects/{diagnostic-ui/src/app/pages => ui/src/app/pages/diagnostic-routes}/logs/logs.page.scss (100%) rename web/projects/{diagnostic-ui/src/app/pages => ui/src/app/pages/diagnostic-routes}/logs/logs.page.ts (93%) create mode 100644 web/projects/ui/src/app/pages/init/init.module.ts create mode 100644 web/projects/ui/src/app/pages/init/init.page.html create mode 100644 web/projects/ui/src/app/pages/init/init.page.scss create mode 100644 web/projects/ui/src/app/pages/init/init.page.ts create mode 100644 web/projects/ui/src/app/pages/init/init.service.ts create mode 100644 web/projects/ui/src/app/pages/init/logs/logs.component.ts create mode 100644 web/projects/ui/src/app/pages/init/logs/logs.module.ts create mode 100644 web/projects/ui/src/app/pages/init/logs/logs.service.ts create mode 100644 web/projects/ui/src/app/pages/init/logs/logs.template.html create mode 100644 web/projects/ui/src/app/services/network.service.ts create mode 100644 web/projects/ui/src/app/services/state.service.ts diff --git a/Makefile b/Makefile index 34064e799..4915101e3 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ PLATFORM := $(shell if [ -f ./PLATFORM.txt ]; then cat ./PLATFORM.txt; else echo ARCH := $(shell if [ "$(PLATFORM)" = "raspberrypi" ]; then echo aarch64; else echo $(PLATFORM) | sed 's/-nonfree$$//g'; fi) IMAGE_TYPE=$(shell if [ "$(PLATFORM)" = raspberrypi ]; then echo img; else echo iso; fi) BINS := core/target/$(ARCH)-unknown-linux-musl/release/startbox core/target/$(ARCH)-unknown-linux-musl/release/containerbox -WEB_UIS := web/dist/raw/ui web/dist/raw/setup-wizard web/dist/raw/diagnostic-ui web/dist/raw/install-wizard +WEB_UIS := web/dist/raw/ui web/dist/raw/setup-wizard web/dist/raw/install-wizard FIRMWARE_ROMS := ./firmware/$(PLATFORM) $(shell jq --raw-output '.[] | select(.platform[] | contains("$(PLATFORM)")) | "./firmware/$(PLATFORM)/" + .id + ".rom.gz"' build/lib/firmware.json) BUILD_SRC := $(shell git ls-files build) build/lib/depends build/lib/conflicts $(FIRMWARE_ROMS) DEBIAN_SRC := $(shell git ls-files debian/) @@ -20,7 +20,6 @@ CORE_SRC := $(shell git ls-files core) $(shell git ls-files --recurse-submodules WEB_SHARED_SRC := $(shell git ls-files web/projects/shared) $(shell ls -p web/ | grep -v / | sed 's/^/web\//g') web/node_modules/.package-lock.json web/config.json patch-db/client/dist web/patchdb-ui-seed.json WEB_UI_SRC := $(shell git ls-files web/projects/ui) WEB_SETUP_WIZARD_SRC := $(shell git ls-files web/projects/setup-wizard) -WEB_DIAGNOSTIC_UI_SRC := $(shell git ls-files web/projects/diagnostic-ui) WEB_INSTALL_WIZARD_SRC := $(shell git ls-files web/projects/install-wizard) PATCH_DB_CLIENT_SRC := $(shell git ls-files --recurse-submodules patch-db/client) GZIP_BIN := $(shell which pigz || which gzip) @@ -244,10 +243,6 @@ web/dist/raw/setup-wizard: $(WEB_SETUP_WIZARD_SRC) $(WEB_SHARED_SRC) npm --prefix web run build:setup touch web/dist/raw/setup-wizard -web/dist/raw/diagnostic-ui: $(WEB_DIAGNOSTIC_UI_SRC) $(WEB_SHARED_SRC) - npm --prefix web run build:dui - touch web/dist/raw/diagnostic-ui - web/dist/raw/install-wizard: $(WEB_INSTALL_WIZARD_SRC) $(WEB_SHARED_SRC) npm --prefix web run build:install-wiz touch web/dist/raw/install-wizard diff --git a/core/Cargo.lock b/core/Cargo.lock index cf10152c2..031ec41ab 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -42,7 +42,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.14", + "getrandom 0.2.15", "once_cell", "version_check", ] @@ -54,7 +54,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", - "getrandom 0.2.14", + "getrandom 0.2.15", "once_cell", "version_check", "zerocopy", @@ -137,9 +137,9 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" dependencies = [ "windows-sys 0.52.0", ] @@ -156,9 +156,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.82" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "arrayref" @@ -187,16 +187,16 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ - "concurrent-queue", + "concurrent-queue 2.5.0", "event-listener", "futures-core", ] [[package]] name = "async-compression" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e9eabd7a98fe442131a17c316bd9349c43695e49e730c3c8e12cfb5f4da2693" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" dependencies = [ "brotli", "flate2", @@ -225,7 +225,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -236,7 +236,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -248,6 +248,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "atty" version = "0.2.14" @@ -278,7 +284,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "itoa", "matchit", "memchr", @@ -418,6 +424,17 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "barrage" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be5951c75bdabb58753d140dd5802f12ff3a483cb2e16fb5276e111b94b19e87" +dependencies = [ + "concurrent-queue 1.2.4", + "event-listener", + "spin 0.9.8", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -562,9 +579,9 @@ dependencies = [ [[package]] name = "brotli" -version = "5.0.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19483b140a7ac7174d34b5a581b406c64f84da5409d3e09cf4fff604f9270e67" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -573,9 +590,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -600,10 +617,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] -name = "cc" -version = "1.0.96" +name = "cache-padded" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "065a29261d53ba54260972629f9ca6bffa69bac13cd1fed61420f7fa68b9f8bd" +checksum = "981520c98f422fcc584dc1a95c334e6953900b9106bc47a9839b81790009eb21" + +[[package]] +name = "cc" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" dependencies = [ "jobserver", "libc", @@ -688,9 +711,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.4" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" dependencies = [ "clap_builder", "clap_derive", @@ -698,9 +721,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.2" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" dependencies = [ "anstream", "anstyle", @@ -710,21 +733,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.4" +version = "4.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] name = "clap_lex" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" [[package]] name = "color-eyre" @@ -759,6 +782,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "concurrent-queue" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" +dependencies = [ + "cache-padded", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -945,18 +977,18 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ "crossbeam-utils", ] @@ -991,9 +1023,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crossterm" @@ -1117,14 +1149,14 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] name = "darling" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" +checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" dependencies = [ "darling_core", "darling_macro", @@ -1132,27 +1164,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" +checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", - "syn 2.0.60", + "strsim 0.11.1", + "syn 2.0.66", ] [[package]] name = "darling_macro" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" +checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ "darling_core", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1183,7 +1215,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1206,7 +1238,7 @@ checksum = "5fe87ce4529967e0ba1dcf8450bab64d97dfd5010a6256187ffe2e43e6f0e049" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1274,6 +1306,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "divrem" version = "1.0.0" @@ -1367,9 +1410,9 @@ dependencies = [ [[package]] name = "either" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" dependencies = [ "serde", ] @@ -1407,9 +1450,9 @@ dependencies = [ [[package]] name = "ena" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c533630cf40e9caa44bd91aadc88a75d75a4c3a12b4cfde353cbed41daa1e1f1" +checksum = "3d248bdd43ce613d87415282f69b9bb99d947d290b10962dd6c56233312c2ad5" dependencies = [ "log", ] @@ -1444,7 +1487,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1455,9 +1498,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1517,9 +1560,9 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38793c55593b33412e3ae40c2c9781ffaa6f438f6f8c10f24e71846fbd7ae01e" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "filetime" @@ -1533,12 +1576,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "finl_unicode" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" - [[package]] name = "fixedbitset" version = "0.4.2" @@ -1678,7 +1715,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -1735,9 +1772,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -1794,15 +1831,15 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", "http 1.1.0", "indexmap 2.2.6", "slab", @@ -2005,12 +2042,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", - "futures-core", + "futures-util", "http 1.1.0", "http-body 1.0.0", "pin-project-lite", @@ -2018,9 +2055,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545" [[package]] name = "httpdate" @@ -2036,9 +2073,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" dependencies = [ "bytes", "futures-channel", @@ -2067,7 +2104,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.4", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -2085,7 +2122,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.28", + "hyper 0.14.29", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -2109,9 +2146,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ "bytes", "futures-channel", @@ -2150,6 +2187,124 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "id-pool" version = "0.2.2" @@ -2187,12 +2342,14 @@ dependencies = [ [[package]] name = "idna" -version = "0.5.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "icu_normalizer", + "icu_properties", + "smallvec", + "utf8_iter", ] [[package]] @@ -2302,9 +2459,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", ] @@ -2535,7 +2692,7 @@ dependencies = [ "petgraph", "pico-args", "regex", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", "string_cache", "term", "tiny-keccak", @@ -2549,7 +2706,7 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" dependencies = [ - "regex-automata 0.4.6", + "regex-automata 0.4.7", ] [[package]] @@ -2579,9 +2736,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.154" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libm" @@ -2618,9 +2775,15 @@ checksum = "3e281a65eeba3d4503a2839252f86374528f9ceafe6fed97c1d3b52e1fb625c1" [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" [[package]] name = "lock_api" @@ -2734,9 +2897,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" dependencies = [ "adler", ] @@ -2786,11 +2949,10 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" dependencies = [ - "lazy_static", "libc", "log", "openssl", @@ -2885,9 +3047,9 @@ dependencies = [ [[package]] name = "num" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135b08af27d103b0a51f2ae0f8632117b7b185ccf931445affa8df530576a41" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ "num-bigint", "num-complex", @@ -2899,11 +3061,10 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -2927,9 +3088,9 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", ] @@ -2962,11 +3123,10 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "autocfg", "num-bigint", "num-integer", "num-traits", @@ -3010,7 +3170,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -3076,7 +3236,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -3087,9 +3247,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.2.3+3.2.1" +version = "300.3.1+3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843" +checksum = "7259953d42a81bf137fbbd73bd30a8e1914d6dce43c2b90ed575783a22608b91" dependencies = [ "cc", ] @@ -3159,9 +3319,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -3182,9 +3342,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "patch-db" @@ -3254,9 +3414,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", "indexmap 2.2.6", @@ -3294,7 +3454,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -3400,9 +3560,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.81" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" dependencies = [ "unicode-ident", ] @@ -3421,7 +3581,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", "rusty-fork", "tempfile", "unarray", @@ -3440,9 +3600,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", "prost-derive", @@ -3450,22 +3610,22 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] name = "prost-types" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ "prost", ] @@ -3566,7 +3726,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.14", + "getrandom 0.2.15", ] [[package]] @@ -3655,21 +3815,21 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ - "getrandom 0.2.14", + "getrandom 0.2.15", "libredox", "thiserror", ] [[package]] name = "regex" -version = "1.10.4" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.6", - "regex-syntax 0.8.3", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", ] [[package]] @@ -3683,13 +3843,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", ] [[package]] @@ -3700,9 +3860,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" @@ -3717,7 +3877,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.4", + "h2 0.4.5", "http 1.1.0", "http-body 1.0.0", "http-body-util", @@ -3781,7 +3941,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.14", + "getrandom 0.2.15", "libc", "spin 0.9.8", "untrusted", @@ -3802,7 +3962,7 @@ dependencies = [ [[package]] name = "rpc-toolkit" version = "0.2.3" -source = "git+https://github.com/Start9Labs/rpc-toolkit.git?branch=refactor/no-dyn-ctx#f5566840bb9af0743612fede4034f5a390cd1eee" +source = "git+https://github.com/Start9Labs/rpc-toolkit.git?branch=refactor/no-dyn-ctx#5a24903031e72ac75fd23889215361edc7b20842" dependencies = [ "async-stream", "async-trait", @@ -3871,9 +4031,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" @@ -3923,7 +4083,7 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki 0.102.3", + "rustls-webpki 0.102.4", "subtle", "zeroize", ] @@ -3949,9 +4109,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.5.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" @@ -3965,9 +4125,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.3" +version = "0.102.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ "ring", "rustls-pki-types", @@ -4064,11 +4224,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "core-foundation", "core-foundation-sys", "libc", @@ -4077,9 +4237,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -4087,9 +4247,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" dependencies = [ "serde", ] @@ -4128,7 +4288,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -4155,9 +4315,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" dependencies = [ "serde", ] @@ -4201,7 +4361,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -4396,11 +4556,10 @@ dependencies = [ [[package]] name = "sqlformat" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +checksum = "f895e3734318cc55f1fe66258926c9b910c124d47520339efecbb6c59cec7c1f" dependencies = [ - "itertools 0.12.1", "nom", "unicode_categories", ] @@ -4629,7 +4788,7 @@ dependencies = [ "quote", "regex-syntax 0.6.29", "strsim 0.10.0", - "syn 2.0.60", + "syn 2.0.66", "unicode-width", ] @@ -4675,6 +4834,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "start-os" version = "0.3.5-rev.2" @@ -4686,6 +4851,7 @@ dependencies = [ "axum 0.7.5", "axum-server", "backhand", + "barrage", "base32", "base64 0.21.7", "base64ct", @@ -4783,8 +4949,9 @@ dependencies = [ "tokio-tar", "tokio-tungstenite", "tokio-util", - "toml 0.8.12", + "toml 0.8.14", "torut", + "tower-service", "tracing", "tracing-error", "tracing-futures", @@ -4827,13 +4994,13 @@ dependencies = [ [[package]] name = "stringprep" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ - "finl_unicode", "unicode-bidi", "unicode-normalization", + "unicode-properties", ] [[package]] @@ -4867,9 +5034,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.60" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", @@ -4888,6 +5055,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -4917,9 +5095,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" dependencies = [ "filetime", "libc", @@ -4970,22 +5148,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.59" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.59" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -5049,6 +5227,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -5066,9 +5254,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -5096,13 +5284,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -5180,16 +5368,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -5206,21 +5393,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.12", + "toml_edit 0.22.14", ] [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" dependencies = [ "serde", ] @@ -5251,15 +5438,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.12" +version = "0.22.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef" +checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" dependencies = [ "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.7", + "winnow 0.6.13", ] [[package]] @@ -5276,7 +5463,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "hyper-timeout", "percent-encoding", "pin-project", @@ -5360,7 +5547,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -5512,7 +5699,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", "termcolor", ] @@ -5553,7 +5740,7 @@ checksum = "1f718dfaf347dcb5b983bfc87608144b0bad87970aebcbea5ce44d2a30c08e63" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", ] [[package]] @@ -5598,6 +5785,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" + [[package]] name = "unicode-segmentation" version = "1.11.0" @@ -5606,9 +5799,9 @@ checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" [[package]] name = "unicode-xid" @@ -5630,12 +5823,12 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna 1.0.0", "percent-encoding", "serde", ] @@ -5653,10 +5846,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] -name = "utf8parse" -version = "0.2.1" +name = "utf16_iter" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" @@ -5664,7 +5869,7 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ - "getrandom 0.2.14", + "getrandom 0.2.15", ] [[package]] @@ -5752,7 +5957,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", "wasm-bindgen-shared", ] @@ -5786,7 +5991,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6026,9 +6231,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.7" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b9415ee827af173ebb3f15f9083df5a122eb93572ec28741fb153356ea2578" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" dependencies = [ "memchr", ] @@ -6043,6 +6248,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "wyz" version = "0.5.1" @@ -6106,30 +6323,75 @@ dependencies = [ ] [[package]] -name = "zerocopy" -version = "0.7.32" +name = "yoke" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure", ] [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ "zeroize_derive", ] @@ -6142,7 +6404,29 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.66", +] + +[[package]] +name = "zerovec" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", ] [[package]] diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index 3ad4f9eeb..a8707bf65 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -59,6 +59,7 @@ async-stream = "0.3.5" async-trait = "0.1.74" axum = { version = "0.7.3", features = ["ws"] } axum-server = "0.6.0" +barrage = "0.2.3" backhand = "0.18.0" base32 = "0.4.0" base64 = "0.21.4" @@ -102,7 +103,7 @@ id-pool = { version = "0.2.2", default-features = false, features = [ ] } imbl = "2.0.2" imbl-value = { git = "https://github.com/Start9Labs/imbl-value.git" } -include_dir = "0.7.3" +include_dir = { version = "0.7.3", features = ["metadata"] } indexmap = { version = "2.0.2", features = ["serde"] } indicatif = { version = "0.17.7", features = ["tokio"] } integer-encoding = { version = "4.0.0", features = ["tokio_async"] } @@ -178,6 +179,7 @@ tokio-util = { version = "0.7.9", features = ["io"] } torut = { git = "https://github.com/Start9Labs/torut.git", branch = "update/dependencies", features = [ "serialize", ] } +tower-service = "0.3.2" tracing = "0.1.39" tracing-error = "0.2.0" tracing-futures = "0.2.5" diff --git a/core/startos/src/backup/restore.rs b/core/startos/src/backup/restore.rs index 4753a4290..556f750ec 100644 --- a/core/startos/src/backup/restore.rs +++ b/core/startos/src/backup/restore.rs @@ -4,25 +4,25 @@ use std::sync::Arc; use clap::Parser; use futures::{stream, StreamExt}; use models::PackageId; -use openssl::x509::X509; use patch_db::json_ptr::ROOT; use serde::{Deserialize, Serialize}; -use torut::onion::OnionAddressV3; +use tokio::sync::Mutex; use tracing::instrument; use ts_rs::TS; use super::target::BackupTargetId; use crate::backup::os::OsBackup; +use crate::context::setup::SetupResult; use crate::context::{RpcContext, SetupContext}; use crate::db::model::Database; use crate::disk::mount::backup::BackupMountGuard; use crate::disk::mount::filesystem::ReadWrite; use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard}; -use crate::hostname::Hostname; -use crate::init::init; +use crate::init::{init, InitResult}; use crate::prelude::*; use crate::s9pk::S9pk; use crate::service::service_map::DownloadInstallFuture; +use crate::setup::SetupExecuteProgress; use crate::util::serde::IoFormat; #[derive(Deserialize, Serialize, Parser, TS)] @@ -67,14 +67,21 @@ pub async fn restore_packages_rpc( Ok(()) } -#[instrument(skip(ctx))] +#[instrument(skip_all)] pub async fn recover_full_embassy( - ctx: SetupContext, + ctx: &SetupContext, disk_guid: Arc, start_os_password: String, recovery_source: TmpMountGuard, recovery_password: Option, -) -> Result<(Arc, Hostname, OnionAddressV3, X509), Error> { + SetupExecuteProgress { + init_phases, + restore_phase, + rpc_ctx_phases, + }: SetupExecuteProgress, +) -> Result<(SetupResult, RpcContext), Error> { + let mut restore_phase = restore_phase.or_not_found("restore progress")?; + let backup_guard = BackupMountGuard::mount( recovery_source, recovery_password.as_deref().unwrap_or_default(), @@ -99,10 +106,17 @@ pub async fn recover_full_embassy( db.put(&ROOT, &Database::init(&os_backup.account)?).await?; drop(db); - init(&ctx.config).await?; + let InitResult { net_ctrl } = init(&ctx.config, init_phases).await?; - let rpc_ctx = RpcContext::init(&ctx.config, disk_guid.clone()).await?; + let rpc_ctx = RpcContext::init( + &ctx.config, + disk_guid.clone(), + Some(net_ctrl), + rpc_ctx_phases, + ) + .await?; + restore_phase.start(); let ids: Vec<_> = backup_guard .metadata .package_backups @@ -110,26 +124,26 @@ pub async fn recover_full_embassy( .cloned() .collect(); let tasks = restore_packages(&rpc_ctx, backup_guard, ids).await?; + restore_phase.set_total(tasks.len() as u64); + let restore_phase = Arc::new(Mutex::new(restore_phase)); stream::iter(tasks) - .for_each_concurrent(5, |(id, res)| async move { - match async { res.await?.await }.await { - Ok(_) => (), - Err(err) => { - tracing::error!("Error restoring package {}: {}", id, err); - tracing::debug!("{:?}", err); + .for_each_concurrent(5, |(id, res)| { + let restore_phase = restore_phase.clone(); + async move { + match async { res.await?.await }.await { + Ok(_) => (), + Err(err) => { + tracing::error!("Error restoring package {}: {}", id, err); + tracing::debug!("{:?}", err); + } } + *restore_phase.lock().await += 1; } }) .await; + restore_phase.lock().await.complete(); - rpc_ctx.shutdown().await?; - - Ok(( - disk_guid, - os_backup.account.hostname, - os_backup.account.tor_key.public().get_onion_address(), - os_backup.account.root_ca_cert, - )) + Ok(((&os_backup.account).try_into()?, rpc_ctx)) } #[instrument(skip(ctx, backup_guard))] diff --git a/core/startos/src/bins/registry.rs b/core/startos/src/bins/registry.rs index 3028d1766..132e0984a 100644 --- a/core/startos/src/bins/registry.rs +++ b/core/startos/src/bins/registry.rs @@ -14,7 +14,8 @@ use crate::util::logger::EmbassyLogger; async fn inner_main(config: &RegistryConfig) -> Result<(), Error> { let server = async { let ctx = RegistryContext::init(config).await?; - let server = WebServer::registry(ctx.listen, ctx.clone()); + let mut server = WebServer::new(ctx.listen); + server.serve_registry(ctx.clone()); let mut shutdown_recv = ctx.shutdown.subscribe(); diff --git a/core/startos/src/bins/start_init.rs b/core/startos/src/bins/start_init.rs index 8e60884f1..f4aa411b5 100644 --- a/core/startos/src/bins/start_init.rs +++ b/core/startos/src/bins/start_init.rs @@ -1,47 +1,56 @@ -use std::net::{Ipv6Addr, SocketAddr}; -use std::path::Path; use std::sync::Arc; -use std::time::Duration; -use helpers::NonDetachingJoinHandle; use tokio::process::Command; use tracing::instrument; use crate::context::config::ServerConfig; -use crate::context::{DiagnosticContext, InstallContext, SetupContext}; -use crate::disk::fsck::{RepairStrategy, RequiresReboot}; +use crate::context::rpc::InitRpcContextPhases; +use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext}; +use crate::disk::fsck::RepairStrategy; use crate::disk::main::DEFAULT_PASSWORD; use crate::disk::REPAIR_DISK_PATH; -use crate::firmware::update_firmware; -use crate::init::STANDBY_MODE_PATH; +use crate::firmware::{check_for_firmware_update, update_firmware}; +use crate::init::{InitPhases, InitResult, STANDBY_MODE_PATH}; use crate::net::web_server::WebServer; +use crate::prelude::*; +use crate::progress::FullProgressTracker; use crate::shutdown::Shutdown; -use crate::sound::{BEP, CHIME}; use crate::util::Invoke; -use crate::{Error, ErrorKind, ResultExt, PLATFORM}; +use crate::PLATFORM; #[instrument(skip_all)] -async fn setup_or_init(config: &ServerConfig) -> Result, Error> { - let song = NonDetachingJoinHandle::from(tokio::spawn(async { - loop { - BEP.play().await.unwrap(); - BEP.play().await.unwrap(); - tokio::time::sleep(Duration::from_secs(30)).await; - } - })); +async fn setup_or_init( + server: &mut WebServer, + config: &ServerConfig, +) -> Result, Error> { + if let Some(firmware) = check_for_firmware_update() + .await + .map_err(|e| { + tracing::warn!("Error checking for firmware update: {e}"); + tracing::debug!("{e:?}"); + }) + .ok() + .and_then(|a| a) + { + let init_ctx = InitContext::init(config).await?; + let handle = &init_ctx.progress; + let mut update_phase = handle.add_phase("Updating Firmware".into(), Some(10)); + let mut reboot_phase = handle.add_phase("Rebooting".into(), Some(1)); - match update_firmware().await { - Ok(RequiresReboot(true)) => { - return Ok(Some(Shutdown { - export_args: None, - restart: true, - })) - } - Err(e) => { + server.serve_init(init_ctx); + + update_phase.start(); + if let Err(e) = update_firmware(firmware).await { tracing::warn!("Error performing firmware update: {e}"); tracing::debug!("{e:?}"); + } else { + update_phase.complete(); + reboot_phase.start(); + return Ok(Err(Shutdown { + export_args: None, + restart: true, + })); } - _ => (), } Command::new("ln") @@ -84,14 +93,7 @@ async fn setup_or_init(config: &ServerConfig) -> Result, Error> let ctx = InstallContext::init().await?; - let server = WebServer::install( - SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80), - ctx.clone(), - )?; - - drop(song); - tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this - CHIME.play().await?; + server.serve_install(ctx.clone()); ctx.shutdown .subscribe() @@ -99,33 +101,23 @@ async fn setup_or_init(config: &ServerConfig) -> Result, Error> .await .expect("context dropped"); - server.shutdown().await; + return Ok(Err(Shutdown { + export_args: None, + restart: true, + })); + } - Command::new("reboot") - .invoke(crate::ErrorKind::Unknown) - .await?; - } else if tokio::fs::metadata("/media/startos/config/disk.guid") + if tokio::fs::metadata("/media/startos/config/disk.guid") .await .is_err() { let ctx = SetupContext::init(config)?; - let server = WebServer::setup( - SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80), - ctx.clone(), - )?; - - drop(song); - tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this - CHIME.play().await?; + server.serve_setup(ctx.clone()); let mut shutdown = ctx.shutdown.subscribe(); shutdown.recv().await.expect("context dropped"); - server.shutdown().await; - - drop(shutdown); - tokio::task::yield_now().await; if let Err(e) = Command::new("killall") .arg("firefox-esr") @@ -135,19 +127,40 @@ async fn setup_or_init(config: &ServerConfig) -> Result, Error> tracing::error!("Failed to kill kiosk: {}", e); tracing::debug!("{:?}", e); } + + Ok(Ok(match ctx.result.get() { + Some(Ok((_, rpc_ctx))) => (rpc_ctx.clone(), ctx.progress.clone()), + Some(Err(e)) => return Err(e.clone_output()), + None => { + return Err(Error::new( + eyre!("Setup mode exited before setup completed"), + ErrorKind::Unknown, + )) + } + })) } else { + let init_ctx = InitContext::init(config).await?; + let handle = init_ctx.progress.clone(); + + let mut disk_phase = handle.add_phase("Opening data drive".into(), Some(10)); + let init_phases = InitPhases::new(&handle); + let rpc_ctx_phases = InitRpcContextPhases::new(&handle); + + server.serve_init(init_ctx); + + disk_phase.start(); let guid_string = tokio::fs::read_to_string("/media/startos/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy .await?; - let guid = guid_string.trim(); + let disk_guid = Arc::new(String::from(guid_string.trim())); let requires_reboot = crate::disk::main::import( - guid, + &**disk_guid, config.datadir(), if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { RepairStrategy::Aggressive } else { RepairStrategy::Preen }, - if guid.ends_with("_UNENC") { + if disk_guid.ends_with("_UNENC") { None } else { Some(DEFAULT_PASSWORD) @@ -159,40 +172,31 @@ async fn setup_or_init(config: &ServerConfig) -> Result, Error> .await .with_ctx(|_| (crate::ErrorKind::Filesystem, REPAIR_DISK_PATH))?; } - if requires_reboot.0 { - crate::disk::main::export(guid, config.datadir()).await?; - Command::new("reboot") - .invoke(crate::ErrorKind::Unknown) - .await?; - } + disk_phase.complete(); tracing::info!("Loaded Disk"); - crate::init::init(config).await?; - drop(song); - } - Ok(None) -} - -async fn run_script_if_exists>(path: P) { - let script = path.as_ref(); - if script.exists() { - match Command::new("/bin/bash").arg(script).spawn() { - Ok(mut c) => { - if let Err(e) = c.wait().await { - tracing::error!("Error Running {}: {}", script.display(), e); - tracing::debug!("{:?}", e); - } - } - Err(e) => { - tracing::error!("Error Running {}: {}", script.display(), e); - tracing::debug!("{:?}", e); - } + if requires_reboot.0 { + let mut reboot_phase = handle.add_phase("Rebooting".into(), Some(1)); + reboot_phase.start(); + return Ok(Err(Shutdown { + export_args: Some((disk_guid, config.datadir().to_owned())), + restart: true, + })); } + + let InitResult { net_ctrl } = crate::init::init(config, init_phases).await?; + + let rpc_ctx = RpcContext::init(config, disk_guid, Some(net_ctrl), rpc_ctx_phases).await?; + + Ok(Ok((rpc_ctx, handle))) } } #[instrument(skip_all)] -async fn inner_main(config: &ServerConfig) -> Result, Error> { +pub async fn main( + server: &mut WebServer, + config: &ServerConfig, +) -> Result, Error> { if &*PLATFORM == "raspberrypi" && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() { tokio::fs::remove_file(STANDBY_MODE_PATH).await?; Command::new("sync").invoke(ErrorKind::Filesystem).await?; @@ -200,16 +204,11 @@ async fn inner_main(config: &ServerConfig) -> Result, Error> { futures::future::pending::<()>().await; } - crate::sound::BEP.play().await?; - - run_script_if_exists("/media/startos/config/preinit.sh").await; - - let res = match setup_or_init(config).await { + let res = match setup_or_init(server, config).await { Err(e) => { async move { - tracing::error!("{}", e.source); - tracing::debug!("{}", e.source); - crate::sound::BEETHOVEN.play().await?; + tracing::error!("{e}"); + tracing::debug!("{e:?}"); let ctx = DiagnosticContext::init( config, @@ -229,44 +228,16 @@ async fn inner_main(config: &ServerConfig) -> Result, Error> { e, )?; - let server = WebServer::diagnostic( - SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80), - ctx.clone(), - )?; + server.serve_diagnostic(ctx.clone()); let shutdown = ctx.shutdown.subscribe().recv().await.unwrap(); - server.shutdown().await; - - Ok(shutdown) + Ok(Err(shutdown)) } .await } Ok(s) => Ok(s), }; - run_script_if_exists("/media/startos/config/postinit.sh").await; - res } - -pub fn main(config: &ServerConfig) { - let res = { - let rt = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .expect("failed to initialize runtime"); - rt.block_on(inner_main(config)) - }; - - match res { - Ok(Some(shutdown)) => shutdown.execute(), - Ok(None) => (), - Err(e) => { - eprintln!("{}", e.source); - tracing::debug!("{:?}", e.source); - drop(e.source); - std::process::exit(e.kind as i32) - } - } -} diff --git a/core/startos/src/bins/startd.rs b/core/startos/src/bins/startd.rs index f0bc428be..7576c41e9 100644 --- a/core/startos/src/bins/startd.rs +++ b/core/startos/src/bins/startd.rs @@ -1,6 +1,5 @@ use std::ffi::OsString; use std::net::{Ipv6Addr, SocketAddr}; -use std::path::Path; use std::sync::Arc; use clap::Parser; @@ -10,7 +9,8 @@ use tokio::signal::unix::signal; use tracing::instrument; use crate::context::config::ServerConfig; -use crate::context::{DiagnosticContext, RpcContext}; +use crate::context::rpc::InitRpcContextPhases; +use crate::context::{DiagnosticContext, InitContext, RpcContext}; use crate::net::web_server::WebServer; use crate::shutdown::Shutdown; use crate::system::launch_metrics_task; @@ -18,9 +18,31 @@ use crate::util::logger::EmbassyLogger; use crate::{Error, ErrorKind, ResultExt}; #[instrument(skip_all)] -async fn inner_main(config: &ServerConfig) -> Result, Error> { - let (rpc_ctx, server, shutdown) = async { - let rpc_ctx = RpcContext::init( +async fn inner_main( + server: &mut WebServer, + config: &ServerConfig, +) -> Result, Error> { + let rpc_ctx = if !tokio::fs::metadata("/run/startos/initialized") + .await + .is_ok() + { + let (ctx, handle) = match super::start_init::main(server, &config).await? { + Err(s) => return Ok(Some(s)), + Ok(ctx) => ctx, + }; + tokio::fs::write("/run/startos/initialized", "").await?; + + server.serve_main(ctx.clone()); + handle.complete(); + + ctx + } else { + let init_ctx = InitContext::init(config).await?; + let handle = init_ctx.progress.clone(); + let rpc_ctx_phases = InitRpcContextPhases::new(&handle); + server.serve_init(init_ctx); + + let ctx = RpcContext::init( config, Arc::new( tokio::fs::read_to_string("/media/startos/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy @@ -28,13 +50,19 @@ async fn inner_main(config: &ServerConfig) -> Result, Error> { .trim() .to_owned(), ), + None, + rpc_ctx_phases, ) .await?; + + server.serve_main(ctx.clone()); + handle.complete(); + + ctx + }; + + let (rpc_ctx, shutdown) = async { crate::hostname::sync_hostname(&rpc_ctx.account.read().await.hostname).await?; - let server = WebServer::main( - SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80), - rpc_ctx.clone(), - )?; let mut shutdown_recv = rpc_ctx.shutdown.subscribe(); @@ -74,8 +102,6 @@ async fn inner_main(config: &ServerConfig) -> Result, Error> { .await }); - crate::sound::CHIME.play().await?; - metrics_task .map_err(|e| { Error::new( @@ -93,10 +119,9 @@ async fn inner_main(config: &ServerConfig) -> Result, Error> { sig_handler.abort(); - Ok::<_, Error>((rpc_ctx, server, shutdown)) + Ok::<_, Error>((rpc_ctx, shutdown)) } .await?; - server.shutdown().await; rpc_ctx.shutdown().await?; tracing::info!("RPC Context is dropped"); @@ -109,24 +134,22 @@ pub fn main(args: impl IntoIterator) { let config = ServerConfig::parse_from(args).load().unwrap(); - if !Path::new("/run/embassy/initialized").exists() { - super::start_init::main(&config); - std::fs::write("/run/embassy/initialized", "").unwrap(); - } - let res = { let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .expect("failed to initialize runtime"); rt.block_on(async { - match inner_main(&config).await { - Ok(a) => Ok(a), + let mut server = WebServer::new(SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80)); + match inner_main(&mut server, &config).await { + Ok(a) => { + server.shutdown().await; + Ok(a) + } Err(e) => { async { - tracing::error!("{}", e.source); - tracing::debug!("{:?}", e.source); - crate::sound::BEETHOVEN.play().await?; + tracing::error!("{e}"); + tracing::debug!("{e:?}"); let ctx = DiagnosticContext::init( &config, if tokio::fs::metadata("/media/startos/config/disk.guid") @@ -145,10 +168,7 @@ pub fn main(args: impl IntoIterator) { e, )?; - let server = WebServer::diagnostic( - SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80), - ctx.clone(), - )?; + server.serve_diagnostic(ctx.clone()); let mut shutdown = ctx.shutdown.subscribe(); @@ -157,7 +177,7 @@ pub fn main(args: impl IntoIterator) { server.shutdown().await; - Ok::<_, Error>(shutdown) + Ok::<_, Error>(Some(shutdown)) } .await } diff --git a/core/startos/src/context/cli.rs b/core/startos/src/context/cli.rs index 014457b67..22084bbe1 100644 --- a/core/startos/src/context/cli.rs +++ b/core/startos/src/context/cli.rs @@ -18,7 +18,7 @@ use tracing::instrument; use super::setup::CURRENT_SECRET; use crate::context::config::{local_config_path, ClientConfig}; -use crate::context::{DiagnosticContext, InstallContext, RpcContext, SetupContext}; +use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext}; use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH; use crate::prelude::*; use crate::rpc_continuations::Guid; @@ -271,6 +271,11 @@ impl CallRemote for CliContext { call_remote_http(&self.client, self.rpc_url.clone(), method, params).await } } +impl CallRemote for CliContext { + async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result { + call_remote_http(&self.client, self.rpc_url.clone(), method, params).await + } +} impl CallRemote for CliContext { async fn call_remote(&self, method: &str, params: Value, _: Empty) -> Result { call_remote_http(&self.client, self.rpc_url.clone(), method, params).await diff --git a/core/startos/src/context/diagnostic.rs b/core/startos/src/context/diagnostic.rs index 10379dcf3..0bf67e172 100644 --- a/core/startos/src/context/diagnostic.rs +++ b/core/startos/src/context/diagnostic.rs @@ -14,7 +14,7 @@ use crate::Error; pub struct DiagnosticContextSeed { pub datadir: PathBuf, - pub shutdown: Sender>, + pub shutdown: Sender, pub error: Arc, pub disk_guid: Option>, pub rpc_continuations: RpcContinuations, diff --git a/core/startos/src/context/init.rs b/core/startos/src/context/init.rs new file mode 100644 index 000000000..f5f4a5430 --- /dev/null +++ b/core/startos/src/context/init.rs @@ -0,0 +1,47 @@ +use std::ops::Deref; +use std::sync::Arc; + +use rpc_toolkit::Context; +use tokio::sync::broadcast::Sender; +use tracing::instrument; + +use crate::context::config::ServerConfig; +use crate::progress::FullProgressTracker; +use crate::rpc_continuations::RpcContinuations; +use crate::Error; + +pub struct InitContextSeed { + pub config: ServerConfig, + pub progress: FullProgressTracker, + pub shutdown: Sender<()>, + pub rpc_continuations: RpcContinuations, +} + +#[derive(Clone)] +pub struct InitContext(Arc); +impl InitContext { + #[instrument(skip_all)] + pub async fn init(cfg: &ServerConfig) -> Result { + let (shutdown, _) = tokio::sync::broadcast::channel(1); + Ok(Self(Arc::new(InitContextSeed { + config: cfg.clone(), + progress: FullProgressTracker::new(), + shutdown, + rpc_continuations: RpcContinuations::new(), + }))) + } +} + +impl AsRef for InitContext { + fn as_ref(&self) -> &RpcContinuations { + &self.rpc_continuations + } +} + +impl Context for InitContext {} +impl Deref for InitContext { + type Target = InitContextSeed; + fn deref(&self) -> &Self::Target { + &*self.0 + } +} diff --git a/core/startos/src/context/install.rs b/core/startos/src/context/install.rs index d4717d2b0..c0c564b34 100644 --- a/core/startos/src/context/install.rs +++ b/core/startos/src/context/install.rs @@ -6,11 +6,13 @@ use tokio::sync::broadcast::Sender; use tracing::instrument; use crate::net::utils::find_eth_iface; +use crate::rpc_continuations::RpcContinuations; use crate::Error; pub struct InstallContextSeed { pub ethernet_interface: String, pub shutdown: Sender<()>, + pub rpc_continuations: RpcContinuations, } #[derive(Clone)] @@ -22,10 +24,17 @@ impl InstallContext { Ok(Self(Arc::new(InstallContextSeed { ethernet_interface: find_eth_iface().await?, shutdown, + rpc_continuations: RpcContinuations::new(), }))) } } +impl AsRef for InstallContext { + fn as_ref(&self) -> &RpcContinuations { + &self.rpc_continuations + } +} + impl Context for InstallContext {} impl Deref for InstallContext { type Target = InstallContextSeed; diff --git a/core/startos/src/context/mod.rs b/core/startos/src/context/mod.rs index 77f54f26c..efe261b0c 100644 --- a/core/startos/src/context/mod.rs +++ b/core/startos/src/context/mod.rs @@ -1,12 +1,14 @@ pub mod cli; pub mod config; pub mod diagnostic; +pub mod init; pub mod install; pub mod rpc; pub mod setup; pub use cli::CliContext; pub use diagnostic::DiagnosticContext; +pub use init::InitContext; pub use install::InstallContext; pub use rpc::RpcContext; pub use setup::SetupContext; diff --git a/core/startos/src/context/rpc.rs b/core/startos/src/context/rpc.rs index a3e77a62c..3dd3354aa 100644 --- a/core/startos/src/context/rpc.rs +++ b/core/startos/src/context/rpc.rs @@ -6,11 +6,12 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; +use imbl_value::InternedString; use josekit::jwk::Jwk; use reqwest::{Client, Proxy}; use rpc_toolkit::yajrc::RpcError; use rpc_toolkit::{CallRemote, Context, Empty}; -use tokio::sync::{broadcast, oneshot, Mutex, RwLock}; +use tokio::sync::{broadcast, Mutex, RwLock}; use tokio::time::Instant; use tracing::instrument; @@ -22,12 +23,12 @@ use crate::dependencies::compute_dependency_config_errs; use crate::disk::OsPartitionInfo; use crate::init::check_time_is_synchronized; use crate::lxc::{ContainerId, LxcContainer, LxcManager}; -use crate::middleware::auth::HashSessionToken; -use crate::net::net_controller::NetController; +use crate::net::net_controller::{NetController, PreInitNetController}; use crate::net::utils::{find_eth_iface, find_wifi_iface}; use crate::net::wifi::WpaCli; use crate::prelude::*; -use crate::rpc_continuations::RpcContinuations; +use crate::progress::{FullProgressTracker, PhaseProgressTrackerHandle}; +use crate::rpc_continuations::{OpenAuthedContinuations, RpcContinuations}; use crate::service::ServiceMap; use crate::shutdown::Shutdown; use crate::system::get_mem_info; @@ -49,7 +50,7 @@ pub struct RpcContextSeed { pub shutdown: broadcast::Sender>, pub tor_socks: SocketAddr, pub lxc_manager: Arc, - pub open_authed_websockets: Mutex>>>, + pub open_authed_continuations: OpenAuthedContinuations, pub rpc_continuations: RpcContinuations, pub wifi_manager: Option>>, pub current_secret: Arc, @@ -68,45 +69,103 @@ pub struct Hardware { pub ram: u64, } +pub struct InitRpcContextPhases { + load_db: PhaseProgressTrackerHandle, + init_net_ctrl: PhaseProgressTrackerHandle, + read_device_info: PhaseProgressTrackerHandle, + cleanup_init: CleanupInitPhases, +} +impl InitRpcContextPhases { + pub fn new(handle: &FullProgressTracker) -> Self { + Self { + load_db: handle.add_phase("Loading database".into(), Some(5)), + init_net_ctrl: handle.add_phase("Initializing network".into(), Some(1)), + read_device_info: handle.add_phase("Reading device information".into(), Some(1)), + cleanup_init: CleanupInitPhases::new(handle), + } + } +} + +pub struct CleanupInitPhases { + init_services: PhaseProgressTrackerHandle, + check_dependencies: PhaseProgressTrackerHandle, +} +impl CleanupInitPhases { + pub fn new(handle: &FullProgressTracker) -> Self { + Self { + init_services: handle.add_phase("Initializing services".into(), Some(10)), + check_dependencies: handle.add_phase("Checking dependencies".into(), Some(1)), + } + } +} + #[derive(Clone)] pub struct RpcContext(Arc); impl RpcContext { #[instrument(skip_all)] - pub async fn init(config: &ServerConfig, disk_guid: Arc) -> Result { - tracing::info!("Loaded Config"); + pub async fn init( + config: &ServerConfig, + disk_guid: Arc, + net_ctrl: Option, + InitRpcContextPhases { + mut load_db, + mut init_net_ctrl, + mut read_device_info, + cleanup_init, + }: InitRpcContextPhases, + ) -> Result { let tor_proxy = config.tor_socks.unwrap_or(SocketAddr::V4(SocketAddrV4::new( Ipv4Addr::new(127, 0, 0, 1), 9050, ))); let (shutdown, _) = tokio::sync::broadcast::channel(1); - let db = TypedPatchDb::::load(config.db().await?).await?; + load_db.start(); + let db = if let Some(net_ctrl) = &net_ctrl { + net_ctrl.db.clone() + } else { + TypedPatchDb::::load(config.db().await?).await? + }; let peek = db.peek().await; let account = AccountInfo::load(&peek)?; + load_db.complete(); tracing::info!("Opened PatchDB"); + + init_net_ctrl.start(); let net_controller = Arc::new( NetController::init( - db.clone(), - config - .tor_control - .unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))), - tor_proxy, + if let Some(net_ctrl) = net_ctrl { + net_ctrl + } else { + PreInitNetController::init( + db.clone(), + config + .tor_control + .unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))), + tor_proxy, + &account.hostname, + account.tor_key.clone(), + ) + .await? + }, config .dns_bind .as_deref() .unwrap_or(&[SocketAddr::from(([127, 0, 0, 1], 53))]), - &account.hostname, - account.tor_key.clone(), ) .await?, ); + init_net_ctrl.complete(); tracing::info!("Initialized Net Controller"); + let services = ServiceMap::default(); let metrics_cache = RwLock::>::new(None); - tracing::info!("Initialized Notification Manager"); let tor_proxy_url = format!("socks5h://{tor_proxy}"); + + read_device_info.start(); let devices = lshw().await?; let ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024; + read_device_info.complete(); if !db .peek() @@ -163,7 +222,7 @@ impl RpcContext { shutdown, tor_socks: tor_proxy, lxc_manager: Arc::new(LxcManager::new()), - open_authed_websockets: Mutex::new(BTreeMap::new()), + open_authed_continuations: OpenAuthedContinuations::new(), rpc_continuations: RpcContinuations::new(), wifi_manager: wifi_interface .clone() @@ -196,7 +255,7 @@ impl RpcContext { }); let res = Self(seed.clone()); - res.cleanup_and_initialize().await?; + res.cleanup_and_initialize(cleanup_init).await?; tracing::info!("Cleaned up transient states"); Ok(res) } @@ -210,11 +269,18 @@ impl RpcContext { Ok(()) } - #[instrument(skip(self))] - pub async fn cleanup_and_initialize(&self) -> Result<(), Error> { - self.services.init(&self).await?; + #[instrument(skip_all)] + pub async fn cleanup_and_initialize( + &self, + CleanupInitPhases { + init_services, + mut check_dependencies, + }: CleanupInitPhases, + ) -> Result<(), Error> { + self.services.init(&self, init_services).await?; tracing::info!("Initialized Package Managers"); + check_dependencies.start(); let mut updated_current_dependents = BTreeMap::new(); let peek = self.db.peek().await; for (package_id, package) in peek.as_public().as_package_data().as_entries()?.into_iter() { @@ -238,6 +304,7 @@ impl RpcContext { Ok(()) }) .await?; + check_dependencies.complete(); Ok(()) } @@ -274,6 +341,11 @@ impl AsRef for RpcContext { &self.rpc_continuations } } +impl AsRef> for RpcContext { + fn as_ref(&self) -> &OpenAuthedContinuations { + &self.open_authed_continuations + } +} impl Context for RpcContext {} impl Deref for RpcContext { type Target = RpcContextSeed; diff --git a/core/startos/src/context/setup.rs b/core/startos/src/context/setup.rs index 013dc060b..6041f49b9 100644 --- a/core/startos/src/context/setup.rs +++ b/core/startos/src/context/setup.rs @@ -1,23 +1,31 @@ use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; +use futures::{Future, StreamExt}; +use helpers::NonDetachingJoinHandle; use josekit::jwk::Jwk; use patch_db::PatchDb; -use rpc_toolkit::yajrc::RpcError; use rpc_toolkit::Context; use serde::{Deserialize, Serialize}; use sqlx::postgres::PgConnectOptions; use sqlx::PgPool; use tokio::sync::broadcast::Sender; -use tokio::sync::RwLock; +use tokio::sync::OnceCell; use tracing::instrument; +use ts_rs::TS; +use crate::account::AccountInfo; use crate::context::config::ServerConfig; +use crate::context::RpcContext; use crate::disk::OsPartitionInfo; use crate::init::init_postgres; use crate::prelude::*; -use crate::setup::SetupStatus; +use crate::progress::FullProgressTracker; +use crate::rpc_continuations::{Guid, RpcContinuation, RpcContinuations}; +use crate::setup::SetupProgress; +use crate::util::net::WebSocketExt; lazy_static::lazy_static! { pub static ref CURRENT_SECRET: Jwk = Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).unwrap_or_else(|e| { @@ -27,30 +35,35 @@ lazy_static::lazy_static! { }); } -#[derive(Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] +#[ts(export)] pub struct SetupResult { pub tor_address: String, pub lan_address: String, pub root_ca: String, } +impl TryFrom<&AccountInfo> for SetupResult { + type Error = Error; + fn try_from(value: &AccountInfo) -> Result { + Ok(Self { + tor_address: format!("https://{}", value.tor_key.public().get_onion_address()), + lan_address: value.hostname.lan_address(), + root_ca: String::from_utf8(value.root_ca_cert.to_pem()?)?, + }) + } +} pub struct SetupContextSeed { pub config: ServerConfig, pub os_partitions: OsPartitionInfo, pub disable_encryption: bool, + pub progress: FullProgressTracker, + pub task: OnceCell>, + pub result: OnceCell>, pub shutdown: Sender<()>, pub datadir: PathBuf, - pub selected_v2_drive: RwLock>, - pub cached_product_key: RwLock>>, - pub setup_status: RwLock>>, - pub setup_result: RwLock, SetupResult)>>, -} - -impl AsRef for SetupContextSeed { - fn as_ref(&self) -> &Jwk { - &*CURRENT_SECRET - } + pub rpc_continuations: RpcContinuations, } #[derive(Clone)] @@ -69,12 +82,12 @@ impl SetupContext { ) })?, disable_encryption: config.disable_encryption.unwrap_or(false), + progress: FullProgressTracker::new(), + task: OnceCell::new(), + result: OnceCell::new(), shutdown, datadir, - selected_v2_drive: RwLock::new(None), - cached_product_key: RwLock::new(None), - setup_status: RwLock::new(None), - setup_result: RwLock::new(None), + rpc_continuations: RpcContinuations::new(), }))) } #[instrument(skip_all)] @@ -97,6 +110,104 @@ impl SetupContext { .with_kind(crate::ErrorKind::Database)?; Ok(secret_store) } + + pub fn run_setup(&self, f: F) -> Result<(), Error> + where + F: FnOnce() -> Fut + Send + 'static, + Fut: Future> + Send, + { + let local_ctx = self.clone(); + self.task + .set( + tokio::spawn(async move { + local_ctx + .result + .get_or_init(|| async { + match f().await { + Ok(res) => { + tracing::info!("Setup complete!"); + Ok(res) + } + Err(e) => { + tracing::error!("Setup failed: {e}"); + tracing::debug!("{e:?}"); + Err(e) + } + } + }) + .await; + local_ctx.progress.complete(); + }) + .into(), + ) + .map_err(|_| { + if self.result.initialized() { + Error::new(eyre!("Setup already complete"), ErrorKind::InvalidRequest) + } else { + Error::new( + eyre!("Setup already in progress"), + ErrorKind::InvalidRequest, + ) + } + })?; + Ok(()) + } + + pub async fn progress(&self) -> SetupProgress { + use axum::extract::ws; + + let guid = Guid::new(); + let progress_tracker = self.progress.clone(); + let progress = progress_tracker.snapshot(); + self.rpc_continuations + .add( + guid.clone(), + RpcContinuation::ws( + |mut ws| async move { + if let Err(e) = async { + let mut stream = + progress_tracker.stream(Some(Duration::from_millis(100))); + while let Some(progress) = stream.next().await { + ws.send(ws::Message::Text( + serde_json::to_string(&progress) + .with_kind(ErrorKind::Serialization)?, + )) + .await + .with_kind(ErrorKind::Network)?; + if progress.overall.is_complete() { + break; + } + } + + ws.normal_close("complete").await?; + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("Error in setup progress websocket: {e}"); + tracing::debug!("{e:?}"); + } + }, + Duration::from_secs(30), + ), + ) + .await; + + SetupProgress { progress, guid } + } +} + +impl AsRef for SetupContext { + fn as_ref(&self) -> &Jwk { + &*CURRENT_SECRET + } +} + +impl AsRef for SetupContext { + fn as_ref(&self) -> &RpcContinuations { + &self.rpc_continuations + } } impl Context for SetupContext {} diff --git a/core/startos/src/db/mod.rs b/core/startos/src/db/mod.rs index 0bb8a23db..e59161e9b 100644 --- a/core/startos/src/db/mod.rs +++ b/core/startos/src/db/mod.rs @@ -3,175 +3,40 @@ pub mod prelude; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; -use axum::extract::ws::{self, WebSocket}; -use axum::extract::WebSocketUpgrade; -use axum::response::Response; +use axum::extract::ws; use clap::Parser; -use futures::{FutureExt, StreamExt}; -use http::header::COOKIE; -use http::HeaderMap; +use imbl_value::InternedString; use itertools::Itertools; use patch_db::json_ptr::{JsonPointer, ROOT}; use patch_db::{Dump, Revision}; use rpc_toolkit::yajrc::RpcError; use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; use serde::{Deserialize, Serialize}; -use serde_json::Value; -use tokio::sync::oneshot; use tracing::instrument; use ts_rs::TS; use crate::context::{CliContext, RpcContext}; -use crate::middleware::auth::{HasValidSession, HashSessionToken}; use crate::prelude::*; +use crate::rpc_continuations::{Guid, RpcContinuation}; +use crate::util::net::WebSocketExt; use crate::util::serde::{apply_expr, HandlerExtSerde}; lazy_static::lazy_static! { static ref PUBLIC: JsonPointer = "/public".parse().unwrap(); } -#[instrument(skip_all)] -async fn ws_handler( - ctx: RpcContext, - session: Option<(HasValidSession, HashSessionToken)>, - mut stream: WebSocket, -) -> Result<(), Error> { - let (dump, sub) = ctx.db.dump_and_sub(PUBLIC.clone()).await; - - if let Some((session, token)) = session { - let kill = subscribe_to_session_kill(&ctx, token).await; - send_dump(session.clone(), &mut stream, dump).await?; - - deal_with_messages(session, kill, sub, stream).await?; - } else { - stream - .send(ws::Message::Close(Some(ws::CloseFrame { - code: ws::close_code::ERROR, - reason: "UNAUTHORIZED".into(), - }))) - .await - .with_kind(ErrorKind::Network)?; - drop(stream); - } - - Ok(()) -} - -async fn subscribe_to_session_kill( - ctx: &RpcContext, - token: HashSessionToken, -) -> oneshot::Receiver<()> { - let (send, recv) = oneshot::channel(); - let mut guard = ctx.open_authed_websockets.lock().await; - if !guard.contains_key(&token) { - guard.insert(token, vec![send]); - } else { - guard.get_mut(&token).unwrap().push(send); - } - recv -} - -#[instrument(skip_all)] -async fn deal_with_messages( - _has_valid_authentication: HasValidSession, - mut kill: oneshot::Receiver<()>, - mut sub: patch_db::Subscriber, - mut stream: WebSocket, -) -> Result<(), Error> { - let mut timer = tokio::time::interval(tokio::time::Duration::from_secs(5)); - - loop { - futures::select! { - _ = (&mut kill).fuse() => { - tracing::info!("Closing WebSocket: Reason: Session Terminated"); - stream - .send(ws::Message::Close(Some(ws::CloseFrame { - code: ws::close_code::ERROR, - reason: "UNAUTHORIZED".into(), - }))).await - .with_kind(ErrorKind::Network)?; - drop(stream); - return Ok(()) - } - new_rev = sub.recv().fuse() => { - let rev = new_rev.expect("UNREACHABLE: patch-db is dropped"); - stream - .send(ws::Message::Text(serde_json::to_string(&rev).with_kind(ErrorKind::Serialization)?)) - .await - .with_kind(ErrorKind::Network)?; - } - message = stream.next().fuse() => { - let message = message.transpose().with_kind(ErrorKind::Network)?; - match message { - None => { - tracing::info!("Closing WebSocket: Stream Finished"); - return Ok(()) - } - _ => (), - } - } - // This is trying to give a health checks to the home to keep the ui alive. - _ = timer.tick().fuse() => { - stream - .send(ws::Message::Ping(vec![])) - .await - .with_kind(crate::ErrorKind::Network)?; - } - } - } -} - -async fn send_dump( - _has_valid_authentication: HasValidSession, - stream: &mut WebSocket, - dump: Dump, -) -> Result<(), Error> { - stream - .send(ws::Message::Text( - serde_json::to_string(&dump).with_kind(ErrorKind::Serialization)?, - )) - .await - .with_kind(ErrorKind::Network)?; - Ok(()) -} - -pub async fn subscribe( - ctx: RpcContext, - headers: HeaderMap, - ws: WebSocketUpgrade, -) -> Result { - let session = match async { - let token = HashSessionToken::from_header(headers.get(COOKIE))?; - let session = HasValidSession::from_header(headers.get(COOKIE), &ctx).await?; - Ok::<_, Error>((session, token)) - } - .await - { - Ok(a) => Some(a), - Err(e) => { - if e.kind != ErrorKind::Authorization { - tracing::error!("Error Authenticating Websocket: {}", e); - tracing::debug!("{:?}", e); - } - None - } - }; - Ok(ws.on_upgrade(|ws| async move { - match ws_handler(ctx, session, ws).await { - Ok(()) => (), - Err(e) => { - tracing::error!("WebSocket Closed: {}", e); - tracing::debug!("{:?}", e); - } - } - })) -} - pub fn db() -> ParentHandler { ParentHandler::new() .subcommand("dump", from_fn_async(cli_dump).with_display_serializable()) .subcommand("dump", from_fn_async(dump).no_cli()) + .subcommand( + "subscribe", + from_fn_async(subscribe) + .with_metadata("get_session", Value::Bool(true)) + .no_cli(), + ) .subcommand("put", put::()) .subcommand("apply", from_fn_async(cli_apply).no_display()) .subcommand("apply", from_fn_async(apply).no_cli()) @@ -215,7 +80,13 @@ async fn cli_dump( context .call_remote::( &method, - imbl_value::json!({ "includePrivate":include_private }), + imbl_value::json!({ + "pointer": if include_private { + AsRef::::as_ref(&ROOT) + } else { + AsRef::::as_ref(&*PUBLIC) + } + }), ) .await?, )? @@ -224,25 +95,76 @@ async fn cli_dump( Ok(dump) } -#[derive(Deserialize, Serialize, Parser, TS)] +#[derive(Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] -#[command(rename_all = "kebab-case")] pub struct DumpParams { - #[arg(long = "include-private", short = 'p')] - #[serde(default)] - #[ts(skip)] - include_private: bool, + #[ts(type = "string | null")] + pointer: Option, } -pub async fn dump( +pub async fn dump(ctx: RpcContext, DumpParams { pointer }: DumpParams) -> Result { + Ok(ctx.db.dump(pointer.as_ref().unwrap_or(&*PUBLIC)).await) +} + +#[derive(Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +pub struct SubscribeParams { + #[ts(type = "string | null")] + pointer: Option, + #[ts(skip)] + #[serde(rename = "__auth_session")] + session: InternedString, +} + +#[derive(Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +pub struct SubscribeRes { + #[ts(type = "{ id: number; value: unknown }")] + pub dump: Dump, + pub guid: Guid, +} + +pub async fn subscribe( ctx: RpcContext, - DumpParams { include_private }: DumpParams, -) -> Result { - Ok(if include_private { - ctx.db.dump(&ROOT).await - } else { - ctx.db.dump(&PUBLIC).await - }) + SubscribeParams { pointer, session }: SubscribeParams, +) -> Result { + let (dump, mut sub) = ctx + .db + .dump_and_sub(pointer.unwrap_or_else(|| PUBLIC.clone())) + .await; + let guid = Guid::new(); + ctx.rpc_continuations + .add( + guid.clone(), + RpcContinuation::ws_authed( + &ctx, + session, + |mut ws| async move { + if let Err(e) = async { + while let Some(rev) = sub.recv().await { + ws.send(ws::Message::Text( + serde_json::to_string(&rev).with_kind(ErrorKind::Serialization)?, + )) + .await + .with_kind(ErrorKind::Network)?; + } + + ws.normal_close("complete").await?; + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("Error in db websocket: {e}"); + tracing::debug!("{e:?}"); + } + }, + Duration::from_secs(30), + ), + ) + .await; + + Ok(SubscribeRes { dump, guid }) } #[derive(Deserialize, Serialize, Parser)] diff --git a/core/startos/src/diagnostic.rs b/core/startos/src/diagnostic.rs index 485f2359f..5e99580e9 100644 --- a/core/startos/src/diagnostic.rs +++ b/core/startos/src/diagnostic.rs @@ -27,10 +27,6 @@ pub fn diagnostic() -> ParentHandler { "kernel-logs", from_fn_async(crate::logs::cli_logs::).no_display(), ) - .subcommand( - "exit", - from_fn(exit).no_display().with_call_remote::(), - ) .subcommand( "restart", from_fn(restart) @@ -51,20 +47,15 @@ pub fn error(ctx: DiagnosticContext) -> Result, Error> { Ok(ctx.error.clone()) } -pub fn exit(ctx: DiagnosticContext) -> Result<(), Error> { - ctx.shutdown.send(None).expect("receiver dropped"); - Ok(()) -} - pub fn restart(ctx: DiagnosticContext) -> Result<(), Error> { ctx.shutdown - .send(Some(Shutdown { + .send(Shutdown { export_args: ctx .disk_guid .clone() .map(|guid| (guid, ctx.datadir.clone())), restart: true, - })) + }) .expect("receiver dropped"); Ok(()) } diff --git a/core/startos/src/disk/main.rs b/core/startos/src/disk/main.rs index d414c247e..ee807a938 100644 --- a/core/startos/src/disk/main.rs +++ b/core/startos/src/disk/main.rs @@ -13,7 +13,7 @@ use crate::disk::mount::util::unmount; use crate::util::Invoke; use crate::{Error, ErrorKind, ResultExt}; -pub const PASSWORD_PATH: &'static str = "/run/embassy/password"; +pub const PASSWORD_PATH: &'static str = "/run/startos/password"; pub const DEFAULT_PASSWORD: &'static str = "password"; pub const MAIN_FS_SIZE: FsSize = FsSize::Gigabytes(8); diff --git a/core/startos/src/firmware.rs b/core/startos/src/firmware.rs index 20347bcff..a9d5ced79 100644 --- a/core/startos/src/firmware.rs +++ b/core/startos/src/firmware.rs @@ -9,6 +9,7 @@ use tokio::process::Command; use crate::disk::fsck::RequiresReboot; use crate::prelude::*; +use crate::progress::PhaseProgressTrackerHandle; use crate::util::Invoke; use crate::PLATFORM; @@ -49,12 +50,7 @@ pub fn display_firmware_update_result(result: RequiresReboot) { } } -/// We wanted to make sure during every init -/// that the firmware was the correct and updated for -/// systems like the Pure System that a new firmware -/// was released and the updates where pushed through the pure os. -// #[command(rename = "update-firmware", display(display_firmware_update_result))] -pub async fn update_firmware() -> Result { +pub async fn check_for_firmware_update() -> Result, Error> { let system_product_name = String::from_utf8( Command::new("dmidecode") .arg("-s") @@ -74,22 +70,21 @@ pub async fn update_firmware() -> Result { .trim() .to_owned(); if system_product_name.is_empty() || bios_version.is_empty() { - return Ok(RequiresReboot(false)); + return Ok(None); } - let firmware_dir = Path::new("/usr/lib/startos/firmware"); - for firmware in serde_json::from_str::>( &tokio::fs::read_to_string("/usr/lib/startos/firmware.json").await?, ) .with_kind(ErrorKind::Deserialization)? { - let id = firmware.id; let matches_product_name = firmware .system_product_name - .map_or(true, |spn| spn == system_product_name); + .as_ref() + .map_or(true, |spn| spn == &system_product_name); let matches_bios_version = firmware .bios_version + .as_ref() .map_or(Some(true), |bv| { let mut semver_str = bios_version.as_str(); if let Some(prefix) = &bv.semver_prefix { @@ -113,35 +108,45 @@ pub async fn update_firmware() -> Result { }) .unwrap_or(false); if firmware.platform.contains(&*PLATFORM) && matches_product_name && matches_bios_version { - let filename = format!("{id}.rom.gz"); - let firmware_path = firmware_dir.join(&filename); - Command::new("sha256sum") - .arg("-c") - .input(Some(&mut std::io::Cursor::new(format!( - "{} {}", - firmware.shasum, - firmware_path.display() - )))) - .invoke(ErrorKind::Filesystem) - .await?; - let mut rdr = if tokio::fs::metadata(&firmware_path).await.is_ok() { - GzipDecoder::new(BufReader::new(File::open(&firmware_path).await?)) - } else { - return Err(Error::new( - eyre!("Firmware {id}.rom.gz not found in {firmware_dir:?}"), - ErrorKind::NotFound, - )); - }; - Command::new("flashrom") - .arg("-p") - .arg("internal") - .arg("-w-") - .input(Some(&mut rdr)) - .invoke(ErrorKind::Firmware) - .await?; - return Ok(RequiresReboot(true)); + return Ok(Some(firmware)); } } - Ok(RequiresReboot(false)) + Ok(None) +} + +/// We wanted to make sure during every init +/// that the firmware was the correct and updated for +/// systems like the Pure System that a new firmware +/// was released and the updates where pushed through the pure os. +pub async fn update_firmware(firmware: Firmware) -> Result<(), Error> { + let id = &firmware.id; + let firmware_dir = Path::new("/usr/lib/startos/firmware"); + let filename = format!("{id}.rom.gz"); + let firmware_path = firmware_dir.join(&filename); + Command::new("sha256sum") + .arg("-c") + .input(Some(&mut std::io::Cursor::new(format!( + "{} {}", + firmware.shasum, + firmware_path.display() + )))) + .invoke(ErrorKind::Filesystem) + .await?; + let mut rdr = if tokio::fs::metadata(&firmware_path).await.is_ok() { + GzipDecoder::new(BufReader::new(File::open(&firmware_path).await?)) + } else { + return Err(Error::new( + eyre!("Firmware {id}.rom.gz not found in {firmware_dir:?}"), + ErrorKind::NotFound, + )); + }; + Command::new("flashrom") + .arg("-p") + .arg("internal") + .arg("-w-") + .input(Some(&mut rdr)) + .invoke(ErrorKind::Firmware) + .await?; + Ok(()) } diff --git a/core/startos/src/init.rs b/core/startos/src/init.rs index 97c674ac5..cdc444c32 100644 --- a/core/startos/src/init.rs +++ b/core/startos/src/init.rs @@ -1,25 +1,40 @@ use std::fs::Permissions; +use std::io::Cursor; +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::os::unix::fs::PermissionsExt; use std::path::Path; use std::time::{Duration, SystemTime}; +use axum::extract::ws::{self, CloseFrame}; use color_eyre::eyre::eyre; +use futures::{StreamExt, TryStreamExt}; +use itertools::Itertools; use models::ResultExt; use rand::random; +use rpc_toolkit::{from_fn_async, Context, Empty, HandlerArgs, HandlerExt, ParentHandler}; +use serde::{Deserialize, Serialize}; use tokio::process::Command; use tracing::instrument; +use ts_rs::TS; use crate::account::AccountInfo; use crate::context::config::ServerConfig; +use crate::context::{CliContext, InitContext}; use crate::db::model::public::ServerStatus; use crate::db::model::Database; use crate::disk::mount::util::unmount; use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH; +use crate::net::net_controller::PreInitNetController; use crate::prelude::*; +use crate::progress::{ + FullProgress, FullProgressTracker, PhaseProgressTrackerHandle, PhasedProgressBar, +}; +use crate::rpc_continuations::{Guid, RpcContinuation}; use crate::ssh::SSH_AUTHORIZED_KEYS_FILE; -use crate::util::cpupower::{get_available_governors, get_preferred_governor, set_governor}; -use crate::util::Invoke; -use crate::{Error, ARCH}; +use crate::util::io::IOHook; +use crate::util::net::WebSocketExt; +use crate::util::{cpupower, Invoke}; +use crate::Error; pub const SYSTEM_REBUILD_PATH: &str = "/media/startos/config/system-rebuild"; pub const STANDBY_MODE_PATH: &str = "/media/startos/config/standby"; @@ -180,14 +195,114 @@ pub async fn init_postgres(datadir: impl AsRef) -> Result<(), Error> { } pub struct InitResult { - pub db: TypedPatchDb, + pub net_ctrl: PreInitNetController, +} + +pub struct InitPhases { + preinit: Option, + local_auth: PhaseProgressTrackerHandle, + load_database: PhaseProgressTrackerHandle, + load_ssh_keys: PhaseProgressTrackerHandle, + start_net: PhaseProgressTrackerHandle, + mount_logs: PhaseProgressTrackerHandle, + load_ca_cert: PhaseProgressTrackerHandle, + load_wifi: PhaseProgressTrackerHandle, + init_tmp: PhaseProgressTrackerHandle, + set_governor: PhaseProgressTrackerHandle, + sync_clock: PhaseProgressTrackerHandle, + enable_zram: PhaseProgressTrackerHandle, + update_server_info: PhaseProgressTrackerHandle, + launch_service_network: PhaseProgressTrackerHandle, + run_migrations: PhaseProgressTrackerHandle, + validate_db: PhaseProgressTrackerHandle, + postinit: Option, +} +impl InitPhases { + pub fn new(handle: &FullProgressTracker) -> Self { + Self { + preinit: if Path::new("/media/startos/config/preinit.sh").exists() { + Some(handle.add_phase("Running preinit.sh".into(), Some(5))) + } else { + None + }, + local_auth: handle.add_phase("Enabling local authentication".into(), Some(1)), + load_database: handle.add_phase("Loading database".into(), Some(5)), + load_ssh_keys: handle.add_phase("Loading SSH Keys".into(), Some(1)), + start_net: handle.add_phase("Starting network controller".into(), Some(1)), + mount_logs: handle.add_phase("Switching logs to write to data drive".into(), Some(1)), + load_ca_cert: handle.add_phase("Loading CA certificate".into(), Some(1)), + load_wifi: handle.add_phase("Loading WiFi configuration".into(), Some(1)), + init_tmp: handle.add_phase("Initializing temporary files".into(), Some(1)), + set_governor: handle.add_phase("Setting CPU performance profile".into(), Some(1)), + sync_clock: handle.add_phase("Synchronizing system clock".into(), Some(10)), + enable_zram: handle.add_phase("Enabling ZRAM".into(), Some(1)), + update_server_info: handle.add_phase("Updating server info".into(), Some(1)), + launch_service_network: handle.add_phase("Launching service intranet".into(), Some(10)), + run_migrations: handle.add_phase("Running migrations".into(), Some(10)), + validate_db: handle.add_phase("Validating database".into(), Some(1)), + postinit: if Path::new("/media/startos/config/postinit.sh").exists() { + Some(handle.add_phase("Running postinit.sh".into(), Some(5))) + } else { + None + }, + } + } +} + +pub async fn run_script>(path: P, mut progress: PhaseProgressTrackerHandle) { + let script = path.as_ref(); + progress.start(); + if let Err(e) = async { + let script = tokio::fs::read_to_string(script).await?; + progress.set_total(script.as_bytes().iter().filter(|b| **b == b'\n').count() as u64); + let mut reader = IOHook::new(Cursor::new(script.as_bytes())); + reader.post_read(|buf| progress += buf.iter().filter(|b| **b == b'\n').count() as u64); + Command::new("/bin/bash") + .input(Some(&mut reader)) + .invoke(ErrorKind::Unknown) + .await?; + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("Error Running {}: {}", script.display(), e); + tracing::debug!("{:?}", e); + } + progress.complete(); } #[instrument(skip_all)] -pub async fn init(cfg: &ServerConfig) -> Result { - tokio::fs::create_dir_all("/run/embassy") +pub async fn init( + cfg: &ServerConfig, + InitPhases { + preinit, + mut local_auth, + mut load_database, + mut load_ssh_keys, + mut start_net, + mut mount_logs, + mut load_ca_cert, + mut load_wifi, + mut init_tmp, + mut set_governor, + mut sync_clock, + mut enable_zram, + mut update_server_info, + mut launch_service_network, + run_migrations, + mut validate_db, + postinit, + }: InitPhases, +) -> Result { + if let Some(progress) = preinit { + run_script("/media/startos/config/preinit.sh", progress).await; + } + + local_auth.start(); + tokio::fs::create_dir_all("/run/startos") .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, "mkdir -p /run/embassy"))?; + .with_ctx(|_| (crate::ErrorKind::Filesystem, "mkdir -p /run/startos"))?; if tokio::fs::metadata(LOCAL_AUTH_COOKIE_PATH).await.is_err() { tokio::fs::write( LOCAL_AUTH_COOKIE_PATH, @@ -207,43 +322,41 @@ pub async fn init(cfg: &ServerConfig) -> Result { .invoke(crate::ErrorKind::Filesystem) .await?; } + local_auth.complete(); + load_database.start(); let db = TypedPatchDb::::load_unchecked(cfg.db().await?); let peek = db.peek().await; + load_database.complete(); tracing::info!("Opened PatchDB"); + load_ssh_keys.start(); crate::ssh::sync_keys( &peek.as_private().as_ssh_pubkeys().de()?, SSH_AUTHORIZED_KEYS_FILE, ) .await?; + load_ssh_keys.complete(); tracing::info!("Synced SSH Keys"); let account = AccountInfo::load(&peek)?; - let mut server_info = peek.as_public().as_server_info().de()?; - - // write to ca cert store - tokio::fs::write( - "/usr/local/share/ca-certificates/startos-root-ca.crt", - account.root_ca_cert.to_pem()?, + start_net.start(); + let net_ctrl = PreInitNetController::init( + db.clone(), + cfg.tor_control + .unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))), + cfg.tor_socks.unwrap_or(SocketAddr::V4(SocketAddrV4::new( + Ipv4Addr::new(127, 0, 0, 1), + 9050, + ))), + &account.hostname, + account.tor_key, ) .await?; - Command::new("update-ca-certificates") - .invoke(crate::ErrorKind::OpenSsl) - .await?; - - crate::net::wifi::synchronize_wpa_supplicant_conf( - &cfg.datadir().join("main"), - &mut server_info.wifi, - ) - .await?; - tracing::info!("Synchronized WiFi"); - - let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok() - || &*server_info.version < &emver::Version::new(0, 3, 2, 0) - || (ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0)); + start_net.complete(); + mount_logs.start(); let log_dir = cfg.datadir().join("main/logs"); if tokio::fs::metadata(&log_dir).await.is_err() { tokio::fs::create_dir_all(&log_dir).await?; @@ -272,10 +385,35 @@ pub async fn init(cfg: &ServerConfig) -> Result { .arg("systemd-journald") .invoke(crate::ErrorKind::Journald) .await?; + mount_logs.complete(); tracing::info!("Mounted Logs"); + let mut server_info = peek.as_public().as_server_info().de()?; + + load_ca_cert.start(); + // write to ca cert store + tokio::fs::write( + "/usr/local/share/ca-certificates/startos-root-ca.crt", + account.root_ca_cert.to_pem()?, + ) + .await?; + Command::new("update-ca-certificates") + .invoke(crate::ErrorKind::OpenSsl) + .await?; + load_ca_cert.complete(); + + load_wifi.start(); + crate::net::wifi::synchronize_wpa_supplicant_conf( + &cfg.datadir().join("main"), + &mut server_info.wifi, + ) + .await?; + load_wifi.complete(); + tracing::info!("Synchronized WiFi"); + + init_tmp.start(); let tmp_dir = cfg.datadir().join("package-data/tmp"); - if should_rebuild && tokio::fs::metadata(&tmp_dir).await.is_ok() { + if tokio::fs::metadata(&tmp_dir).await.is_ok() { tokio::fs::remove_dir_all(&tmp_dir).await?; } if tokio::fs::metadata(&tmp_dir).await.is_err() { @@ -286,23 +424,30 @@ pub async fn init(cfg: &ServerConfig) -> Result { tokio::fs::remove_dir_all(&tmp_var).await?; } crate::disk::mount::util::bind(&tmp_var, "/var/tmp", false).await?; + init_tmp.complete(); + set_governor.start(); let governor = if let Some(governor) = &server_info.governor { - if get_available_governors().await?.contains(governor) { + if cpupower::get_available_governors() + .await? + .contains(governor) + { Some(governor) } else { tracing::warn!("CPU Governor \"{governor}\" Not Available"); None } } else { - get_preferred_governor().await? + cpupower::get_preferred_governor().await? }; if let Some(governor) = governor { tracing::info!("Setting CPU Governor to \"{governor}\""); - set_governor(governor).await?; + cpupower::set_governor(governor).await?; tracing::info!("Set CPU Governor"); } + set_governor.complete(); + sync_clock.start(); server_info.ntp_synced = false; let mut not_made_progress = 0u32; for _ in 0..1800 { @@ -329,10 +474,15 @@ pub async fn init(cfg: &ServerConfig) -> Result { } else { tracing::info!("Syncronized system clock"); } + sync_clock.complete(); + enable_zram.start(); if server_info.zram { crate::system::enable_zram().await? } + enable_zram.complete(); + + update_server_info.start(); server_info.ip_info = crate::net::dhcp::init_ips().await?; server_info.status_info = ServerStatus { updated: false, @@ -341,36 +491,129 @@ pub async fn init(cfg: &ServerConfig) -> Result { shutting_down: false, restarting: false, }; - db.mutate(|v| { v.as_public_mut().as_server_info_mut().ser(&server_info)?; Ok(()) }) .await?; + update_server_info.complete(); + launch_service_network.start(); Command::new("systemctl") .arg("start") .arg("lxc-net.service") .invoke(ErrorKind::Lxc) .await?; + launch_service_network.complete(); - crate::version::init(&db).await?; + crate::version::init(&db, run_migrations).await?; + validate_db.start(); db.mutate(|d| { let model = d.de()?; d.ser(&model) }) .await?; + validate_db.complete(); - if should_rebuild { - match tokio::fs::remove_file(SYSTEM_REBUILD_PATH).await { - Ok(()) => Ok(()), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(e) => Err(e), - }?; + if let Some(progress) = postinit { + run_script("/media/startos/config/postinit.sh", progress).await; } tracing::info!("System initialized."); - Ok(InitResult { db }) + Ok(InitResult { net_ctrl }) +} + +pub fn init_api() -> ParentHandler { + ParentHandler::new() + .subcommand("logs", crate::system::logs::()) + .subcommand( + "logs", + from_fn_async(crate::logs::cli_logs::).no_display(), + ) + .subcommand("kernel-logs", crate::system::kernel_logs::()) + .subcommand( + "kernel-logs", + from_fn_async(crate::logs::cli_logs::).no_display(), + ) + .subcommand("subscribe", from_fn_async(init_progress).no_cli()) + .subcommand("subscribe", from_fn_async(cli_init_progress).no_display()) +} + +#[derive(Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct InitProgressRes { + pub progress: FullProgress, + pub guid: Guid, +} + +pub async fn init_progress(ctx: InitContext) -> Result { + let progress_tracker = ctx.progress.clone(); + let progress = progress_tracker.snapshot(); + let guid = Guid::new(); + ctx.rpc_continuations + .add( + guid.clone(), + RpcContinuation::ws( + |mut ws| async move { + if let Err(e) = async { + let mut stream = progress_tracker.stream(Some(Duration::from_millis(100))); + while let Some(progress) = stream.next().await { + ws.send(ws::Message::Text( + serde_json::to_string(&progress) + .with_kind(ErrorKind::Serialization)?, + )) + .await + .with_kind(ErrorKind::Network)?; + if progress.overall.is_complete() { + break; + } + } + + ws.normal_close("complete").await?; + + Ok::<_, Error>(()) + } + .await + { + tracing::error!("error in init progress websocket: {e}"); + tracing::debug!("{e:?}"); + } + }, + Duration::from_secs(30), + ), + ) + .await; + Ok(InitProgressRes { progress, guid }) +} + +pub async fn cli_init_progress( + HandlerArgs { + context: ctx, + parent_method, + method, + raw_params, + .. + }: HandlerArgs, +) -> Result<(), Error> { + let res: InitProgressRes = from_value( + ctx.call_remote::( + &parent_method + .into_iter() + .chain(method.into_iter()) + .join("."), + raw_params, + ) + .await?, + )?; + let mut ws = ctx.ws_continuation(res.guid).await?; + let mut bar = PhasedProgressBar::new("Initializing..."); + while let Some(msg) = ws.try_next().await.with_kind(ErrorKind::Network)? { + if let tokio_tungstenite::tungstenite::Message::Text(msg) = msg { + bar.update(&serde_json::from_str(&msg).with_kind(ErrorKind::Deserialization)?); + } + } + Ok(()) } diff --git a/core/startos/src/install/mod.rs b/core/startos/src/install/mod.rs index 707503615..5d50da27d 100644 --- a/core/startos/src/install/mod.rs +++ b/core/startos/src/install/mod.rs @@ -6,7 +6,8 @@ use clap::builder::ValueParserFactory; use clap::{value_parser, CommandFactory, FromArgMatches, Parser}; use color_eyre::eyre::eyre; use emver::VersionRange; -use futures::{FutureExt, StreamExt}; +use futures::StreamExt; +use imbl_value::InternedString; use itertools::Itertools; use patch_db::json_ptr::JsonPointer; use reqwest::header::{HeaderMap, CONTENT_LENGTH}; @@ -29,6 +30,7 @@ use crate::s9pk::merkle_archive::source::http::HttpSource; use crate::s9pk::S9pk; use crate::upload::upload; use crate::util::clap::FromStrParser; +use crate::util::net::WebSocketExt; use crate::util::Never; pub const PKG_ARCHIVE_DIR: &str = "package-data/archive"; @@ -170,7 +172,15 @@ pub async fn install( Ok(()) } -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +pub struct SideloadParams { + #[ts(skip)] + #[serde(rename = "__auth_session")] + session: InternedString, +} + +#[derive(Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] pub struct SideloadResponse { pub upload: Guid, @@ -178,8 +188,11 @@ pub struct SideloadResponse { } #[instrument(skip_all)] -pub async fn sideload(ctx: RpcContext) -> Result { - let (upload, file) = upload(&ctx).await?; +pub async fn sideload( + ctx: RpcContext, + SideloadParams { session }: SideloadParams, +) -> Result { + let (upload, file) = upload(&ctx, session.clone()).await?; let (id_send, id_recv) = oneshot::channel(); let (err_send, err_recv) = oneshot::channel(); let progress = Guid::new(); @@ -193,8 +206,8 @@ pub async fn sideload(ctx: RpcContext) -> Result { .await; ctx.rpc_continuations.add( progress.clone(), - RpcContinuation::ws( - Box::new(|mut ws| { + RpcContinuation::ws_authed(&ctx, session, + |mut ws| { use axum::extract::ws::Message; async move { if let Err(e) = async { @@ -251,7 +264,7 @@ pub async fn sideload(ctx: RpcContext) -> Result { } } - ws.close().await.with_kind(ErrorKind::Network)?; + ws.normal_close("complete").await?; Ok::<_, Error>(()) } @@ -261,8 +274,7 @@ pub async fn sideload(ctx: RpcContext) -> Result { tracing::debug!("{e:?}"); } } - .boxed() - }), + }, Duration::from_secs(600), ), ) diff --git a/core/startos/src/lib.rs b/core/startos/src/lib.rs index 0e125af98..6e7cc8f8b 100644 --- a/core/startos/src/lib.rs +++ b/core/startos/src/lib.rs @@ -1,8 +1,5 @@ pub const DEFAULT_MARKETPLACE: &str = "https://registry.start9.com"; // pub const COMMUNITY_MARKETPLACE: &str = "https://community-registry.start9.com"; -pub const CAP_1_KiB: usize = 1024; -pub const CAP_1_MiB: usize = CAP_1_KiB * CAP_1_KiB; -pub const CAP_10_MiB: usize = 10 * CAP_1_MiB; pub const HOST_IP: [u8; 4] = [172, 18, 0, 1]; pub use std::env::consts::ARCH; lazy_static::lazy_static! { @@ -18,6 +15,15 @@ lazy_static::lazy_static! { }; } +mod cap { + #![allow(non_upper_case_globals)] + + pub const CAP_1_KiB: usize = 1024; + pub const CAP_1_MiB: usize = CAP_1_KiB * CAP_1_KiB; + pub const CAP_10_MiB: usize = 10 * CAP_1_MiB; +} +pub use cap::*; + pub mod account; pub mod action; pub mod auth; @@ -75,13 +81,17 @@ use rpc_toolkit::{ use serde::{Deserialize, Serialize}; use ts_rs::TS; -use crate::context::{CliContext, DiagnosticContext, InstallContext, RpcContext, SetupContext}; +use crate::context::{ + CliContext, DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext, +}; +use crate::disk::fsck::RequiresReboot; use crate::registry::context::{RegistryContext, RegistryUrlParams}; use crate::util::serde::HandlerExtSerde; #[derive(Deserialize, Serialize, Parser, TS)] #[serde(rename_all = "camelCase")] #[command(rename_all = "kebab-case")] +#[ts(export)] pub struct EchoParams { message: String, } @@ -90,6 +100,20 @@ pub fn echo(_: C, EchoParams { message }: EchoParams) -> Result) -> std::fmt::Result { + std::fmt::Debug::fmt(&self, f) + } +} + pub fn main_api() -> ParentHandler { ParentHandler::new() .subcommand::("git-info", from_fn(version::git_info)) @@ -99,6 +123,12 @@ pub fn main_api() -> ParentHandler { .with_metadata("authenticated", Value::Bool(false)) .with_call_remote::(), ) + .subcommand( + "state", + from_fn(|_: RpcContext| Ok::<_, Error>(ApiState::Running)) + .with_metadata("authenticated", Value::Bool(false)) + .with_call_remote::(), + ) .subcommand("server", server::()) .subcommand("package", package::()) .subcommand("net", net::net::()) @@ -179,11 +209,18 @@ pub fn server() -> ParentHandler { ) .subcommand( "update-firmware", - from_fn_async(|_: RpcContext| firmware::update_firmware()) - .with_custom_display_fn(|_handle, result| { - Ok(firmware::display_firmware_update_result(result)) - }) - .with_call_remote::(), + from_fn_async(|_: RpcContext| async { + if let Some(firmware) = firmware::check_for_firmware_update().await? { + firmware::update_firmware(firmware).await?; + Ok::<_, Error>(RequiresReboot(true)) + } else { + Ok(RequiresReboot(false)) + } + }) + .with_custom_display_fn(|_handle, result| { + Ok(firmware::display_firmware_update_result(result)) + }) + .with_call_remote::(), ) } @@ -204,7 +241,12 @@ pub fn package() -> ParentHandler { .with_metadata("sync_db", Value::Bool(true)) .no_cli(), ) - .subcommand("sideload", from_fn_async(install::sideload).no_cli()) + .subcommand( + "sideload", + from_fn_async(install::sideload) + .with_metadata("get_session", Value::Bool(true)) + .no_cli(), + ) .subcommand("install", from_fn_async(install::cli_install).no_display()) .subcommand( "uninstall", @@ -273,9 +315,34 @@ pub fn diagnostic_api() -> ParentHandler { "echo", from_fn(echo::).with_call_remote::(), ) + .subcommand( + "state", + from_fn(|_: DiagnosticContext| Ok::<_, Error>(ApiState::Error)) + .with_metadata("authenticated", Value::Bool(false)) + .with_call_remote::(), + ) .subcommand("diagnostic", diagnostic::diagnostic::()) } +pub fn init_api() -> ParentHandler { + ParentHandler::new() + .subcommand::( + "git-info", + from_fn(version::git_info).with_metadata("authenticated", Value::Bool(false)), + ) + .subcommand( + "echo", + from_fn(echo::).with_call_remote::(), + ) + .subcommand( + "state", + from_fn(|_: InitContext| Ok::<_, Error>(ApiState::Initializing)) + .with_metadata("authenticated", Value::Bool(false)) + .with_call_remote::(), + ) + .subcommand("init", init::init_api::()) +} + pub fn setup_api() -> ParentHandler { ParentHandler::new() .subcommand::( diff --git a/core/startos/src/lxc/mod.rs b/core/startos/src/lxc/mod.rs index 8d37120ba..1f1424338 100644 --- a/core/startos/src/lxc/mod.rs +++ b/core/startos/src/lxc/mod.rs @@ -7,7 +7,7 @@ use std::time::Duration; use clap::builder::ValueParserFactory; use clap::Parser; -use futures::{AsyncWriteExt, FutureExt, StreamExt}; +use futures::{AsyncWriteExt, StreamExt}; use imbl_value::{InOMap, InternedString}; use models::InvalidId; use rpc_toolkit::yajrc::{RpcError, RpcResponse}; @@ -456,51 +456,49 @@ pub async fn connect(ctx: &RpcContext, container: &LxcContainer) -> Result break, - Some(Ok(Message::Text(txt))) => { - let mut id = None; - let result = async { - let req: RpcRequest = serde_json::from_str(&txt) - .map_err(|e| RpcError { - data: Some(serde_json::Value::String( - e.to_string(), - )), - ..rpc_toolkit::yajrc::PARSE_ERROR - })?; - id = req.id; - rpc.request(req.method, req.params).await - } - .await; - ws.send(Message::Text( - serde_json::to_string( - &RpcResponse:: { id, result }, - ) - .with_kind(ErrorKind::Serialization)?, - )) - .await - .with_kind(ErrorKind::Network)?; - } - Some(Ok(_)) => (), - Some(Err(e)) => { - return Err(Error::new(e, ErrorKind::Network)); + |mut ws| async move { + if let Err(e) = async { + loop { + match ws.next().await { + None => break, + Some(Ok(Message::Text(txt))) => { + let mut id = None; + let result = async { + let req: RpcRequest = + serde_json::from_str(&txt).map_err(|e| RpcError { + data: Some(serde_json::Value::String( + e.to_string(), + )), + ..rpc_toolkit::yajrc::PARSE_ERROR + })?; + id = req.id; + rpc.request(req.method, req.params).await } + .await; + ws.send(Message::Text( + serde_json::to_string(&RpcResponse:: { + id, + result, + }) + .with_kind(ErrorKind::Serialization)?, + )) + .await + .with_kind(ErrorKind::Network)?; + } + Some(Ok(_)) => (), + Some(Err(e)) => { + return Err(Error::new(e, ErrorKind::Network)); } } - Ok::<_, Error>(()) - } - .await - { - tracing::error!("{e}"); - tracing::debug!("{e:?}"); } + Ok::<_, Error>(()) } - .boxed() - }), + .await + { + tracing::error!("{e}"); + tracing::debug!("{e:?}"); + } + }, Duration::from_secs(30), ), ) diff --git a/core/startos/src/middleware/auth.rs b/core/startos/src/middleware/auth.rs index 1852c4890..ecf88dba7 100644 --- a/core/startos/src/middleware/auth.rs +++ b/core/startos/src/middleware/auth.rs @@ -23,7 +23,7 @@ use tokio::sync::Mutex; use crate::context::RpcContext; use crate::prelude::*; -pub const LOCAL_AUTH_COOKIE_PATH: &str = "/run/embassy/rpc.authcookie"; +pub const LOCAL_AUTH_COOKIE_PATH: &str = "/run/startos/rpc.authcookie"; #[derive(Deserialize, Serialize)] #[serde(rename_all = "camelCase")] @@ -48,19 +48,9 @@ impl HasLoggedOutSessions { .into_iter() .map(|s| s.as_logout_session_id()) .collect(); - ctx.open_authed_websockets - .lock() - .await - .retain(|session, sockets| { - if to_log_out.contains(session.hashed()) { - for socket in std::mem::take(sockets) { - let _ = socket.send(()); - } - false - } else { - true - } - }); + for sid in &to_log_out { + ctx.open_authed_continuations.kill(sid) + } ctx.db .mutate(|db| { let sessions = db.as_private_mut().as_sessions_mut(); diff --git a/core/startos/src/middleware/diagnostic.rs b/core/startos/src/middleware/diagnostic.rs deleted file mode 100644 index 2a7467e34..000000000 --- a/core/startos/src/middleware/diagnostic.rs +++ /dev/null @@ -1,42 +0,0 @@ -use rpc_toolkit::yajrc::RpcMethod; -use rpc_toolkit::{Empty, Middleware, RpcRequest, RpcResponse}; - -use crate::context::DiagnosticContext; -use crate::prelude::*; - -#[derive(Clone)] -pub struct DiagnosticMode { - method: Option, -} -impl DiagnosticMode { - pub fn new() -> Self { - Self { method: None } - } -} - -impl Middleware for DiagnosticMode { - type Metadata = Empty; - async fn process_rpc_request( - &mut self, - _: &DiagnosticContext, - _: Self::Metadata, - request: &mut RpcRequest, - ) -> Result<(), RpcResponse> { - self.method = Some(request.method.as_str().to_owned()); - Ok(()) - } - async fn process_rpc_response(&mut self, _: &DiagnosticContext, response: &mut RpcResponse) { - if let Err(e) = &mut response.result { - if e.code == -32601 { - *e = Error::new( - eyre!( - "{} is not available on the Diagnostic API", - self.method.as_ref().map(|s| s.as_str()).unwrap_or_default() - ), - crate::ErrorKind::DiagnosticMode, - ) - .into(); - } - } - } -} diff --git a/core/startos/src/middleware/mod.rs b/core/startos/src/middleware/mod.rs index 3af0cb5a4..3438dc3db 100644 --- a/core/startos/src/middleware/mod.rs +++ b/core/startos/src/middleware/mod.rs @@ -1,4 +1,3 @@ pub mod auth; pub mod cors; pub mod db; -pub mod diagnostic; diff --git a/core/startos/src/net/net_controller.rs b/core/startos/src/net/net_controller.rs index 69c5c7940..270c7ca09 100644 --- a/core/startos/src/net/net_controller.rs +++ b/core/startos/src/net/net_controller.rs @@ -23,22 +23,18 @@ use crate::prelude::*; use crate::util::serde::MaybeUtf8String; use crate::HOST_IP; -pub struct NetController { - db: TypedPatchDb, - pub(super) tor: TorController, - pub(super) vhost: VHostController, - pub(super) dns: DnsController, - pub(super) forward: LanPortForwardController, - pub(super) os_bindings: Vec>, +pub struct PreInitNetController { + pub db: TypedPatchDb, + tor: TorController, + vhost: VHostController, + os_bindings: Vec>, } - -impl NetController { +impl PreInitNetController { #[instrument(skip_all)] pub async fn init( db: TypedPatchDb, tor_control: SocketAddr, tor_socks: SocketAddr, - dns_bind: &[SocketAddr], hostname: &Hostname, os_tor_key: TorSecretKeyV3, ) -> Result { @@ -46,8 +42,6 @@ impl NetController { db: db.clone(), tor: TorController::new(tor_control, tor_socks), vhost: VHostController::new(db), - dns: DnsController::init(dns_bind).await?, - forward: LanPortForwardController::new(), os_bindings: Vec::new(), }; res.add_os_bindings(hostname, os_tor_key).await?; @@ -73,8 +67,6 @@ impl NetController { alpn.clone(), ) .await?; - self.os_bindings - .push(self.dns.add(None, HOST_IP.into()).await?); // LAN IP self.os_bindings.push( @@ -142,6 +134,39 @@ impl NetController { Ok(()) } +} + +pub struct NetController { + db: TypedPatchDb, + pub(super) tor: TorController, + pub(super) vhost: VHostController, + pub(super) dns: DnsController, + pub(super) forward: LanPortForwardController, + pub(super) os_bindings: Vec>, +} + +impl NetController { + pub async fn init( + PreInitNetController { + db, + tor, + vhost, + os_bindings, + }: PreInitNetController, + dns_bind: &[SocketAddr], + ) -> Result { + let mut res = Self { + db, + tor, + vhost, + dns: DnsController::init(dns_bind).await?, + forward: LanPortForwardController::new(), + os_bindings, + }; + res.os_bindings + .push(res.dns.add(None, HOST_IP.into()).await?); + Ok(res) + } #[instrument(skip_all)] pub async fn create_service( diff --git a/core/startos/src/net/refresher.html b/core/startos/src/net/refresher.html new file mode 100644 index 000000000..445c6b5be --- /dev/null +++ b/core/startos/src/net/refresher.html @@ -0,0 +1,11 @@ + + + StartOS: Loading... + + + + Loading... + + \ No newline at end of file diff --git a/core/startos/src/net/static_server.rs b/core/startos/src/net/static_server.rs index fff1731ce..7e8034d99 100644 --- a/core/startos/src/net/static_server.rs +++ b/core/startos/src/net/static_server.rs @@ -1,4 +1,3 @@ -use std::fs::Metadata; use std::future::Future; use std::path::{Path, PathBuf}; use std::time::UNIX_EPOCH; @@ -13,25 +12,26 @@ use digest::Digest; use futures::future::ready; use http::header::ACCEPT_ENCODING; use http::request::Parts as RequestParts; -use http::{HeaderMap, Method, StatusCode}; +use http::{Method, StatusCode}; +use imbl_value::InternedString; use include_dir::Dir; use new_mime_guess::MimeGuess; use openssl::hash::MessageDigest; use openssl::x509::X509; -use rpc_toolkit::Server; +use rpc_toolkit::{Context, HttpServer, Server}; use tokio::fs::File; use tokio::io::BufReader; use tokio_util::io::ReaderStream; -use crate::context::{DiagnosticContext, InstallContext, RpcContext, SetupContext}; -use crate::db::subscribe; +use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext}; use crate::hostname::Hostname; use crate::middleware::auth::{Auth, HasValidSession}; use crate::middleware::cors::Cors; use crate::middleware::db::SyncDb; -use crate::middleware::diagnostic::DiagnosticMode; -use crate::rpc_continuations::Guid; -use crate::{diagnostic_api, install_api, main_api, setup_api, Error, ErrorKind, ResultExt}; +use crate::rpc_continuations::{Guid, RpcContinuations}; +use crate::{ + diagnostic_api, init_api, install_api, main_api, setup_api, Error, ErrorKind, ResultExt, +}; const NOT_FOUND: &[u8] = b"Not Found"; const METHOD_NOT_ALLOWED: &[u8] = b"Method Not Allowed"; @@ -49,7 +49,6 @@ const PROXY_STRIP_HEADERS: &[&str] = &["cookie", "host", "origin", "referer", "u #[derive(Clone)] pub enum UiMode { Setup, - Diag, Install, Main, } @@ -58,128 +57,46 @@ impl UiMode { fn path(&self, path: &str) -> PathBuf { match self { Self::Setup => Path::new("setup-wizard").join(path), - Self::Diag => Path::new("diagnostic-ui").join(path), Self::Install => Path::new("install-wizard").join(path), Self::Main => Path::new("ui").join(path), } } } -pub fn setup_ui_file_router(ctx: SetupContext) -> Router { - Router::new() - .route_service( - "/rpc/*path", - post(Server::new(move || ready(Ok(ctx.clone())), setup_api()).middleware(Cors::new())), - ) - .fallback(any(|request: Request| async move { - alt_ui(request, UiMode::Setup) - .await - .unwrap_or_else(server_error) - })) -} - -pub fn diag_ui_file_router(ctx: DiagnosticContext) -> Router { +pub fn rpc_router>( + ctx: C, + server: HttpServer, +) -> Router { Router::new() + .route("/rpc/*path", post(server)) .route( - "/rpc/*path", - post( - Server::new(move || ready(Ok(ctx.clone())), diagnostic_api()) - .middleware(Cors::new()) - .middleware(DiagnosticMode::new()), - ), - ) - .fallback(any(|request: Request| async move { - alt_ui(request, UiMode::Diag) - .await - .unwrap_or_else(server_error) - })) -} - -pub fn install_ui_file_router(ctx: InstallContext) -> Router { - Router::new() - .route("/rpc/*path", { - let ctx = ctx.clone(); - post(Server::new(move || ready(Ok(ctx.clone())), install_api()).middleware(Cors::new())) - }) - .fallback(any(|request: Request| async move { - alt_ui(request, UiMode::Install) - .await - .unwrap_or_else(server_error) - })) -} - -pub fn main_ui_server_router(ctx: RpcContext) -> Router { - Router::new() - .route("/rpc/*path", { - let ctx = ctx.clone(); - post( - Server::new(move || ready(Ok(ctx.clone())), main_api::()) - .middleware(Cors::new()) - .middleware(Auth::new()) - .middleware(SyncDb::new()), - ) - }) - .route( - "/ws/db", - any({ - let ctx = ctx.clone(); - move |headers: HeaderMap, ws: x::WebSocketUpgrade| async move { - subscribe(ctx, headers, ws) - .await - .unwrap_or_else(server_error) - } - }), - ) - .route( - "/ws/rpc/*path", + "/ws/rpc/:guid", get({ let ctx = ctx.clone(); - move |x::Path(path): x::Path, + move |x::Path(guid): x::Path, ws: axum::extract::ws::WebSocketUpgrade| async move { - match Guid::from(&path) { - None => { - tracing::debug!("No Guid Path"); - bad_request() - } - Some(guid) => match ctx.rpc_continuations.get_ws_handler(&guid).await { - Some(cont) => ws.on_upgrade(cont), - _ => not_found(), - }, + match AsRef::::as_ref(&ctx).get_ws_handler(&guid).await { + Some(cont) => ws.on_upgrade(cont), + _ => not_found(), } } }), ) .route( - "/rest/rpc/*path", + "/rest/rpc/:guid", any({ let ctx = ctx.clone(); - move |request: x::Request| async move { - let path = request - .uri() - .path() - .strip_prefix("/rest/rpc/") - .unwrap_or_default(); - match Guid::from(&path) { - None => { - tracing::debug!("No Guid Path"); - bad_request() - } - Some(guid) => match ctx.rpc_continuations.get_rest_handler(&guid).await { - None => not_found(), - Some(cont) => cont(request).await.unwrap_or_else(server_error), - }, + move |x::Path(guid): x::Path, request: x::Request| async move { + match AsRef::::as_ref(&ctx).get_rest_handler(&guid).await { + None => not_found(), + Some(cont) => cont(request).await.unwrap_or_else(server_error), } } }), ) - .fallback(any(move |request: Request| async move { - main_start_os_ui(request, ctx) - .await - .unwrap_or_else(server_error) - })) } -async fn alt_ui(req: Request, ui_mode: UiMode) -> Result { +fn serve_ui(req: Request, ui_mode: UiMode) -> Result { let (request_parts, _body) = req.into_parts(); match &request_parts.method { &Method::GET => { @@ -196,9 +113,7 @@ async fn alt_ui(req: Request, ui_mode: UiMode) -> Result { .or_else(|| EMBEDDED_UIS.get_file(&*ui_mode.path("index.html"))); if let Some(file) = file { - FileData::from_embedded(&request_parts, file) - .into_response(&request_parts) - .await + FileData::from_embedded(&request_parts, file).into_response(&request_parts) } else { Ok(not_found()) } @@ -207,6 +122,75 @@ async fn alt_ui(req: Request, ui_mode: UiMode) -> Result { } } +pub fn setup_ui_router(ctx: SetupContext) -> Router { + rpc_router( + ctx.clone(), + Server::new(move || ready(Ok(ctx.clone())), setup_api()).middleware(Cors::new()), + ) + .fallback(any(|request: Request| async move { + serve_ui(request, UiMode::Setup).unwrap_or_else(server_error) + })) +} + +pub fn diagnostic_ui_router(ctx: DiagnosticContext) -> Router { + rpc_router( + ctx.clone(), + Server::new(move || ready(Ok(ctx.clone())), diagnostic_api()).middleware(Cors::new()), + ) + .fallback(any(|request: Request| async move { + serve_ui(request, UiMode::Main).unwrap_or_else(server_error) + })) +} + +pub fn install_ui_router(ctx: InstallContext) -> Router { + rpc_router( + ctx.clone(), + Server::new(move || ready(Ok(ctx.clone())), install_api()).middleware(Cors::new()), + ) + .fallback(any(|request: Request| async move { + serve_ui(request, UiMode::Install).unwrap_or_else(server_error) + })) +} + +pub fn init_ui_router(ctx: InitContext) -> Router { + rpc_router( + ctx.clone(), + Server::new(move || ready(Ok(ctx.clone())), init_api()).middleware(Cors::new()), + ) + .fallback(any(|request: Request| async move { + serve_ui(request, UiMode::Main).unwrap_or_else(server_error) + })) +} + +pub fn main_ui_router(ctx: RpcContext) -> Router { + rpc_router( + ctx.clone(), + Server::new(move || ready(Ok(ctx.clone())), main_api::()) + .middleware(Cors::new()) + .middleware(Auth::new()) + .middleware(SyncDb::new()), + ) + // TODO: cert + .fallback(any(|request: Request| async move { + serve_ui(request, UiMode::Main).unwrap_or_else(server_error) + })) +} + +pub fn refresher() -> Router { + Router::new().fallback(get(|request: Request| async move { + let res = include_bytes!("./refresher.html"); + FileData { + data: Body::from(&res[..]), + e_tag: None, + encoding: None, + len: Some(res.len() as u64), + mime: Some("text/html".into()), + } + .into_response(&request.into_parts().0) + .unwrap_or_else(server_error) + })) +} + async fn if_authorized< F: FnOnce() -> Fut, Fut: Future> + Send + Sync, @@ -223,89 +207,6 @@ async fn if_authorized< } } -async fn main_start_os_ui(req: Request, ctx: RpcContext) -> Result { - let (request_parts, _body) = req.into_parts(); - match ( - &request_parts.method, - request_parts - .uri - .path() - .strip_prefix('/') - .unwrap_or(request_parts.uri.path()) - .split_once('/'), - ) { - (&Method::GET, Some(("public", path))) => { - todo!("pull directly from s9pk") - } - (&Method::GET, Some(("proxy", target))) => { - if_authorized(&ctx, &request_parts, || async { - let target = urlencoding::decode(target)?; - let res = ctx - .client - .get(target.as_ref()) - .headers( - request_parts - .headers - .iter() - .filter(|(h, _)| { - !PROXY_STRIP_HEADERS - .iter() - .any(|bad| h.as_str().eq_ignore_ascii_case(bad)) - }) - .flat_map(|(h, v)| { - Some(( - reqwest::header::HeaderName::from_lowercase( - h.as_str().as_bytes(), - ) - .ok()?, - reqwest::header::HeaderValue::from_bytes(v.as_bytes()).ok()?, - )) - }) - .collect(), - ) - .send() - .await - .with_kind(crate::ErrorKind::Network)?; - let mut hres = Response::builder().status(res.status().as_u16()); - for (h, v) in res.headers().clone() { - if let Some(h) = h { - hres = hres.header(h.to_string(), v.as_bytes()); - } - } - hres.body(Body::from_stream(res.bytes_stream())) - .with_kind(crate::ErrorKind::Network) - }) - .await - } - (&Method::GET, Some(("eos", "local.crt"))) => { - let account = ctx.account.read().await; - cert_send(&account.root_ca_cert, &account.hostname) - } - (&Method::GET, _) => { - let uri_path = UiMode::Main.path( - request_parts - .uri - .path() - .strip_prefix('/') - .unwrap_or(request_parts.uri.path()), - ); - - let file = EMBEDDED_UIS - .get_file(&*uri_path) - .or_else(|| EMBEDDED_UIS.get_file(&*UiMode::Main.path("index.html"))); - - if let Some(file) = file { - FileData::from_embedded(&request_parts, file) - .into_response(&request_parts) - .await - } else { - Ok(not_found()) - } - } - _ => Ok(method_not_allowed()), - } -} - pub fn unauthorized(err: Error, path: &str) -> Response { tracing::warn!("unauthorized for {} @{:?}", err, path); tracing::debug!("{:?}", err); @@ -373,8 +274,8 @@ struct FileData { data: Body, len: Option, encoding: Option<&'static str>, - e_tag: String, - mime: Option, + e_tag: Option, + mime: Option, } impl FileData { fn from_embedded(req: &RequestParts, file: &'static include_dir::File<'static>) -> Self { @@ -407,10 +308,23 @@ impl FileData { len: Some(data.len() as u64), encoding, data: data.into(), - e_tag: e_tag(path, None), + e_tag: file.metadata().map(|metadata| { + e_tag( + path, + format!( + "{}", + metadata + .modified() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or_else(|e| e.duration().as_secs() as i64 * -1), + ) + .as_bytes(), + ) + }), mime: MimeGuess::from_path(path) .first() - .map(|m| m.essence_str().to_owned()), + .map(|m| m.essence_str().into()), } } @@ -434,7 +348,18 @@ impl FileData { .await .with_ctx(|_| (ErrorKind::Filesystem, path.display().to_string()))?; - let e_tag = e_tag(path, Some(&metadata)); + let e_tag = Some(e_tag( + path, + format!( + "{}", + metadata + .modified()? + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or_else(|e| e.duration().as_secs() as i64 * -1) + ) + .as_bytes(), + )); let (len, data) = if encoding == Some("gzip") { ( @@ -455,16 +380,18 @@ impl FileData { e_tag, mime: MimeGuess::from_path(path) .first() - .map(|m| m.essence_str().to_owned()), + .map(|m| m.essence_str().into()), }) } - async fn into_response(self, req: &RequestParts) -> Result { + fn into_response(self, req: &RequestParts) -> Result { let mut builder = Response::builder(); if let Some(mime) = self.mime { builder = builder.header(http::header::CONTENT_TYPE, &*mime); } - builder = builder.header(http::header::ETAG, &*self.e_tag); + if let Some(e_tag) = &self.e_tag { + builder = builder.header(http::header::ETAG, &**e_tag); + } builder = builder.header( http::header::CACHE_CONTROL, "public, max-age=21000000, immutable", @@ -481,11 +408,12 @@ impl FileData { builder = builder.header(http::header::CONNECTION, "keep-alive"); } - if req - .headers - .get("if-none-match") - .and_then(|h| h.to_str().ok()) - == Some(self.e_tag.as_ref()) + if self.e_tag.is_some() + && req + .headers + .get("if-none-match") + .and_then(|h| h.to_str().ok()) + == self.e_tag.as_deref() { builder = builder.status(StatusCode::NOT_MODIFIED); builder.body(Body::empty()) @@ -503,21 +431,14 @@ impl FileData { } } -fn e_tag(path: &Path, metadata: Option<&Metadata>) -> String { +lazy_static::lazy_static! { + static ref INSTANCE_NONCE: u64 = rand::random(); +} + +fn e_tag(path: &Path, modified: impl AsRef<[u8]>) -> String { let mut hasher = sha2::Sha256::new(); hasher.update(format!("{:?}", path).as_bytes()); - if let Some(modified) = metadata.and_then(|m| m.modified().ok()) { - hasher.update( - format!( - "{}", - modified - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - ) - .as_bytes(), - ); - } + hasher.update(modified.as_ref()); let res = hasher.finalize(); format!( "\"{}\"", diff --git a/core/startos/src/net/web_server.rs b/core/startos/src/net/web_server.rs index a89aae92f..a9cfdf046 100644 --- a/core/startos/src/net/web_server.rs +++ b/core/startos/src/net/web_server.rs @@ -1,23 +1,84 @@ +use std::convert::Infallible; use std::net::SocketAddr; +use std::task::Poll; use std::time::Duration; +use axum::extract::Request; use axum::Router; use axum_server::Handle; +use bytes::Bytes; +use futures::future::ready; +use futures::FutureExt; use helpers::NonDetachingJoinHandle; -use tokio::sync::oneshot; +use tokio::sync::{oneshot, watch}; -use crate::context::{DiagnosticContext, InstallContext, RpcContext, SetupContext}; +use crate::context::{DiagnosticContext, InitContext, InstallContext, RpcContext, SetupContext}; use crate::net::static_server::{ - diag_ui_file_router, install_ui_file_router, main_ui_server_router, setup_ui_file_router, + diagnostic_ui_router, init_ui_router, install_ui_router, main_ui_router, refresher, + setup_ui_router, }; -use crate::Error; +use crate::prelude::*; + +#[derive(Clone)] +pub struct SwappableRouter(watch::Sender); +impl SwappableRouter { + pub fn new(router: Router) -> Self { + Self(watch::channel(router).0) + } + pub fn swap(&self, router: Router) { + let _ = self.0.send_replace(router); + } +} + +#[derive(Clone)] +pub struct SwappableRouterService(watch::Receiver); +impl tower_service::Service> for SwappableRouterService +where + B: axum::body::HttpBody + Send + 'static, + B::Error: Into, +{ + type Response = >>::Response; + type Error = >>::Error; + type Future = >>::Future; + #[inline] + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + let mut changed = self.0.changed().boxed(); + if changed.poll_unpin(cx).is_ready() { + return Poll::Ready(Ok(())); + } + drop(changed); + tower_service::Service::>::poll_ready(&mut self.0.borrow().clone(), cx) + } + fn call(&mut self, req: Request) -> Self::Future { + self.0.borrow().clone().call(req) + } +} + +impl tower_service::Service for SwappableRouter { + type Response = SwappableRouterService; + type Error = Infallible; + type Future = futures::future::Ready>; + #[inline] + fn poll_ready( + &mut self, + _: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, _: T) -> Self::Future { + ready(Ok(SwappableRouterService(self.0.subscribe()))) + } +} pub struct WebServer { shutdown: oneshot::Sender<()>, + router: SwappableRouter, thread: NonDetachingJoinHandle<()>, } impl WebServer { - pub fn new(bind: SocketAddr, router: Router) -> Self { + pub fn new(bind: SocketAddr) -> Self { + let router = SwappableRouter::new(refresher()); + let thread_router = router.clone(); let (shutdown, shutdown_recv) = oneshot::channel(); let thread = NonDetachingJoinHandle::from(tokio::spawn(async move { let handle = Handle::new(); @@ -25,14 +86,18 @@ impl WebServer { server.http_builder().http1().preserve_header_case(true); server.http_builder().http1().title_case_headers(true); - if let (Err(e), _) = tokio::join!(server.serve(router.into_make_service()), async { + if let (Err(e), _) = tokio::join!(server.serve(thread_router), async { let _ = shutdown_recv.await; handle.graceful_shutdown(Some(Duration::from_secs(0))); }) { tracing::error!("Spawning hyper server error: {}", e); } })); - Self { shutdown, thread } + Self { + shutdown, + router, + thread, + } } pub async fn shutdown(self) { @@ -40,19 +105,27 @@ impl WebServer { self.thread.await.unwrap() } - pub fn main(bind: SocketAddr, ctx: RpcContext) -> Result { - Ok(Self::new(bind, main_ui_server_router(ctx))) + pub fn serve_router(&mut self, router: Router) { + self.router.swap(router) } - pub fn setup(bind: SocketAddr, ctx: SetupContext) -> Result { - Ok(Self::new(bind, setup_ui_file_router(ctx))) + pub fn serve_main(&mut self, ctx: RpcContext) { + self.serve_router(main_ui_router(ctx)) } - pub fn diagnostic(bind: SocketAddr, ctx: DiagnosticContext) -> Result { - Ok(Self::new(bind, diag_ui_file_router(ctx))) + pub fn serve_setup(&mut self, ctx: SetupContext) { + self.serve_router(setup_ui_router(ctx)) } - pub fn install(bind: SocketAddr, ctx: InstallContext) -> Result { - Ok(Self::new(bind, install_ui_file_router(ctx))) + pub fn serve_diagnostic(&mut self, ctx: DiagnosticContext) { + self.serve_router(diagnostic_ui_router(ctx)) + } + + pub fn serve_install(&mut self, ctx: InstallContext) { + self.serve_router(install_ui_router(ctx)) + } + + pub fn serve_init(&mut self, ctx: InitContext) { + self.serve_router(init_ui_router(ctx)) } } diff --git a/core/startos/src/progress.rs b/core/startos/src/progress.rs index 9a22405ca..f70a5adbc 100644 --- a/core/startos/src/progress.rs +++ b/core/startos/src/progress.rs @@ -1,14 +1,16 @@ use std::panic::UnwindSafe; -use std::sync::Arc; use std::time::Duration; -use futures::Future; +use futures::future::pending; +use futures::stream::BoxStream; +use futures::{Future, FutureExt, StreamExt, TryFutureExt}; +use helpers::NonDetachingJoinHandle; use imbl_value::{InOMap, InternedString}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use itertools::Itertools; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncSeek, AsyncWrite}; -use tokio::sync::{mpsc, watch}; +use tokio::sync::watch; use ts_rs::TS; use crate::db::model::{Database, DatabaseModel}; @@ -168,39 +170,23 @@ impl FullProgress { } } +#[derive(Clone)] pub struct FullProgressTracker { - overall: Arc>, - overall_recv: watch::Receiver, - phases: InOMap>, - new_phase: ( - mpsc::UnboundedSender<(InternedString, watch::Receiver)>, - mpsc::UnboundedReceiver<(InternedString, watch::Receiver)>, - ), + overall: watch::Sender, + phases: watch::Sender>>, } impl FullProgressTracker { pub fn new() -> Self { - let (overall, overall_recv) = watch::channel(Progress::new()); - Self { - overall: Arc::new(overall), - overall_recv, - phases: InOMap::new(), - new_phase: mpsc::unbounded_channel(), - } + let (overall, _) = watch::channel(Progress::new()); + let (phases, _) = watch::channel(InOMap::new()); + Self { overall, phases } } - fn fill_phases(&mut self) -> bool { - let mut changed = false; - while let Ok((name, phase)) = self.new_phase.1.try_recv() { - self.phases.insert(name, phase); - changed = true; - } - changed - } - pub fn snapshot(&mut self) -> FullProgress { - self.fill_phases(); + pub fn snapshot(&self) -> FullProgress { FullProgress { overall: *self.overall.borrow(), phases: self .phases + .borrow() .iter() .map(|(name, progress)| NamedProgress { name: name.clone(), @@ -209,28 +195,75 @@ impl FullProgressTracker { .collect(), } } - pub async fn changed(&mut self) { - if self.fill_phases() { - return; - } - let phases = self - .phases - .iter_mut() - .map(|(_, p)| Box::pin(p.changed())) - .collect_vec(); - tokio::select! { - _ = self.overall_recv.changed() => (), - _ = futures::future::select_all(phases) => (), - } - } - pub fn handle(&self) -> FullProgressTrackerHandle { - FullProgressTrackerHandle { - overall: self.overall.clone(), - new_phase: self.new_phase.0.clone(), + pub fn stream(&self, min_interval: Option) -> BoxStream<'static, FullProgress> { + struct StreamState { + overall: watch::Receiver, + phases_recv: watch::Receiver>>, + phases: InOMap>, } + let mut overall = self.overall.subscribe(); + overall.mark_changed(); // make sure stream starts with a value + let phases_recv = self.phases.subscribe(); + let phases = phases_recv.borrow().clone(); + let state = StreamState { + overall, + phases_recv, + phases, + }; + futures::stream::unfold( + state, + move |StreamState { + mut overall, + mut phases_recv, + mut phases, + }| async move { + let changed = phases + .iter_mut() + .map(|(_, p)| async move { p.changed().or_else(|_| pending()).await }.boxed()) + .chain([overall.changed().boxed()]) + .chain([phases_recv.changed().boxed()]) + .map(|fut| fut.map(|r| r.unwrap_or_default())) + .collect_vec(); + if let Some(min_interval) = min_interval { + tokio::join!( + tokio::time::sleep(min_interval), + futures::future::select_all(changed), + ); + } else { + futures::future::select_all(changed).await; + } + + for (name, phase) in &*phases_recv.borrow_and_update() { + if !phases.contains_key(name) { + phases.insert(name.clone(), phase.clone()); + } + } + + let o = *overall.borrow_and_update(); + + Some(( + FullProgress { + overall: o, + phases: phases + .iter_mut() + .map(|(name, progress)| NamedProgress { + name: name.clone(), + progress: *progress.borrow_and_update(), + }) + .collect(), + }, + StreamState { + overall, + phases_recv, + phases, + }, + )) + }, + ) + .boxed() } pub fn sync_to_db( - mut self, + &self, db: TypedPatchDb, deref: DerefFn, min_interval: Option, @@ -239,9 +272,9 @@ impl FullProgressTracker { DerefFn: Fn(&mut DatabaseModel) -> Option<&mut Model> + 'static, for<'a> &'a DerefFn: UnwindSafe + Send, { + let mut stream = self.stream(min_interval); async move { - loop { - let progress = self.snapshot(); + while let Some(progress) = stream.next().await { if db .mutate(|v| { if let Some(p) = deref(v) { @@ -255,25 +288,23 @@ impl FullProgressTracker { { break; } - tokio::join!(self.changed(), async { - if let Some(interval) = min_interval { - tokio::time::sleep(interval).await - } else { - futures::future::ready(()).await - } - }); } Ok(()) } } -} - -#[derive(Clone)] -pub struct FullProgressTrackerHandle { - overall: Arc>, - new_phase: mpsc::UnboundedSender<(InternedString, watch::Receiver)>, -} -impl FullProgressTrackerHandle { + pub fn progress_bar_task(&self, name: &str) -> NonDetachingJoinHandle<()> { + let mut stream = self.stream(None); + let mut bar = PhasedProgressBar::new(name); + tokio::spawn(async move { + while let Some(progress) = stream.next().await { + bar.update(&progress); + if progress.overall.is_complete() { + break; + } + } + }) + .into() + } pub fn add_phase( &self, name: InternedString, @@ -284,7 +315,9 @@ impl FullProgressTrackerHandle { .send_modify(|o| o.add_total(overall_contribution)); } let (send, recv) = watch::channel(Progress::new()); - let _ = self.new_phase.send((name, recv)); + self.phases.send_modify(|p| { + p.insert(name, recv); + }); PhaseProgressTrackerHandle { overall: self.overall.clone(), overall_contribution, @@ -298,7 +331,7 @@ impl FullProgressTrackerHandle { } pub struct PhaseProgressTrackerHandle { - overall: Arc>, + overall: watch::Sender, overall_contribution: Option, contributed: u64, progress: watch::Sender, diff --git a/core/startos/src/registry/context.rs b/core/startos/src/registry/context.rs index 99d60307b..e5beabca7 100644 --- a/core/startos/src/registry/context.rs +++ b/core/startos/src/registry/context.rs @@ -169,7 +169,8 @@ impl CallRemote for CliContext { &AnySigningKey::Ed25519(self.developer_key()?.clone()), &body, &host, - )?.to_header(), + )? + .to_header(), ) .body(body) .send() diff --git a/core/startos/src/registry/mod.rs b/core/startos/src/registry/mod.rs index 656edf337..9a53d6338 100644 --- a/core/startos/src/registry/mod.rs +++ b/core/startos/src/registry/mod.rs @@ -70,7 +70,7 @@ pub fn registry_api() -> ParentHandler { .subcommand("db", db::db_api::()) } -pub fn registry_server_router(ctx: RegistryContext) -> Router { +pub fn registry_router(ctx: RegistryContext) -> Router { use axum::extract as x; use axum::routing::{any, get, post}; Router::new() @@ -128,7 +128,7 @@ pub fn registry_server_router(ctx: RegistryContext) -> Router { } impl WebServer { - pub fn registry(bind: SocketAddr, ctx: RegistryContext) -> Self { - Self::new(bind, registry_server_router(ctx)) + pub fn serve_registry(&mut self, ctx: RegistryContext) { + self.serve_router(registry_router(ctx)) } } diff --git a/core/startos/src/registry/os/asset/add.rs b/core/startos/src/registry/os/asset/add.rs index 6ca495547..33f5ef90f 100644 --- a/core/startos/src/registry/os/asset/add.rs +++ b/core/startos/src/registry/os/asset/add.rs @@ -186,29 +186,16 @@ pub async fn cli_add_asset( let file = MultiCursorFile::from(tokio::fs::File::open(&path).await?); - let mut progress = FullProgressTracker::new(); - let progress_handle = progress.handle(); - let mut sign_phase = - progress_handle.add_phase(InternedString::intern("Signing File"), Some(10)); - let mut verify_phase = - progress_handle.add_phase(InternedString::intern("Verifying URL"), Some(100)); - let mut index_phase = progress_handle.add_phase( + let progress = FullProgressTracker::new(); + let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(10)); + let mut verify_phase = progress.add_phase(InternedString::intern("Verifying URL"), Some(100)); + let mut index_phase = progress.add_phase( InternedString::intern("Adding File to Registry Index"), Some(1), ); - let progress_task: NonDetachingJoinHandle<()> = tokio::spawn(async move { - let mut bar = PhasedProgressBar::new(&format!("Adding {} to registry...", path.display())); - loop { - let snap = progress.snapshot(); - bar.update(&snap); - if snap.overall.is_complete() { - break; - } - progress.changed().await - } - }) - .into(); + let progress_task = + progress.progress_bar_task(&format!("Adding {} to registry...", path.display())); sign_phase.start(); let blake3 = file.blake3_mmap().await?; @@ -252,7 +239,7 @@ pub async fn cli_add_asset( .await?; index_phase.complete(); - progress_handle.complete(); + progress.complete(); progress_task.await.with_kind(ErrorKind::Unknown)?; diff --git a/core/startos/src/registry/os/asset/get.rs b/core/startos/src/registry/os/asset/get.rs index e099a50cf..29ff24da6 100644 --- a/core/startos/src/registry/os/asset/get.rs +++ b/core/startos/src/registry/os/asset/get.rs @@ -3,7 +3,7 @@ use std::panic::UnwindSafe; use std::path::{Path, PathBuf}; use clap::Parser; -use helpers::{AtomicFile, NonDetachingJoinHandle}; +use helpers::AtomicFile; use imbl_value::{json, InternedString}; use itertools::Itertools; use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; @@ -12,7 +12,7 @@ use ts_rs::TS; use crate::context::CliContext; use crate::prelude::*; -use crate::progress::{FullProgressTracker, PhasedProgressBar}; +use crate::progress::FullProgressTracker; use crate::registry::asset::RegistryAsset; use crate::registry::context::RegistryContext; use crate::registry::os::index::OsVersionInfo; @@ -135,29 +135,17 @@ async fn cli_get_os_asset( .await .with_kind(ErrorKind::Filesystem)?; - let mut progress = FullProgressTracker::new(); - let progress_handle = progress.handle(); + let progress = FullProgressTracker::new(); let mut download_phase = - progress_handle.add_phase(InternedString::intern("Downloading File"), Some(100)); + progress.add_phase(InternedString::intern("Downloading File"), Some(100)); download_phase.set_total(res.commitment.size); let reverify_phase = if reverify { - Some(progress_handle.add_phase(InternedString::intern("Reverifying File"), Some(10))) + Some(progress.add_phase(InternedString::intern("Reverifying File"), Some(10))) } else { None }; - let progress_task: NonDetachingJoinHandle<()> = tokio::spawn(async move { - let mut bar = PhasedProgressBar::new("Downloading..."); - loop { - let snap = progress.snapshot(); - bar.update(&snap); - if snap.overall.is_complete() { - break; - } - progress.changed().await - } - }) - .into(); + let progress_task = progress.progress_bar_task("Downloading..."); download_phase.start(); let mut download_writer = download_phase.writer(&mut *file); @@ -177,7 +165,7 @@ async fn cli_get_os_asset( reverify_phase.complete(); } - progress_handle.complete(); + progress.complete(); progress_task.await.with_kind(ErrorKind::Unknown)?; } diff --git a/core/startos/src/registry/os/asset/sign.rs b/core/startos/src/registry/os/asset/sign.rs index 0cb657bef..50c583593 100644 --- a/core/startos/src/registry/os/asset/sign.rs +++ b/core/startos/src/registry/os/asset/sign.rs @@ -3,7 +3,6 @@ use std::panic::UnwindSafe; use std::path::PathBuf; use clap::Parser; -use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; use itertools::Itertools; use rpc_toolkit::{from_fn_async, Context, HandlerArgs, HandlerExt, ParentHandler}; @@ -12,7 +11,7 @@ use ts_rs::TS; use crate::context::CliContext; use crate::prelude::*; -use crate::progress::{FullProgressTracker, PhasedProgressBar}; +use crate::progress::FullProgressTracker; use crate::registry::asset::RegistryAsset; use crate::registry::context::RegistryContext; use crate::registry::os::index::OsVersionInfo; @@ -169,27 +168,15 @@ pub async fn cli_sign_asset( let file = MultiCursorFile::from(tokio::fs::File::open(&path).await?); - let mut progress = FullProgressTracker::new(); - let progress_handle = progress.handle(); - let mut sign_phase = - progress_handle.add_phase(InternedString::intern("Signing File"), Some(10)); - let mut index_phase = progress_handle.add_phase( + let progress = FullProgressTracker::new(); + let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(10)); + let mut index_phase = progress.add_phase( InternedString::intern("Adding Signature to Registry Index"), Some(1), ); - let progress_task: NonDetachingJoinHandle<()> = tokio::spawn(async move { - let mut bar = PhasedProgressBar::new(&format!("Adding {} to registry...", path.display())); - loop { - let snap = progress.snapshot(); - bar.update(&snap); - if snap.overall.is_complete() { - break; - } - progress.changed().await - } - }) - .into(); + let progress_task = + progress.progress_bar_task(&format!("Adding {} to registry...", path.display())); sign_phase.start(); let blake3 = file.blake3_mmap().await?; @@ -220,7 +207,7 @@ pub async fn cli_sign_asset( .await?; index_phase.complete(); - progress_handle.complete(); + progress.complete(); progress_task.await.with_kind(ErrorKind::Unknown)?; diff --git a/core/startos/src/registry/package/add.rs b/core/startos/src/registry/package/add.rs index d28aeaaa4..9bc772f78 100644 --- a/core/startos/src/registry/package/add.rs +++ b/core/startos/src/registry/package/add.rs @@ -2,7 +2,6 @@ use std::path::PathBuf; use std::sync::Arc; use clap::Parser; -use helpers::NonDetachingJoinHandle; use imbl_value::InternedString; use itertools::Itertools; use rpc_toolkit::HandlerArgs; @@ -12,7 +11,7 @@ use url::Url; use crate::context::CliContext; use crate::prelude::*; -use crate::progress::{FullProgressTracker, PhasedProgressBar}; +use crate::progress::FullProgressTracker; use crate::registry::context::RegistryContext; use crate::registry::package::index::PackageVersionInfo; use crate::registry::signer::commitment::merkle_archive::MerkleArchiveCommitment; @@ -110,28 +109,16 @@ pub async fn cli_add_package( ) -> Result<(), Error> { let s9pk = S9pk::open(&file, None).await?; - let mut progress = FullProgressTracker::new(); - let progress_handle = progress.handle(); - let mut sign_phase = progress_handle.add_phase(InternedString::intern("Signing File"), Some(1)); - let mut verify_phase = - progress_handle.add_phase(InternedString::intern("Verifying URL"), Some(100)); - let mut index_phase = progress_handle.add_phase( + let progress = FullProgressTracker::new(); + let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(1)); + let mut verify_phase = progress.add_phase(InternedString::intern("Verifying URL"), Some(100)); + let mut index_phase = progress.add_phase( InternedString::intern("Adding File to Registry Index"), Some(1), ); - let progress_task: NonDetachingJoinHandle<()> = tokio::spawn(async move { - let mut bar = PhasedProgressBar::new(&format!("Adding {} to registry...", file.display())); - loop { - let snap = progress.snapshot(); - bar.update(&snap); - if snap.overall.is_complete() { - break; - } - progress.changed().await - } - }) - .into(); + let progress_task = + progress.progress_bar_task(&format!("Adding {} to registry...", file.display())); sign_phase.start(); let commitment = s9pk.as_archive().commitment().await?; @@ -160,7 +147,7 @@ pub async fn cli_add_package( .await?; index_phase.complete(); - progress_handle.complete(); + progress.complete(); progress_task.await.with_kind(ErrorKind::Unknown)?; diff --git a/core/startos/src/registry/signer/commitment/request.rs b/core/startos/src/registry/signer/commitment/request.rs index 62d59163f..ce60b7f88 100644 --- a/core/startos/src/registry/signer/commitment/request.rs +++ b/core/startos/src/registry/signer/commitment/request.rs @@ -1,5 +1,5 @@ -use std::time::{SystemTime, UNIX_EPOCH}; use std::collections::BTreeMap; +use std::time::{SystemTime, UNIX_EPOCH}; use axum::body::Body; use axum::extract::Request; diff --git a/core/startos/src/rpc_continuations.rs b/core/startos/src/rpc_continuations.rs index e6b823ef9..043130b69 100644 --- a/core/startos/src/rpc_continuations.rs +++ b/core/startos/src/rpc_continuations.rs @@ -1,5 +1,8 @@ use std::collections::BTreeMap; +use std::pin::Pin; use std::str::FromStr; +use std::sync::Mutex as SyncMutex; +use std::task::{Context, Poll}; use std::time::Duration; use axum::extract::ws::WebSocket; @@ -7,9 +10,10 @@ use axum::extract::Request; use axum::response::Response; use clap::builder::ValueParserFactory; use futures::future::BoxFuture; +use futures::{Future, FutureExt}; use helpers::TimedResource; use imbl_value::InternedString; -use tokio::sync::Mutex; +use tokio::sync::{broadcast, Mutex as AsyncMutex}; use ts_rs::TS; #[allow(unused_imports)] @@ -73,21 +77,103 @@ impl std::fmt::Display for Guid { } } -pub type RestHandler = - Box BoxFuture<'static, Result> + Send>; +pub struct RestFuture { + kill: Option>, + fut: BoxFuture<'static, Result>, +} +impl Future for RestFuture { + type Output = Result; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if self.kill.as_ref().map_or(false, |k| !k.is_empty()) { + Poll::Ready(Err(Error::new( + eyre!("session killed"), + ErrorKind::Authorization, + ))) + } else { + self.fut.poll_unpin(cx) + } + } +} +pub type RestHandler = Box RestFuture + Send>; -pub type WebSocketHandler = Box BoxFuture<'static, ()> + Send>; +pub struct WebSocketFuture { + kill: Option>, + fut: BoxFuture<'static, ()>, +} +impl Future for WebSocketFuture { + type Output = (); + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if self.kill.as_ref().map_or(false, |k| !k.is_empty()) { + Poll::Ready(()) + } else { + self.fut.poll_unpin(cx) + } + } +} +pub type WebSocketHandler = Box WebSocketFuture + Send>; pub enum RpcContinuation { Rest(TimedResource), WebSocket(TimedResource), } impl RpcContinuation { - pub fn rest(handler: RestHandler, timeout: Duration) -> Self { - RpcContinuation::Rest(TimedResource::new(handler, timeout)) + pub fn rest(handler: F, timeout: Duration) -> Self + where + F: FnOnce(Request) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + RpcContinuation::Rest(TimedResource::new( + Box::new(|req| RestFuture { + kill: None, + fut: handler(req).boxed(), + }), + timeout, + )) } - pub fn ws(handler: WebSocketHandler, timeout: Duration) -> Self { - RpcContinuation::WebSocket(TimedResource::new(handler, timeout)) + pub fn ws(handler: F, timeout: Duration) -> Self + where + F: FnOnce(WebSocket) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + { + RpcContinuation::WebSocket(TimedResource::new( + Box::new(|ws| WebSocketFuture { + kill: None, + fut: handler(ws).boxed(), + }), + timeout, + )) + } + pub fn rest_authed(ctx: Ctx, session: T, handler: F, timeout: Duration) -> Self + where + Ctx: AsRef>, + T: Eq + Ord, + F: FnOnce(Request) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let kill = Some(ctx.as_ref().subscribe_to_kill(session)); + RpcContinuation::Rest(TimedResource::new( + Box::new(|req| RestFuture { + kill, + fut: handler(req).boxed(), + }), + timeout, + )) + } + pub fn ws_authed(ctx: Ctx, session: T, handler: F, timeout: Duration) -> Self + where + Ctx: AsRef>, + T: Eq + Ord, + F: FnOnce(WebSocket) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + { + let kill = Some(ctx.as_ref().subscribe_to_kill(session)); + RpcContinuation::WebSocket(TimedResource::new( + Box::new(|ws| WebSocketFuture { + kill, + fut: handler(ws).boxed(), + }), + timeout, + )) } pub fn is_timed_out(&self) -> bool { match self { @@ -97,10 +183,10 @@ impl RpcContinuation { } } -pub struct RpcContinuations(Mutex>); +pub struct RpcContinuations(AsyncMutex>); impl RpcContinuations { pub fn new() -> Self { - RpcContinuations(Mutex::new(BTreeMap::new())) + RpcContinuations(AsyncMutex::new(BTreeMap::new())) } #[instrument(skip_all)] @@ -146,3 +232,28 @@ impl RpcContinuations { x.get().await } } + +pub struct OpenAuthedContinuations(SyncMutex>>); +impl OpenAuthedContinuations +where + T: Eq + Ord, +{ + pub fn new() -> Self { + Self(SyncMutex::new(BTreeMap::new())) + } + pub fn kill(&self, session: &T) { + if let Some(channel) = self.0.lock().unwrap().remove(session) { + channel.send(()).ok(); + } + } + fn subscribe_to_kill(&self, session: T) -> broadcast::Receiver<()> { + let mut map = self.0.lock().unwrap(); + if let Some(send) = map.get(&session) { + send.subscribe() + } else { + let (send, recv) = broadcast::channel(1); + map.insert(session, send); + recv + } + } +} diff --git a/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs b/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs index 92eb40f9d..4b7d5736d 100644 --- a/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs +++ b/core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs @@ -97,6 +97,7 @@ impl ArchiveSource for MultiCursorFile { .ok() .map(|m| m.len()) } + #[allow(refining_impl_trait)] async fn fetch_all(&self) -> Result { use tokio::io::AsyncSeekExt; diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index 6588de836..2eecc565e 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -354,7 +354,7 @@ impl Service { .with_kind(ErrorKind::MigrationFailed)?; // TODO: handle cancellation if let Some(mut progress) = progress { progress.finalization_progress.complete(); - progress.progress_handle.complete(); + progress.progress.complete(); tokio::task::yield_now().await; } ctx.db diff --git a/core/startos/src/service/service_map.rs b/core/startos/src/service/service_map.rs index 1474ea35e..f8874d21a 100644 --- a/core/startos/src/service/service_map.rs +++ b/core/startos/src/service/service_map.rs @@ -18,10 +18,7 @@ use crate::disk::mount::guard::GenericMountGuard; use crate::install::PKG_ARCHIVE_DIR; use crate::notifications::{notify, NotificationLevel}; use crate::prelude::*; -use crate::progress::{ - FullProgressTracker, FullProgressTrackerHandle, PhaseProgressTrackerHandle, - ProgressTrackerWriter, -}; +use crate::progress::{FullProgressTracker, PhaseProgressTrackerHandle, ProgressTrackerWriter}; use crate::s9pk::manifest::PackageId; use crate::s9pk::merkle_archive::source::FileSource; use crate::s9pk::S9pk; @@ -34,7 +31,7 @@ pub type InstallFuture = BoxFuture<'static, Result<(), Error>>; pub struct InstallProgressHandles { pub finalization_progress: PhaseProgressTrackerHandle, - pub progress_handle: FullProgressTrackerHandle, + pub progress: FullProgressTracker, } /// This is the structure to contain all the services @@ -59,13 +56,22 @@ impl ServiceMap { } #[instrument(skip_all)] - pub async fn init(&self, ctx: &RpcContext) -> Result<(), Error> { - for id in ctx.db.peek().await.as_public().as_package_data().keys()? { + pub async fn init( + &self, + ctx: &RpcContext, + mut progress: PhaseProgressTrackerHandle, + ) -> Result<(), Error> { + progress.start(); + let ids = ctx.db.peek().await.as_public().as_package_data().keys()?; + progress.set_total(ids.len() as u64); + for id in ids { if let Err(e) = self.load(ctx, &id, LoadDisposition::Retry).await { tracing::error!("Error loading installed package as service: {e}"); tracing::debug!("{e:?}"); } + progress += 1; } + progress.complete(); Ok(()) } @@ -112,17 +118,16 @@ impl ServiceMap { }; let size = s9pk.size(); - let mut progress = FullProgressTracker::new(); + let progress = FullProgressTracker::new(); let download_progress_contribution = size.unwrap_or(60); - let progress_handle = progress.handle(); - let mut download_progress = progress_handle.add_phase( + let mut download_progress = progress.add_phase( InternedString::intern("Download"), Some(download_progress_contribution), ); if let Some(size) = size { download_progress.set_total(size); } - let mut finalization_progress = progress_handle.add_phase( + let mut finalization_progress = progress.add_phase( InternedString::intern(op_name), Some(download_progress_contribution / 2), ); @@ -194,7 +199,7 @@ impl ServiceMap { let deref_id = id.clone(); let sync_progress_task = - NonDetachingJoinHandle::from(tokio::spawn(progress.sync_to_db( + NonDetachingJoinHandle::from(tokio::spawn(progress.clone().sync_to_db( ctx.db.clone(), move |v| { v.as_public_mut() @@ -248,7 +253,7 @@ impl ServiceMap { service .uninstall(Some(s9pk.as_manifest().version.clone())) .await?; - progress_handle.complete(); + progress.complete(); Some(version) } else { None @@ -261,7 +266,7 @@ impl ServiceMap { recovery_source, Some(InstallProgressHandles { finalization_progress, - progress_handle, + progress, }), ) .await? @@ -275,7 +280,7 @@ impl ServiceMap { prev, Some(InstallProgressHandles { finalization_progress, - progress_handle, + progress, }), ) .await? diff --git a/core/startos/src/setup.rs b/core/startos/src/setup.rs index a035e932f..2b701c01f 100644 --- a/core/startos/src/setup.rs +++ b/core/startos/src/setup.rs @@ -4,7 +4,6 @@ use std::time::Duration; use color_eyre::eyre::eyre; use josekit::jwk::Jwk; -use openssl::x509::X509; use patch_db::json_ptr::ROOT; use rpc_toolkit::yajrc::RpcError; use rpc_toolkit::{from_fn_async, Context, HandlerExt, ParentHandler}; @@ -12,15 +11,15 @@ use serde::{Deserialize, Serialize}; use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio::try_join; -use torut::onion::OnionAddressV3; use tracing::instrument; use ts_rs::TS; use crate::account::AccountInfo; use crate::backup::restore::recover_full_embassy; use crate::backup::target::BackupTargetFS; +use crate::context::rpc::InitRpcContextPhases; use crate::context::setup::SetupResult; -use crate::context::SetupContext; +use crate::context::{RpcContext, SetupContext}; use crate::db::model::Database; use crate::disk::fsck::RepairStrategy; use crate::disk::main::DEFAULT_PASSWORD; @@ -29,10 +28,12 @@ use crate::disk::mount::filesystem::ReadWrite; use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard}; use crate::disk::util::{pvscan, recovery_info, DiskInfo, EmbassyOsRecoveryInfo}; use crate::disk::REPAIR_DISK_PATH; -use crate::hostname::Hostname; -use crate::init::{init, InitResult}; +use crate::init::{init, InitPhases, InitResult}; +use crate::net::net_controller::PreInitNetController; use crate::net::ssl::root_ca_start_time; use crate::prelude::*; +use crate::progress::{FullProgress, PhaseProgressTrackerHandle}; +use crate::rpc_continuations::Guid; use crate::util::crypto::EncryptedWire; use crate::util::io::{dir_copy, dir_size, Counter}; use crate::{Error, ErrorKind, ResultExt}; @@ -75,10 +76,12 @@ pub async fn list_disks(ctx: SetupContext) -> Result, Error> { async fn setup_init( ctx: &SetupContext, password: Option, -) -> Result<(Hostname, OnionAddressV3, X509), Error> { - let InitResult { db } = init(&ctx.config).await?; + init_phases: InitPhases, +) -> Result<(AccountInfo, PreInitNetController), Error> { + let InitResult { net_ctrl } = init(&ctx.config, init_phases).await?; - let account = db + let account = net_ctrl + .db .mutate(|m| { let mut account = AccountInfo::load(m)?; if let Some(password) = password { @@ -93,15 +96,12 @@ async fn setup_init( }) .await?; - Ok(( - account.hostname, - account.tor_key.public().get_onion_address(), - account.root_ca_cert, - )) + Ok((account, net_ctrl)) } #[derive(Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] +#[ts(export)] pub struct AttachParams { #[serde(rename = "startOsPassword")] password: Option, @@ -110,25 +110,20 @@ pub struct AttachParams { pub async fn attach( ctx: SetupContext, - AttachParams { password, guid }: AttachParams, -) -> Result<(), Error> { - let mut status = ctx.setup_status.write().await; - if status.is_some() { - return Err(Error::new( - eyre!("Setup already in progress"), - ErrorKind::InvalidRequest, - )); - } - *status = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: None, - complete: false, - })); - drop(status); - tokio::task::spawn(async move { - if let Err(e) = async { + AttachParams { + password, + guid: disk_guid, + }: AttachParams, +) -> Result { + let setup_ctx = ctx.clone(); + ctx.run_setup(|| async move { + let progress = &setup_ctx.progress; + let mut disk_phase = progress.add_phase("Opening data drive".into(), Some(10)); + let init_phases = InitPhases::new(&progress); + let rpc_ctx_phases = InitRpcContextPhases::new(&progress); + let password: Option = match password { - Some(a) => match a.decrypt(&*ctx) { + Some(a) => match a.decrypt(&setup_ctx) { a @ Some(_) => a, None => { return Err(Error::new( @@ -139,15 +134,17 @@ pub async fn attach( }, None => None, }; + + disk_phase.start(); let requires_reboot = crate::disk::main::import( - &*guid, - &ctx.datadir, + &*disk_guid, + &setup_ctx.datadir, if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { RepairStrategy::Aggressive } else { RepairStrategy::Preen }, - if guid.ends_with("_UNENC") { None } else { Some(DEFAULT_PASSWORD) }, + if disk_guid.ends_with("_UNENC") { None } else { Some(DEFAULT_PASSWORD) }, ) .await?; if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { @@ -156,7 +153,7 @@ pub async fn attach( .with_ctx(|_| (ErrorKind::Filesystem, REPAIR_DISK_PATH))?; } if requires_reboot.0 { - crate::disk::main::export(&*guid, &ctx.datadir).await?; + crate::disk::main::export(&*disk_guid, &setup_ctx.datadir).await?; return Err(Error::new( eyre!( "Errors were corrected with your disk, but the server must be restarted in order to proceed" @@ -164,37 +161,48 @@ pub async fn attach( ErrorKind::DiskManagement, )); } - let (hostname, tor_addr, root_ca) = setup_init(&ctx, password).await?; - *ctx.setup_result.write().await = Some((guid, SetupResult { - tor_address: format!("https://{}", tor_addr), - lan_address: hostname.lan_address(), - root_ca: String::from_utf8(root_ca.to_pem()?)?, - })); - *ctx.setup_status.write().await = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: None, - complete: true, - })); - Ok(()) - }.await { - tracing::error!("Error Setting Up Embassy: {}", e); - tracing::debug!("{:?}", e); - *ctx.setup_status.write().await = Some(Err(e.into())); - } - }); - Ok(()) + disk_phase.complete(); + + let (account, net_ctrl) = setup_init(&setup_ctx, password, init_phases).await?; + + let rpc_ctx = RpcContext::init(&setup_ctx.config, disk_guid, Some(net_ctrl), rpc_ctx_phases).await?; + + Ok(((&account).try_into()?, rpc_ctx)) + })?; + + Ok(ctx.progress().await) } -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] -pub struct SetupStatus { - pub bytes_transferred: u64, - pub total_bytes: Option, - pub complete: bool, +#[ts(export)] +#[serde(tag = "status")] +pub enum SetupStatusRes { + Complete(SetupResult), + Running(SetupProgress), } -pub async fn status(ctx: SetupContext) -> Result, RpcError> { - ctx.setup_status.read().await.clone().transpose() +#[derive(Debug, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct SetupProgress { + pub progress: FullProgress, + pub guid: Guid, +} + +pub async fn status(ctx: SetupContext) -> Result, Error> { + if let Some(res) = ctx.result.get() { + match res { + Ok((res, _)) => Ok(Some(SetupStatusRes::Complete(res.clone()))), + Err(e) => Err(e.clone_output()), + } + } else { + if ctx.task.initialized() { + Ok(Some(SetupStatusRes::Running(ctx.progress().await))) + } else { + Ok(None) + } + } } /// We want to be able to get a secret, a shared private key with the frontend @@ -202,7 +210,7 @@ pub async fn status(ctx: SetupContext) -> Result, RpcError> /// without knowing the password over clearnet. We use the public key shared across the network /// since it is fine to share the public, and encrypt against the public. pub async fn get_pubkey(ctx: SetupContext) -> Result { - let secret = ctx.as_ref().clone(); + let secret = AsRef::::as_ref(&ctx).clone(); let pub_key = secret.to_public_key()?; Ok(pub_key) } @@ -213,6 +221,7 @@ pub fn cifs() -> ParentHandler { #[derive(Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] +#[ts(export)] pub struct VerifyCifsParams { hostname: String, path: PathBuf, @@ -230,7 +239,7 @@ pub async fn verify_cifs( password, }: VerifyCifsParams, ) -> Result { - let password: Option = password.map(|x| x.decrypt(&*ctx)).flatten(); + let password: Option = password.map(|x| x.decrypt(&ctx)).flatten(); let guard = TmpMountGuard::mount( &Cifs { hostname, @@ -256,7 +265,8 @@ pub enum RecoverySource { #[derive(Deserialize, Serialize, TS)] #[serde(rename_all = "camelCase")] -pub struct ExecuteParams { +#[ts(export)] +pub struct SetupExecuteParams { start_os_logicalname: PathBuf, start_os_password: EncryptedWire, recovery_source: Option, @@ -266,104 +276,65 @@ pub struct ExecuteParams { // #[command(rpc_only)] pub async fn execute( ctx: SetupContext, - ExecuteParams { + SetupExecuteParams { start_os_logicalname, start_os_password, recovery_source, recovery_password, - }: ExecuteParams, -) -> Result<(), Error> { - let start_os_password = match start_os_password.decrypt(&*ctx) { + }: SetupExecuteParams, +) -> Result { + let start_os_password = match start_os_password.decrypt(&ctx) { Some(a) => a, None => { return Err(Error::new( - color_eyre::eyre::eyre!("Couldn't decode embassy-password"), + color_eyre::eyre::eyre!("Couldn't decode startOsPassword"), crate::ErrorKind::Unknown, )) } }; let recovery_password: Option = match recovery_password { - Some(a) => match a.decrypt(&*ctx) { + Some(a) => match a.decrypt(&ctx) { Some(a) => Some(a), None => { return Err(Error::new( - color_eyre::eyre::eyre!("Couldn't decode recovery-password"), + color_eyre::eyre::eyre!("Couldn't decode recoveryPassword"), crate::ErrorKind::Unknown, )) } }, None => None, }; - let mut status = ctx.setup_status.write().await; - if status.is_some() { - return Err(Error::new( - eyre!("Setup already in progress"), - ErrorKind::InvalidRequest, - )); - } - *status = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: None, - complete: false, - })); - drop(status); - tokio::task::spawn({ - async move { - let ctx = ctx.clone(); - match execute_inner( - ctx.clone(), - start_os_logicalname, - start_os_password, - recovery_source, - recovery_password, - ) - .await - { - Ok((guid, hostname, tor_addr, root_ca)) => { - tracing::info!("Setup Complete!"); - *ctx.setup_result.write().await = Some(( - guid, - SetupResult { - tor_address: format!("https://{}", tor_addr), - lan_address: hostname.lan_address(), - root_ca: String::from_utf8( - root_ca.to_pem().expect("failed to serialize root ca"), - ) - .expect("invalid pem string"), - }, - )); - *ctx.setup_status.write().await = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: None, - complete: true, - })); - } - Err(e) => { - tracing::error!("Error Setting Up Server: {}", e); - tracing::debug!("{:?}", e); - *ctx.setup_status.write().await = Some(Err(e.into())); - } - } - } - }); - Ok(()) + + let setup_ctx = ctx.clone(); + ctx.run_setup(|| { + execute_inner( + setup_ctx, + start_os_logicalname, + start_os_password, + recovery_source, + recovery_password, + ) + })?; + + Ok(ctx.progress().await) } #[instrument(skip_all)] // #[command(rpc_only)] pub async fn complete(ctx: SetupContext) -> Result { - let (guid, setup_result) = if let Some((guid, setup_result)) = &*ctx.setup_result.read().await { - (guid.clone(), setup_result.clone()) - } else { - return Err(Error::new( + match ctx.result.get() { + Some(Ok((res, ctx))) => { + let mut guid_file = File::create("/media/startos/config/disk.guid").await?; + guid_file.write_all(ctx.disk_guid.as_bytes()).await?; + guid_file.sync_all().await?; + Ok(res.clone()) + } + Some(Err(e)) => Err(e.clone_output()), + None => Err(Error::new( eyre!("setup.execute has not completed successfully"), crate::ErrorKind::InvalidRequest, - )); - }; - let mut guid_file = File::create("/media/startos/config/disk.guid").await?; - guid_file.write_all(guid.as_bytes()).await?; - guid_file.sync_all().await?; - Ok(setup_result) + )), + } } #[instrument(skip_all)] @@ -380,7 +351,22 @@ pub async fn execute_inner( start_os_password: String, recovery_source: Option, recovery_password: Option, -) -> Result<(Arc, Hostname, OnionAddressV3, X509), Error> { +) -> Result<(SetupResult, RpcContext), Error> { + let progress = &ctx.progress; + let mut disk_phase = progress.add_phase("Formatting data drive".into(), Some(10)); + let restore_phase = match &recovery_source { + Some(RecoverySource::Backup { .. }) => { + Some(progress.add_phase("Restoring backup".into(), Some(100))) + } + Some(RecoverySource::Migrate { .. }) => { + Some(progress.add_phase("Transferring data".into(), Some(100))) + } + None => None, + }; + let init_phases = InitPhases::new(&progress); + let rpc_ctx_phases = InitRpcContextPhases::new(&progress); + + disk_phase.start(); let encryption_password = if ctx.disable_encryption { None } else { @@ -402,41 +388,70 @@ pub async fn execute_inner( encryption_password, ) .await?; + disk_phase.complete(); - if let Some(RecoverySource::Backup { target }) = recovery_source { - recover(ctx, guid, start_os_password, target, recovery_password).await - } else if let Some(RecoverySource::Migrate { guid: old_guid }) = recovery_source { - migrate(ctx, guid, &old_guid, start_os_password).await - } else { - let (hostname, tor_addr, root_ca) = fresh_setup(&ctx, &start_os_password).await?; - Ok((guid, hostname, tor_addr, root_ca)) + let progress = SetupExecuteProgress { + init_phases, + restore_phase, + rpc_ctx_phases, + }; + + match recovery_source { + Some(RecoverySource::Backup { target }) => { + recover( + &ctx, + guid, + start_os_password, + target, + recovery_password, + progress, + ) + .await + } + Some(RecoverySource::Migrate { guid: old_guid }) => { + migrate(&ctx, guid, &old_guid, start_os_password, progress).await + } + None => fresh_setup(&ctx, guid, &start_os_password, progress).await, } } +pub struct SetupExecuteProgress { + pub init_phases: InitPhases, + pub restore_phase: Option, + pub rpc_ctx_phases: InitRpcContextPhases, +} + async fn fresh_setup( ctx: &SetupContext, + guid: Arc, start_os_password: &str, -) -> Result<(Hostname, OnionAddressV3, X509), Error> { + SetupExecuteProgress { + init_phases, + rpc_ctx_phases, + .. + }: SetupExecuteProgress, +) -> Result<(SetupResult, RpcContext), Error> { let account = AccountInfo::new(start_os_password, root_ca_start_time().await?)?; let db = ctx.db().await?; db.put(&ROOT, &Database::init(&account)?).await?; drop(db); - init(&ctx.config).await?; - Ok(( - account.hostname, - account.tor_key.public().get_onion_address(), - account.root_ca_cert, - )) + + let InitResult { net_ctrl } = init(&ctx.config, init_phases).await?; + + let rpc_ctx = RpcContext::init(&ctx.config, guid, Some(net_ctrl), rpc_ctx_phases).await?; + + Ok(((&account).try_into()?, rpc_ctx)) } #[instrument(skip_all)] async fn recover( - ctx: SetupContext, + ctx: &SetupContext, guid: Arc, start_os_password: String, recovery_source: BackupTargetFS, recovery_password: Option, -) -> Result<(Arc, Hostname, OnionAddressV3, X509), Error> { + progress: SetupExecuteProgress, +) -> Result<(SetupResult, RpcContext), Error> { let recovery_source = TmpMountGuard::mount(&recovery_source, ReadWrite).await?; recover_full_embassy( ctx, @@ -444,23 +459,26 @@ async fn recover( start_os_password, recovery_source, recovery_password, + progress, ) .await } #[instrument(skip_all)] async fn migrate( - ctx: SetupContext, + ctx: &SetupContext, guid: Arc, old_guid: &str, start_os_password: String, -) -> Result<(Arc, Hostname, OnionAddressV3, X509), Error> { - *ctx.setup_status.write().await = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: None, - complete: false, - })); + SetupExecuteProgress { + init_phases, + restore_phase, + rpc_ctx_phases, + }: SetupExecuteProgress, +) -> Result<(SetupResult, RpcContext), Error> { + let mut restore_phase = restore_phase.or_not_found("restore progress")?; + restore_phase.start(); let _ = crate::disk::main::import( &old_guid, "/media/startos/migrate", @@ -500,20 +518,12 @@ async fn migrate( res = async { loop { tokio::time::sleep(Duration::from_secs(1)).await; - *ctx.setup_status.write().await = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: Some(main_transfer_size.load() + package_data_transfer_size.load()), - complete: false, - })); + restore_phase.set_total(main_transfer_size.load() + package_data_transfer_size.load()); } } => res, }; - *ctx.setup_status.write().await = Some(Ok(SetupStatus { - bytes_transferred: 0, - total_bytes: Some(size), - complete: false, - })); + restore_phase.set_total(size); let main_transfer_progress = Counter::new(0, ordering); let package_data_transfer_progress = Counter::new(0, ordering); @@ -529,18 +539,17 @@ async fn migrate( res = async { loop { tokio::time::sleep(Duration::from_secs(1)).await; - *ctx.setup_status.write().await = Some(Ok(SetupStatus { - bytes_transferred: main_transfer_progress.load() + package_data_transfer_progress.load(), - total_bytes: Some(size), - complete: false, - })); + restore_phase.set_done(main_transfer_progress.load() + package_data_transfer_progress.load()); } } => res, } - let (hostname, tor_addr, root_ca) = setup_init(&ctx, Some(start_os_password)).await?; - crate::disk::main::export(&old_guid, "/media/startos/migrate").await?; + restore_phase.complete(); - Ok((guid, hostname, tor_addr, root_ca)) + let (account, net_ctrl) = setup_init(&ctx, Some(start_os_password), init_phases).await?; + + let rpc_ctx = RpcContext::init(&ctx.config, guid, Some(net_ctrl), rpc_ctx_phases).await?; + + Ok(((&account).try_into()?, rpc_ctx)) } diff --git a/core/startos/src/update/mod.rs b/core/startos/src/update/mod.rs index be908e776..dc164d2cd 100644 --- a/core/startos/src/update/mod.rs +++ b/core/startos/src/update/mod.rs @@ -20,9 +20,7 @@ use ts_rs::TS; use crate::context::{CliContext, RpcContext}; use crate::notifications::{notify, NotificationLevel}; use crate::prelude::*; -use crate::progress::{ - FullProgressTracker, FullProgressTrackerHandle, PhaseProgressTrackerHandle, PhasedProgressBar, -}; +use crate::progress::{FullProgressTracker, PhaseProgressTrackerHandle, PhasedProgressBar}; use crate::registry::asset::RegistryAsset; use crate::registry::context::{RegistryContext, RegistryUrlParams}; use crate::registry::os::index::OsVersionInfo; @@ -34,6 +32,7 @@ use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; use crate::sound::{ CIRCLE_OF_5THS_SHORT, UPDATE_FAILED_1, UPDATE_FAILED_2, UPDATE_FAILED_3, UPDATE_FAILED_4, }; +use crate::util::net::WebSocketExt; use crate::util::Invoke; use crate::PLATFORM; @@ -91,50 +90,47 @@ pub async fn update_system( .add( guid.clone(), RpcContinuation::ws( - Box::new(|mut ws| { - async move { - if let Err(e) = async { - let mut sub = ctx + |mut ws| async move { + if let Err(e) = async { + let mut sub = ctx + .db + .subscribe( + "/public/serverInfo/statusInfo/updateProgress" + .parse::() + .with_kind(ErrorKind::Database)?, + ) + .await; + while { + let progress = ctx .db - .subscribe( - "/public/serverInfo/statusInfo/updateProgress" - .parse::() - .with_kind(ErrorKind::Database)?, - ) - .await; - while { - let progress = ctx - .db - .peek() - .await - .into_public() - .into_server_info() - .into_status_info() - .into_update_progress() - .de()?; - ws.send(axum::extract::ws::Message::Text( - serde_json::to_string(&progress) - .with_kind(ErrorKind::Serialization)?, - )) + .peek() .await - .with_kind(ErrorKind::Network)?; - progress.is_some() - } { - sub.recv().await; - } - - ws.close().await.with_kind(ErrorKind::Network)?; - - Ok::<_, Error>(()) - } - .await - { - tracing::error!("Error returning progress of update: {e}"); - tracing::debug!("{e:?}") + .into_public() + .into_server_info() + .into_status_info() + .into_update_progress() + .de()?; + ws.send(axum::extract::ws::Message::Text( + serde_json::to_string(&progress) + .with_kind(ErrorKind::Serialization)?, + )) + .await + .with_kind(ErrorKind::Network)?; + progress.is_some() + } { + sub.recv().await; } + + ws.normal_close("complete").await?; + + Ok::<_, Error>(()) } - .boxed() - }), + .await + { + tracing::error!("Error returning progress of update: {e}"); + tracing::debug!("{e:?}") + } + }, Duration::from_secs(30), ), ) @@ -250,13 +246,12 @@ async fn maybe_do_update( asset.validate(SIG_CONTEXT, asset.all_signers())?; - let mut progress = FullProgressTracker::new(); - let progress_handle = progress.handle(); - let mut download_phase = progress_handle.add_phase("Downloading File".into(), Some(100)); + let progress = FullProgressTracker::new(); + let mut download_phase = progress.add_phase("Downloading File".into(), Some(100)); download_phase.set_total(asset.commitment.size); - let reverify_phase = progress_handle.add_phase("Reverifying File".into(), Some(10)); - let sync_boot_phase = progress_handle.add_phase("Syncing Boot Files".into(), Some(1)); - let finalize_phase = progress_handle.add_phase("Finalizing Update".into(), Some(1)); + let reverify_phase = progress.add_phase("Reverifying File".into(), Some(10)); + let sync_boot_phase = progress.add_phase("Syncing Boot Files".into(), Some(1)); + let finalize_phase = progress.add_phase("Finalizing Update".into(), Some(1)); let start_progress = progress.snapshot(); @@ -287,7 +282,7 @@ async fn maybe_do_update( )); } - let progress_task = NonDetachingJoinHandle::from(tokio::spawn(progress.sync_to_db( + let progress_task = NonDetachingJoinHandle::from(tokio::spawn(progress.clone().sync_to_db( ctx.db.clone(), |db| { db.as_public_mut() @@ -304,7 +299,7 @@ async fn maybe_do_update( ctx.clone(), asset, UpdateProgressHandles { - progress_handle, + progress, download_phase, reverify_phase, sync_boot_phase, @@ -373,7 +368,7 @@ async fn maybe_do_update( } struct UpdateProgressHandles { - progress_handle: FullProgressTrackerHandle, + progress: FullProgressTracker, download_phase: PhaseProgressTrackerHandle, reverify_phase: PhaseProgressTrackerHandle, sync_boot_phase: PhaseProgressTrackerHandle, @@ -385,7 +380,7 @@ async fn do_update( ctx: RpcContext, asset: RegistryAsset, UpdateProgressHandles { - progress_handle, + progress, mut download_phase, mut reverify_phase, mut sync_boot_phase, @@ -436,7 +431,7 @@ async fn do_update( .await?; finalize_phase.complete(); - progress_handle.complete(); + progress.complete(); Ok(()) } diff --git a/core/startos/src/upload.rs b/core/startos/src/upload.rs index f28d81799..b922ab9d2 100644 --- a/core/startos/src/upload.rs +++ b/core/startos/src/upload.rs @@ -5,9 +5,10 @@ use std::time::Duration; use axum::body::Body; use axum::response::Response; -use futures::{FutureExt, StreamExt}; +use futures::StreamExt; use http::header::CONTENT_LENGTH; use http::StatusCode; +use imbl_value::InternedString; use tokio::fs::File; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tokio::sync::watch; @@ -19,68 +20,70 @@ use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile; use crate::s9pk::merkle_archive::source::ArchiveSource; use crate::util::io::TmpDir; -pub async fn upload(ctx: &RpcContext) -> Result<(Guid, UploadingFile), Error> { +pub async fn upload( + ctx: &RpcContext, + session: InternedString, +) -> Result<(Guid, UploadingFile), Error> { let guid = Guid::new(); let (mut handle, file) = UploadingFile::new().await?; ctx.rpc_continuations .add( guid.clone(), - RpcContinuation::rest( - Box::new(|request| { - async move { - let headers = request.headers(); - let content_length = match headers.get(CONTENT_LENGTH).map(|a| a.to_str()) { - None => { - return Response::builder() - .status(StatusCode::BAD_REQUEST) - .body(Body::from("Content-Length is required")) - .with_kind(ErrorKind::Network) - } - Some(Err(_)) => { + RpcContinuation::rest_authed( + ctx, + session, + |request| async move { + let headers = request.headers(); + let content_length = match headers.get(CONTENT_LENGTH).map(|a| a.to_str()) { + None => { + return Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(Body::from("Content-Length is required")) + .with_kind(ErrorKind::Network) + } + Some(Err(_)) => { + return Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(Body::from("Invalid Content-Length")) + .with_kind(ErrorKind::Network) + } + Some(Ok(a)) => match a.parse::() { + Err(_) => { return Response::builder() .status(StatusCode::BAD_REQUEST) .body(Body::from("Invalid Content-Length")) .with_kind(ErrorKind::Network) } - Some(Ok(a)) => match a.parse::() { - Err(_) => { - return Response::builder() - .status(StatusCode::BAD_REQUEST) - .body(Body::from("Invalid Content-Length")) - .with_kind(ErrorKind::Network) - } - Ok(a) => a, - }, - }; + Ok(a) => a, + }, + }; - handle - .progress - .send_modify(|p| p.expected_size = Some(content_length)); + handle + .progress + .send_modify(|p| p.expected_size = Some(content_length)); - let mut body = request.into_body().into_data_stream(); - while let Some(next) = body.next().await { - if let Err(e) = async { - handle - .write_all(&next.map_err(|e| { - std::io::Error::new(std::io::ErrorKind::Other, e) - })?) - .await?; - Ok(()) - } - .await - { - handle.progress.send_if_modified(|p| p.handle_error(&e)); - break; - } + let mut body = request.into_body().into_data_stream(); + while let Some(next) = body.next().await { + if let Err(e) = async { + handle + .write_all(&next.map_err(|e| { + std::io::Error::new(std::io::ErrorKind::Other, e) + })?) + .await?; + Ok(()) + } + .await + { + handle.progress.send_if_modified(|p| p.handle_error(&e)); + break; } - - Response::builder() - .status(StatusCode::NO_CONTENT) - .body(Body::empty()) - .with_kind(ErrorKind::Network) } - .boxed() - }), + + Response::builder() + .status(StatusCode::NO_CONTENT) + .body(Body::empty()) + .with_kind(ErrorKind::Network) + }, Duration::from_secs(30), ), ) diff --git a/core/startos/src/util/io.rs b/core/startos/src/util/io.rs index f4476ee2b..9a6bab64b 100644 --- a/core/startos/src/util/io.rs +++ b/core/startos/src/util/io.rs @@ -274,6 +274,81 @@ pub fn response_to_reader(response: reqwest::Response) -> impl AsyncRead + Unpin })) } +#[pin_project::pin_project] +pub struct IOHook<'a, T> { + #[pin] + pub io: T, + pre_write: Option Result<(), std::io::Error> + Send + 'a>>, + post_write: Option>, + post_read: Option>, +} +impl<'a, T> IOHook<'a, T> { + pub fn new(io: T) -> Self { + Self { + io, + pre_write: None, + post_write: None, + post_read: None, + } + } + pub fn into_inner(self) -> T { + self.io + } + pub fn pre_write Result<(), std::io::Error> + Send + 'a>(&mut self, f: F) { + self.pre_write = Some(Box::new(f)) + } + pub fn post_write(&mut self, f: F) { + self.post_write = Some(Box::new(f)) + } + pub fn post_read(&mut self, f: F) { + self.post_read = Some(Box::new(f)) + } +} +impl<'a, T: AsyncWrite> AsyncWrite for IOHook<'a, T> { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> Poll> { + let this = self.project(); + if let Some(pre_write) = this.pre_write { + pre_write(buf)?; + } + let written = futures::ready!(this.io.poll_write(cx, buf)?); + if let Some(post_write) = this.post_write { + post_write(&buf[..written]); + } + Poll::Ready(Ok(written)) + } + fn poll_flush( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + self.project().io.poll_flush(cx) + } + fn poll_shutdown( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + self.project().io.poll_shutdown(cx) + } +} +impl<'a, T: AsyncRead> AsyncRead for IOHook<'a, T> { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + let this = self.project(); + let start = buf.filled().len(); + futures::ready!(this.io.poll_read(cx, buf)?); + if let Some(post_read) = this.post_read { + post_read(&buf.filled()[start..]); + } + Poll::Ready(Ok(())) + } +} + #[pin_project::pin_project] pub struct BufferedWriteReader { #[pin] @@ -768,7 +843,7 @@ fn poll_flush_prefix( flush_writer: bool, ) -> Poll> { while let Some(mut cur) = prefix.pop_front() { - let buf = cur.remaining_slice(); + let buf = CursorExt::remaining_slice(&cur); if !buf.is_empty() { match writer.as_mut().poll_write(cx, buf)? { Poll::Ready(n) if n == buf.len() => (), diff --git a/core/startos/src/util/mod.rs b/core/startos/src/util/mod.rs index 4346a0b1e..f2334632e 100644 --- a/core/startos/src/util/mod.rs +++ b/core/startos/src/util/mod.rs @@ -36,6 +36,7 @@ pub mod http_reader; pub mod io; pub mod logger; pub mod lshw; +pub mod net; pub mod rpc; pub mod rpc_client; pub mod serde; diff --git a/core/startos/src/util/net.rs b/core/startos/src/util/net.rs new file mode 100644 index 000000000..93131f16e --- /dev/null +++ b/core/startos/src/util/net.rs @@ -0,0 +1,24 @@ +use std::borrow::Cow; + +use axum::extract::ws::{self, CloseFrame}; +use futures::Future; + +use crate::prelude::*; + +pub trait WebSocketExt { + fn normal_close( + self, + msg: impl Into>, + ) -> impl Future>; +} + +impl WebSocketExt for ws::WebSocket { + async fn normal_close(mut self, msg: impl Into>) -> Result<(), Error> { + self.send(ws::Message::Close(Some(CloseFrame { + code: 1000, + reason: msg.into(), + }))) + .await + .with_kind(ErrorKind::Network) + } +} diff --git a/core/startos/src/util/serde.rs b/core/startos/src/util/serde.rs index 382ef2814..5d1289e7e 100644 --- a/core/startos/src/util/serde.rs +++ b/core/startos/src/util/serde.rs @@ -22,8 +22,8 @@ use ts_rs::TS; use super::IntoDoubleEndedIterator; use crate::prelude::*; -use crate::util::Apply; use crate::util::clap::FromStrParser; +use crate::util::Apply; pub fn deserialize_from_str< 'de, @@ -1040,15 +1040,19 @@ impl> std::fmt::Display for Base64 { f.write_str(&base64::encode(self.0.as_ref())) } } -impl>> FromStr for Base64 -{ +impl>> FromStr for Base64 { type Err = Error; fn from_str(s: &str) -> Result { base64::decode(&s) .with_kind(ErrorKind::Deserialization)? .apply(TryFrom::try_from) .map(Self) - .map_err(|_| Error::new(eyre!("failed to create from buffer"), ErrorKind::Deserialization)) + .map_err(|_| { + Error::new( + eyre!("failed to create from buffer"), + ErrorKind::Deserialization, + ) + }) } } impl<'de, T: TryFrom>> Deserialize<'de> for Base64 { diff --git a/core/startos/src/version/mod.rs b/core/startos/src/version/mod.rs index 7212b8801..d063558e2 100644 --- a/core/startos/src/version/mod.rs +++ b/core/startos/src/version/mod.rs @@ -7,6 +7,7 @@ use imbl_value::InternedString; use crate::db::model::Database; use crate::prelude::*; +use crate::progress::PhaseProgressTrackerHandle; use crate::Error; mod v0_3_5; @@ -85,11 +86,12 @@ where &self, version: &V, db: &TypedPatchDb, + progress: &mut PhaseProgressTrackerHandle, ) -> impl Future> + Send { async { match self.semver().cmp(&version.semver()) { - Ordering::Greater => self.rollback_to_unchecked(version, db).await, - Ordering::Less => version.migrate_from_unchecked(self, db).await, + Ordering::Greater => self.rollback_to_unchecked(version, db, progress).await, + Ordering::Less => version.migrate_from_unchecked(self, db, progress).await, Ordering::Equal => Ok(()), } } @@ -98,11 +100,15 @@ where &'a self, version: &'a V, db: &'a TypedPatchDb, + progress: &'a mut PhaseProgressTrackerHandle, ) -> BoxFuture<'a, Result<(), Error>> { + progress.add_total(1); async { let previous = Self::Previous::new(); if version.semver() < previous.semver() { - previous.migrate_from_unchecked(version, db).await?; + previous + .migrate_from_unchecked(version, db, progress) + .await?; } else if version.semver() > previous.semver() { return Err(Error::new( eyre!( @@ -115,6 +121,7 @@ where tracing::info!("{} -> {}", previous.semver(), self.semver(),); self.up(db).await?; self.commit(db).await?; + *progress += 1; Ok(()) } .boxed() @@ -123,14 +130,18 @@ where &'a self, version: &'a V, db: &'a TypedPatchDb, + progress: &'a mut PhaseProgressTrackerHandle, ) -> BoxFuture<'a, Result<(), Error>> { async { let previous = Self::Previous::new(); tracing::info!("{} -> {}", self.semver(), previous.semver(),); self.down(db).await?; previous.commit(db).await?; + *progress += 1; if version.semver() < previous.semver() { - previous.rollback_to_unchecked(version, db).await?; + previous + .rollback_to_unchecked(version, db, progress) + .await?; } else if version.semver() > previous.semver() { return Err(Error::new( eyre!( @@ -196,7 +207,11 @@ where } } -pub async fn init(db: &TypedPatchDb) -> Result<(), Error> { +pub async fn init( + db: &TypedPatchDb, + mut progress: PhaseProgressTrackerHandle, +) -> Result<(), Error> { + progress.start(); let version = Version::from_util_version( db.peek() .await @@ -213,10 +228,10 @@ pub async fn init(db: &TypedPatchDb) -> Result<(), Error> { ErrorKind::MigrationFailed, )); } - Version::V0_3_5(v) => v.0.migrate_to(&Current::new(), &db).await?, - Version::V0_3_5_1(v) => v.0.migrate_to(&Current::new(), &db).await?, - Version::V0_3_5_2(v) => v.0.migrate_to(&Current::new(), &db).await?, - Version::V0_3_6(v) => v.0.migrate_to(&Current::new(), &db).await?, + Version::V0_3_5(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?, + Version::V0_3_5_1(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?, + Version::V0_3_5_2(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?, + Version::V0_3_6(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?, Version::Other(_) => { return Err(Error::new( eyre!("Cannot downgrade"), @@ -224,6 +239,7 @@ pub async fn init(db: &TypedPatchDb) -> Result<(), Error> { )) } } + progress.complete(); Ok(()) } diff --git a/core/startos/src/volume.rs b/core/startos/src/volume.rs index e801da79b..c7165b202 100644 --- a/core/startos/src/volume.rs +++ b/core/startos/src/volume.rs @@ -20,7 +20,11 @@ pub fn data_dir>(datadir: P, pkg_id: &PackageId, volume_id: &Volu .join(volume_id) } -pub fn asset_dir>(datadir: P, pkg_id: &PackageId, version: &VersionString) -> PathBuf { +pub fn asset_dir>( + datadir: P, + pkg_id: &PackageId, + version: &VersionString, +) -> PathBuf { datadir .as_ref() .join(PKG_VOLUME_DIR) diff --git a/patch-db b/patch-db index 88a804f56..7aa53249f 160000 --- a/patch-db +++ b/patch-db @@ -1 +1 @@ -Subproject commit 88a804f56f446d34896ef331d915b821a581cf01 +Subproject commit 7aa53249f9353162475ea347abac92abcfba5493 diff --git a/sdk/lib/osBindings/AttachParams.ts b/sdk/lib/osBindings/AttachParams.ts new file mode 100644 index 000000000..048151d2f --- /dev/null +++ b/sdk/lib/osBindings/AttachParams.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { EncryptedWire } from "./EncryptedWire" + +export type AttachParams = { + startOsPassword: EncryptedWire | null + guid: string +} diff --git a/sdk/lib/osBindings/BackupTargetFS.ts b/sdk/lib/osBindings/BackupTargetFS.ts new file mode 100644 index 000000000..0cff2cc4e --- /dev/null +++ b/sdk/lib/osBindings/BackupTargetFS.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BlockDev } from "./BlockDev" +import type { Cifs } from "./Cifs" + +export type BackupTargetFS = + | ({ type: "disk" } & BlockDev) + | ({ type: "cifs" } & Cifs) diff --git a/sdk/lib/osBindings/BlockDev.ts b/sdk/lib/osBindings/BlockDev.ts new file mode 100644 index 000000000..46db81011 --- /dev/null +++ b/sdk/lib/osBindings/BlockDev.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type BlockDev = { logicalname: string } diff --git a/sdk/lib/osBindings/Cifs.ts b/sdk/lib/osBindings/Cifs.ts new file mode 100644 index 000000000..f7099bd7f --- /dev/null +++ b/sdk/lib/osBindings/Cifs.ts @@ -0,0 +1,8 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type Cifs = { + hostname: string + path: string + username: string + password: string | null +} diff --git a/sdk/lib/osBindings/InitProgressRes.ts b/sdk/lib/osBindings/InitProgressRes.ts new file mode 100644 index 000000000..38caf7bdb --- /dev/null +++ b/sdk/lib/osBindings/InitProgressRes.ts @@ -0,0 +1,5 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { FullProgress } from "./FullProgress" +import type { Guid } from "./Guid" + +export type InitProgressRes = { progress: FullProgress; guid: Guid } diff --git a/sdk/lib/osBindings/RecoverySource.ts b/sdk/lib/osBindings/RecoverySource.ts new file mode 100644 index 000000000..c40ec5132 --- /dev/null +++ b/sdk/lib/osBindings/RecoverySource.ts @@ -0,0 +1,6 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BackupTargetFS } from "./BackupTargetFS" + +export type RecoverySource = + | { type: "migrate"; guid: string } + | { type: "backup"; target: BackupTargetFS } diff --git a/sdk/lib/osBindings/SetupExecuteParams.ts b/sdk/lib/osBindings/SetupExecuteParams.ts new file mode 100644 index 000000000..4593e7667 --- /dev/null +++ b/sdk/lib/osBindings/SetupExecuteParams.ts @@ -0,0 +1,10 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { EncryptedWire } from "./EncryptedWire" +import type { RecoverySource } from "./RecoverySource" + +export type SetupExecuteParams = { + startOsLogicalname: string + startOsPassword: EncryptedWire + recoverySource: RecoverySource | null + recoveryPassword: EncryptedWire | null +} diff --git a/sdk/lib/osBindings/SetupProgress.ts b/sdk/lib/osBindings/SetupProgress.ts new file mode 100644 index 000000000..845636da3 --- /dev/null +++ b/sdk/lib/osBindings/SetupProgress.ts @@ -0,0 +1,5 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { FullProgress } from "./FullProgress" +import type { Guid } from "./Guid" + +export type SetupProgress = { progress: FullProgress; guid: Guid } diff --git a/sdk/lib/osBindings/SetupResult.ts b/sdk/lib/osBindings/SetupResult.ts new file mode 100644 index 000000000..464aeb4b7 --- /dev/null +++ b/sdk/lib/osBindings/SetupResult.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type SetupResult = { + torAddress: string + lanAddress: string + rootCa: string +} diff --git a/sdk/lib/osBindings/SetupStatusRes.ts b/sdk/lib/osBindings/SetupStatusRes.ts new file mode 100644 index 000000000..93d10c59b --- /dev/null +++ b/sdk/lib/osBindings/SetupStatusRes.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { SetupProgress } from "./SetupProgress" +import type { SetupResult } from "./SetupResult" + +export type SetupStatusRes = + | ({ status: "complete" } & SetupResult) + | ({ status: "running" } & SetupProgress) diff --git a/sdk/lib/osBindings/VerifyCifsParams.ts b/sdk/lib/osBindings/VerifyCifsParams.ts new file mode 100644 index 000000000..407e6caaa --- /dev/null +++ b/sdk/lib/osBindings/VerifyCifsParams.ts @@ -0,0 +1,9 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { EncryptedWire } from "./EncryptedWire" + +export type VerifyCifsParams = { + hostname: string + path: string + username: string + password: EncryptedWire | null +} diff --git a/sdk/lib/osBindings/index.ts b/sdk/lib/osBindings/index.ts index 06a4bed7e..ac2e19e45 100644 --- a/sdk/lib/osBindings/index.ts +++ b/sdk/lib/osBindings/index.ts @@ -15,17 +15,21 @@ export { AlpnInfo } from "./AlpnInfo" export { AnySignature } from "./AnySignature" export { AnySigningKey } from "./AnySigningKey" export { AnyVerifyingKey } from "./AnyVerifyingKey" +export { AttachParams } from "./AttachParams" export { BackupProgress } from "./BackupProgress" +export { BackupTargetFS } from "./BackupTargetFS" export { Base64 } from "./Base64" export { BindInfo } from "./BindInfo" export { BindOptions } from "./BindOptions" export { BindParams } from "./BindParams" export { Blake3Commitment } from "./Blake3Commitment" +export { BlockDev } from "./BlockDev" export { Callback } from "./Callback" export { Category } from "./Category" export { CheckDependenciesParam } from "./CheckDependenciesParam" export { CheckDependenciesResult } from "./CheckDependenciesResult" export { ChrootParams } from "./ChrootParams" +export { Cifs } from "./Cifs" export { ContactInfo } from "./ContactInfo" export { CreateOverlayedImageParams } from "./CreateOverlayedImageParams" export { CurrentDependencies } from "./CurrentDependencies" @@ -73,6 +77,7 @@ export { ImageConfig } from "./ImageConfig" export { ImageId } from "./ImageId" export { ImageMetadata } from "./ImageMetadata" export { ImageSource } from "./ImageSource" +export { InitProgressRes } from "./InitProgressRes" export { InstalledState } from "./InstalledState" export { InstallingInfo } from "./InstallingInfo" export { InstallingState } from "./InstallingState" @@ -105,6 +110,7 @@ export { ParamsPackageId } from "./ParamsPackageId" export { PasswordType } from "./PasswordType" export { Progress } from "./Progress" export { Public } from "./Public" +export { RecoverySource } from "./RecoverySource" export { RegistryAsset } from "./RegistryAsset" export { RemoveActionParams } from "./RemoveActionParams" export { RemoveAddressParams } from "./RemoveAddressParams" @@ -127,10 +133,15 @@ export { SetMainStatusStatus } from "./SetMainStatusStatus" export { SetMainStatus } from "./SetMainStatus" export { SetStoreParams } from "./SetStoreParams" export { SetSystemSmtpParams } from "./SetSystemSmtpParams" +export { SetupExecuteParams } from "./SetupExecuteParams" +export { SetupProgress } from "./SetupProgress" +export { SetupResult } from "./SetupResult" +export { SetupStatusRes } from "./SetupStatusRes" export { SignAssetParams } from "./SignAssetParams" export { SignerInfo } from "./SignerInfo" export { Status } from "./Status" export { UpdatingState } from "./UpdatingState" +export { VerifyCifsParams } from "./VerifyCifsParams" export { VersionSignerParams } from "./VersionSignerParams" export { Version } from "./Version" export { VolumeId } from "./VolumeId" diff --git a/web/package-lock.json b/web/package-lock.json index 68156eb21..81b612ac1 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -31,6 +31,7 @@ "@taiga-ui/core": "3.20.0", "@taiga-ui/icons": "3.20.0", "@taiga-ui/kit": "3.20.0", + "@tinkoff/ng-dompurify": "4.0.0", "angular-svg-round-progressbar": "^9.0.0", "ansi-to-html": "^0.7.2", "base64-js": "^1.5.1", @@ -1973,7 +1974,7 @@ }, "../sdk/dist": { "name": "@start9labs/start-sdk", - "version": "0.4.0-rev0.lib0.rc8.beta10", + "version": "0.3.6-alpha1", "license": "MIT", "dependencies": { "isomorphic-fetch": "^3.0.0", @@ -5432,6 +5433,20 @@ "rxjs": ">=6.0.0" } }, + "node_modules/@tinkoff/ng-dompurify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@tinkoff/ng-dompurify/-/ng-dompurify-4.0.0.tgz", + "integrity": "sha512-BjKUweWLrOx8UOZw+Tl+Dae5keYuSbeMkppcXQdsvwASMrPfmP7d3Q206Q6HDqOV2WnpnFqGUB95IMbLAeRRuw==", + "dependencies": { + "tslib": "^2.0.0" + }, + "peerDependencies": { + "@angular/core": ">=12.0.0", + "@angular/platform-browser": ">=12.0.0", + "@types/dompurify": ">=2.3.0", + "dompurify": ">= 2.3.0" + } + }, "node_modules/@tinkoff/ng-event-plugins": { "version": "3.1.0", "license": "Apache-2.0", @@ -5549,7 +5564,6 @@ }, "node_modules/@types/dompurify": { "version": "2.3.4", - "dev": true, "license": "MIT", "dependencies": { "@types/trusted-types": "*" @@ -5726,7 +5740,6 @@ }, "node_modules/@types/trusted-types": { "version": "2.0.2", - "dev": true, "license": "MIT" }, "node_modules/@types/uuid": { diff --git a/web/package.json b/web/package.json index 9e842455e..3ea29c6fd 100644 --- a/web/package.json +++ b/web/package.json @@ -13,7 +13,6 @@ "check:setup": "tsc --project projects/setup-wizard/tsconfig.json --noEmit --skipLibCheck", "check:ui": "tsc --project projects/ui/tsconfig.json --noEmit --skipLibCheck", "build:deps": "rm -rf .angular/cache && (cd ../patch-db/client && npm ci && npm run build) && (cd ../sdk && make bundle)", - "build:dui": "ng run diagnostic-ui:build", "build:install-wiz": "ng run install-wizard:build", "build:setup": "ng run setup-wizard:build", "build:ui": "ng run ui:build", @@ -25,7 +24,6 @@ "analyze:ui": "webpack-bundle-analyzer dist/raw/ui/stats.json", "publish:shared": "npm run build:shared && npm publish ./dist/shared --access public", "publish:marketplace": "npm run build:marketplace && npm publish ./dist/marketplace --access public", - "start:dui": "npm run-script build-config && ionic serve --project diagnostic-ui --host 0.0.0.0", "start:install-wiz": "npm run-script build-config && ionic serve --project install-wizard --host 0.0.0.0", "start:setup": "npm run-script build-config && ionic serve --project setup-wizard --host 0.0.0.0", "start:ui": "npm run-script build-config && ionic serve --project ui --ip --host 0.0.0.0", @@ -56,6 +54,7 @@ "@taiga-ui/core": "3.20.0", "@taiga-ui/icons": "3.20.0", "@taiga-ui/kit": "3.20.0", + "@tinkoff/ng-dompurify": "4.0.0", "angular-svg-round-progressbar": "^9.0.0", "ansi-to-html": "^0.7.2", "base64-js": "^1.5.1", diff --git a/web/projects/diagnostic-ui/src/app/app-routing.module.ts b/web/projects/diagnostic-ui/src/app/app-routing.module.ts deleted file mode 100644 index f9f009b48..000000000 --- a/web/projects/diagnostic-ui/src/app/app-routing.module.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { NgModule } from '@angular/core' -import { PreloadAllModules, RouterModule, Routes } from '@angular/router' - -const routes: Routes = [ - { - path: '', - loadChildren: () => - import('./pages/home/home.module').then(m => m.HomePageModule), - }, - { - path: 'logs', - loadChildren: () => - import('./pages/logs/logs.module').then(m => m.LogsPageModule), - }, -] - -@NgModule({ - imports: [ - RouterModule.forRoot(routes, { - scrollPositionRestoration: 'enabled', - preloadingStrategy: PreloadAllModules, - useHash: true, - }), - ], - exports: [RouterModule], -}) -export class AppRoutingModule {} diff --git a/web/projects/diagnostic-ui/src/app/app.component.html b/web/projects/diagnostic-ui/src/app/app.component.html deleted file mode 100644 index cd28a7e80..000000000 --- a/web/projects/diagnostic-ui/src/app/app.component.html +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/web/projects/diagnostic-ui/src/app/app.component.scss b/web/projects/diagnostic-ui/src/app/app.component.scss deleted file mode 100644 index b528fd9bd..000000000 --- a/web/projects/diagnostic-ui/src/app/app.component.scss +++ /dev/null @@ -1,8 +0,0 @@ -:host { - display: block; - height: 100%; -} - -tui-root { - height: 100%; -} diff --git a/web/projects/diagnostic-ui/src/app/app.component.ts b/web/projects/diagnostic-ui/src/app/app.component.ts deleted file mode 100644 index 5ac82a652..000000000 --- a/web/projects/diagnostic-ui/src/app/app.component.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Component } from '@angular/core' - -@Component({ - selector: 'app-root', - templateUrl: 'app.component.html', - styleUrls: ['app.component.scss'], -}) -export class AppComponent { - constructor() {} -} diff --git a/web/projects/diagnostic-ui/src/app/app.module.ts b/web/projects/diagnostic-ui/src/app/app.module.ts deleted file mode 100644 index 1abde53a3..000000000 --- a/web/projects/diagnostic-ui/src/app/app.module.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { NgModule } from '@angular/core' -import { BrowserAnimationsModule } from '@angular/platform-browser/animations' -import { RouteReuseStrategy } from '@angular/router' -import { IonicModule, IonicRouteStrategy } from '@ionic/angular' -import { TuiRootModule } from '@taiga-ui/core' -import { AppComponent } from './app.component' -import { AppRoutingModule } from './app-routing.module' -import { HttpClientModule } from '@angular/common/http' -import { ApiService } from './services/api/api.service' -import { MockApiService } from './services/api/mock-api.service' -import { LiveApiService } from './services/api/live-api.service' -import { RELATIVE_URL, WorkspaceConfig } from '@start9labs/shared' - -const { - useMocks, - ui: { api }, -} = require('../../../../config.json') as WorkspaceConfig - -@NgModule({ - declarations: [AppComponent], - imports: [ - HttpClientModule, - BrowserAnimationsModule, - IonicModule.forRoot({ - mode: 'md', - }), - AppRoutingModule, - TuiRootModule, - ], - providers: [ - { provide: RouteReuseStrategy, useClass: IonicRouteStrategy }, - { - provide: ApiService, - useClass: useMocks ? MockApiService : LiveApiService, - }, - { - provide: RELATIVE_URL, - useValue: `/${api.url}/${api.version}`, - }, - ], - bootstrap: [AppComponent], -}) -export class AppModule {} diff --git a/web/projects/diagnostic-ui/src/app/pages/home/home-routing.module.ts b/web/projects/diagnostic-ui/src/app/pages/home/home-routing.module.ts deleted file mode 100644 index efb1977dc..000000000 --- a/web/projects/diagnostic-ui/src/app/pages/home/home-routing.module.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { NgModule } from '@angular/core' -import { RouterModule, Routes } from '@angular/router' -import { HomePage } from './home.page' - -const routes: Routes = [ - { - path: '', - component: HomePage, - }, -] - -@NgModule({ - imports: [RouterModule.forChild(routes)], - exports: [RouterModule], -}) -export class HomePageRoutingModule {} diff --git a/web/projects/diagnostic-ui/src/app/services/api/api.service.ts b/web/projects/diagnostic-ui/src/app/services/api/api.service.ts deleted file mode 100644 index 562d486c3..000000000 --- a/web/projects/diagnostic-ui/src/app/services/api/api.service.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { LogsRes, ServerLogsReq } from '@start9labs/shared' - -export abstract class ApiService { - abstract getError(): Promise - abstract restart(): Promise - abstract forgetDrive(): Promise - abstract repairDisk(): Promise - abstract systemRebuild(): Promise - abstract getLogs(params: ServerLogsReq): Promise -} - -export interface GetErrorRes { - code: number - message: string - data: { details: string } -} diff --git a/web/projects/diagnostic-ui/src/app/services/api/live-api.service.ts b/web/projects/diagnostic-ui/src/app/services/api/live-api.service.ts deleted file mode 100644 index bbde6e5ba..000000000 --- a/web/projects/diagnostic-ui/src/app/services/api/live-api.service.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { Injectable } from '@angular/core' -import { - HttpService, - isRpcError, - RpcError, - RPCOptions, -} from '@start9labs/shared' -import { ApiService, GetErrorRes } from './api.service' -import { LogsRes, ServerLogsReq } from '@start9labs/shared' - -@Injectable() -export class LiveApiService implements ApiService { - constructor(private readonly http: HttpService) {} - - async getError(): Promise { - return this.rpcRequest({ - method: 'diagnostic.error', - params: {}, - }) - } - - async restart(): Promise { - return this.rpcRequest({ - method: 'diagnostic.restart', - params: {}, - }) - } - - async forgetDrive(): Promise { - return this.rpcRequest({ - method: 'diagnostic.disk.forget', - params: {}, - }) - } - - async repairDisk(): Promise { - return this.rpcRequest({ - method: 'diagnostic.disk.repair', - params: {}, - }) - } - - async systemRebuild(): Promise { - return this.rpcRequest({ - method: 'diagnostic.rebuild', - params: {}, - }) - } - - async getLogs(params: ServerLogsReq): Promise { - return this.rpcRequest({ - method: 'diagnostic.logs', - params, - }) - } - - private async rpcRequest(opts: RPCOptions): Promise { - const res = await this.http.rpcRequest(opts) - - const rpcRes = res.body - - if (isRpcError(rpcRes)) { - throw new RpcError(rpcRes.error) - } - - return rpcRes.result - } -} diff --git a/web/projects/diagnostic-ui/src/app/services/api/mock-api.service.ts b/web/projects/diagnostic-ui/src/app/services/api/mock-api.service.ts deleted file mode 100644 index d991edd32..000000000 --- a/web/projects/diagnostic-ui/src/app/services/api/mock-api.service.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { Injectable } from '@angular/core' -import { pauseFor } from '@start9labs/shared' -import { ApiService, GetErrorRes } from './api.service' -import { LogsRes, ServerLogsReq, Log } from '@start9labs/shared' - -@Injectable() -export class MockApiService implements ApiService { - async getError(): Promise { - await pauseFor(1000) - return { - code: 15, - message: 'Unknown server', - data: { details: 'Some details about the error here' }, - } - } - - async restart(): Promise { - await pauseFor(1000) - } - - async forgetDrive(): Promise { - await pauseFor(1000) - } - - async repairDisk(): Promise { - await pauseFor(1000) - } - - async systemRebuild(): Promise { - await pauseFor(1000) - } - - async getLogs(params: ServerLogsReq): Promise { - await pauseFor(1000) - let entries: Log[] - if (Math.random() < 0.2) { - entries = packageLogs - } else { - const arrLength = params.limit - ? Math.ceil(params.limit / packageLogs.length) - : 10 - entries = new Array(arrLength) - .fill(packageLogs) - .reduce((acc, val) => acc.concat(val), []) - } - return { - entries, - startCursor: 'start-cursor', - endCursor: 'end-cursor', - } - } -} - -const packageLogs = [ - { - timestamp: '2019-12-26T14:20:30.872Z', - message: '****** START *****', - }, - { - timestamp: '2019-12-26T14:21:30.872Z', - message: 'ServerLogs ServerLogs ServerLogs ServerLogs ServerLogs', - }, - { - timestamp: '2019-12-26T14:22:30.872Z', - message: '****** FINISH *****', - }, -] diff --git a/web/projects/diagnostic-ui/src/environments/environment.prod.ts b/web/projects/diagnostic-ui/src/environments/environment.prod.ts deleted file mode 100644 index 970e25bd7..000000000 --- a/web/projects/diagnostic-ui/src/environments/environment.prod.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const environment = { - production: true, -} diff --git a/web/projects/diagnostic-ui/src/environments/environment.ts b/web/projects/diagnostic-ui/src/environments/environment.ts deleted file mode 100644 index 5c68c17ab..000000000 --- a/web/projects/diagnostic-ui/src/environments/environment.ts +++ /dev/null @@ -1,16 +0,0 @@ -// This file can be replaced during build by using the `fileReplacements` array. -// `ng build --prod` replaces `environment.ts` with `environment.prod.ts`. -// The list of file replacements can be found in `angular.json`. - -export const environment = { - production: false, -} - -/* - * For easier debugging in development mode, you can import the following file - * to ignore zone related error stack frames such as `zone.run`, `zoneDelegate.invokeTask`. - * - * This import should be commented out in production mode because it will have a negative impact - * on performance if an error is thrown. - */ -// import 'zone.js/dist/zone-error'; // Included with Angular CLI. diff --git a/web/projects/diagnostic-ui/src/index.html b/web/projects/diagnostic-ui/src/index.html deleted file mode 100644 index 1822018f3..000000000 --- a/web/projects/diagnostic-ui/src/index.html +++ /dev/null @@ -1,23 +0,0 @@ - - - - - StartOS Diagnostic UI - - - - - - - - - - - - - - - diff --git a/web/projects/diagnostic-ui/src/main.ts b/web/projects/diagnostic-ui/src/main.ts deleted file mode 100644 index 21499c3cd..000000000 --- a/web/projects/diagnostic-ui/src/main.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { enableProdMode } from '@angular/core' -import { platformBrowserDynamic } from '@angular/platform-browser-dynamic' -import { AppModule } from './app/app.module' -import { environment } from './environments/environment' - -if (environment.production) { - enableProdMode() -} - -platformBrowserDynamic() - .bootstrapModule(AppModule) - .catch(err => console.error(err)) diff --git a/web/projects/diagnostic-ui/src/polyfills.ts b/web/projects/diagnostic-ui/src/polyfills.ts deleted file mode 100644 index 4437ced44..000000000 --- a/web/projects/diagnostic-ui/src/polyfills.ts +++ /dev/null @@ -1,64 +0,0 @@ -/** - * This file includes polyfills needed by Angular and is loaded before the app. - * You can add your own extra polyfills to this file. - * - * This file is divided into 2 sections: - * 1. Browser polyfills. These are applied before loading ZoneJS and are sorted by browsers. - * 2. Application imports. Files imported after ZoneJS that should be loaded before your main - * file. - * - * The current setup is for so-called "evergreen" browsers; the last versions of browsers that - * automatically update themselves. This includes Safari >= 10, Chrome >= 55 (including Opera), - * Edge >= 13 on the desktop, and iOS 10 and Chrome on mobile. - * - * Learn more in https://angular.io/guide/browser-support - */ - -/*************************************************************************************************** - * BROWSER POLYFILLS - */ - -/** IE11 requires the following for NgClass support on SVG elements */ -// import 'classlist.js'; // Run `npm install --save classlist.js`. - -/** - * Web Animations `@angular/platform-browser/animations` - * Only required if AnimationBuilder is used within the application and using IE/Edge or Safari. - * Standard animation support in Angular DOES NOT require any polyfills (as of Angular 6.0). - */ -// import 'web-animations-js'; // Run `npm install --save web-animations-js`. - -/** - * By default, zone.js will patch all possible macroTask and DomEvents - * user can disable parts of macroTask/DomEvents patch by setting following flags - * because those flags need to be set before `zone.js` being loaded, and webpack - * will put import in the top of bundle, so user need to create a separate file - * in this directory (for example: zone-flags.ts), and put the following flags - * into that file, and then add the following code before importing zone.js. - * import './zone-flags'; - * - * The flags allowed in zone-flags.ts are listed here. - * - * The following flags will work for all browsers. - * - * (window as any).__Zone_disable_requestAnimationFrame = true; // disable patch requestAnimationFrame - * (window as any).__Zone_disable_on_property = true; // disable patch onProperty such as onclick - * (window as any).__zone_symbol__UNPATCHED_EVENTS = ['scroll', 'mousemove']; // disable patch specified eventNames - * - * in IE/Edge developer tools, the addEventListener will also be wrapped by zone.js - * with the following flag, it will bypass `zone.js` patch for IE/Edge - * - * (window as any).__Zone_enable_cross_context_check = true; - * - */ - -import './zone-flags' - -/*************************************************************************************************** - * Zone JS is required by default for Angular itself. - */ -import 'zone.js/dist/zone' // Included with Angular CLI. - -/*************************************************************************************************** - * APPLICATION IMPORTS - */ diff --git a/web/projects/diagnostic-ui/src/styles.scss b/web/projects/diagnostic-ui/src/styles.scss deleted file mode 100644 index ac0aadb69..000000000 --- a/web/projects/diagnostic-ui/src/styles.scss +++ /dev/null @@ -1,41 +0,0 @@ -@font-face { - font-family: 'Montserrat'; - font-style: normal; - font-weight: normal; - src: url('/assets/fonts/Montserrat/Montserrat-Regular.ttf'); -} - -/** Ionic CSS Variables overrides **/ -:root { - --ion-font-family: 'Montserrat'; - - --ion-color-primary: #0075e1; - - --ion-color-medium: #989aa2; - --ion-color-medium-rgb: 152,154,162; - --ion-color-medium-contrast: #000000; - --ion-color-medium-contrast-rgb: 0,0,0; - --ion-color-medium-shade: #86888f; - --ion-color-medium-tint: #a2a4ab; - - --ion-color-light: #222428; - --ion-color-light-rgb: 34,36,40; - --ion-color-light-contrast: #ffffff; - --ion-color-light-contrast-rgb: 255,255,255; - --ion-color-light-shade: #1e2023; - --ion-color-light-tint: #383a3e; - - --ion-item-background: #2b2b2b; - --ion-toolbar-background: #2b2b2b; - --ion-card-background: #2b2b2b; - - --ion-background-color: #282828; - --ion-background-color-rgb: 30,30,30; - --ion-text-color: var(--ion-color-dark); - --ion-text-color-rgb: var(--ion-color-dark-rgb); -} - -.loader { - --spinner-color: var(--ion-color-warning) !important; - z-index: 40000 !important; -} diff --git a/web/projects/diagnostic-ui/src/zone-flags.ts b/web/projects/diagnostic-ui/src/zone-flags.ts deleted file mode 100644 index 24ca60fe2..000000000 --- a/web/projects/diagnostic-ui/src/zone-flags.ts +++ /dev/null @@ -1,6 +0,0 @@ -/** - * Prevents Angular change detection from - * running with certain Web Component callbacks - */ -// eslint-disable-next-line no-underscore-dangle -(window as any).__Zone_disable_customElements = true diff --git a/web/projects/diagnostic-ui/tsconfig.json b/web/projects/diagnostic-ui/tsconfig.json deleted file mode 100644 index f642f09b3..000000000 --- a/web/projects/diagnostic-ui/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -/* To learn more about this file see: https://angular.io/config/tsconfig. */ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "baseUrl": "./" - }, - "files": ["src/main.ts", "src/polyfills.ts"], - "include": ["src/**/*.d.ts"] -} diff --git a/web/projects/setup-wizard/src/app/app.component.ts b/web/projects/setup-wizard/src/app/app.component.ts index e4bf41f5c..e10d672e5 100644 --- a/web/projects/setup-wizard/src/app/app.component.ts +++ b/web/projects/setup-wizard/src/app/app.component.ts @@ -21,7 +21,7 @@ export class AppComponent { let route = '/home' if (inProgress) { - route = inProgress.complete ? '/success' : '/loading' + route = inProgress.status === 'complete' ? '/success' : '/loading' } await this.navCtrl.navigateForward(route) diff --git a/web/projects/setup-wizard/src/app/pages/embassy/embassy.page.ts b/web/projects/setup-wizard/src/app/pages/embassy/embassy.page.ts index 1a5dd042d..7f7ee6241 100644 --- a/web/projects/setup-wizard/src/app/pages/embassy/embassy.page.ts +++ b/web/projects/setup-wizard/src/app/pages/embassy/embassy.page.ts @@ -5,12 +5,7 @@ import { ModalController, NavController, } from '@ionic/angular' -import { - ApiService, - BackupRecoverySource, - DiskRecoverySource, - DiskMigrateSource, -} from 'src/app/services/api/api.service' +import { ApiService } from 'src/app/services/api/api.service' import { DiskInfo, ErrorToastService, GuidPipe } from '@start9labs/shared' import { StateService } from 'src/app/services/state.service' import { PasswordPage } from '../../modals/password/password.page' @@ -58,18 +53,17 @@ export class EmbassyPage { } else if (this.stateService.setupType === 'restore') { this.storageDrives = disks.filter( d => + this.stateService.recoverySource?.type === 'backup' && + this.stateService.recoverySource.target?.type === 'disk' && !d.partitions .map(p => p.logicalname) - .includes( - ( - (this.stateService.recoverySource as BackupRecoverySource) - ?.target as DiskRecoverySource - )?.logicalname, - ), + .includes(this.stateService.recoverySource.target.logicalname), ) - } else if (this.stateService.setupType === 'transfer') { - const guid = (this.stateService.recoverySource as DiskMigrateSource) - .guid + } else if ( + this.stateService.setupType === 'transfer' && + this.stateService.recoverySource?.type === 'migrate' + ) { + const guid = this.stateService.recoverySource.guid this.storageDrives = disks.filter(d => { return ( d.guid !== guid && !d.partitions.map(p => p.guid).includes(guid) diff --git a/web/projects/setup-wizard/src/app/pages/loading/loading.module.ts b/web/projects/setup-wizard/src/app/pages/loading/loading.module.ts index e937a7e19..2f3507941 100644 --- a/web/projects/setup-wizard/src/app/pages/loading/loading.module.ts +++ b/web/projects/setup-wizard/src/app/pages/loading/loading.module.ts @@ -2,11 +2,11 @@ import { NgModule } from '@angular/core' import { CommonModule } from '@angular/common' import { IonicModule } from '@ionic/angular' import { FormsModule } from '@angular/forms' -import { LoadingPage, ToMessagePipe } from './loading.page' +import { LoadingPage } from './loading.page' import { LoadingPageRoutingModule } from './loading-routing.module' @NgModule({ imports: [CommonModule, FormsModule, IonicModule, LoadingPageRoutingModule], - declarations: [LoadingPage, ToMessagePipe], + declarations: [LoadingPage], }) export class LoadingPageModule {} diff --git a/web/projects/setup-wizard/src/app/pages/loading/loading.page.html b/web/projects/setup-wizard/src/app/pages/loading/loading.page.html index fd7fcc24c..6c9ca41ab 100644 --- a/web/projects/setup-wizard/src/app/pages/loading/loading.page.html +++ b/web/projects/setup-wizard/src/app/pages/loading/loading.page.html @@ -1,39 +1,17 @@ - - - - - - - Initializing StartOS -
- - {{ progress.transferred | toMessage }} - -
- +
+

+ Setting up your server +

+
+ Progress: {{ (progress.total * 100).toFixed(0) }}% +
- - -

- - - Progress: {{ (transferred * 100).toFixed() }}% - - - {{ (progress.totalBytes / 1073741824).toFixed(2) }} GB - - -

- - - - - - + +

{{ progress.message }}

+
diff --git a/web/projects/setup-wizard/src/app/pages/loading/loading.page.scss b/web/projects/setup-wizard/src/app/pages/loading/loading.page.scss index 87bfffa33..e69de29bb 100644 --- a/web/projects/setup-wizard/src/app/pages/loading/loading.page.scss +++ b/web/projects/setup-wizard/src/app/pages/loading/loading.page.scss @@ -1,3 +0,0 @@ -ion-card-title { - font-size: 42px; -} \ No newline at end of file diff --git a/web/projects/setup-wizard/src/app/pages/loading/loading.page.ts b/web/projects/setup-wizard/src/app/pages/loading/loading.page.ts index ce1a1b3c0..459be5c7a 100644 --- a/web/projects/setup-wizard/src/app/pages/loading/loading.page.ts +++ b/web/projects/setup-wizard/src/app/pages/loading/loading.page.ts @@ -1,15 +1,23 @@ import { Component } from '@angular/core' import { NavController } from '@ionic/angular' -import { StateService } from 'src/app/services/state.service' import { Pipe, PipeTransform } from '@angular/core' -import { BehaviorSubject } from 'rxjs' +import { + EMPTY, + Observable, + catchError, + filter, + from, + interval, + map, + of, + startWith, + switchMap, + take, + tap, +} from 'rxjs' import { ApiService } from 'src/app/services/api/api.service' -import { ErrorToastService, pauseFor } from '@start9labs/shared' - -type Progress = { - totalBytes: number | null - transferred: number -} +import { ErrorToastService } from '@start9labs/shared' +import { T } from '@start9labs/start-sdk' @Component({ selector: 'app-loading', @@ -17,10 +25,46 @@ type Progress = { styleUrls: ['loading.page.scss'], }) export class LoadingPage { - readonly progress$ = new BehaviorSubject({ - totalBytes: null, - transferred: 0, - }) + readonly progress$ = this.getRunningStatus$().pipe( + switchMap(res => + this.api.openProgressWebsocket$(res.guid).pipe( + startWith(res.progress), + catchError((_, watch$) => { + return interval(2000).pipe( + switchMap(() => + from(this.api.getStatus()).pipe(catchError(() => EMPTY)), + ), + take(1), + switchMap(() => watch$), + ) + }), + tap(progress => { + if (progress.overall === true) { + this.getStatus() + } + }), + ), + ), + map(({ phases, overall }) => { + return { + total: getDecimal(overall), + message: phases + .filter( + ( + p, + ): p is { + name: string + progress: { + done: number + total: number | null + } + } => p.progress !== true && p.progress !== null, + ) + .map(p => `${p.name}${getPhaseBytes(p.progress)}`) + .join(','), + } + }), + ) constructor( private readonly navCtrl: NavController, @@ -28,55 +72,55 @@ export class LoadingPage { private readonly errorToastService: ErrorToastService, ) {} - ngOnInit() { - this.poll() - } + private async getStatus(): Promise<{ + status: 'running' + guid: string + progress: T.FullProgress + } | void> { + const res = await this.api.getStatus() - async poll() { - try { - const progress = await this.api.getStatus() - - if (!progress) return - - const { totalBytes, bytesTransferred } = progress - - this.progress$.next({ - totalBytes, - transferred: totalBytes ? bytesTransferred / totalBytes : 0, - }) - - if (progress.complete) { - this.navCtrl.navigateForward(`/success`) - this.progress$.complete() - return - } - - await pauseFor(250) - - setTimeout(() => this.poll(), 0) // prevent call stack from growing - } catch (e: any) { - this.errorToastService.present(e) - } - } -} - -@Pipe({ - name: 'toMessage', -}) -export class ToMessagePipe implements PipeTransform { - constructor(private readonly stateService: StateService) {} - - transform(progress: number | null): string { - if (['fresh', 'attach'].includes(this.stateService.setupType || '')) { - return 'Setting up your server' - } - - if (!progress) { - return 'Calculating size' - } else if (progress < 1) { - return 'Copying data' + if (!res) { + this.navCtrl.navigateRoot('/home') + } else if (res.status === 'complete') { + this.navCtrl.navigateForward(`/success`) } else { - return 'Finalizing' + return res } } + + private getRunningStatus$(): Observable<{ + status: 'running' + guid: string + progress: T.FullProgress + }> { + return from(this.getStatus()).pipe( + filter(Boolean), + catchError(e => { + this.errorToastService.present(e) + return of(e) + }), + take(1), + ) + } +} + +function getDecimal(progress: T.Progress): number { + if (progress === true) { + return 1 + } else if (!progress || !progress.total) { + return 0 + } else { + return progress.total && progress.done / progress.total + } +} + +function getPhaseBytes( + progress: + | false + | { + done: number + total: number | null + }, +): string { + return progress === false ? '' : `: (${progress.done}/${progress.total})` } diff --git a/web/projects/setup-wizard/src/app/services/api/api.service.ts b/web/projects/setup-wizard/src/app/services/api/api.service.ts index 375e64a78..6719ce859 100644 --- a/web/projects/setup-wizard/src/app/services/api/api.service.ts +++ b/web/projects/setup-wizard/src/app/services/api/api.service.ts @@ -1,16 +1,21 @@ import * as jose from 'node-jose' import { DiskListResponse, StartOSDiskInfo } from '@start9labs/shared' +import { T } from '@start9labs/start-sdk' +import { WebSocketSubjectConfig } from 'rxjs/webSocket' +import { Observable } from 'rxjs' + export abstract class ApiService { pubkey?: jose.JWK.Key - abstract getStatus(): Promise // setup.status + abstract getStatus(): Promise // setup.status abstract getPubKey(): Promise // setup.get-pubkey abstract getDrives(): Promise // setup.disk.list - abstract verifyCifs(cifs: CifsRecoverySource): Promise // setup.cifs.verify - abstract attach(importInfo: AttachReq): Promise // setup.attach - abstract execute(setupInfo: ExecuteReq): Promise // setup.execute - abstract complete(): Promise // setup.complete + abstract verifyCifs(cifs: T.VerifyCifsParams): Promise // setup.cifs.verify + abstract attach(importInfo: T.AttachParams): Promise // setup.attach + abstract execute(setupInfo: T.SetupExecuteParams): Promise // setup.execute + abstract complete(): Promise // setup.complete abstract exit(): Promise // setup.exit + abstract openProgressWebsocket$(guid: string): Observable async encrypt(toEncrypt: string): Promise { if (!this.pubkey) throw new Error('No pubkey found!') @@ -27,29 +32,7 @@ type Encrypted = { encrypted: string } -export type StatusRes = { - bytesTransferred: number - totalBytes: number | null - complete: boolean -} | null - -export type AttachReq = { - guid: string - startOsPassword: Encrypted -} - -export type ExecuteReq = { - startOsLogicalname: string - startOsPassword: Encrypted - recoverySource: RecoverySource | null - recoveryPassword: Encrypted | null -} - -export type CompleteRes = { - torAddress: string - lanAddress: string - rootCa: string -} +export type WebsocketConfig = Omit, 'url'> export type DiskBackupTarget = { vendor: string | null @@ -68,27 +51,3 @@ export type CifsBackupTarget = { mountable: boolean startOs: StartOSDiskInfo | null } - -export type DiskRecoverySource = { - type: 'disk' - logicalname: string // partition logicalname -} - -export type BackupRecoverySource = { - type: 'backup' - target: CifsRecoverySource | DiskRecoverySource -} -export type RecoverySource = BackupRecoverySource | DiskMigrateSource - -export type DiskMigrateSource = { - type: 'migrate' - guid: string -} - -export type CifsRecoverySource = { - type: 'cifs' - hostname: string - path: string - username: string - password: Encrypted | null -} diff --git a/web/projects/setup-wizard/src/app/services/api/live-api.service.ts b/web/projects/setup-wizard/src/app/services/api/live-api.service.ts index 566cc84cf..f431f5151 100644 --- a/web/projects/setup-wizard/src/app/services/api/live-api.service.ts +++ b/web/projects/setup-wizard/src/app/services/api/live-api.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@angular/core' +import { Inject, Injectable } from '@angular/core' import { DiskListResponse, StartOSDiskInfo, @@ -8,27 +8,35 @@ import { RpcError, RPCOptions, } from '@start9labs/shared' -import { - ApiService, - CifsRecoverySource, - DiskRecoverySource, - StatusRes, - AttachReq, - ExecuteReq, - CompleteRes, -} from './api.service' +import { T } from '@start9labs/start-sdk' +import { ApiService, WebsocketConfig } from './api.service' import * as jose from 'node-jose' +import { Observable } from 'rxjs' +import { DOCUMENT } from '@angular/common' +import { webSocket } from 'rxjs/webSocket' @Injectable({ providedIn: 'root', }) export class LiveApiService extends ApiService { - constructor(private readonly http: HttpService) { + constructor( + private readonly http: HttpService, + @Inject(DOCUMENT) private readonly document: Document, + ) { super() } - async getStatus() { - return this.rpcRequest({ + openProgressWebsocket$(guid: string): Observable { + const { location } = this.document.defaultView! + const host = location.host + + return webSocket({ + url: `ws://${host}/ws/rpc/${guid}`, + }) + } + + async getStatus(): Promise { + return this.rpcRequest({ method: 'setup.status', params: {}, }) @@ -41,7 +49,7 @@ export class LiveApiService extends ApiService { * this wil all public/private key, which means that there is no information loss * through the network. */ - async getPubKey() { + async getPubKey(): Promise { const response: jose.JWK.Key = await this.rpcRequest({ method: 'setup.get-pubkey', params: {}, @@ -50,14 +58,14 @@ export class LiveApiService extends ApiService { this.pubkey = response } - async getDrives() { + async getDrives(): Promise { return this.rpcRequest({ method: 'setup.disk.list', params: {}, }) } - async verifyCifs(source: CifsRecoverySource) { + async verifyCifs(source: T.VerifyCifsParams): Promise { source.path = source.path.replace('/\\/g', '/') return this.rpcRequest({ method: 'setup.cifs.verify', @@ -65,14 +73,14 @@ export class LiveApiService extends ApiService { }) } - async attach(params: AttachReq) { - await this.rpcRequest({ + async attach(params: T.AttachParams): Promise { + return this.rpcRequest({ method: 'setup.attach', params, }) } - async execute(setupInfo: ExecuteReq) { + async execute(setupInfo: T.SetupExecuteParams): Promise { if (setupInfo.recoverySource?.type === 'backup') { if (isCifsSource(setupInfo.recoverySource.target)) { setupInfo.recoverySource.target.path = @@ -80,14 +88,14 @@ export class LiveApiService extends ApiService { } } - await this.rpcRequest({ + return this.rpcRequest({ method: 'setup.execute', params: setupInfo, }) } - async complete() { - const res = await this.rpcRequest({ + async complete(): Promise { + const res = await this.rpcRequest({ method: 'setup.complete', params: {}, }) @@ -98,7 +106,7 @@ export class LiveApiService extends ApiService { } } - async exit() { + async exit(): Promise { await this.rpcRequest({ method: 'setup.exit', params: {}, @@ -119,7 +127,7 @@ export class LiveApiService extends ApiService { } function isCifsSource( - source: CifsRecoverySource | DiskRecoverySource | null, -): source is CifsRecoverySource { - return !!(source as CifsRecoverySource)?.hostname + source: T.BackupTargetFS | null, +): source is T.Cifs & { type: 'cifs' } { + return !!(source as T.Cifs)?.hostname } diff --git a/web/projects/setup-wizard/src/app/services/api/mock-api.service.ts b/web/projects/setup-wizard/src/app/services/api/mock-api.service.ts index df32bd09e..0a1c221f7 100644 --- a/web/projects/setup-wizard/src/app/services/api/mock-api.service.ts +++ b/web/projects/setup-wizard/src/app/services/api/mock-api.service.ts @@ -1,42 +1,151 @@ import { Injectable } from '@angular/core' -import { encodeBase64, pauseFor } from '@start9labs/shared' import { - ApiService, - AttachReq, - CifsRecoverySource, - CompleteRes, - ExecuteReq, -} from './api.service' + DiskListResponse, + StartOSDiskInfo, + encodeBase64, + pauseFor, +} from '@start9labs/shared' +import { ApiService } from './api.service' import * as jose from 'node-jose' - -let tries: number +import { T } from '@start9labs/start-sdk' +import { + Observable, + concatMap, + delay, + from, + interval, + map, + mergeScan, + of, + startWith, + switchMap, + switchScan, + takeWhile, +} from 'rxjs' @Injectable({ providedIn: 'root', }) export class MockApiService extends ApiService { - async getStatus() { - const restoreOrMigrate = true + // fullProgress$(): Observable { + // const phases = [ + // { + // name: 'Preparing Data', + // progress: null, + // }, + // { + // name: 'Transferring Data', + // progress: null, + // }, + // { + // name: 'Finalizing Setup', + // progress: null, + // }, + // ] + + // return from(phases).pipe( + // switchScan((acc, val, i) => {}, { overall: null, phases }), + // ) + // } + + // namedProgress$(namedProgress: T.NamedProgress): Observable { + // return of(namedProgress).pipe(startWith(namedProgress)) + // } + + // progress$(progress: T.Progress): Observable {} + + // websocket + + openProgressWebsocket$(guid: string): Observable { + return of(PROGRESS) + // const numPhases = PROGRESS.phases.length + + // return of(PROGRESS).pipe( + // switchMap(full => + // from(PROGRESS.phases).pipe( + // mergeScan((full, phase, i) => { + // if ( + // !phase.progress || + // typeof phase.progress !== 'object' || + // !phase.progress.total + // ) { + // full.phases[i].progress = true + + // if ( + // full.overall && + // typeof full.overall === 'object' && + // full.overall.total + // ) { + // const step = full.overall.total / numPhases + // full.overall.done += step + // } + + // return of(full).pipe(delay(2000)) + // } else { + // const total = phase.progress.total + // const step = total / 4 + // let done = phase.progress.done + + // return interval(1000).pipe( + // takeWhile(() => done < total), + // map(() => { + // done += step + + // console.error(done) + + // if ( + // full.overall && + // typeof full.overall === 'object' && + // full.overall.total + // ) { + // const step = full.overall.total / numPhases / 4 + + // full.overall.done += step + // } + + // if (done === total) { + // full.phases[i].progress = true + + // if (i === numPhases - 1) { + // full.overall = true + // } + // } + // return full + // }), + // ) + // } + // }, full), + // ), + // ), + // ) + } + + private statusIndex = 0 + async getStatus(): Promise { await pauseFor(1000) - if (tries === undefined) { - tries = 0 - return null - } + this.statusIndex++ - tries++ - - const total = tries <= 4 ? tries * 268435456 : 1073741824 - const progress = tries > 4 ? (tries - 4) * 268435456 : 0 - - return { - bytesTransferred: restoreOrMigrate ? progress : 0, - totalBytes: restoreOrMigrate ? total : null, - complete: progress === total, + switch (this.statusIndex) { + case 2: + return { + status: 'running', + progress: PROGRESS, + guid: 'progress-guid', + } + case 3: + return { + status: 'complete', + torAddress: 'https://asdafsadasdasasdasdfasdfasdf.onion', + lanAddress: 'https://adjective-noun.local', + rootCa: encodeBase64(rootCA), + } + default: + return null } } - async getPubKey() { + async getPubKey(): Promise { await pauseFor(1000) // randomly generated @@ -52,7 +161,7 @@ export class MockApiService extends ApiService { }) } - async getDrives() { + async getDrives(): Promise { await pauseFor(1000) return [ { @@ -127,7 +236,7 @@ export class MockApiService extends ApiService { ] } - async verifyCifs(params: CifsRecoverySource) { + async verifyCifs(params: T.VerifyCifsParams): Promise { await pauseFor(1000) return { version: '0.3.0', @@ -138,15 +247,25 @@ export class MockApiService extends ApiService { } } - async attach(params: AttachReq) { + async attach(params: T.AttachParams): Promise { await pauseFor(1000) + + return { + progress: PROGRESS, + guid: 'progress-guid', + } } - async execute(setupInfo: ExecuteReq) { + async execute(setupInfo: T.SetupExecuteParams): Promise { await pauseFor(1000) + + return { + progress: PROGRESS, + guid: 'progress-guid', + } } - async complete(): Promise { + async complete(): Promise { await pauseFor(1000) return { torAddress: 'https://asdafsadasdasasdasdfasdfasdf.onion', @@ -155,7 +274,7 @@ export class MockApiService extends ApiService { } } - async exit() { + async exit(): Promise { await pauseFor(1000) } } @@ -182,3 +301,8 @@ Rf3ZOPm9QP92YpWyYDkfAU04xdDo1vR0MYjKPkl4LjRqSU/tcCJnPMbJiwq+bWpX 2WJoEBXB/p15Kn6JxjI0ze2SnSI48JZ8it4fvxrhOo0VoLNIuCuNXJOwU17Rdl1W YJidaq7je6k18AdgPA0Kh8y1XtfUH3fTaVw4 -----END CERTIFICATE-----` + +const PROGRESS = { + overall: null, + phases: [], +} diff --git a/web/projects/setup-wizard/src/app/services/state.service.ts b/web/projects/setup-wizard/src/app/services/state.service.ts index 916b066ee..8c653a088 100644 --- a/web/projects/setup-wizard/src/app/services/state.service.ts +++ b/web/projects/setup-wizard/src/app/services/state.service.ts @@ -1,13 +1,13 @@ import { Injectable } from '@angular/core' -import { ApiService, RecoverySource } from './api/api.service' +import { ApiService } from './api/api.service' +import { T } from '@start9labs/start-sdk' @Injectable({ providedIn: 'root', }) export class StateService { setupType?: 'fresh' | 'restore' | 'attach' | 'transfer' - - recoverySource?: RecoverySource + recoverySource?: T.RecoverySource recoveryPassword?: string constructor(private readonly api: ApiService) {} diff --git a/web/projects/shared/src/types/api.ts b/web/projects/shared/src/types/api.ts index 26cd00218..743ea6ac8 100644 --- a/web/projects/shared/src/types/api.ts +++ b/web/projects/shared/src/types/api.ts @@ -13,6 +13,7 @@ export type LogsRes = { export interface Log { timestamp: string message: string + bootId: string } export type DiskListResponse = DiskInfo[] diff --git a/web/projects/ui/src/app/app-routing.module.ts b/web/projects/ui/src/app/app-routing.module.ts index ddb66d3c5..e7a2036ec 100644 --- a/web/projects/ui/src/app/app-routing.module.ts +++ b/web/projects/ui/src/app/app-routing.module.ts @@ -1,5 +1,6 @@ import { NgModule } from '@angular/core' import { PreloadAllModules, RouterModule, Routes } from '@angular/router' +import { stateNot } from 'src/app/services/state.service' import { AuthGuard } from './guards/auth.guard' import { UnauthGuard } from './guards/unauth.guard' @@ -15,15 +16,29 @@ const routes: Routes = [ loadChildren: () => import('./pages/login/login.module').then(m => m.LoginPageModule), }, + { + path: 'diagnostic', + canActivate: [stateNot(['initializing', 'running'])], + loadChildren: () => + import('./pages/diagnostic-routes/diagnostic-routing.module').then( + m => m.DiagnosticModule, + ), + }, + { + path: 'initializing', + canActivate: [stateNot(['error', 'running'])], + loadChildren: () => + import('./pages/init/init.module').then(m => m.InitPageModule), + }, { path: 'home', - canActivate: [AuthGuard], + canActivate: [AuthGuard, stateNot(['error', 'initializing'])], loadChildren: () => import('./pages/home/home.module').then(m => m.HomePageModule), }, { path: 'system', - canActivate: [AuthGuard], + canActivate: [AuthGuard, stateNot(['error', 'initializing'])], canActivateChild: [AuthGuard], loadChildren: () => import('./pages/server-routes/server-routing.module').then( @@ -32,14 +47,14 @@ const routes: Routes = [ }, { path: 'updates', - canActivate: [AuthGuard], + canActivate: [AuthGuard, stateNot(['error', 'initializing'])], canActivateChild: [AuthGuard], loadChildren: () => import('./pages/updates/updates.module').then(m => m.UpdatesPageModule), }, { path: 'marketplace', - canActivate: [AuthGuard], + canActivate: [AuthGuard, stateNot(['error', 'initializing'])], canActivateChild: [AuthGuard], loadChildren: () => import('./pages/marketplace-routes/marketplace-routing.module').then( @@ -48,7 +63,7 @@ const routes: Routes = [ }, { path: 'notifications', - canActivate: [AuthGuard], + canActivate: [AuthGuard, stateNot(['error', 'initializing'])], loadChildren: () => import('./pages/notifications/notifications.module').then( m => m.NotificationsPageModule, @@ -56,7 +71,7 @@ const routes: Routes = [ }, { path: 'services', - canActivate: [AuthGuard], + canActivate: [AuthGuard, stateNot(['error', 'initializing'])], canActivateChild: [AuthGuard], loadChildren: () => import('./pages/apps-routes/apps-routing.module').then( diff --git a/web/projects/ui/src/app/app.component.html b/web/projects/ui/src/app/app.component.html index 0506d5214..0d9fb860f 100644 --- a/web/projects/ui/src/app/app.component.html +++ b/web/projects/ui/src/app/app.component.html @@ -15,6 +15,7 @@ type="overlay" side="start" class="left-menu" + [class.left-menu_hidden]="withoutMenu" > diff --git a/web/projects/ui/src/app/app.component.scss b/web/projects/ui/src/app/app.component.scss index 55135b1e5..aedbdc6c4 100644 --- a/web/projects/ui/src/app/app.component.scss +++ b/web/projects/ui/src/app/app.component.scss @@ -9,11 +9,15 @@ tui-root { .left-menu { --side-max-width: 280px; + + &_hidden { + display: none; + } } .menu { :host-context(body[data-theme='Light']) & { - --ion-color-base: #F4F4F5 !important; + --ion-color-base: #f4f4f5 !important; } } diff --git a/web/projects/ui/src/app/app.component.ts b/web/projects/ui/src/app/app.component.ts index 1210eba7a..ddf8c074f 100644 --- a/web/projects/ui/src/app/app.component.ts +++ b/web/projects/ui/src/app/app.component.ts @@ -1,4 +1,5 @@ import { Component, inject, OnDestroy } from '@angular/core' +import { IsActiveMatchOptions, Router } from '@angular/router' import { combineLatest, map, merge, startWith } from 'rxjs' import { AuthService } from './services/auth.service' import { SplitPaneTracker } from './services/split-pane.service' @@ -15,6 +16,13 @@ import { THEME } from '@start9labs/shared' import { PatchDB } from 'patch-db-client' import { DataModel } from './services/patch-db/data-model' +const OPTIONS: IsActiveMatchOptions = { + paths: 'subset', + queryParams: 'exact', + fragment: 'ignored', + matrixParams: 'ignored', +} + @Component({ selector: 'app-root', templateUrl: 'app.component.html', @@ -27,7 +35,7 @@ export class AppComponent implements OnDestroy { readonly theme$ = inject(THEME) readonly offline$ = combineLatest([ this.authService.isVerified$, - this.connection.connected$, + this.connection$, this.patch .watch$('serverInfo', 'statusInfo') .pipe(startWith({ restarting: false, shuttingDown: false })), @@ -44,8 +52,9 @@ export class AppComponent implements OnDestroy { private readonly patchMonitor: PatchMonitorService, private readonly splitPane: SplitPaneTracker, private readonly patch: PatchDB, + private readonly router: Router, readonly authService: AuthService, - readonly connection: ConnectionService, + readonly connection$: ConnectionService, readonly clientStorageService: ClientStorageService, readonly themeSwitcher: ThemeSwitcherService, ) {} @@ -56,6 +65,13 @@ export class AppComponent implements OnDestroy { .subscribe(name => this.titleService.setTitle(name || 'StartOS')) } + get withoutMenu(): boolean { + return ( + this.router.isActive('initializing', OPTIONS) || + this.router.isActive('diagnostic', OPTIONS) + ) + } + splitPaneVisible({ detail }: any) { this.splitPane.sidebarOpen$.next(detail.visible) } diff --git a/web/projects/ui/src/app/app.module.ts b/web/projects/ui/src/app/app.module.ts index 324300851..c0264f064 100644 --- a/web/projects/ui/src/app/app.module.ts +++ b/web/projects/ui/src/app/app.module.ts @@ -1,4 +1,5 @@ import { + TuiAlertModule, TuiDialogModule, TuiModeModule, TuiRootModule, @@ -58,6 +59,7 @@ import { environment } from '../environments/environment' ConnectionBarComponentModule, TuiRootModule, TuiDialogModule, + TuiAlertModule, TuiModeModule, TuiThemeNightModule, WidgetsPageModule, diff --git a/web/projects/ui/src/app/app.providers.ts b/web/projects/ui/src/app/app.providers.ts index 5d0ccc4f2..bf26a8cb9 100644 --- a/web/projects/ui/src/app/app.providers.ts +++ b/web/projects/ui/src/app/app.providers.ts @@ -10,6 +10,7 @@ import { AuthService } from './services/auth.service' import { ClientStorageService } from './services/client-storage.service' import { FilterPackagesPipe } from '../../../marketplace/src/pipes/filter-packages.pipe' import { ThemeSwitcherService } from './services/theme-switcher.service' +import { StorageService } from './services/storage.service' const { useMocks, @@ -30,7 +31,7 @@ export const APP_PROVIDERS: Provider[] = [ }, { provide: APP_INITIALIZER, - deps: [AuthService, ClientStorageService, Router], + deps: [StorageService, AuthService, ClientStorageService, Router], useFactory: appInitializer, multi: true, }, @@ -45,13 +46,15 @@ export const APP_PROVIDERS: Provider[] = [ ] export function appInitializer( + storage: StorageService, auth: AuthService, localStorage: ClientStorageService, router: Router, ): () => void { return () => { + storage.migrate036() auth.init() - localStorage.init() + localStorage.init() // @TODO pretty sure we can navigate before this step router.initialNavigation() } } diff --git a/web/projects/ui/src/app/app/menu/menu.component.ts b/web/projects/ui/src/app/app/menu/menu.component.ts index 5c1fbe8bd..b2ab62368 100644 --- a/web/projects/ui/src/app/app/menu/menu.component.ts +++ b/web/projects/ui/src/app/app/menu/menu.component.ts @@ -70,7 +70,7 @@ export class MenuComponent { readonly showEOSUpdate$ = this.eosService.showUpdate$ - private readonly local$ = this.connectionService.connected$.pipe( + private readonly local$ = this.connection$.pipe( filter(Boolean), switchMap(() => this.patch.watch$('packageData').pipe(first())), switchMap(outer => @@ -126,6 +126,6 @@ export class MenuComponent { private readonly marketplaceService: MarketplaceService, private readonly splitPane: SplitPaneTracker, private readonly emver: Emver, - private readonly connectionService: ConnectionService, + private readonly connection$: ConnectionService, ) {} } diff --git a/web/projects/ui/src/app/components/connection-bar/connection-bar.component.ts b/web/projects/ui/src/app/components/connection-bar/connection-bar.component.ts index da28f805f..cf0eab598 100644 --- a/web/projects/ui/src/app/components/connection-bar/connection-bar.component.ts +++ b/web/projects/ui/src/app/components/connection-bar/connection-bar.component.ts @@ -1,8 +1,9 @@ import { ChangeDetectionStrategy, Component } from '@angular/core' import { PatchDB } from 'patch-db-client' import { combineLatest, map, Observable, startWith } from 'rxjs' -import { ConnectionService } from 'src/app/services/connection.service' +import { NetworkService } from 'src/app/services/network.service' import { DataModel } from 'src/app/services/patch-db/data-model' +import { StateService } from 'src/app/services/state.service' @Component({ selector: 'connection-bar', @@ -11,16 +12,14 @@ import { DataModel } from 'src/app/services/patch-db/data-model' changeDetection: ChangeDetectionStrategy.OnPush, }) export class ConnectionBarComponent { - private readonly websocket$ = this.connectionService.websocketConnected$ - readonly connection$: Observable<{ message: string color: string icon: string dots: boolean }> = combineLatest([ - this.connectionService.networkConnected$, - this.websocket$.pipe(startWith(false)), + this.network$, + this.state$.pipe(map(Boolean)), this.patch .watch$('serverInfo', 'statusInfo') .pipe(startWith({ restarting: false, shuttingDown: false })), @@ -65,7 +64,8 @@ export class ConnectionBarComponent { ) constructor( - private readonly connectionService: ConnectionService, + private readonly network$: NetworkService, + private readonly state$: StateService, private readonly patch: PatchDB, ) {} } diff --git a/web/projects/ui/src/app/components/logs/logs.component.ts b/web/projects/ui/src/app/components/logs/logs.component.ts index 5d033fc83..3d31313cb 100644 --- a/web/projects/ui/src/app/components/logs/logs.component.ts +++ b/web/projects/ui/src/app/components/logs/logs.component.ts @@ -11,7 +11,6 @@ import { takeUntil, tap, } from 'rxjs' -import { WebSocketSubjectConfig } from 'rxjs/webSocket' import { LogsRes, ServerLogsReq, @@ -72,7 +71,7 @@ export class LogsComponent { private readonly api: ApiService, private readonly loadingCtrl: LoadingController, private readonly downloadHtml: DownloadHTMLService, - private readonly connectionService: ConnectionService, + private readonly connection$: ConnectionService, ) {} async ngOnInit() { @@ -149,43 +148,42 @@ export class LogsComponent { private reconnect$(): Observable { return from(this.followLogs({})).pipe( tap(_ => this.recordConnectionChange()), - switchMap(({ guid }) => this.connect$(guid, true)), + switchMap(({ guid }) => this.connect$(guid)), ) } - private connect$(guid: string, reconnect = false) { - const config: WebSocketSubjectConfig = { - url: `/rpc/${guid}`, - openObserver: { - next: () => { - this.websocketStatus = 'connected' + private connect$(guid: string) { + return this.api + .openWebsocket$(guid, { + openObserver: { + next: () => { + this.websocketStatus = 'connected' + }, }, - }, - } - - return this.api.openLogsWebsocket$(config).pipe( - tap(_ => this.count++), - bufferTime(1000), - tap(msgs => { - this.loading = false - this.processRes({ entries: msgs }) - if (this.infiniteStatus === 0 && this.count >= this.limit) - this.infiniteStatus = 1 - }), - catchError(() => { - this.recordConnectionChange(false) - return this.connectionService.connected$.pipe( - tap( - connected => - (this.websocketStatus = connected - ? 'reconnecting' - : 'disconnected'), - ), - filter(Boolean), - switchMap(() => this.reconnect$()), - ) - }), - ) + }) + .pipe( + tap(_ => this.count++), + bufferTime(1000), + tap(msgs => { + this.loading = false + this.processRes({ entries: msgs }) + if (this.infiniteStatus === 0 && this.count >= this.limit) + this.infiniteStatus = 1 + }), + catchError(() => { + this.recordConnectionChange(false) + return this.connection$.pipe( + tap( + connected => + (this.websocketStatus = connected + ? 'reconnecting' + : 'disconnected'), + ), + filter(Boolean), + switchMap(() => this.reconnect$()), + ) + }), + ) } private recordConnectionChange(success = true) { diff --git a/web/projects/ui/src/app/components/status/status.component.html b/web/projects/ui/src/app/components/status/status.component.html index db9e8f8f7..65c142f4a 100644 --- a/web/projects/ui/src/app/components/status/status.component.html +++ b/web/projects/ui/src/app/components/status/status.component.html @@ -1,12 +1,12 @@

- {{ (connected$ | async) ? rendering.display : 'Unknown' }} + {{ (connection$ | async) ? rendering.display : 'Unknown' }} . This may take a while diff --git a/web/projects/ui/src/app/components/status/status.component.ts b/web/projects/ui/src/app/components/status/status.component.ts index c9fec4968..45f66f291 100644 --- a/web/projects/ui/src/app/components/status/status.component.ts +++ b/web/projects/ui/src/app/components/status/status.component.ts @@ -21,7 +21,5 @@ export class StatusComponent { @Input() installingInfo?: InstallingInfo @Input() sigtermTimeout?: string | null = null - readonly connected$ = this.connectionService.connected$ - - constructor(private readonly connectionService: ConnectionService) {} + constructor(readonly connection$: ConnectionService) {} } diff --git a/web/projects/ui/src/app/modals/os-update/os-update.page.ts b/web/projects/ui/src/app/modals/os-update/os-update.page.ts index 9c49b72bf..9900bbb47 100644 --- a/web/projects/ui/src/app/modals/os-update/os-update.page.ts +++ b/web/projects/ui/src/app/modals/os-update/os-update.page.ts @@ -20,9 +20,8 @@ export class OSUpdatePage { private readonly embassyApi: ApiService, private readonly eosService: EOSService, ) {} - ngOnInit() { - const releaseNotes = this.eosService.eos?.releaseNotes! + const releaseNotes = this.eosService.osUpdate?.releaseNotes! this.versions = Object.keys(releaseNotes) .sort() diff --git a/web/projects/ui/src/app/pages/apps-routes/app-list/app-list-icon/app-list-icon.component.html b/web/projects/ui/src/app/pages/apps-routes/app-list/app-list-icon/app-list-icon.component.html index 8f1af1470..b0bfbbb24 100644 --- a/web/projects/ui/src/app/pages/apps-routes/app-list/app-list-icon/app-list-icon.component.html +++ b/web/projects/ui/src/app/pages/apps-routes/app-list/app-list-icon/app-list-icon.component.html @@ -1,4 +1,4 @@ - + Health Checks - + diff --git a/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-health-checks/app-show-health-checks.component.ts b/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-health-checks/app-show-health-checks.component.ts index f3db08063..fef84a5ba 100644 --- a/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-health-checks/app-show-health-checks.component.ts +++ b/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-health-checks/app-show-health-checks.component.ts @@ -12,9 +12,7 @@ export class AppShowHealthChecksComponent { @Input() healthChecks!: Record - readonly connected$ = this.connectionService.connected$ - - constructor(private readonly connectionService: ConnectionService) {} + constructor(readonly connection$: ConnectionService) {} isLoading(result: T.HealthCheckResult['result']): boolean { return result === 'starting' || result === 'loading' diff --git a/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.html b/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.html index f9db34c73..be7bc1c30 100644 --- a/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.html +++ b/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.html @@ -11,7 +11,7 @@ - + diff --git a/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.ts b/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.ts index 4bc1bc464..ba60c331d 100644 --- a/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.ts +++ b/web/projects/ui/src/app/pages/apps-routes/app-show/components/app-show-status/app-show-status.component.ts @@ -39,8 +39,6 @@ export class AppShowStatusComponent { isInstalled = isInstalled - readonly connected$ = this.connectionService.connected$ - constructor( private readonly alertCtrl: AlertController, private readonly errToast: ErrorToastService, @@ -48,7 +46,7 @@ export class AppShowStatusComponent { private readonly embassyApi: ApiService, private readonly launcherService: UiLauncherService, private readonly modalService: ModalService, - private readonly connectionService: ConnectionService, + readonly connection$: ConnectionService, private readonly patch: PatchDB, ) {} diff --git a/web/projects/ui/src/app/pages/diagnostic-routes/diagnostic-routing.module.ts b/web/projects/ui/src/app/pages/diagnostic-routes/diagnostic-routing.module.ts new file mode 100644 index 000000000..4409288c1 --- /dev/null +++ b/web/projects/ui/src/app/pages/diagnostic-routes/diagnostic-routing.module.ts @@ -0,0 +1,21 @@ +import { NgModule } from '@angular/core' +import { RouterModule, Routes } from '@angular/router' + +const ROUTES: Routes = [ + { + path: '', + loadChildren: () => + import('./home/home.module').then(m => m.HomePageModule), + }, + { + path: 'logs', + loadChildren: () => + import('./logs/logs.module').then(m => m.LogsPageModule), + }, +] + +@NgModule({ + imports: [RouterModule.forChild(ROUTES)], + exports: [RouterModule], +}) +export class DiagnosticModule {} diff --git a/web/projects/diagnostic-ui/src/app/pages/home/home.module.ts b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.module.ts similarity index 55% rename from web/projects/diagnostic-ui/src/app/pages/home/home.module.ts rename to web/projects/ui/src/app/pages/diagnostic-routes/home/home.module.ts index 1664b7c72..9565220ae 100644 --- a/web/projects/diagnostic-ui/src/app/pages/home/home.module.ts +++ b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.module.ts @@ -1,12 +1,18 @@ import { NgModule } from '@angular/core' import { CommonModule } from '@angular/common' +import { Routes, RouterModule } from '@angular/router' import { IonicModule } from '@ionic/angular' -import { FormsModule } from '@angular/forms' import { HomePage } from './home.page' -import { HomePageRoutingModule } from './home-routing.module' + +const routes: Routes = [ + { + path: '', + component: HomePage, + }, +] @NgModule({ - imports: [CommonModule, FormsModule, IonicModule, HomePageRoutingModule], + imports: [CommonModule, IonicModule, RouterModule.forChild(routes)], declarations: [HomePage], }) export class HomePageModule {} diff --git a/web/projects/diagnostic-ui/src/app/pages/home/home.page.html b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.html similarity index 92% rename from web/projects/diagnostic-ui/src/app/pages/home/home.page.html rename to web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.html index 9cba08258..69a58a3aa 100644 --- a/web/projects/diagnostic-ui/src/app/pages/home/home.page.html +++ b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.html @@ -51,12 +51,6 @@ }} -

- - System Rebuild - -
-
Repair Drive diff --git a/web/projects/diagnostic-ui/src/app/pages/home/home.page.scss b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.scss similarity index 100% rename from web/projects/diagnostic-ui/src/app/pages/home/home.page.scss rename to web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.scss diff --git a/web/projects/diagnostic-ui/src/app/pages/home/home.page.ts b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.ts similarity index 74% rename from web/projects/diagnostic-ui/src/app/pages/home/home.page.ts rename to web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.ts index bbda6939f..9bb7376bc 100644 --- a/web/projects/diagnostic-ui/src/app/pages/home/home.page.ts +++ b/web/projects/ui/src/app/pages/diagnostic-routes/home/home.page.ts @@ -1,9 +1,9 @@ import { Component } from '@angular/core' import { AlertController, LoadingController } from '@ionic/angular' -import { ApiService } from 'src/app/services/api/api.service' +import { ApiService } from 'src/app/services/api/embassy-api.service' @Component({ - selector: 'app-home', + selector: 'diagnostic-home', templateUrl: 'home.page.html', styleUrls: ['home.page.scss'], }) @@ -25,7 +25,7 @@ export class HomePage { async ngOnInit() { try { - const error = await this.api.getError() + const error = await this.api.diagnosticGetError() // incorrect drive if (error.code === 15) { this.error = { @@ -92,7 +92,7 @@ export class HomePage { await loader.present() try { - await this.api.restart() + await this.api.diagnosticRestart() this.restarted = true } catch (e) { console.error(e) @@ -108,8 +108,8 @@ export class HomePage { await loader.present() try { - await this.api.forgetDrive() - await this.api.restart() + await this.api.diagnosticForgetDrive() + await this.api.diagnosticRestart() this.restarted = true } catch (e) { console.error(e) @@ -118,32 +118,6 @@ export class HomePage { } } - async presentAlertSystemRebuild() { - const alert = await this.alertCtrl.create({ - header: 'Warning', - message: - '

This action will tear down all service containers and rebuild them from scratch. No data will be deleted.

A system rebuild can be useful if your system gets into a bad state, and it should only be performed if you are experiencing general performance or reliability issues.

It may take up to an hour to complete. During this time, you will lose all connectivity to your Start9 server.

', - buttons: [ - { - text: 'Cancel', - role: 'cancel', - }, - { - text: 'Rebuild', - handler: () => { - try { - this.systemRebuild() - } catch (e) { - console.error(e) - } - }, - }, - ], - cssClass: 'alert-warning-message', - }) - await alert.present() - } - async presentAlertRepairDisk() { const alert = await this.alertCtrl.create({ header: 'Warning', @@ -174,23 +148,6 @@ export class HomePage { window.location.reload() } - private async systemRebuild(): Promise { - const loader = await this.loadingCtrl.create({ - cssClass: 'loader', - }) - await loader.present() - - try { - await this.api.systemRebuild() - await this.api.restart() - this.restarted = true - } catch (e) { - console.error(e) - } finally { - loader.dismiss() - } - } - private async repairDisk(): Promise { const loader = await this.loadingCtrl.create({ cssClass: 'loader', @@ -198,8 +155,8 @@ export class HomePage { await loader.present() try { - await this.api.repairDisk() - await this.api.restart() + await this.api.diagnosticRepairDisk() + await this.api.diagnosticRestart() this.restarted = true } catch (e) { console.error(e) diff --git a/web/projects/diagnostic-ui/src/app/pages/logs/logs.module.ts b/web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.module.ts similarity index 100% rename from web/projects/diagnostic-ui/src/app/pages/logs/logs.module.ts rename to web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.module.ts diff --git a/web/projects/diagnostic-ui/src/app/pages/logs/logs.page.html b/web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.html similarity index 100% rename from web/projects/diagnostic-ui/src/app/pages/logs/logs.page.html rename to web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.html diff --git a/web/projects/diagnostic-ui/src/app/pages/logs/logs.page.scss b/web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.scss similarity index 100% rename from web/projects/diagnostic-ui/src/app/pages/logs/logs.page.scss rename to web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.scss diff --git a/web/projects/diagnostic-ui/src/app/pages/logs/logs.page.ts b/web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.ts similarity index 93% rename from web/projects/diagnostic-ui/src/app/pages/logs/logs.page.ts rename to web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.ts index 44c314a96..5119b9d93 100644 --- a/web/projects/diagnostic-ui/src/app/pages/logs/logs.page.ts +++ b/web/projects/ui/src/app/pages/diagnostic-routes/logs/logs.page.ts @@ -1,6 +1,6 @@ import { Component, ViewChild } from '@angular/core' import { IonContent } from '@ionic/angular' -import { ApiService } from 'src/app/services/api/api.service' +import { ApiService } from 'src/app/services/api/embassy-api.service' import { ErrorToastService, toLocalIsoString } from '@start9labs/shared' var Convert = require('ansi-to-html') @@ -49,7 +49,7 @@ export class LogsPage { private async getLogs() { try { - const { startCursor, entries } = await this.api.getLogs({ + const { startCursor, entries } = await this.api.diagnosticGetLogs({ cursor: this.startCursor, before: !!this.startCursor, limit: this.limit, diff --git a/web/projects/ui/src/app/pages/init/init.module.ts b/web/projects/ui/src/app/pages/init/init.module.ts new file mode 100644 index 000000000..07dd71185 --- /dev/null +++ b/web/projects/ui/src/app/pages/init/init.module.ts @@ -0,0 +1,24 @@ +import { CommonModule } from '@angular/common' +import { NgModule } from '@angular/core' +import { RouterModule, Routes } from '@angular/router' +import { TuiProgressModule } from '@taiga-ui/kit' +import { LogsModule } from 'src/app/pages/init/logs/logs.module' +import { InitPage } from './init.page' + +const routes: Routes = [ + { + path: '', + component: InitPage, + }, +] + +@NgModule({ + imports: [ + CommonModule, + LogsModule, + TuiProgressModule, + RouterModule.forChild(routes), + ], + declarations: [InitPage], +}) +export class InitPageModule {} diff --git a/web/projects/ui/src/app/pages/init/init.page.html b/web/projects/ui/src/app/pages/init/init.page.html new file mode 100644 index 000000000..bd3467bbb --- /dev/null +++ b/web/projects/ui/src/app/pages/init/init.page.html @@ -0,0 +1,18 @@ +
+

+ Initializing StartOS +

+
+ Progress: {{ (progress.total * 100).toFixed(0) }}% +
+ + +

+
+ diff --git a/web/projects/ui/src/app/pages/init/init.page.scss b/web/projects/ui/src/app/pages/init/init.page.scss new file mode 100644 index 000000000..9fbf7098a --- /dev/null +++ b/web/projects/ui/src/app/pages/init/init.page.scss @@ -0,0 +1,23 @@ +section { + border-radius: 0.25rem; + padding: 1rem; + margin: 1.5rem; + text-align: center; + /* TODO: Theme */ + background: #e0e0e0; + color: #333; + --tui-clear-inverse: rgba(0, 0, 0, 0.1); +} + +logs-window { + display: flex; + flex-direction: column; + height: 18rem; + padding: 1rem; + margin: 0 1.5rem auto; + text-align: left; + overflow: hidden; + border-radius: 2rem; + /* TODO: Theme */ + background: #181818; +} diff --git a/web/projects/ui/src/app/pages/init/init.page.ts b/web/projects/ui/src/app/pages/init/init.page.ts new file mode 100644 index 000000000..318881223 --- /dev/null +++ b/web/projects/ui/src/app/pages/init/init.page.ts @@ -0,0 +1,11 @@ +import { Component, inject } from '@angular/core' +import { InitService } from 'src/app/pages/init/init.service' + +@Component({ + selector: 'init-page', + templateUrl: 'init.page.html', + styleUrls: ['init.page.scss'], +}) +export class InitPage { + readonly progress$ = inject(InitService) +} diff --git a/web/projects/ui/src/app/pages/init/init.service.ts b/web/projects/ui/src/app/pages/init/init.service.ts new file mode 100644 index 000000000..3cca42a58 --- /dev/null +++ b/web/projects/ui/src/app/pages/init/init.service.ts @@ -0,0 +1,91 @@ +import { inject, Injectable } from '@angular/core' +import { ErrorToastService } from '@start9labs/shared' +import { T } from '@start9labs/start-sdk' +import { + catchError, + defer, + EMPTY, + from, + map, + Observable, + startWith, + switchMap, + tap, +} from 'rxjs' +import { ApiService } from 'src/app/services/api/embassy-api.service' +import { StateService } from 'src/app/services/state.service' + +interface MappedProgress { + readonly total: number | null + readonly message: string +} + +@Injectable({ providedIn: 'root' }) +export class InitService extends Observable { + private readonly state = inject(StateService) + private readonly api = inject(ApiService) + private readonly errorService = inject(ErrorToastService) + private readonly progress$ = defer(() => + from(this.api.initGetProgress()), + ).pipe( + switchMap(({ guid, progress }) => + this.api + .openWebsocket$(guid, {}) + .pipe(startWith(progress)), + ), + map(({ phases, overall }) => { + return { + total: getOverallDecimal(overall), + message: phases + .filter( + ( + p, + ): p is { + name: string + progress: { + done: number + total: number | null + } + } => p.progress !== true && p.progress !== null, + ) + .map(p => `${p.name}${getPhaseBytes(p.progress)}`) + .join(', '), + } + }), + tap(({ total }) => { + if (total === 1) { + this.state.syncState() + } + }), + catchError(e => { + this.errorService.present(e) + + return EMPTY + }), + ) + + constructor() { + super(subscriber => this.progress$.subscribe(subscriber)) + } +} + +function getOverallDecimal(progress: T.Progress): number { + if (progress === true) { + return 1 + } else if (!progress || !progress.total) { + return 0 + } else { + return progress.total && progress.done / progress.total + } +} + +function getPhaseBytes( + progress: + | false + | { + done: number + total: number | null + }, +): string { + return progress === false ? '' : `: (${progress.done}/${progress.total})` +} diff --git a/web/projects/ui/src/app/pages/init/logs/logs.component.ts b/web/projects/ui/src/app/pages/init/logs/logs.component.ts new file mode 100644 index 000000000..edce1f282 --- /dev/null +++ b/web/projects/ui/src/app/pages/init/logs/logs.component.ts @@ -0,0 +1,33 @@ +import { Component, ElementRef, inject } from '@angular/core' +import { INTERSECTION_ROOT } from '@ng-web-apis/intersection-observer' +import { LogsService } from 'src/app/pages/init/logs/logs.service' + +@Component({ + selector: 'logs-window', + templateUrl: 'logs.template.html', + styles: [ + ` + pre { + margin: 0; + } + `, + ], + providers: [ + { + provide: INTERSECTION_ROOT, + useExisting: ElementRef, + }, + ], +}) +export class LogsComponent { + readonly logs$ = inject(LogsService) + scroll = true + + scrollTo(bottom: HTMLElement) { + if (this.scroll) bottom.scrollIntoView({ behavior: 'smooth' }) + } + + onBottom([{ isIntersecting }]: readonly IntersectionObserverEntry[]) { + this.scroll = isIntersecting + } +} diff --git a/web/projects/ui/src/app/pages/init/logs/logs.module.ts b/web/projects/ui/src/app/pages/init/logs/logs.module.ts new file mode 100644 index 000000000..ee4a1bc1d --- /dev/null +++ b/web/projects/ui/src/app/pages/init/logs/logs.module.ts @@ -0,0 +1,20 @@ +import { CommonModule } from '@angular/common' +import { NgModule } from '@angular/core' +import { IntersectionObserverModule } from '@ng-web-apis/intersection-observer' +import { MutationObserverModule } from '@ng-web-apis/mutation-observer' +import { TuiScrollbarModule } from '@taiga-ui/core' +import { NgDompurifyModule } from '@tinkoff/ng-dompurify' +import { LogsComponent } from './logs.component' + +@NgModule({ + imports: [ + CommonModule, + MutationObserverModule, + IntersectionObserverModule, + NgDompurifyModule, + TuiScrollbarModule, + ], + declarations: [LogsComponent], + exports: [LogsComponent], +}) +export class LogsModule {} diff --git a/web/projects/ui/src/app/pages/init/logs/logs.service.ts b/web/projects/ui/src/app/pages/init/logs/logs.service.ts new file mode 100644 index 000000000..e06d56b42 --- /dev/null +++ b/web/projects/ui/src/app/pages/init/logs/logs.service.ts @@ -0,0 +1,49 @@ +import { inject, Injectable } from '@angular/core' +import { Log, toLocalIsoString } from '@start9labs/shared' +import { + bufferTime, + defer, + filter, + map, + Observable, + scan, + switchMap, +} from 'rxjs' +import { ApiService } from 'src/app/services/api/embassy-api.service' + +var Convert = require('ansi-to-html') +var convert = new Convert({ + newline: true, + bg: 'transparent', + colors: { + 4: 'Cyan', + }, + escapeXML: true, +}) + +function convertAnsi(entries: readonly any[]): string { + return entries + .map( + ({ timestamp, message }) => + `${toLocalIsoString( + new Date(timestamp), + )}  ${convert.toHtml(message)}`, + ) + .join('
') +} + +@Injectable({ providedIn: 'root' }) +export class LogsService extends Observable { + private readonly api = inject(ApiService) + private readonly log$ = defer(() => this.api.initFollowLogs({})).pipe( + switchMap(({ guid }) => this.api.openWebsocket$(guid, {})), + bufferTime(250), + filter(logs => !!logs.length), + map(convertAnsi), + scan((logs: readonly string[], log) => [...logs, log], []), + ) + + constructor() { + super(subscriber => this.log$.subscribe(subscriber)) + } +} diff --git a/web/projects/ui/src/app/pages/init/logs/logs.template.html b/web/projects/ui/src/app/pages/init/logs/logs.template.html new file mode 100644 index 000000000..24ea6d0c1 --- /dev/null +++ b/web/projects/ui/src/app/pages/init/logs/logs.template.html @@ -0,0 +1,9 @@ + + 

+  

+
diff --git a/web/projects/ui/src/app/pages/login/ca-wizard/ca-wizard.component.ts b/web/projects/ui/src/app/pages/login/ca-wizard/ca-wizard.component.ts
index fde1c968f..2c0e9c7fe 100644
--- a/web/projects/ui/src/app/pages/login/ca-wizard/ca-wizard.component.ts
+++ b/web/projects/ui/src/app/pages/login/ca-wizard/ca-wizard.component.ts
@@ -42,7 +42,7 @@ export class CAWizardComponent {
 
   private async testHttps() {
     const url = `https://${this.document.location.host}${this.relativeUrl}`
-    await this.api.echo({ message: 'ping' }, url).then(() => {
+    await this.api.getState().then(() => {
       this.caTrusted = true
     })
   }
diff --git a/web/projects/ui/src/app/pages/server-routes/server-metrics/server-metrics.page.html b/web/projects/ui/src/app/pages/server-routes/server-metrics/server-metrics.page.html
index 70b977ecc..859d48129 100644
--- a/web/projects/ui/src/app/pages/server-routes/server-metrics/server-metrics.page.html
+++ b/web/projects/ui/src/app/pages/server-routes/server-metrics/server-metrics.page.html
@@ -74,7 +74,7 @@
       Memory
       
         Percentage Used
-        {{ memory.percentageUsed }} %
+        {{ memory.percentageUsed.value }} %
       
       
         Total
@@ -98,7 +98,7 @@
       
       
         zram Total
-        {{ memory.zramTotal }} MiB
+        {{ memory.zramTotal.value }} MiB
       
       
         zram Available
diff --git a/web/projects/ui/src/app/pages/server-routes/server-show/server-show.page.ts b/web/projects/ui/src/app/pages/server-routes/server-show/server-show.page.ts
index 73763578e..f375b0e86 100644
--- a/web/projects/ui/src/app/pages/server-routes/server-show/server-show.page.ts
+++ b/web/projects/ui/src/app/pages/server-routes/server-show/server-show.page.ts
@@ -319,30 +319,6 @@ export class ServerShowPage {
     await alert.present()
   }
 
-  async presentAlertSystemRebuild() {
-    const localPkgs = await getAllPackages(this.patch)
-    const minutes = Object.keys(localPkgs).length * 2
-    const alert = await this.alertCtrl.create({
-      header: 'Warning',
-      message: `This action will tear down all service containers and rebuild them from scratch. No data will be deleted. This action is useful if your system gets into a bad state, and it should only be performed if you are experiencing general performance or reliability issues. It may take up to ${minutes} minutes to complete. During this time, you will lose all connectivity to your server.`,
-      buttons: [
-        {
-          text: 'Cancel',
-          role: 'cancel',
-        },
-        {
-          text: 'Rebuild',
-          handler: () => {
-            this.systemRebuild()
-          },
-          cssClass: 'enter-click',
-        },
-      ],
-      cssClass: 'alert-warning-message',
-    })
-    await alert.present()
-  }
-
   async presentAlertRepairDisk() {
     const alert = await this.alertCtrl.create({
       header: 'Warning',
@@ -437,23 +413,6 @@ export class ServerShowPage {
     }
   }
 
-  private async systemRebuild() {
-    const action = 'System Rebuild'
-
-    const loader = await this.loadingCtrl.create({
-      message: `Beginning ${action}...`,
-    })
-    await loader.present()
-
-    try {
-      await this.embassyApi.systemRebuild({})
-    } catch (e: any) {
-      this.errToast.present(e)
-    } finally {
-      loader.dismiss()
-    }
-  }
-
   private async checkForEosUpdate(): Promise {
     const loader = await this.loadingCtrl.create({
       message: 'Checking for updates',
@@ -718,14 +677,6 @@ export class ServerShowPage {
         detail: false,
         disabled$: of(false),
       },
-      {
-        title: 'System Rebuild',
-        description: '',
-        icon: 'construct-outline',
-        action: () => this.presentAlertSystemRebuild(),
-        detail: false,
-        disabled$: of(false),
-      },
       {
         title: 'Repair Disk',
         description: '',
diff --git a/web/projects/ui/src/app/services/api/api.fixures.ts b/web/projects/ui/src/app/services/api/api.fixures.ts
index 5642eece5..38ee4774b 100644
--- a/web/projects/ui/src/app/services/api/api.fixures.ts
+++ b/web/projects/ui/src/app/services/api/api.fixures.ts
@@ -16,7 +16,7 @@ export module Mock {
     restarting: false,
     shuttingDown: false,
   }
-  export const MarketplaceEos: RR.GetMarketplaceEosRes = {
+  export const MarketplaceEos: RR.CheckOSUpdateRes = {
     version: '0.3.5.2',
     headline: 'Our biggest release ever.',
     releaseNotes: {
@@ -493,30 +493,23 @@ export module Mock {
     {
       timestamp: '2022-07-28T03:52:54.808769Z',
       message: '****** START *****',
+      bootId: 'hsjnfdklasndhjasvbjamsksajbndjn',
     },
     {
       timestamp: '2019-12-26T14:21:30.872Z',
       message:
         '\u001b[34mPOST \u001b[0;32;49m200\u001b[0m photoview.startos/api/graphql \u001b[0;36;49m1.169406ms\u001b',
+      bootId: 'hsjnfdklasndhjasvbjamsksajbndjn',
     },
     {
       timestamp: '2019-12-26T14:22:30.872Z',
       message: '****** FINISH *****',
-    },
-  ]
-
-  export const PackageLogs: Log[] = [
-    {
-      timestamp: '2022-07-28T03:52:54.808769Z',
-      message: '****** START *****',
+      bootId: 'gvbwfiuasokdasjndasnjdmfvbahjdmdkfm',
     },
     {
-      timestamp: '2019-12-26T14:21:30.872Z',
-      message: 'PackageLogs PackageLogs PackageLogs PackageLogs PackageLogs',
-    },
-    {
-      timestamp: '2019-12-26T14:22:30.872Z',
-      message: '****** FINISH *****',
+      timestamp: '2019-12-26T15:22:30.872Z',
+      message: '****** AGAIN *****',
+      bootId: 'gvbwfiuasokdasjndasnjdmfvbahjdmdkfm',
     },
   ]
 
diff --git a/web/projects/ui/src/app/services/api/api.types.ts b/web/projects/ui/src/app/services/api/api.types.ts
index 3539164f5..a5c52e9a2 100644
--- a/web/projects/ui/src/app/services/api/api.types.ts
+++ b/web/projects/ui/src/app/services/api/api.types.ts
@@ -1,17 +1,28 @@
-import { Dump, Revision } from 'patch-db-client'
+import { Dump } from 'patch-db-client'
 import { MarketplacePkg, StoreInfo } from '@start9labs/marketplace'
 import { PackagePropertiesVersioned } from 'src/app/util/properties.util'
 import { ConfigSpec } from 'src/app/pkg-config/config-types'
 import { DataModel } from 'src/app/services/patch-db/data-model'
 import { StartOSDiskInfo, LogsRes, ServerLogsReq } from '@start9labs/shared'
 import { T } from '@start9labs/start-sdk'
+import { WebSocketSubjectConfig } from 'rxjs/webSocket'
 
 export module RR {
+  // websocket
+
+  export type WebsocketConfig = Omit, 'url'>
+
+  // server state
+
+  export type ServerState = 'initializing' | 'error' | 'running'
+
   // DB
 
-  export type GetRevisionsRes = Revision[] | Dump
-
-  export type GetDumpRes = Dump
+  export type SubscribePatchReq = {}
+  export type SubscribePatchRes = {
+    dump: Dump
+    guid: string
+  }
 
   export type SetDBValueReq = { pointer: string; value: T } // db.put.ui
   export type SetDBValueRes = null
@@ -33,10 +44,22 @@ export module RR {
   } // auth.reset-password
   export type ResetPasswordRes = null
 
-  // server
+  // diagnostic
 
-  export type EchoReq = { message: string; timeout?: number } // server.echo
-  export type EchoRes = string
+  export type DiagnosticErrorRes = {
+    code: number
+    message: string
+    data: { details: string }
+  }
+
+  // init
+
+  export type InitGetProgressRes = {
+    progress: T.FullProgress
+    guid: string
+  }
+
+  // server
 
   export type GetSystemTimeReq = {} // server.time
   export type GetSystemTimeRes = {
@@ -65,8 +88,8 @@ export module RR {
   export type ShutdownServerReq = {} // server.shutdown
   export type ShutdownServerRes = null
 
-  export type SystemRebuildReq = {} // server.rebuild
-  export type SystemRebuildRes = null
+  export type DiskRepairReq = {} // server.disk.repair
+  export type DiskRepairRes = null
 
   export type ResetTorReq = {
     wipeState: boolean
@@ -254,8 +277,8 @@ export module RR {
   export type GetMarketplaceInfoReq = { serverId: string }
   export type GetMarketplaceInfoRes = StoreInfo
 
-  export type GetMarketplaceEosReq = { serverId: string }
-  export type GetMarketplaceEosRes = MarketplaceEOS
+  export type CheckOSUpdateReq = { serverId: string }
+  export type CheckOSUpdateRes = OSUpdate
 
   export type GetMarketplacePackagesReq = {
     ids?: { id: string; version: string }[]
@@ -271,7 +294,7 @@ export module RR {
   export type GetReleaseNotesRes = { [version: string]: string }
 }
 
-export interface MarketplaceEOS {
+export interface OSUpdate {
   version: string
   headline: string
   releaseNotes: { [version: string]: string }
diff --git a/web/projects/ui/src/app/services/api/embassy-api.service.ts b/web/projects/ui/src/app/services/api/embassy-api.service.ts
index 3f1d9881d..d6bb11632 100644
--- a/web/projects/ui/src/app/services/api/embassy-api.service.ts
+++ b/web/projects/ui/src/app/services/api/embassy-api.service.ts
@@ -1,9 +1,5 @@
 import { Observable } from 'rxjs'
-import { Update } from 'patch-db-client'
 import { RR } from './api.types'
-import { DataModel } from 'src/app/services/patch-db/data-model'
-import { Log } from '@start9labs/shared'
-import { WebSocketSubjectConfig } from 'rxjs/webSocket'
 
 export abstract class ApiService {
   // http
@@ -14,8 +10,23 @@ export abstract class ApiService {
   // for sideloading packages
   abstract uploadPackage(guid: string, body: Blob): Promise
 
+  // websocket
+
+  abstract openWebsocket$(
+    guid: string,
+    config: RR.WebsocketConfig,
+  ): Observable
+
+  // server state
+
+  abstract getState(): Promise
+
   // db
 
+  abstract subscribeToPatchDB(
+    params: RR.SubscribePatchReq,
+  ): Promise
+
   abstract setDbValue(
     pathArr: Array,
     value: T,
@@ -35,16 +46,26 @@ export abstract class ApiService {
     params: RR.ResetPasswordReq,
   ): Promise
 
+  // diagnostic
+
+  abstract diagnosticGetError(): Promise
+  abstract diagnosticRestart(): Promise
+  abstract diagnosticForgetDrive(): Promise
+  abstract diagnosticRepairDisk(): Promise
+  abstract diagnosticGetLogs(
+    params: RR.GetServerLogsReq,
+  ): Promise
+
+  // init
+
+  abstract initGetProgress(): Promise
+
+  abstract initFollowLogs(
+    params: RR.FollowServerLogsReq,
+  ): Promise
+
   // server
 
-  abstract echo(params: RR.EchoReq, urlOverride?: string): Promise
-
-  abstract openPatchWebsocket$(): Observable>
-
-  abstract openLogsWebsocket$(
-    config: WebSocketSubjectConfig,
-  ): Observable
-
   abstract getSystemTime(
     params: RR.GetSystemTimeReq,
   ): Promise
@@ -89,11 +110,7 @@ export abstract class ApiService {
     params: RR.ShutdownServerReq,
   ): Promise
 
-  abstract systemRebuild(
-    params: RR.SystemRebuildReq,
-  ): Promise
-
-  abstract repairDisk(params: RR.SystemRebuildReq): Promise
+  abstract repairDisk(params: RR.DiskRepairReq): Promise
 
   abstract resetTor(params: RR.ResetTorReq): Promise
 
@@ -105,7 +122,7 @@ export abstract class ApiService {
     url: string,
   ): Promise
 
-  abstract getEos(): Promise
+  abstract checkOSUpdate(qp: RR.CheckOSUpdateReq): Promise
 
   // notification
 
diff --git a/web/projects/ui/src/app/services/api/embassy-live-api.service.ts b/web/projects/ui/src/app/services/api/embassy-live-api.service.ts
index 631c12e6b..8826b2883 100644
--- a/web/projects/ui/src/app/services/api/embassy-live-api.service.ts
+++ b/web/projects/ui/src/app/services/api/embassy-live-api.service.ts
@@ -3,7 +3,6 @@ import {
   HttpOptions,
   HttpService,
   isRpcError,
-  Log,
   Method,
   RpcError,
   RPCOptions,
@@ -12,13 +11,12 @@ import { ApiService } from './embassy-api.service'
 import { RR } from './api.types'
 import { parsePropertiesPermissive } from 'src/app/util/properties.util'
 import { ConfigService } from '../config.service'
-import { webSocket, WebSocketSubjectConfig } from 'rxjs/webSocket'
+import { webSocket } from 'rxjs/webSocket'
 import { Observable, filter, firstValueFrom } from 'rxjs'
 import { AuthService } from '../auth.service'
 import { DOCUMENT } from '@angular/common'
 import { DataModel } from '../patch-db/data-model'
-import { PatchDB, pathFromArray, Update } from 'patch-db-client'
-import { getServerInfo } from 'src/app/util/get-server-info'
+import { PatchDB, pathFromArray } from 'patch-db-client'
 
 @Injectable()
 export class LiveApiService extends ApiService {
@@ -30,10 +28,11 @@ export class LiveApiService extends ApiService {
     private readonly patch: PatchDB,
   ) {
     super()
-    ;(window as any).rpcClient = this
+    ; (window as any).rpcClient = this
   }
 
   // for getting static files: ex icons, instructions, licenses
+
   async getStatic(url: string): Promise {
     return this.httpRequest({
       method: Method.GET,
@@ -43,6 +42,7 @@ export class LiveApiService extends ApiService {
   }
 
   // for sideloading packages
+
   async uploadPackage(guid: string, body: Blob): Promise {
     return this.httpRequest({
       method: Method.POST,
@@ -52,8 +52,36 @@ export class LiveApiService extends ApiService {
     })
   }
 
+  // websocket
+
+  openWebsocket$(
+    guid: string,
+    config: RR.WebsocketConfig,
+  ): Observable {
+    const { location } = this.document.defaultView!
+    const protocol = location.protocol === 'http:' ? 'ws' : 'wss'
+    const host = location.host
+
+    return webSocket({
+      url: `${protocol}://${host}/ws/rpc/${guid}`,
+      ...config,
+    })
+  }
+
+  // state
+
+  async getState(): Promise {
+    return this.rpcRequest({ method: 'state', params: {} })
+  }
+
   // db
 
+  async subscribeToPatchDB(
+    params: RR.SubscribePatchReq,
+  ): Promise {
+    return this.rpcRequest({ method: 'db.subscribe', params })
+  }
+
   async setDbValue(
     pathArr: Array,
     value: T,
@@ -87,29 +115,57 @@ export class LiveApiService extends ApiService {
     return this.rpcRequest({ method: 'auth.reset-password', params })
   }
 
+  // diagnostic
+
+  async diagnosticGetError(): Promise {
+    return this.rpcRequest({
+      method: 'diagnostic.error',
+      params: {},
+    })
+  }
+
+  async diagnosticRestart(): Promise {
+    return this.rpcRequest({
+      method: 'diagnostic.restart',
+      params: {},
+    })
+  }
+
+  async diagnosticForgetDrive(): Promise {
+    return this.rpcRequest({
+      method: 'diagnostic.disk.forget',
+      params: {},
+    })
+  }
+
+  async diagnosticRepairDisk(): Promise {
+    return this.rpcRequest({
+      method: 'diagnostic.disk.repair',
+      params: {},
+    })
+  }
+
+  async diagnosticGetLogs(
+    params: RR.GetServerLogsReq,
+  ): Promise {
+    return this.rpcRequest({
+      method: 'diagnostic.logs',
+      params,
+    })
+  }
+
+  // init
+
+  async initGetProgress(): Promise {
+    return this.rpcRequest({ method: 'init.subscribe', params: {} })
+  }
+
+  async initFollowLogs(): Promise {
+    return this.rpcRequest({ method: 'init.logs.follow', params: {} })
+  }
+
   // server
 
-  async echo(params: RR.EchoReq, urlOverride?: string): Promise {
-    return this.rpcRequest({ method: 'echo', params }, urlOverride)
-  }
-
-  openPatchWebsocket$(): Observable> {
-    const config: WebSocketSubjectConfig> = {
-      url: `/db`,
-      closeObserver: {
-        next: val => {
-          if (val.reason === 'UNAUTHORIZED') this.auth.setUnverified()
-        },
-      },
-    }
-
-    return this.openWebsocket(config)
-  }
-
-  openLogsWebsocket$(config: WebSocketSubjectConfig): Observable {
-    return this.openWebsocket(config)
-  }
-
   async getSystemTime(
     params: RR.GetSystemTimeReq,
   ): Promise {
@@ -175,12 +231,6 @@ export class LiveApiService extends ApiService {
     return this.rpcRequest({ method: 'server.shutdown', params })
   }
 
-  async systemRebuild(
-    params: RR.RestartServerReq,
-  ): Promise {
-    return this.rpcRequest({ method: 'server.rebuild', params })
-  }
-
   async repairDisk(params: RR.RestartServerReq): Promise {
     return this.rpcRequest({ method: 'disk.repair', params })
   }
@@ -203,10 +253,7 @@ export class LiveApiService extends ApiService {
     })
   }
 
-  async getEos(): Promise {
-    const { id } = await getServerInfo(this.patch)
-    const qp: RR.GetMarketplaceEosReq = { serverId: id }
-
+  async checkOSUpdate(qp: RR.CheckOSUpdateReq): Promise {
     return this.marketplaceProxy(
       '/eos/v0/latest',
       qp,
@@ -417,16 +464,6 @@ export class LiveApiService extends ApiService {
     })
   }
 
-  private openWebsocket(config: WebSocketSubjectConfig): Observable {
-    const { location } = this.document.defaultView!
-    const protocol = location.protocol === 'http:' ? 'ws' : 'wss'
-    const host = location.host
-
-    config.url = `${protocol}://${host}/ws${config.url}`
-
-    return webSocket(config)
-  }
-
   private async rpcRequest(
     options: RPCOptions,
     urlOverride?: string,
@@ -445,9 +482,7 @@ export class LiveApiService extends ApiService {
     const patchSequence = res.headers.get('x-patch-sequence')
     if (patchSequence)
       await firstValueFrom(
-        this.patch.cache$.pipe(
-          filter(({ sequence }) => sequence >= Number(patchSequence)),
-        ),
+        this.patch.cache$.pipe(filter(({ id }) => id >= Number(patchSequence))),
       )
 
     return body.result
diff --git a/web/projects/ui/src/app/services/api/embassy-mock-api.service.ts b/web/projects/ui/src/app/services/api/embassy-mock-api.service.ts
index f623f6db3..728b4ff35 100644
--- a/web/projects/ui/src/app/services/api/embassy-mock-api.service.ts
+++ b/web/projects/ui/src/app/services/api/embassy-mock-api.service.ts
@@ -1,15 +1,14 @@
 import { Injectable } from '@angular/core'
-import { Log, pauseFor } from '@start9labs/shared'
+import { Log, RPCErrorDetails, pauseFor } from '@start9labs/shared'
 import { ApiService } from './embassy-api.service'
 import {
   Operation,
   PatchOp,
   pathFromArray,
   RemoveOperation,
-  Update,
+  Revision,
 } from 'patch-db-client'
 import {
-  DataModel,
   InstallingState,
   PackageDataEntry,
   StateInfo,
@@ -20,22 +19,17 @@ import { parsePropertiesPermissive } from 'src/app/util/properties.util'
 import { Mock } from './api.fixures'
 import markdown from 'raw-loader!../../../../../shared/assets/markdown/md-sample.md'
 import {
-  EMPTY,
-  iif,
+  from,
   interval,
   map,
   Observable,
   shareReplay,
+  startWith,
   Subject,
-  switchMap,
   tap,
-  timer,
 } from 'rxjs'
-import { LocalStorageBootstrap } from '../patch-db/local-storage-bootstrap'
 import { mockPatchData } from './mock-patch'
-import { WebSocketSubjectConfig } from 'rxjs/webSocket'
 import { AuthService } from '../auth.service'
-import { ConnectionService } from '../connection.service'
 import { StoreInfo } from '@start9labs/marketplace'
 import { T } from '@start9labs/start-sdk'
 
@@ -71,32 +65,17 @@ const PROGRESS: T.FullProgress = {
 
 @Injectable()
 export class MockApiService extends ApiService {
-  readonly mockWsSource$ = new Subject>()
+  readonly mockWsSource$ = new Subject()
   private readonly revertTime = 1800
   sequence = 0
 
-  constructor(
-    private readonly bootstrapper: LocalStorageBootstrap,
-    private readonly connectionService: ConnectionService,
-    private readonly auth: AuthService,
-  ) {
+  constructor(private readonly auth: AuthService) {
     super()
     this.auth.isVerified$
       .pipe(
         tap(() => {
           this.sequence = 0
         }),
-        switchMap(verified =>
-          iif(
-            () => verified,
-            timer(2000).pipe(
-              tap(() => {
-                this.connectionService.websocketConnected$.next(true)
-              }),
-            ),
-            EMPTY,
-          ),
-        ),
       )
       .subscribe()
   }
@@ -111,8 +90,57 @@ export class MockApiService extends ApiService {
     return 'success'
   }
 
+  // websocket
+
+  openWebsocket$(
+    guid: string,
+    config: RR.WebsocketConfig,
+  ): Observable {
+    if (guid === 'db-guid') {
+      return this.mockWsSource$.pipe(
+        shareReplay({ bufferSize: 1, refCount: true }),
+      )
+    } else if (guid === 'logs-guid') {
+      return interval(50).pipe(
+        map((_, index) => {
+          // mock fire open observer
+          if (index === 0) config.openObserver?.next(new Event(''))
+          if (index === 100) throw new Error('HAAHHA')
+          return Mock.ServerLogs[0]
+        }),
+      )
+    } else if (guid === 'init-progress-guid') {
+      return from(this.initProgress()).pipe(
+        startWith(PROGRESS),
+      ) as Observable
+    } else {
+      throw new Error('invalid guid type')
+    }
+  }
+
+  // server state
+
+  private stateIndex = 0
+  async getState(): Promise {
+    await pauseFor(1000)
+
+    this.stateIndex++
+
+    return this.stateIndex === 1 ? 'initializing' : 'running'
+  }
+
   // db
 
+  async subscribeToPatchDB(
+    params: RR.SubscribePatchReq,
+  ): Promise {
+    await pauseFor(2000)
+    return {
+      dump: { id: 1, value: mockPatchData },
+      guid: 'db-guid',
+    }
+  }
+
   async setDbValue(
     pathArr: Array,
     value: T,
@@ -136,11 +164,6 @@ export class MockApiService extends ApiService {
 
   async login(params: RR.LoginReq): Promise {
     await pauseFor(2000)
-
-    setTimeout(() => {
-      this.mockWsSource$.next({ id: 1, value: mockPatchData })
-    }, 2000)
-
     return null
   }
 
@@ -166,34 +189,63 @@ export class MockApiService extends ApiService {
     return null
   }
 
-  // server
+  // diagnostic
 
-  async echo(params: RR.EchoReq, url?: string): Promise {
-    if (url) {
-      const num = Math.floor(Math.random() * 10) + 1
-      if (num > 8) return params.message
-      throw new Error()
+  async getError(): Promise {
+    await pauseFor(1000)
+    return {
+      code: 15,
+      message: 'Unknown server',
+      data: { details: 'Some details about the error here' },
     }
+  }
+
+  async diagnosticGetError(): Promise {
+    await pauseFor(1000)
+    return {
+      code: 15,
+      message: 'Unknown server',
+      data: { details: 'Some details about the error here' },
+    }
+  }
+
+  async diagnosticRestart(): Promise {
+    await pauseFor(1000)
+  }
+
+  async diagnosticForgetDrive(): Promise {
+    await pauseFor(1000)
+  }
+
+  async diagnosticRepairDisk(): Promise {
+    await pauseFor(1000)
+  }
+
+  async diagnosticGetLogs(
+    params: RR.GetServerLogsReq,
+  ): Promise {
+    return this.getServerLogs(params)
+  }
+
+  // init
+
+  async initGetProgress(): Promise {
+    await pauseFor(250)
+    return {
+      progress: PROGRESS,
+      guid: 'init-progress-guid',
+    }
+  }
+
+  async initFollowLogs(): Promise {
     await pauseFor(2000)
-    return params.message
+    return {
+      startCursor: 'start-cursor',
+      guid: 'logs-guid',
+    }
   }
 
-  openPatchWebsocket$(): Observable> {
-    return this.mockWsSource$.pipe(
-      shareReplay({ bufferSize: 1, refCount: true }),
-    )
-  }
-
-  openLogsWebsocket$(config: WebSocketSubjectConfig): Observable {
-    return interval(50).pipe(
-      map((_, index) => {
-        // mock fire open observer
-        if (index === 0) config.openObserver?.next(new Event(''))
-        if (index === 100) throw new Error('HAAHHA')
-        return Mock.ServerLogs[0]
-      }),
-    )
-  }
+  // server
 
   async getSystemTime(
     params: RR.GetSystemTimeReq,
@@ -248,7 +300,7 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
     return {
       startCursor: 'start-cursor',
-      guid: '7251d5be-645f-4362-a51b-3a85be92b31e',
+      guid: 'logs-guid',
     }
   }
 
@@ -258,7 +310,7 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
     return {
       startCursor: 'start-cursor',
-      guid: '7251d5be-645f-4362-a51b-3a85be92b31e',
+      guid: 'logs-guid',
     }
   }
 
@@ -268,11 +320,11 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
     return {
       startCursor: 'start-cursor',
-      guid: '7251d5be-645f-4362-a51b-3a85be92b31e',
+      guid: 'logs-guid',
     }
   }
 
-  randomLogs(limit = 1): Log[] {
+  private randomLogs(limit = 1): Log[] {
     const arrLength = Math.ceil(limit / Mock.ServerLogs.length)
     const logs = new Array(arrLength)
       .fill(Mock.ServerLogs)
@@ -374,12 +426,6 @@ export class MockApiService extends ApiService {
     return null
   }
 
-  async systemRebuild(
-    params: RR.SystemRebuildReq,
-  ): Promise {
-    return this.restartServer(params)
-  }
-
   async repairDisk(params: RR.RestartServerReq): Promise {
     await pauseFor(2000)
     return null
@@ -422,7 +468,7 @@ export class MockApiService extends ApiService {
     }
   }
 
-  async getEos(): Promise {
+  async checkOSUpdate(qp: RR.CheckOSUpdateReq): Promise {
     await pauseFor(2000)
     return Mock.MarketplaceEos
   }
@@ -641,13 +687,13 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
     let entries
     if (Math.random() < 0.2) {
-      entries = Mock.PackageLogs
+      entries = Mock.ServerLogs
     } else {
       const arrLength = params.limit
-        ? Math.ceil(params.limit / Mock.PackageLogs.length)
+        ? Math.ceil(params.limit / Mock.ServerLogs.length)
         : 10
       entries = new Array(arrLength)
-        .fill(Mock.PackageLogs)
+        .fill(Mock.ServerLogs)
         .reduce((acc, val) => acc.concat(val), [])
     }
     return {
@@ -663,7 +709,7 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
     return {
       startCursor: 'start-cursor',
-      guid: '7251d5be-645f-4362-a51b-3a85be92b31e',
+      guid: 'logs-guid',
     }
   }
 
@@ -673,7 +719,7 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
 
     setTimeout(async () => {
-      this.updateProgress(params.id)
+      this.installProgress(params.id)
     }, 1000)
 
     const patch: Operation<
@@ -745,7 +791,7 @@ export class MockApiService extends ApiService {
     await pauseFor(2000)
     const patch: Operation[] = params.ids.map(id => {
       setTimeout(async () => {
-        this.updateProgress(id)
+        this.installProgress(id)
       }, 2000)
 
       return {
@@ -1013,7 +1059,57 @@ export class MockApiService extends ApiService {
     return '4120e092-05ab-4de2-9fbd-c3f1f4b1df9e' // no significance, randomly generated
   }
 
-  private async updateProgress(id: string): Promise {
+  private async initProgress(): Promise {
+    const progress = JSON.parse(JSON.stringify(PROGRESS))
+
+    for (let [i, phase] of progress.phases.entries()) {
+      if (
+        !phase.progress ||
+        typeof phase.progress !== 'object' ||
+        !phase.progress.total
+      ) {
+        await pauseFor(2000)
+
+        progress.phases[i].progress = true
+
+        if (
+          progress.overall &&
+          typeof progress.overall === 'object' &&
+          progress.overall.total
+        ) {
+          const step = progress.overall.total / progress.phases.length
+          progress.overall.done += step
+        }
+      } else {
+        const step = phase.progress.total / 4
+
+        while (phase.progress.done < phase.progress.total) {
+          await pauseFor(200)
+
+          phase.progress.done += step
+
+          if (
+            progress.overall &&
+            typeof progress.overall === 'object' &&
+            progress.overall.total
+          ) {
+            const step = progress.overall.total / progress.phases.length / 4
+
+            progress.overall.done += step
+          }
+
+          if (phase.progress.done === phase.progress.total) {
+            await pauseFor(250)
+
+            progress.phases[i].progress = true
+          }
+        }
+      }
+    }
+    return progress
+  }
+
+  private async installProgress(id: string): Promise {
     const progress = JSON.parse(JSON.stringify(PROGRESS))
 
     for (let [i, phase] of progress.phases.entries()) {
@@ -1194,10 +1290,6 @@ export class MockApiService extends ApiService {
   }
 
   private async mockRevision(patch: Operation[]): Promise {
-    if (!this.sequence) {
-      const { sequence } = this.bootstrapper.init()
-      this.sequence = sequence
-    }
     const revision = {
       id: ++this.sequence,
       patch,
diff --git a/web/projects/ui/src/app/services/auth.service.ts b/web/projects/ui/src/app/services/auth.service.ts
index 5d755aa98..9c16d0e26 100644
--- a/web/projects/ui/src/app/services/auth.service.ts
+++ b/web/projects/ui/src/app/services/auth.service.ts
@@ -12,7 +12,7 @@ export enum AuthState {
   providedIn: 'root',
 })
 export class AuthService {
-  private readonly LOGGED_IN_KEY = 'loggedInKey'
+  private readonly LOGGED_IN_KEY = 'loggedIn'
   private readonly authState$ = new ReplaySubject(1)
 
   readonly isVerified$ = this.authState$.pipe(
diff --git a/web/projects/ui/src/app/services/connection.service.ts b/web/projects/ui/src/app/services/connection.service.ts
index a45d5ec4c..7d3328503 100644
--- a/web/projects/ui/src/app/services/connection.service.ts
+++ b/web/projects/ui/src/app/services/connection.service.ts
@@ -1,25 +1,23 @@
-import { Injectable } from '@angular/core'
-import { combineLatest, fromEvent, merge, ReplaySubject } from 'rxjs'
-import { distinctUntilChanged, map, startWith } from 'rxjs/operators'
+import { inject, Injectable } from '@angular/core'
+import { combineLatest, Observable, shareReplay } from 'rxjs'
+import { distinctUntilChanged, map } from 'rxjs/operators'
+import { NetworkService } from 'src/app/services/network.service'
+import { StateService } from 'src/app/services/state.service'
 
 @Injectable({
   providedIn: 'root',
 })
-export class ConnectionService {
-  readonly networkConnected$ = merge(
-    fromEvent(window, 'online'),
-    fromEvent(window, 'offline'),
-  ).pipe(
-    startWith(null),
-    map(() => navigator.onLine),
-    distinctUntilChanged(),
-  )
-  readonly websocketConnected$ = new ReplaySubject(1)
-  readonly connected$ = combineLatest([
-    this.networkConnected$,
-    this.websocketConnected$.pipe(distinctUntilChanged()),
+export class ConnectionService extends Observable {
+  private readonly stream$ = combineLatest([
+    inject(NetworkService),
+    inject(StateService).pipe(map(Boolean)),
   ]).pipe(
     map(([network, websocket]) => network && websocket),
     distinctUntilChanged(),
+    shareReplay(1),
   )
+
+  constructor() {
+    super(subscriber => this.stream$.subscribe(subscriber))
+  }
 }
diff --git a/web/projects/ui/src/app/services/eos.service.ts b/web/projects/ui/src/app/services/eos.service.ts
index dcd1d0de0..81b97bf11 100644
--- a/web/projects/ui/src/app/services/eos.service.ts
+++ b/web/projects/ui/src/app/services/eos.service.ts
@@ -2,7 +2,7 @@ import { Injectable } from '@angular/core'
 import { Emver } from '@start9labs/shared'
 import { BehaviorSubject, combineLatest } from 'rxjs'
 import { distinctUntilChanged, map } from 'rxjs/operators'
-import { MarketplaceEOS } from 'src/app/services/api/api.types'
+import { OSUpdate } from 'src/app/services/api/api.types'
 import { ApiService } from 'src/app/services/api/embassy-api.service'
 import { PatchDB } from 'patch-db-client'
 import { getServerInfo } from 'src/app/util/get-server-info'
@@ -12,7 +12,7 @@ import { DataModel } from './patch-db/data-model'
   providedIn: 'root',
 })
 export class EOSService {
-  eos?: MarketplaceEOS
+  osUpdate?: OSUpdate
   updateAvailable$ = new BehaviorSubject(false)
 
   readonly updating$ = this.patch.watch$('serverInfo', 'statusInfo').pipe(
@@ -52,9 +52,10 @@ export class EOSService {
   ) {}
 
   async loadEos(): Promise {
-    const { version } = await getServerInfo(this.patch)
-    this.eos = await this.api.getEos()
-    const updateAvailable = this.emver.compare(this.eos.version, version) === 1
+    const { version, id } = await getServerInfo(this.patch)
+    this.osUpdate = await this.api.checkOSUpdate({ serverId: id })
+    const updateAvailable =
+      this.emver.compare(this.osUpdate.version, version) === 1
     this.updateAvailable$.next(updateAvailable)
   }
 }
diff --git a/web/projects/ui/src/app/services/network.service.ts b/web/projects/ui/src/app/services/network.service.ts
new file mode 100644
index 000000000..e1568603d
--- /dev/null
+++ b/web/projects/ui/src/app/services/network.service.ts
@@ -0,0 +1,22 @@
+import { inject, Injectable } from '@angular/core'
+import { WINDOW } from '@ng-web-apis/common'
+import { fromEvent, merge, Observable, shareReplay } from 'rxjs'
+import { distinctUntilChanged, map, startWith } from 'rxjs/operators'
+
+@Injectable({ providedIn: 'root' })
+export class NetworkService extends Observable {
+  private readonly win = inject(WINDOW)
+  private readonly stream$ = merge(
+    fromEvent(this.win, 'online'),
+    fromEvent(this.win, 'offline'),
+  ).pipe(
+    startWith(null),
+    map(() => this.win.navigator.onLine),
+    distinctUntilChanged(),
+    shareReplay(1),
+  )
+
+  constructor() {
+    super(subscriber => this.stream$.subscribe(subscriber))
+  }
+}
diff --git a/web/projects/ui/src/app/services/patch-data.service.ts b/web/projects/ui/src/app/services/patch-data.service.ts
index 5372a665c..50023c1f1 100644
--- a/web/projects/ui/src/app/services/patch-data.service.ts
+++ b/web/projects/ui/src/app/services/patch-data.service.ts
@@ -1,9 +1,9 @@
 import { Inject, Injectable } from '@angular/core'
 import { ModalController } from '@ionic/angular'
 import { Observable } from 'rxjs'
-import { filter, share, switchMap, take, tap } from 'rxjs/operators'
+import { filter, map, share, switchMap, take, tap } from 'rxjs/operators'
 import { PatchDB } from 'patch-db-client'
-import { DataModel, UIData } from 'src/app/services/patch-db/data-model'
+import { DataModel } from 'src/app/services/patch-db/data-model'
 import { EOSService } from 'src/app/services/eos.service'
 import { OSWelcomePage } from 'src/app/modals/os-welcome/os-welcome.page'
 import { ConfigService } from 'src/app/services/config.service'
@@ -11,21 +11,25 @@ import { ApiService } from 'src/app/services/api/embassy-api.service'
 import { MarketplaceService } from 'src/app/services/marketplace.service'
 import { AbstractMarketplaceService } from '@start9labs/marketplace'
 import { ConnectionService } from 'src/app/services/connection.service'
+import { LocalStorageBootstrap } from './patch-db/local-storage-bootstrap'
 
 // Get data from PatchDb after is starts and act upon it
 @Injectable({
   providedIn: 'root',
 })
-export class PatchDataService extends Observable {
-  private readonly stream$ = this.connectionService.connected$.pipe(
+export class PatchDataService extends Observable {
+  private readonly stream$ = this.connection$.pipe(
     filter(Boolean),
     switchMap(() => this.patch.watch$()),
-    take(1),
-    tap(({ ui }) => {
-      // check for updates to eOS and services
-      this.checkForUpdates()
-      // show eos welcome message
-      this.showEosWelcome(ui.ackWelcome)
+    map((cache, index) => {
+      this.bootstrapper.update(cache)
+
+      if (index === 0) {
+        // check for updates to StartOS and services
+        this.checkForUpdates()
+        // show eos welcome message
+        this.showEosWelcome(cache.ui.ackWelcome)
+      }
     }),
     share(),
   )
@@ -38,7 +42,8 @@ export class PatchDataService extends Observable {
     private readonly embassyApi: ApiService,
     @Inject(AbstractMarketplaceService)
     private readonly marketplaceService: MarketplaceService,
-    private readonly connectionService: ConnectionService,
+    private readonly connection$: ConnectionService,
+    private readonly bootstrapper: LocalStorageBootstrap,
   ) {
     super(subscriber => this.stream$.subscribe(subscriber))
   }
diff --git a/web/projects/ui/src/app/services/patch-db/local-storage-bootstrap.ts b/web/projects/ui/src/app/services/patch-db/local-storage-bootstrap.ts
index 2ea5bef02..079def855 100644
--- a/web/projects/ui/src/app/services/patch-db/local-storage-bootstrap.ts
+++ b/web/projects/ui/src/app/services/patch-db/local-storage-bootstrap.ts
@@ -1,4 +1,4 @@
-import { Bootstrapper, DBCache } from 'patch-db-client'
+import { Dump } from 'patch-db-client'
 import { DataModel } from 'src/app/services/patch-db/data-model'
 import { Injectable } from '@angular/core'
 import { StorageService } from '../storage.service'
@@ -6,20 +6,18 @@ import { StorageService } from '../storage.service'
 @Injectable({
   providedIn: 'root',
 })
-export class LocalStorageBootstrap implements Bootstrapper {
-  static CONTENT_KEY = 'patch-db-cache'
+export class LocalStorageBootstrap {
+  static CONTENT_KEY = 'patchDB'
 
   constructor(private readonly storage: StorageService) {}
 
-  init(): DBCache {
-    const cache = this.storage.get>(
-      LocalStorageBootstrap.CONTENT_KEY,
-    )
+  init(): Dump {
+    const cache = this.storage.get(LocalStorageBootstrap.CONTENT_KEY)
 
-    return cache || { sequence: 0, data: {} as DataModel }
+    return cache ? { id: 1, value: cache } : { id: 0, value: {} as DataModel }
   }
 
-  update(cache: DBCache): void {
+  update(cache: DataModel): void {
     this.storage.set(LocalStorageBootstrap.CONTENT_KEY, cache)
   }
 }
diff --git a/web/projects/ui/src/app/services/patch-db/patch-db.factory.ts b/web/projects/ui/src/app/services/patch-db/patch-db.factory.ts
index 51d29edc8..bb2dcdf57 100644
--- a/web/projects/ui/src/app/services/patch-db/patch-db.factory.ts
+++ b/web/projects/ui/src/app/services/patch-db/patch-db.factory.ts
@@ -1,19 +1,19 @@
 import { InjectionToken, Injector } from '@angular/core'
+import { Revision, Update } from 'patch-db-client'
+import { defer, EMPTY, from, Observable } from 'rxjs'
 import {
   bufferTime,
   catchError,
   filter,
+  startWith,
   switchMap,
   take,
-  tap,
 } from 'rxjs/operators'
-import { Update } from 'patch-db-client'
-import { DataModel } from './data-model'
-import { defer, EMPTY, from, interval, Observable } from 'rxjs'
-import { AuthService } from '../auth.service'
-import { ConnectionService } from '../connection.service'
+import { StateService } from 'src/app/services/state.service'
 import { ApiService } from '../api/embassy-api.service'
-import { ConfigService } from '../config.service'
+import { AuthService } from '../auth.service'
+import { DataModel } from './data-model'
+import { LocalStorageBootstrap } from './local-storage-bootstrap'
 
 export const PATCH_SOURCE = new InjectionToken[]>>(
   '',
@@ -25,33 +25,31 @@ export function sourceFactory(
   // defer() needed to avoid circular dependency with ApiService, since PatchDB is needed there
   return defer(() => {
     const api = injector.get(ApiService)
-    const authService = injector.get(AuthService)
-    const connectionService = injector.get(ConnectionService)
-    const configService = injector.get(ConfigService)
-    const isTor = configService.isTor()
-    const timeout = isTor ? 16000 : 4000
+    const auth = injector.get(AuthService)
+    const state = injector.get(StateService)
+    const bootstrapper = injector.get(LocalStorageBootstrap)
 
-    const websocket$ = api.openPatchWebsocket$().pipe(
-      bufferTime(250),
-      filter(updates => !!updates.length),
-      catchError((_, watch$) => {
-        connectionService.websocketConnected$.next(false)
+    return auth.isVerified$.pipe(
+      switchMap(verified =>
+        verified ? from(api.subscribeToPatchDB({})) : EMPTY,
+      ),
+      switchMap(({ dump, guid }) =>
+        api.openWebsocket$(guid, {}).pipe(
+          bufferTime(250),
+          filter(revisions => !!revisions.length),
+          startWith([dump]),
+        ),
+      ),
+      catchError((_, original$) => {
+        state.retrigger()
 
-        return interval(timeout).pipe(
-          switchMap(() =>
-            from(api.echo({ message: 'ping', timeout })).pipe(
-              catchError(() => EMPTY),
-            ),
-          ),
+        return state.pipe(
+          filter(current => current === 'running'),
           take(1),
-          switchMap(() => watch$),
+          switchMap(() => original$),
         )
       }),
-      tap(() => connectionService.websocketConnected$.next(true)),
-    )
-
-    return authService.isVerified$.pipe(
-      switchMap(verified => (verified ? websocket$ : EMPTY)),
+      startWith([bootstrapper.init()]),
     )
   })
 }
diff --git a/web/projects/ui/src/app/services/patch-monitor.service.ts b/web/projects/ui/src/app/services/patch-monitor.service.ts
index cafb9f0fe..675531dda 100644
--- a/web/projects/ui/src/app/services/patch-monitor.service.ts
+++ b/web/projects/ui/src/app/services/patch-monitor.service.ts
@@ -4,24 +4,19 @@ import { tap } from 'rxjs/operators'
 import { PatchDB } from 'patch-db-client'
 import { AuthService } from 'src/app/services/auth.service'
 import { DataModel } from './patch-db/data-model'
-import { LocalStorageBootstrap } from './patch-db/local-storage-bootstrap'
 
 // Start and stop PatchDb upon verification
 @Injectable({
   providedIn: 'root',
 })
-export class PatchMonitorService extends Observable {
-  // @TODO not happy with Observable
+export class PatchMonitorService extends Observable {
   private readonly stream$ = this.authService.isVerified$.pipe(
-    tap(verified =>
-      verified ? this.patch.start(this.bootstrapper) : this.patch.stop(),
-    ),
+    tap(verified => (verified ? this.patch.start() : this.patch.stop())),
   )
 
   constructor(
     private readonly authService: AuthService,
     private readonly patch: PatchDB,
-    private readonly bootstrapper: LocalStorageBootstrap,
   ) {
     super(subscriber => this.stream$.subscribe(subscriber))
   }
diff --git a/web/projects/ui/src/app/services/state.service.ts b/web/projects/ui/src/app/services/state.service.ts
new file mode 100644
index 000000000..33569a751
--- /dev/null
+++ b/web/projects/ui/src/app/services/state.service.ts
@@ -0,0 +1,136 @@
+import { inject, Injectable } from '@angular/core'
+import { CanActivateFn, IsActiveMatchOptions, Router } from '@angular/router'
+import { ALWAYS_TRUE_HANDLER } from '@taiga-ui/cdk'
+import { TuiAlertService, TuiNotification } from '@taiga-ui/core'
+import {
+  BehaviorSubject,
+  combineLatest,
+  concat,
+  EMPTY,
+  exhaustMap,
+  from,
+  merge,
+  Observable,
+  startWith,
+  Subject,
+  timer,
+} from 'rxjs'
+import {
+  catchError,
+  filter,
+  map,
+  shareReplay,
+  skip,
+  switchMap,
+  take,
+  takeUntil,
+  tap,
+} from 'rxjs/operators'
+import { RR } from 'src/app/services/api/api.types'
+import { ApiService } from 'src/app/services/api/embassy-api.service'
+import { NetworkService } from 'src/app/services/network.service'
+
+const OPTIONS: IsActiveMatchOptions = {
+  paths: 'subset',
+  queryParams: 'exact',
+  fragment: 'ignored',
+  matrixParams: 'ignored',
+}
+
+@Injectable({
+  providedIn: 'root',
+})
+export class StateService extends Observable {
+  private readonly alerts = inject(TuiAlertService)
+  private readonly api = inject(ApiService)
+  private readonly router = inject(Router)
+  private readonly network$ = inject(NetworkService)
+
+  private readonly single$ = new Subject()
+
+  private readonly trigger$ = new BehaviorSubject(undefined)
+  private readonly poll$ = this.trigger$.pipe(
+    switchMap(() =>
+      timer(0, 2000).pipe(
+        switchMap(() =>
+          from(this.api.getState()).pipe(catchError(() => EMPTY)),
+        ),
+        take(1),
+      ),
+    ),
+  )
+
+  private readonly stream$ = merge(this.single$, this.poll$).pipe(
+    tap(state => {
+      switch (state) {
+        case 'initializing':
+          this.router.navigate(['initializing'], { replaceUrl: true })
+          break
+        case 'error':
+          this.router.navigate(['diagnostic'], { replaceUrl: true })
+          break
+        case 'running':
+          if (
+            this.router.isActive('initializing', OPTIONS) ||
+            this.router.isActive('diagnostic', OPTIONS)
+          ) {
+            this.router.navigate([''], { replaceUrl: true })
+          }
+
+          break
+      }
+    }),
+    startWith(null),
+    shareReplay(1),
+  )
+
+  private readonly alert = merge(
+    this.trigger$.pipe(skip(1)),
+    this.network$.pipe(filter(v => !v)),
+  )
+    .pipe(
+      exhaustMap(() =>
+        concat(
+          this.alerts
+            .open('Trying to reach server', {
+              label: 'State unknown',
+              autoClose: false,
+              status: TuiNotification.Error,
+            })
+            .pipe(
+              takeUntil(
+                combineLatest([this.stream$, this.network$]).pipe(
+                  filter(state => state.every(Boolean)),
+                ),
+              ),
+            ),
+          this.alerts.open('Connection restored', {
+            label: 'Server reached',
+            status: TuiNotification.Success,
+          }),
+        ),
+      ),
+    )
+    .subscribe() // @TODO shouldn't this be subscribed in app component with the others? Do we ever need to unsubscribe?
+
+  constructor() {
+    super(subscriber => this.stream$.subscribe(subscriber))
+  }
+
+  retrigger() {
+    this.trigger$.next()
+  }
+
+  async syncState() {
+    const state = await this.api.getState()
+    this.single$.next(state)
+  }
+}
+
+export function stateNot(state: RR.ServerState[]): CanActivateFn {
+  return () =>
+    inject(StateService).pipe(
+      filter(current => !current || !state.includes(current)),
+      map(ALWAYS_TRUE_HANDLER),
+    )
+}
diff --git a/web/projects/ui/src/app/services/storage.service.ts b/web/projects/ui/src/app/services/storage.service.ts
index ec87864b5..e59eba439 100644
--- a/web/projects/ui/src/app/services/storage.service.ts
+++ b/web/projects/ui/src/app/services/storage.service.ts
@@ -1,7 +1,7 @@
 import { Inject, Injectable } from '@angular/core'
 import { DOCUMENT } from '@angular/common'
 
-const PREFIX = '_embassystorage/_embassykv/'
+const PREFIX = '_startos/'
 
 @Injectable({
   providedIn: 'root',
@@ -15,16 +15,21 @@ export class StorageService {
     return JSON.parse(String(this.storage.getItem(`${PREFIX}${key}`)))
   }
 
-  set(key: string, value: T) {
+  set(key: string, value: any) {
     this.storage.setItem(`${PREFIX}${key}`, JSON.stringify(value))
   }
 
   clear() {
-    Array.from(
-      { length: this.storage.length },
-      (_, i) => this.storage.key(i) || '',
-    )
-      .filter(key => key.startsWith(PREFIX))
-      .forEach(key => this.storage.removeItem(key))
+    this.storage.clear()
+  }
+
+  migrate036() {
+    const oldPrefix = '_embassystorage/_embassykv/'
+    if (!!this.storage.getItem(`${oldPrefix}loggedInKey`)) {
+      const cache = this.storage.getItem(`${oldPrefix}patch-db-cache`)
+      this.clear()
+      this.set('loggedIn', true)
+      this.set('patchDB', cache)
+    }
   }
 }

From 355452cdb30c73fe3a9b134987310414a86f830b Mon Sep 17 00:00:00 2001
From: Jade <2364004+Blu-J@users.noreply.github.com>
Date: Wed, 19 Jun 2024 17:30:05 -0600
Subject: [PATCH 11/17] Feat/next packages (#2646)

* fix mac build

* wip

* chore: Update the effects to get rid of bad pattern

* chore: Some small changes

* wip

* fix: Health checks don't show during race

* fix: Restart working

---------

Co-authored-by: Aiden McClelland 
---
 .../src/Adapters/Systems/SystemForStartOs.ts  |  6 ++-
 core/startos/src/service/mod.rs               | 34 ++++++++++++++-
 .../src/service/service_effect_handler.rs     |  6 +--
 sdk/Makefile                                  |  4 +-
 sdk/lib/mainFn/Daemons.ts                     |  4 +-
 sdk/lib/mainFn/HealthDaemon.ts                |  4 +-
 sdk/lib/util/Overlay.ts                       | 41 +++++++++----------
 7 files changed, 65 insertions(+), 34 deletions(-)

diff --git a/container-runtime/src/Adapters/Systems/SystemForStartOs.ts b/container-runtime/src/Adapters/Systems/SystemForStartOs.ts
index e10434032..e7f09952a 100644
--- a/container-runtime/src/Adapters/Systems/SystemForStartOs.ts
+++ b/container-runtime/src/Adapters/Systems/SystemForStartOs.ts
@@ -123,9 +123,9 @@ export class SystemForStartOs implements System {
         return this.abi.uninit({ effects, nextVersion })
       }
       case "/main/start": {
+        if (this.onTerm) await this.onTerm()
         const started = async (onTerm: () => Promise) => {
           await effects.setMainStatus({ status: "running" })
-          if (this.onTerm) await this.onTerm()
           this.onTerm = onTerm
         }
         const daemons = await (
@@ -135,10 +135,11 @@ export class SystemForStartOs implements System {
           })
         ).build()
         this.onTerm = daemons.term
+        return
       }
       case "/main/stop": {
-        await effects.setMainStatus({ status: "stopped" })
         if (this.onTerm) await this.onTerm()
+        await effects.setMainStatus({ status: "stopped" })
         delete this.onTerm
         return duration(30, "s")
       }
@@ -183,6 +184,7 @@ export class SystemForStartOs implements System {
             return dependencyConfig.update(options.input as any) // TODO
           }
         }
+        return
     }
     throw new Error(`Method ${options.procedure} not implemented.`)
   }
diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs
index 2eecc565e..92564b8e4 100644
--- a/core/startos/src/service/mod.rs
+++ b/core/startos/src/service/mod.rs
@@ -485,7 +485,7 @@ impl Actor for ServiceActor {
             let mut current = seed.persistent_container.state.subscribe();
 
             loop {
-                let kinds = dbg!(current.borrow().kinds());
+                let kinds = current.borrow().kinds();
 
                 if let Err(e) = async {
                     let main_status = match (
@@ -493,6 +493,14 @@ impl Actor for ServiceActor {
                         kinds.desired_state,
                         kinds.running_status,
                     ) {
+                        (Some(TransitionKind::Restarting), StartStop::Stop, Some(_)) => {
+                            seed.persistent_container.stop().await?;
+                            MainStatus::Restarting
+                        }
+                        (Some(TransitionKind::Restarting), StartStop::Start, _) => {
+                            seed.persistent_container.start().await?;
+                            MainStatus::Restarting
+                        }
                         (Some(TransitionKind::Restarting), _, _) => MainStatus::Restarting,
                         (Some(TransitionKind::Restoring), _, _) => MainStatus::Restoring,
                         (Some(TransitionKind::BackingUp), _, Some(status)) => {
@@ -523,6 +531,30 @@ impl Actor for ServiceActor {
                         .mutate(|d| {
                             if let Some(i) = d.as_public_mut().as_package_data_mut().as_idx_mut(&id)
                             {
+                                let previous = i.as_status().as_main().de()?;
+                                let previous_health = previous.health();
+                                let previous_started = previous.started();
+                                let mut main_status = main_status;
+                                match &mut main_status {
+                                    &mut MainStatus::Running { ref mut health, .. }
+                                    | &mut MainStatus::BackingUp { ref mut health, .. } => {
+                                        *health = previous_health.unwrap_or(health).clone();
+                                    }
+                                    _ => (),
+                                };
+                                match &mut main_status {
+                                    MainStatus::Running {
+                                        ref mut started, ..
+                                    } => {
+                                        *started = previous_started.unwrap_or(*started);
+                                    }
+                                    MainStatus::BackingUp {
+                                        ref mut started, ..
+                                    } => {
+                                        *started = previous_started.map(Some).unwrap_or(*started);
+                                    }
+                                    _ => (),
+                                };
                                 i.as_status_mut().as_main_mut().ser(&main_status)?;
                             }
                             Ok(())
diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs
index 28a611854..bcd6da7d1 100644
--- a/core/startos/src/service/service_effect_handler.rs
+++ b/core/startos/src/service/service_effect_handler.rs
@@ -936,7 +936,6 @@ async fn stopped(context: EffectContext, params: ParamsMaybePackageId) -> Result
     Ok(json!(matches!(package, MainStatus::Stopped)))
 }
 async fn running(context: EffectContext, params: ParamsPackageId) -> Result {
-    dbg!("Starting the running {params:?}");
     let context = context.deref()?;
     let peeked = context.seed.ctx.db.peek().await;
     let package_id = params.package_id;
@@ -956,9 +955,7 @@ async fn restart(
     WithProcedureId { procedure_id, .. }: WithProcedureId,
 ) -> Result<(), Error> {
     let context = context.deref()?;
-    dbg!("here");
     context.restart(procedure_id).await?;
-    dbg!("here");
     Ok(())
 }
 
@@ -1032,12 +1029,11 @@ struct SetMainStatus {
     status: SetMainStatusStatus,
 }
 async fn set_main_status(context: EffectContext, params: SetMainStatus) -> Result {
-    dbg!(format!("Status for main will be is {params:?}"));
     let context = context.deref()?;
     match params.status {
         SetMainStatusStatus::Running => context.seed.started(),
         SetMainStatusStatus::Stopped => context.seed.stopped(),
-        SetMainStatusStatus::Starting => context.seed.stopped(),
+        SetMainStatusStatus::Starting => context.seed.started(),
     }
     Ok(Value::Null)
 }
diff --git a/sdk/Makefile b/sdk/Makefile
index 091f4c7bb..4d01fa3d7 100644
--- a/sdk/Makefile
+++ b/sdk/Makefile
@@ -25,7 +25,7 @@ dist: $(TS_FILES) package.json node_modules README.md LICENSE
 	cp LICENSE dist/LICENSE
 	touch dist
 
-full-bundle: clean bundle
+full-bundle: bundle
 
 check:
 	npm run check
@@ -36,7 +36,7 @@ fmt: node_modules
 node_modules: package.json
 	npm ci
 
-publish: clean bundle package.json README.md LICENSE
+publish: bundle package.json README.md LICENSE
 	cd dist && npm publish --access=public
 
 link: bundle
diff --git a/sdk/lib/mainFn/Daemons.ts b/sdk/lib/mainFn/Daemons.ts
index d51d444a8..f48dd51c0 100644
--- a/sdk/lib/mainFn/Daemons.ts
+++ b/sdk/lib/mainFn/Daemons.ts
@@ -154,7 +154,7 @@ export class Daemons {
     this.healthDaemons.forEach((x) =>
       x.addWatcher(() => this.updateMainHealth()),
     )
-    return {
+    const built = {
       term: async (options?: { signal?: Signals; timeout?: number }) => {
         try {
           await Promise.all(this.healthDaemons.map((x) => x.term(options)))
@@ -163,6 +163,8 @@ export class Daemons {
         }
       },
     }
+    this.started(() => built.term())
+    return built
   }
 
   private updateMainHealth() {
diff --git a/sdk/lib/mainFn/HealthDaemon.ts b/sdk/lib/mainFn/HealthDaemon.ts
index 84e9e34d7..48f3fab55 100644
--- a/sdk/lib/mainFn/HealthDaemon.ts
+++ b/sdk/lib/mainFn/HealthDaemon.ts
@@ -132,14 +132,14 @@ export class HealthDaemon {
       this.effects.setHealth({
         result: status,
         message: health.message,
-        id: display,
+        id: this.id,
         name: display,
       })
     } else {
       this.effects.setHealth({
         result: health.status,
         message: health.message || "",
-        id: display,
+        id: this.id,
         name: display,
       })
     }
diff --git a/sdk/lib/util/Overlay.ts b/sdk/lib/util/Overlay.ts
index fbc854c57..4d6d36b34 100644
--- a/sdk/lib/util/Overlay.ts
+++ b/sdk/lib/util/Overlay.ts
@@ -27,12 +27,11 @@ export class Overlay {
     }
 
     for (const dirPart of shared) {
-      await fs.mkdir(`${rootfs}/${dirPart}`, { recursive: true })
-      await execFile("mount", [
-        "--rbind",
-        `/${dirPart}`,
-        `${rootfs}/${dirPart}`,
-      ])
+      const from = `/${dirPart}`
+      const to = `${rootfs}/${dirPart}`
+      await fs.mkdir(from, { recursive: true })
+      await fs.mkdir(to, { recursive: true })
+      await execFile("mount", ["--rbind", from, to])
     }
 
     return new Overlay(effects, id, rootfs, guid)
@@ -48,22 +47,22 @@ export class Overlay {
           ? options.subpath
           : `/${options.subpath}`
         : "/"
-      await execFile("mount", [
-        "--bind",
-        `/media/startos/volumes/${options.id}${subpath}`,
-        path,
-      ])
+      const from = `/media/startos/volumes/${options.id}${subpath}`
+
+      await fs.mkdir(from, { recursive: true })
+      await fs.mkdir(path, { recursive: true })
+      await await execFile("mount", ["--bind", from, path])
     } else if (options.type === "assets") {
       const subpath = options.subpath
         ? options.subpath.startsWith("/")
           ? options.subpath
           : `/${options.subpath}`
         : "/"
-      await execFile("mount", [
-        "--bind",
-        `/media/startos/assets/${options.id}${subpath}`,
-        path,
-      ])
+      const from = `/media/startos/assets/${options.id}${subpath}`
+
+      await fs.mkdir(from, { recursive: true })
+      await fs.mkdir(path, { recursive: true })
+      await execFile("mount", ["--bind", from, path])
     } else if (options.type === "pointer") {
       await this.effects.mount({ location: path, target: options })
     } else if (options.type === "backup") {
@@ -72,11 +71,11 @@ export class Overlay {
           ? options.subpath
           : `/${options.subpath}`
         : "/"
-      await execFile("mount", [
-        "--bind",
-        `/media/startos/backup${subpath}`,
-        path,
-      ])
+      const from = `/media/startos/backup${subpath}`
+
+      await fs.mkdir(from, { recursive: true })
+      await fs.mkdir(path, { recursive: true })
+      await execFile("mount", ["--bind", from, path])
     } else {
       throw new Error(`unknown type ${(options as any).type}`)
     }

From 07104b18f579e96a7fdea27a613355176eb710ca Mon Sep 17 00:00:00 2001
From: Mariusz Kogen 
Date: Thu, 20 Jun 2024 20:59:16 +0200
Subject: [PATCH 12/17] Update workflows actions (#2628)

* Update workflows actions to the latest versions
---
 .github/workflows/startos-iso.yaml | 20 ++++++++++----------
 .github/workflows/test.yaml        |  4 ++--
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/startos-iso.yaml b/.github/workflows/startos-iso.yaml
index 30b2bec0f..c51eafc5c 100644
--- a/.github/workflows/startos-iso.yaml
+++ b/.github/workflows/startos-iso.yaml
@@ -71,27 +71,27 @@ jobs:
           sudo mount -t tmpfs tmpfs .
         if: ${{ github.event.inputs.runner == 'fast' }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODEJS_VERSION }}
 
       - name: Set up docker QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up system dependencies
         run: sudo apt-get update && sudo apt-get install -y qemu-user-static systemd-container
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Make
         run: make ARCH=${{ matrix.arch }} compiled-${{ matrix.arch }}.tar
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: compiled-${{ matrix.arch }}.tar
           path: compiled-${{ matrix.arch }}.tar
@@ -144,7 +144,7 @@ jobs:
         run: rm -rf /opt/hostedtoolcache*
         if: ${{ github.event.inputs.runner != 'fast' }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
 
@@ -166,7 +166,7 @@ jobs:
         if: ${{ github.event.inputs.runner == 'fast' && (matrix.platform == 'x86_64' || matrix.platform == 'x86_64-nonfree') }}
 
       - name: Download compiled artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: compiled-${{ env.ARCH }}.tar
 
@@ -190,18 +190,18 @@ jobs:
         run: PLATFORM=${{ matrix.platform }} make img
         if: ${{ matrix.platform == 'raspberrypi' }}
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.platform }}.squashfs
           path: results/*.squashfs
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.platform }}.iso
           path: results/*.iso
         if: ${{ matrix.platform != 'raspberrypi' }}
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.platform }}.img
           path: results/*.img
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index c6082ac25..0a5eb38e9 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -19,11 +19,11 @@ jobs:
     name: Run Automated Tests
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODEJS_VERSION }}
 

From e6abf4e33b841961ff37c701daa2eb4e203d7e9f Mon Sep 17 00:00:00 2001
From: waterplea 
Date: Fri, 21 Jun 2024 15:51:04 +0500
Subject: [PATCH 13/17] feat: get rid of cyclic dep between patch-db and api
 service

Signed-off-by: waterplea 
---
 web/projects/ui/src/app/app.module.ts         |  2 -
 web/projects/ui/src/app/app.providers.ts      | 10 ++++
 .../services/api/embassy-live-api.service.ts  |  7 ++-
 .../app/services/patch-db/patch-db-source.ts  | 56 +++++++++++++++++++
 .../app/services/patch-db/patch-db.factory.ts | 55 ------------------
 .../app/services/patch-db/patch-db.module.ts  | 20 -------
 6 files changed, 70 insertions(+), 80 deletions(-)
 create mode 100644 web/projects/ui/src/app/services/patch-db/patch-db-source.ts
 delete mode 100644 web/projects/ui/src/app/services/patch-db/patch-db.factory.ts
 delete mode 100644 web/projects/ui/src/app/services/patch-db/patch-db.module.ts

diff --git a/web/projects/ui/src/app/app.module.ts b/web/projects/ui/src/app/app.module.ts
index c0264f064..f13b4fbfa 100644
--- a/web/projects/ui/src/app/app.module.ts
+++ b/web/projects/ui/src/app/app.module.ts
@@ -28,7 +28,6 @@ import { PreloaderModule } from './app/preloader/preloader.module'
 import { FooterModule } from './app/footer/footer.module'
 import { MenuModule } from './app/menu/menu.module'
 import { APP_PROVIDERS } from './app.providers'
-import { PatchDbModule } from './services/patch-db/patch-db.module'
 import { ToastContainerModule } from './components/toast-container/toast-container.module'
 import { ConnectionBarComponentModule } from './components/connection-bar/connection-bar.component.module'
 import { WidgetsPageModule } from './pages/widgets/widgets.module'
@@ -54,7 +53,6 @@ import { environment } from '../environments/environment'
     MonacoEditorModule,
     SharedPipesModule,
     MarketplaceModule,
-    PatchDbModule,
     ToastContainerModule,
     ConnectionBarComponentModule,
     TuiRootModule,
diff --git a/web/projects/ui/src/app/app.providers.ts b/web/projects/ui/src/app/app.providers.ts
index bf26a8cb9..c6a8d756e 100644
--- a/web/projects/ui/src/app/app.providers.ts
+++ b/web/projects/ui/src/app/app.providers.ts
@@ -3,6 +3,11 @@ import { UntypedFormBuilder } from '@angular/forms'
 import { Router, RouteReuseStrategy } from '@angular/router'
 import { IonicRouteStrategy, IonNav } from '@ionic/angular'
 import { RELATIVE_URL, THEME, WorkspaceConfig } from '@start9labs/shared'
+import { PatchDB } from 'patch-db-client'
+import {
+  PATCH_CACHE,
+  PatchDbSource,
+} from 'src/app/services/patch-db/patch-db-source'
 import { ApiService } from './services/api/embassy-api.service'
 import { MockApiService } from './services/api/embassy-mock-api.service'
 import { LiveApiService } from './services/api/embassy-live-api.service'
@@ -29,6 +34,11 @@ export const APP_PROVIDERS: Provider[] = [
     provide: ApiService,
     useClass: useMocks ? MockApiService : LiveApiService,
   },
+  {
+    provide: PatchDB,
+    deps: [PatchDbSource, PATCH_CACHE],
+    useClass: PatchDB,
+  },
   {
     provide: APP_INITIALIZER,
     deps: [StorageService, AuthService, ClientStorageService, Router],
diff --git a/web/projects/ui/src/app/services/api/embassy-live-api.service.ts b/web/projects/ui/src/app/services/api/embassy-live-api.service.ts
index 8826b2883..954a4475e 100644
--- a/web/projects/ui/src/app/services/api/embassy-live-api.service.ts
+++ b/web/projects/ui/src/app/services/api/embassy-live-api.service.ts
@@ -7,6 +7,7 @@ import {
   RpcError,
   RPCOptions,
 } from '@start9labs/shared'
+import { PATCH_CACHE } from 'src/app/services/patch-db/patch-db-source'
 import { ApiService } from './embassy-api.service'
 import { RR } from './api.types'
 import { parsePropertiesPermissive } from 'src/app/util/properties.util'
@@ -16,7 +17,7 @@ import { Observable, filter, firstValueFrom } from 'rxjs'
 import { AuthService } from '../auth.service'
 import { DOCUMENT } from '@angular/common'
 import { DataModel } from '../patch-db/data-model'
-import { PatchDB, pathFromArray } from 'patch-db-client'
+import { Dump, pathFromArray } from 'patch-db-client'
 
 @Injectable()
 export class LiveApiService extends ApiService {
@@ -25,7 +26,7 @@ export class LiveApiService extends ApiService {
     private readonly http: HttpService,
     private readonly config: ConfigService,
     private readonly auth: AuthService,
-    private readonly patch: PatchDB,
+    @Inject(PATCH_CACHE) private readonly cache$: Observable>,
   ) {
     super()
     ; (window as any).rpcClient = this
@@ -482,7 +483,7 @@ export class LiveApiService extends ApiService {
     const patchSequence = res.headers.get('x-patch-sequence')
     if (patchSequence)
       await firstValueFrom(
-        this.patch.cache$.pipe(filter(({ id }) => id >= Number(patchSequence))),
+        this.cache$.pipe(filter(({ id }) => id >= Number(patchSequence))),
       )
 
     return body.result
diff --git a/web/projects/ui/src/app/services/patch-db/patch-db-source.ts b/web/projects/ui/src/app/services/patch-db/patch-db-source.ts
new file mode 100644
index 000000000..803fbce2c
--- /dev/null
+++ b/web/projects/ui/src/app/services/patch-db/patch-db-source.ts
@@ -0,0 +1,56 @@
+import { inject, Injectable, InjectionToken } from '@angular/core'
+import { Dump, Revision, Update } from 'patch-db-client'
+import { BehaviorSubject, EMPTY, Observable } from 'rxjs'
+import {
+  bufferTime,
+  catchError,
+  filter,
+  startWith,
+  switchMap,
+  take,
+} from 'rxjs/operators'
+import { StateService } from 'src/app/services/state.service'
+import { ApiService } from '../api/embassy-api.service'
+import { AuthService } from '../auth.service'
+import { DataModel } from './data-model'
+import { LocalStorageBootstrap } from './local-storage-bootstrap'
+
+export const PATCH_CACHE = new InjectionToken('', {
+  factory: () =>
+    new BehaviorSubject>({
+      id: 0,
+      value: {} as DataModel,
+    }),
+})
+
+@Injectable({
+  providedIn: 'root',
+})
+export class PatchDbSource extends Observable[]> {
+  private readonly api = inject(ApiService)
+  private readonly state = inject(StateService)
+  private readonly stream$ = inject(AuthService).isVerified$.pipe(
+    switchMap(verified => (verified ? this.api.subscribeToPatchDB({}) : EMPTY)),
+    switchMap(({ dump, guid }) =>
+      this.api.openWebsocket$(guid, {}).pipe(
+        bufferTime(250),
+        filter(revisions => !!revisions.length),
+        startWith([dump]),
+      ),
+    ),
+    catchError((_, original$) => {
+      this.state.retrigger()
+
+      return this.state.pipe(
+        filter(current => current === 'running'),
+        take(1),
+        switchMap(() => original$),
+      )
+    }),
+    startWith([inject(LocalStorageBootstrap).init()]),
+  )
+
+  constructor() {
+    super(subscriber => this.stream$.subscribe(subscriber))
+  }
+}
diff --git a/web/projects/ui/src/app/services/patch-db/patch-db.factory.ts b/web/projects/ui/src/app/services/patch-db/patch-db.factory.ts
deleted file mode 100644
index bb2dcdf57..000000000
--- a/web/projects/ui/src/app/services/patch-db/patch-db.factory.ts
+++ /dev/null
@@ -1,55 +0,0 @@
-import { InjectionToken, Injector } from '@angular/core'
-import { Revision, Update } from 'patch-db-client'
-import { defer, EMPTY, from, Observable } from 'rxjs'
-import {
-  bufferTime,
-  catchError,
-  filter,
-  startWith,
-  switchMap,
-  take,
-} from 'rxjs/operators'
-import { StateService } from 'src/app/services/state.service'
-import { ApiService } from '../api/embassy-api.service'
-import { AuthService } from '../auth.service'
-import { DataModel } from './data-model'
-import { LocalStorageBootstrap } from './local-storage-bootstrap'
-
-export const PATCH_SOURCE = new InjectionToken[]>>(
-  '',
-)
-
-export function sourceFactory(
-  injector: Injector,
-): Observable[]> {
-  // defer() needed to avoid circular dependency with ApiService, since PatchDB is needed there
-  return defer(() => {
-    const api = injector.get(ApiService)
-    const auth = injector.get(AuthService)
-    const state = injector.get(StateService)
-    const bootstrapper = injector.get(LocalStorageBootstrap)
-
-    return auth.isVerified$.pipe(
-      switchMap(verified =>
-        verified ? from(api.subscribeToPatchDB({})) : EMPTY,
-      ),
-      switchMap(({ dump, guid }) =>
-        api.openWebsocket$(guid, {}).pipe(
-          bufferTime(250),
-          filter(revisions => !!revisions.length),
-          startWith([dump]),
-        ),
-      ),
-      catchError((_, original$) => {
-        state.retrigger()
-
-        return state.pipe(
-          filter(current => current === 'running'),
-          take(1),
-          switchMap(() => original$),
-        )
-      }),
-      startWith([bootstrapper.init()]),
-    )
-  })
-}
diff --git a/web/projects/ui/src/app/services/patch-db/patch-db.module.ts b/web/projects/ui/src/app/services/patch-db/patch-db.module.ts
deleted file mode 100644
index 3c816e339..000000000
--- a/web/projects/ui/src/app/services/patch-db/patch-db.module.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-import { PatchDB } from 'patch-db-client'
-import { Injector, NgModule } from '@angular/core'
-import { PATCH_SOURCE, sourceFactory } from './patch-db.factory'
-
-// This module is purely for providers organization purposes
-@NgModule({
-  providers: [
-    {
-      provide: PATCH_SOURCE,
-      deps: [Injector],
-      useFactory: sourceFactory,
-    },
-    {
-      provide: PatchDB,
-      deps: [PATCH_SOURCE],
-      useClass: PatchDB,
-    },
-  ],
-})
-export class PatchDbModule {}

From e0d23f4436cb7bc28106702efaf45a5c581fca9e Mon Sep 17 00:00:00 2001
From: Matt Hill 
Date: Sat, 22 Jun 2024 11:33:30 -0600
Subject: [PATCH 14/17] bump patchDB dep

---
 patch-db | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/patch-db b/patch-db
index 7aa53249f..c537a07ea 160000
--- a/patch-db
+++ b/patch-db
@@ -1 +1 @@
-Subproject commit 7aa53249f9353162475ea347abac92abcfba5493
+Subproject commit c537a07ea937e69b66841d903c70fd75623e5457

From 68ed1c80ce5edf546081e86e9cb62309fff4112a Mon Sep 17 00:00:00 2001
From: Matt Hill 
Date: Sat, 22 Jun 2024 21:47:18 -0600
Subject: [PATCH 15/17] update todos

---
 web/projects/ui/src/app/app.providers.ts                 | 2 +-
 .../ui/src/app/modals/generic-form/generic-form.page.ts  | 1 -
 .../pages/apps-routes/app-actions/app-actions.page.ts    | 9 ++-------
 web/projects/ui/src/app/pages/init/init.service.ts       | 1 +
 web/projects/ui/src/app/services/api/api.fixures.ts      | 6 +++---
 web/projects/ui/src/app/services/api/mock-patch.ts       | 2 +-
 .../ui/src/app/services/patch-db/patch-db-source.ts      | 1 +
 7 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/web/projects/ui/src/app/app.providers.ts b/web/projects/ui/src/app/app.providers.ts
index c6a8d756e..e93c323d0 100644
--- a/web/projects/ui/src/app/app.providers.ts
+++ b/web/projects/ui/src/app/app.providers.ts
@@ -64,7 +64,7 @@ export function appInitializer(
   return () => {
     storage.migrate036()
     auth.init()
-    localStorage.init() // @TODO pretty sure we can navigate before this step
+    localStorage.init()
     router.initialNavigation()
   }
 }
diff --git a/web/projects/ui/src/app/modals/generic-form/generic-form.page.ts b/web/projects/ui/src/app/modals/generic-form/generic-form.page.ts
index eb690a787..e74c65d47 100644
--- a/web/projects/ui/src/app/modals/generic-form/generic-form.page.ts
+++ b/web/projects/ui/src/app/modals/generic-form/generic-form.page.ts
@@ -54,7 +54,6 @@ export class GenericFormPage {
       return
     }
 
-    // @TODO make this more like generic input component dismissal
     const success = await handler(this.formGroup.value)
     if (success !== false) this.modalCtrl.dismiss()
   }
diff --git a/web/projects/ui/src/app/pages/apps-routes/app-actions/app-actions.page.ts b/web/projects/ui/src/app/pages/apps-routes/app-actions/app-actions.page.ts
index a48ac8185..fc008a17a 100644
--- a/web/projects/ui/src/app/pages/apps-routes/app-actions/app-actions.page.ts
+++ b/web/projects/ui/src/app/pages/apps-routes/app-actions/app-actions.page.ts
@@ -44,12 +44,7 @@ export class AppActionsPage {
     status: T.Status,
     action: { key: string; value: T.ActionMetadata },
   ) {
-    if (
-      status &&
-      action.value.allowedStatuses.includes(
-        status.main.status, // @TODO
-      )
-    ) {
+    if (status && action.value.allowedStatuses.includes(status.main.status)) {
       if (!isEmptyObject(action.value.input || {})) {
         const modal = await this.modalCtrl.create({
           component: GenericFormPage,
@@ -91,7 +86,7 @@ export class AppActionsPage {
         await alert.present()
       }
     } else {
-      const statuses = [...action.value.allowedStatuses] // @TODO
+      const statuses = [...action.value.allowedStatuses]
       const last = statuses.pop()
       let statusesStr = statuses.join(', ')
       let error = ''
diff --git a/web/projects/ui/src/app/pages/init/init.service.ts b/web/projects/ui/src/app/pages/init/init.service.ts
index 3cca42a58..7102c9db5 100644
--- a/web/projects/ui/src/app/pages/init/init.service.ts
+++ b/web/projects/ui/src/app/pages/init/init.service.ts
@@ -58,6 +58,7 @@ export class InitService extends Observable {
       }
     }),
     catchError(e => {
+      // @TODO this toast is presenting when we navigate away from init page. It seems other websockets exhibit the same behavior, but we never noticed because the error were not being caught and presented in this manner
       this.errorService.present(e)
 
       return EMPTY
diff --git a/web/projects/ui/src/app/services/api/api.fixures.ts b/web/projects/ui/src/app/services/api/api.fixures.ts
index 38ee4774b..acebdfdfb 100644
--- a/web/projects/ui/src/app/services/api/api.fixures.ts
+++ b/web/projects/ui/src/app/services/api/api.fixures.ts
@@ -714,7 +714,7 @@ export module Mock {
         value: 'https://guessagain.com',
       },
     },
-  } as any // @TODO why is this necessary?
+  }
 
   export const ConfigSpec: RR.GetPackageConfigRes['spec'] = {
     bitcoin: {
@@ -1419,7 +1419,7 @@ export module Mock {
         health: {},
       },
     },
-    actions: {}, // @TODO need mocks
+    actions: {},
     serviceInterfaces: {
       ui: {
         id: 'ui',
@@ -1618,7 +1618,7 @@ export module Mock {
         icon: 'assets/img/service-icons/btc-rpc-proxy.png',
         kind: 'exists',
         registryUrl: 'https://community-registry.start9.com',
-        versionSpec: '>2.0.0', // @TODO
+        versionSpec: '>2.0.0',
         configSatisfied: false,
       },
     },
diff --git a/web/projects/ui/src/app/services/api/mock-patch.ts b/web/projects/ui/src/app/services/api/mock-patch.ts
index ee96f9fe2..755799522 100644
--- a/web/projects/ui/src/app/services/api/mock-patch.ts
+++ b/web/projects/ui/src/app/services/api/mock-patch.ts
@@ -127,7 +127,7 @@ export const mockPatchData: DataModel = {
           },
         },
       },
-      actions: {}, // @TODO
+      actions: {},
       serviceInterfaces: {
         ui: {
           id: 'ui',
diff --git a/web/projects/ui/src/app/services/patch-db/patch-db-source.ts b/web/projects/ui/src/app/services/patch-db/patch-db-source.ts
index 803fbce2c..0b80370d0 100644
--- a/web/projects/ui/src/app/services/patch-db/patch-db-source.ts
+++ b/web/projects/ui/src/app/services/patch-db/patch-db-source.ts
@@ -41,6 +41,7 @@ export class PatchDbSource extends Observable[]> {
     catchError((_, original$) => {
       this.state.retrigger()
 
+      // @TODO this is returning right away, but we need to wait until state emits again from the retrigger() above.
       return this.state.pipe(
         filter(current => current === 'running'),
         take(1),

From 2c255b6dfefb31e9ef9187cf8f54b8240a176049 Mon Sep 17 00:00:00 2001
From: Jade <2364004+Blu-J@users.noreply.github.com>
Date: Mon, 24 Jun 2024 16:00:31 -0600
Subject: [PATCH 16/17] chore: Do some type cleanups (#2650)

chore: fix the WithProcedureId
---
 .../src/service/service_effect_handler.rs     | 124 ++++++++----------
 sdk/lib/mainFn/Daemons.ts                     |   6 +-
 sdk/lib/osBindings/ExecuteAction.ts           |   2 +
 sdk/lib/osBindings/ProcedureId.ts             |   4 +
 sdk/lib/osBindings/SetDependenciesParams.ts   |   2 +
 sdk/lib/osBindings/SetMainStatusStatus.ts     |   2 +-
 sdk/lib/osBindings/index.ts                   |   1 +
 7 files changed, 69 insertions(+), 72 deletions(-)
 create mode 100644 sdk/lib/osBindings/ProcedureId.ts

diff --git a/core/startos/src/service/service_effect_handler.rs b/core/startos/src/service/service_effect_handler.rs
index bcd6da7d1..a61a00b9b 100644
--- a/core/startos/src/service/service_effect_handler.rs
+++ b/core/startos/src/service/service_effect_handler.rs
@@ -64,56 +64,6 @@ impl EffectContext {
     }
 }
 
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct WithProcedureId {
-    #[serde(default)]
-    procedure_id: Guid,
-    #[serde(flatten)]
-    rest: T,
-}
-impl FromArgMatches for WithProcedureId {
-    fn from_arg_matches(matches: &clap::ArgMatches) -> Result {
-        let rest = T::from_arg_matches(matches)?;
-        Ok(Self {
-            procedure_id: matches.get_one("procedure-id").cloned().unwrap_or_default(),
-            rest,
-        })
-    }
-    fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result {
-        let rest = T::from_arg_matches_mut(matches)?;
-        Ok(Self {
-            procedure_id: matches.get_one("procedure-id").cloned().unwrap_or_default(),
-            rest,
-        })
-    }
-    fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> {
-        self.rest.update_from_arg_matches(matches)?;
-        self.procedure_id = matches.get_one("procedure-id").cloned().unwrap_or_default();
-        Ok(())
-    }
-    fn update_from_arg_matches_mut(
-        &mut self,
-        matches: &mut clap::ArgMatches,
-    ) -> Result<(), clap::Error> {
-        self.rest.update_from_arg_matches_mut(matches)?;
-        self.procedure_id = matches.get_one("procedure-id").cloned().unwrap_or_default();
-        Ok(())
-    }
-}
-impl CommandFactory for WithProcedureId {
-    fn command() -> clap::Command {
-        T::command_for_update().arg(
-            clap::Arg::new("procedure-id")
-                .action(clap::ArgAction::Set)
-                .value_parser(clap::value_parser!(Guid)),
-        )
-    }
-    fn command_for_update() -> clap::Command {
-        Self::command()
-    }
-}
-
 pub fn service_effect_handler() -> ParentHandler {
     ParentHandler::new()
         .subcommand("gitInfo", from_fn(|_: C| crate::version::git_info()))
@@ -877,6 +827,8 @@ async fn exists(context: EffectContext, params: ParamsPackageId) -> Result,
     #[ts(type = "string")]
@@ -886,15 +838,12 @@ struct ExecuteAction {
 }
 async fn execute_action(
     context: EffectContext,
-    WithProcedureId {
+    ExecuteAction {
         procedure_id,
-        rest:
-            ExecuteAction {
-                service_id,
-                action_id,
-                input,
-            },
-    }: WithProcedureId,
+        service_id,
+        action_id,
+        input,
+    }: ExecuteAction,
 ) -> Result {
     let context = context.deref()?;
     let package_id = service_id
@@ -950,9 +899,53 @@ async fn running(context: EffectContext, params: ParamsPackageId) -> Result Result {
+        Ok(Self {
+            procedure_id: matches.get_one("procedure-id").cloned().unwrap_or_default(),
+        })
+    }
+    fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result {
+        Ok(Self {
+            procedure_id: matches.get_one("procedure-id").cloned().unwrap_or_default(),
+        })
+    }
+    fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> {
+        self.procedure_id = matches.get_one("procedure-id").cloned().unwrap_or_default();
+        Ok(())
+    }
+    fn update_from_arg_matches_mut(
+        &mut self,
+        matches: &mut clap::ArgMatches,
+    ) -> Result<(), clap::Error> {
+        self.procedure_id = matches.get_one("procedure-id").cloned().unwrap_or_default();
+        Ok(())
+    }
+}
+impl CommandFactory for ProcedureId {
+    fn command() -> clap::Command {
+        Self::command_for_update().arg(
+            clap::Arg::new("procedure-id")
+                .action(clap::ArgAction::Set)
+                .value_parser(clap::value_parser!(Guid)),
+        )
+    }
+    fn command_for_update() -> clap::Command {
+        Self::command()
+    }
+}
+
 async fn restart(
     context: EffectContext,
-    WithProcedureId { procedure_id, .. }: WithProcedureId,
+    ProcedureId { procedure_id }: ProcedureId,
 ) -> Result<(), Error> {
     let context = context.deref()?;
     context.restart(procedure_id).await?;
@@ -961,7 +954,7 @@ async fn restart(
 
 async fn shutdown(
     context: EffectContext,
-    WithProcedureId { procedure_id, .. }: WithProcedureId,
+    ProcedureId { procedure_id }: ProcedureId,
 ) -> Result<(), Error> {
     let context = context.deref()?;
     context.stop(procedure_id).await?;
@@ -1001,7 +994,6 @@ async fn set_configured(context: EffectContext, params: SetConfigured) -> Result
 enum SetMainStatusStatus {
     Running,
     Stopped,
-    Starting,
 }
 impl FromStr for SetMainStatusStatus {
     type Err = color_eyre::eyre::Report;
@@ -1009,7 +1001,6 @@ impl FromStr for SetMainStatusStatus {
         match s {
             "running" => Ok(Self::Running),
             "stopped" => Ok(Self::Stopped),
-            "starting" => Ok(Self::Starting),
             _ => Err(eyre!("unknown status {s}")),
         }
     }
@@ -1033,7 +1024,6 @@ async fn set_main_status(context: EffectContext, params: SetMainStatus) -> Resul
     match params.status {
         SetMainStatusStatus::Running => context.seed.started(),
         SetMainStatusStatus::Stopped => context.seed.stopped(),
-        SetMainStatusStatus::Starting => context.seed.started(),
     }
     Ok(Value::Null)
 }
@@ -1262,15 +1252,17 @@ impl ValueParserFactory for DependencyRequirement {
 #[command(rename_all = "camelCase")]
 #[ts(export)]
 struct SetDependenciesParams {
+    #[serde(default)]
+    procedure_id: Guid,
     dependencies: Vec,
 }
 
 async fn set_dependencies(
     context: EffectContext,
-    WithProcedureId {
+    SetDependenciesParams {
         procedure_id,
-        rest: SetDependenciesParams { dependencies },
-    }: WithProcedureId,
+        dependencies,
+    }: SetDependenciesParams,
 ) -> Result<(), Error> {
     let context = context.deref()?;
     let id = &context.seed.id;
diff --git a/sdk/lib/mainFn/Daemons.ts b/sdk/lib/mainFn/Daemons.ts
index f48dd51c0..059d148ab 100644
--- a/sdk/lib/mainFn/Daemons.ts
+++ b/sdk/lib/mainFn/Daemons.ts
@@ -168,10 +168,6 @@ export class Daemons {
   }
 
   private updateMainHealth() {
-    if (this.healthDaemons.every((x) => x.health.status === "success")) {
-      this.effects.setMainStatus({ status: "running" })
-    } else {
-      this.effects.setMainStatus({ status: "starting" })
-    }
+    this.effects.setMainStatus({ status: "running" })
   }
 }
diff --git a/sdk/lib/osBindings/ExecuteAction.ts b/sdk/lib/osBindings/ExecuteAction.ts
index abd8c151f..b4eb60949 100644
--- a/sdk/lib/osBindings/ExecuteAction.ts
+++ b/sdk/lib/osBindings/ExecuteAction.ts
@@ -1,6 +1,8 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Guid } from "./Guid"
 
 export type ExecuteAction = {
+  procedureId: Guid
   serviceId: string | null
   actionId: string
   input: any
diff --git a/sdk/lib/osBindings/ProcedureId.ts b/sdk/lib/osBindings/ProcedureId.ts
new file mode 100644
index 000000000..4d9a0debd
--- /dev/null
+++ b/sdk/lib/osBindings/ProcedureId.ts
@@ -0,0 +1,4 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Guid } from "./Guid"
+
+export type ProcedureId = { procedureId: Guid }
diff --git a/sdk/lib/osBindings/SetDependenciesParams.ts b/sdk/lib/osBindings/SetDependenciesParams.ts
index 7b34b50c9..bbc9b325f 100644
--- a/sdk/lib/osBindings/SetDependenciesParams.ts
+++ b/sdk/lib/osBindings/SetDependenciesParams.ts
@@ -1,6 +1,8 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { DependencyRequirement } from "./DependencyRequirement"
+import type { Guid } from "./Guid"
 
 export type SetDependenciesParams = {
+  procedureId: Guid
   dependencies: Array
 }
diff --git a/sdk/lib/osBindings/SetMainStatusStatus.ts b/sdk/lib/osBindings/SetMainStatusStatus.ts
index 6db32e7bf..03bb4a119 100644
--- a/sdk/lib/osBindings/SetMainStatusStatus.ts
+++ b/sdk/lib/osBindings/SetMainStatusStatus.ts
@@ -1,3 +1,3 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export type SetMainStatusStatus = "running" | "stopped" | "starting"
+export type SetMainStatusStatus = "running" | "stopped"
diff --git a/sdk/lib/osBindings/index.ts b/sdk/lib/osBindings/index.ts
index ac2e19e45..22b03a7ca 100644
--- a/sdk/lib/osBindings/index.ts
+++ b/sdk/lib/osBindings/index.ts
@@ -108,6 +108,7 @@ export { PackageVersionInfo } from "./PackageVersionInfo"
 export { ParamsMaybePackageId } from "./ParamsMaybePackageId"
 export { ParamsPackageId } from "./ParamsPackageId"
 export { PasswordType } from "./PasswordType"
+export { ProcedureId } from "./ProcedureId"
 export { Progress } from "./Progress"
 export { Public } from "./Public"
 export { RecoverySource } from "./RecoverySource"

From 9da49be44d0da70c28755dd531ea689813fc10c6 Mon Sep 17 00:00:00 2001
From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>
Date: Mon, 24 Jun 2024 16:15:56 -0600
Subject: [PATCH 17/17] Bugfix/patch db subscriber (#2652)

* fix socket sending empty patches

* do not timeout tcp connections, just poll them more

* switch from poll to tcp keepalive
---
 core/Cargo.lock                      |  2 +
 core/startos/Cargo.toml              |  2 +
 core/startos/src/net/utils.rs        | 18 ------
 core/startos/src/net/vhost.rs        | 85 ++++++++++++++++------------
 core/startos/src/s9pk/v2/manifest.rs |  2 +-
 core/startos/src/util/io.rs          | 36 +++++++++---
 patch-db                             |  2 +-
 7 files changed, 83 insertions(+), 64 deletions(-)

diff --git a/core/Cargo.lock b/core/Cargo.lock
index 031ec41ab..f8db0bd7e 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -4881,6 +4881,7 @@ dependencies = [
  "hmac",
  "http 1.1.0",
  "http-body-util",
+ "hyper-util",
  "id-pool",
  "imbl",
  "imbl-value",
@@ -4936,6 +4937,7 @@ dependencies = [
  "sha2 0.10.8",
  "shell-words",
  "simple-logging",
+ "socket2",
  "sqlx",
  "sscanf",
  "ssh-key",
diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml
index a8707bf65..750bc4953 100644
--- a/core/startos/Cargo.toml
+++ b/core/startos/Cargo.toml
@@ -97,6 +97,7 @@ hex = "0.4.3"
 hmac = "0.12.1"
 http = "1.0.0"
 http-body-util = "0.1"
+hyper-util = { version = "0.1.5", features = ["tokio", "service"] }
 id-pool = { version = "0.2.2", default-features = false, features = [
   "serde",
   "u16",
@@ -159,6 +160,7 @@ serde_yaml = { package = "serde_yml", version = "0.0.10" }
 sha2 = "0.10.2"
 shell-words = "1"
 simple-logging = "2.0.2"
+socket2 = "0.5.7"
 sqlx = { version = "0.7.2", features = [
   "chrono",
   "runtime-tokio-rustls",
diff --git a/core/startos/src/net/utils.rs b/core/startos/src/net/utils.rs
index 6de319a5e..9cba8a0cd 100644
--- a/core/startos/src/net/utils.rs
+++ b/core/startos/src/net/utils.rs
@@ -112,24 +112,6 @@ pub async fn find_eth_iface() -> Result {
     ))
 }
 
-#[pin_project::pin_project]
-pub struct SingleAccept(Option);
-impl SingleAccept {
-    pub fn new(conn: T) -> Self {
-        Self(Some(conn))
-    }
-}
-// impl axum_server::accept::Accept for SingleAccept {
-//     type Conn = T;
-//     type Error = Infallible;
-//     fn poll_accept(
-//         self: std::pin::Pin<&mut Self>,
-//         _cx: &mut std::task::Context<'_>,
-//     ) -> std::task::Poll>> {
-//         std::task::Poll::Ready(self.project().0.take().map(Ok))
-//     }
-// }
-
 pub struct TcpListeners {
     listeners: Vec,
 }
diff --git a/core/startos/src/net/vhost.rs b/core/startos/src/net/vhost.rs
index b4f5715ae..e6a9d5b21 100644
--- a/core/startos/src/net/vhost.rs
+++ b/core/startos/src/net/vhost.rs
@@ -1,10 +1,15 @@
 use std::collections::BTreeMap;
 use std::net::{IpAddr, Ipv6Addr, SocketAddr};
+use std::str::FromStr;
 use std::sync::{Arc, Weak};
 use std::time::Duration;
 
+use axum::body::Body;
+use axum::extract::Request;
+use axum::response::Response;
 use color_eyre::eyre::eyre;
 use helpers::NonDetachingJoinHandle;
+use http::Uri;
 use imbl_value::InternedString;
 use models::ResultExt;
 use serde::{Deserialize, Serialize};
@@ -20,8 +25,9 @@ use tracing::instrument;
 use ts_rs::TS;
 
 use crate::db::model::Database;
+use crate::net::static_server::server_error;
 use crate::prelude::*;
-use crate::util::io::{BackTrackingReader, TimeoutStream};
+use crate::util::io::BackTrackingReader;
 use crate::util::serde::MaybeUtf8String;
 
 // not allowed: <=1024, >=32768, 5355, 5432, 9050, 6010, 9051, 5353
@@ -113,8 +119,16 @@ impl VHostServer {
                 loop {
                     match listener.accept().await {
                         Ok((stream, _)) => {
-                            let stream =
-                                Box::pin(TimeoutStream::new(stream, Duration::from_secs(300)));
+                            if let Err(e) = socket2::SockRef::from(&stream).set_tcp_keepalive(
+                                &socket2::TcpKeepalive::new()
+                                    .with_time(Duration::from_secs(900))
+                                    .with_interval(Duration::from_secs(60))
+                                    .with_retries(5),
+                            ) {
+                                tracing::error!("Failed to set tcp keepalive: {e}");
+                                tracing::debug!("{e:?}");
+                            }
+
                             let mut stream = BackTrackingReader::new(stream);
                             stream.start_buffering();
                             let mapping = mapping.clone();
@@ -129,38 +143,39 @@ impl VHostServer {
                                     {
                                         Ok(a) => a,
                                         Err(_) => {
-                                            // stream.rewind();
-                                            // return hyper::server::Server::builder(
-                                            //     SingleAccept::new(stream),
-                                            // )
-                                            // .serve(make_service_fn(|_| async {
-                                            //     Ok::<_, Infallible>(service_fn(|req| async move {
-                                            //         let host = req
-                                            //             .headers()
-                                            //             .get(http::header::HOST)
-                                            //             .and_then(|host| host.to_str().ok());
-                                            //         let uri = Uri::from_parts({
-                                            //             let mut parts =
-                                            //                 req.uri().to_owned().into_parts();
-                                            //             parts.authority = host
-                                            //                 .map(FromStr::from_str)
-                                            //                 .transpose()?;
-                                            //             parts
-                                            //         })?;
-                                            //         Response::builder()
-                                            //             .status(
-                                            //                 http::StatusCode::TEMPORARY_REDIRECT,
-                                            //             )
-                                            //             .header(
-                                            //                 http::header::LOCATION,
-                                            //                 uri.to_string(),
-                                            //             )
-                                            //             .body(Body::default())
-                                            //     }))
-                                            // }))
-                                            // .await
-                                            // .with_kind(crate::ErrorKind::Network);
-                                            todo!()
+                                            stream.rewind();
+                                            return hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new())
+                                                .serve_connection(
+                                                    hyper_util::rt::TokioIo::new(stream),
+                                                    hyper_util::service::TowerToHyperService::new(axum::Router::new().fallback(
+                                                        axum::routing::method_routing::any(move |req: Request| async move {
+                                                            match async move {
+                                                                let host = req
+                                                                    .headers()
+                                                                    .get(http::header::HOST)
+                                                                    .and_then(|host| host.to_str().ok());
+                                                                let uri = Uri::from_parts({
+                                                                    let mut parts = req.uri().to_owned().into_parts();
+                                                                    parts.authority = host.map(FromStr::from_str).transpose()?;
+                                                                    parts
+                                                                })?;
+                                                                Response::builder()
+                                                                    .status(http::StatusCode::TEMPORARY_REDIRECT)
+                                                                    .header(http::header::LOCATION, uri.to_string())
+                                                                    .body(Body::default())
+                                                            }.await {
+                                                                Ok(a) => a,
+                                                                Err(e) => {
+                                                                    tracing::warn!("Error redirecting http request on ssl port: {e}");
+                                                                    tracing::error!("{e:?}");
+                                                                    server_error(Error::new(e, ErrorKind::Network))
+                                                                }
+                                                            }
+                                                        }),
+                                                    )),
+                                                )
+                                                .await
+                                                .map_err(|e| Error::new(color_eyre::eyre::Report::msg(e), ErrorKind::Network));
                                         }
                                     };
                                     let target_name =
diff --git a/core/startos/src/s9pk/v2/manifest.rs b/core/startos/src/s9pk/v2/manifest.rs
index 77e48c126..9ae8524fa 100644
--- a/core/startos/src/s9pk/v2/manifest.rs
+++ b/core/startos/src/s9pk/v2/manifest.rs
@@ -146,7 +146,7 @@ impl Manifest {
 #[ts(export)]
 pub struct HardwareRequirements {
     #[serde(default)]
-    #[ts(type = "{ [key: string]: string }")]
+    #[ts(type = "{ [key: string]: string }")] // TODO more specific key
     pub device: BTreeMap,
     #[ts(type = "number | null")]
     pub ram: Option,
diff --git a/core/startos/src/util/io.rs b/core/startos/src/util/io.rs
index 9a6bab64b..b5748d1d9 100644
--- a/core/startos/src/util/io.rs
+++ b/core/startos/src/util/io.rs
@@ -1,4 +1,4 @@
-use std::collections::{BTreeSet, VecDeque};
+use std::collections::VecDeque;
 use std::future::Future;
 use std::io::Cursor;
 use std::os::unix::prelude::MetadataExt;
@@ -706,16 +706,16 @@ impl AsyncRead for TimeoutStream {
         buf: &mut tokio::io::ReadBuf<'_>,
     ) -> std::task::Poll> {
         let mut this = self.project();
-        if let std::task::Poll::Ready(_) = this.sleep.as_mut().poll(cx) {
+        let timeout = this.sleep.as_mut().poll(cx);
+        let res = this.stream.poll_read(cx, buf);
+        if res.is_ready() {
+            this.sleep.reset(Instant::now() + *this.timeout);
+        } else if timeout.is_ready() {
             return std::task::Poll::Ready(Err(std::io::Error::new(
                 std::io::ErrorKind::TimedOut,
                 "timed out",
             )));
         }
-        let res = this.stream.poll_read(cx, buf);
-        if res.is_ready() {
-            this.sleep.reset(Instant::now() + *this.timeout);
-        }
         res
     }
 }
@@ -725,10 +725,16 @@ impl AsyncWrite for TimeoutStream {
         cx: &mut std::task::Context<'_>,
         buf: &[u8],
     ) -> std::task::Poll> {
-        let this = self.project();
+        let mut this = self.project();
+        let timeout = this.sleep.as_mut().poll(cx);
         let res = this.stream.poll_write(cx, buf);
         if res.is_ready() {
             this.sleep.reset(Instant::now() + *this.timeout);
+        } else if timeout.is_ready() {
+            return std::task::Poll::Ready(Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                "timed out",
+            )));
         }
         res
     }
@@ -736,10 +742,16 @@ impl AsyncWrite for TimeoutStream {
         self: std::pin::Pin<&mut Self>,
         cx: &mut std::task::Context<'_>,
     ) -> std::task::Poll> {
-        let this = self.project();
+        let mut this = self.project();
+        let timeout = this.sleep.as_mut().poll(cx);
         let res = this.stream.poll_flush(cx);
         if res.is_ready() {
             this.sleep.reset(Instant::now() + *this.timeout);
+        } else if timeout.is_ready() {
+            return std::task::Poll::Ready(Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                "timed out",
+            )));
         }
         res
     }
@@ -747,10 +759,16 @@ impl AsyncWrite for TimeoutStream {
         self: std::pin::Pin<&mut Self>,
         cx: &mut std::task::Context<'_>,
     ) -> std::task::Poll> {
-        let this = self.project();
+        let mut this = self.project();
+        let timeout = this.sleep.as_mut().poll(cx);
         let res = this.stream.poll_shutdown(cx);
         if res.is_ready() {
             this.sleep.reset(Instant::now() + *this.timeout);
+        } else if timeout.is_ready() {
+            return std::task::Poll::Ready(Err(std::io::Error::new(
+                std::io::ErrorKind::TimedOut,
+                "timed out",
+            )));
         }
         res
     }
diff --git a/patch-db b/patch-db
index c537a07ea..99076d349 160000
--- a/patch-db
+++ b/patch-db
@@ -1 +1 @@
-Subproject commit c537a07ea937e69b66841d903c70fd75623e5457
+Subproject commit 99076d349c6768000483ea8d47216d273586552e