Merge branch 'next/major' of github.com:Start9Labs/start-os into feature/nvidia

This commit is contained in:
Aiden McClelland
2026-01-13 12:33:11 -07:00
24 changed files with 332 additions and 34 deletions

View File

@@ -38,7 +38,7 @@
},
"../sdk/dist": {
"name": "@start9labs/start-sdk",
"version": "0.4.0-beta.46",
"version": "0.4.0-beta.47",
"license": "MIT",
"dependencies": {
"@iarna/toml": "^3.0.0",

View File

@@ -178,6 +178,13 @@ export function makeEffects(context: EffectContext): Effects {
T.Effects["getInstalledPackages"]
>
},
getServiceManifest(
...[options]: Parameters<T.Effects["getServiceManifest"]>
) {
return rpcRound("get-service-manifest", options) as ReturnType<
T.Effects["getServiceManifest"]
>
},
subcontainer: {
createFs(options: { imageId: string; name: string }) {
return rpcRound("subcontainer.create-fs", options) as ReturnType<

View File

@@ -184,7 +184,11 @@ async fn cli_login<C: SessionAuthContext>(
where
CliContext: CallRemote<C>,
{
let password = rpassword::prompt_password("Password: ")?;
let password = if let Ok(password) = std::env::var("PASSWORD") {
password
} else {
rpassword::prompt_password("Password: ")?
};
ctx.call_remote::<C>(
&parent_method.into_iter().chain(method).join("."),

View File

@@ -56,8 +56,7 @@ pub async fn restart(ctx: RpcContext, ControlParams { id }: ControlParams) -> Re
.as_idx_mut(&id)
.or_not_found(&id)?
.as_status_info_mut()
.as_desired_mut()
.map_mutate(|s| Ok(s.restart()))
.restart()
})
.await
.result?;

View File

@@ -15,7 +15,7 @@ use crate::util::future::NonDetachingJoinHandle;
pub const DEFAULT_SOCKS_LISTEN: SocketAddr = SocketAddr::V4(SocketAddrV4::new(
Ipv4Addr::new(HOST_IP[0], HOST_IP[1], HOST_IP[2], HOST_IP[3]),
9050,
1080,
));
pub struct SocksController {

View File

@@ -43,7 +43,6 @@ const STARTING_HEALTH_TIMEOUT: u64 = 120; // 2min
const TOR_CONTROL: SocketAddr =
SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 1, 1), 9051));
const TOR_SOCKS: SocketAddr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 1, 1), 9050));
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct OnionAddress(OnionAddressV3);
@@ -402,10 +401,15 @@ fn event_handler(_event: AsyncEvent<'static>) -> BoxFuture<'static, Result<(), C
#[derive(Clone)]
pub struct TorController(Arc<TorControl>);
impl TorController {
const TOR_SOCKS: &[SocketAddr] = &[
SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 9050)),
SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(10, 0, 3, 1), 9050)),
];
pub fn new() -> Result<Self, Error> {
Ok(TorController(Arc::new(TorControl::new(
TOR_CONTROL,
TOR_SOCKS,
Self::TOR_SOCKS,
))))
}
@@ -508,7 +512,7 @@ impl TorController {
}
Ok(Box::new(tcp_stream))
} else {
let mut stream = TcpStream::connect(TOR_SOCKS)
let mut stream = TcpStream::connect(Self::TOR_SOCKS[0])
.await
.with_kind(ErrorKind::Tor)?;
if let Err(e) = socket2::SockRef::from(&stream).set_keepalive(true) {
@@ -595,7 +599,7 @@ enum TorCommand {
#[instrument(skip_all)]
async fn torctl(
tor_control: SocketAddr,
tor_socks: SocketAddr,
tor_socks: &[SocketAddr],
recv: &mut mpsc::UnboundedReceiver<TorCommand>,
services: &mut Watch<
BTreeMap<
@@ -641,10 +645,21 @@ async fn torctl(
tokio::fs::remove_dir_all("/var/lib/tor").await?;
wipe_state.store(false, std::sync::atomic::Ordering::SeqCst);
}
write_file_atomic(
"/etc/tor/torrc",
format!("SocksPort {TOR_SOCKS}\nControlPort {TOR_CONTROL}\nCookieAuthentication 1\n"),
)
write_file_atomic("/etc/tor/torrc", {
use std::fmt::Write;
let mut conf = String::new();
for tor_socks in tor_socks {
writeln!(&mut conf, "SocksPort {tor_socks}").unwrap();
}
writeln!(
&mut conf,
"ControlPort {tor_control}\nCookieAuthentication 1"
)
.unwrap();
conf
})
.await?;
tokio::fs::create_dir_all("/var/lib/tor").await?;
Command::new("chown")
@@ -976,7 +991,10 @@ struct TorControl {
>,
}
impl TorControl {
pub fn new(tor_control: SocketAddr, tor_socks: SocketAddr) -> Self {
pub fn new(
tor_control: SocketAddr,
tor_socks: impl AsRef<[SocketAddr]> + Send + 'static,
) -> Self {
let (send, mut recv) = mpsc::unbounded_channel();
let services = Watch::new(BTreeMap::new());
let mut thread_services = services.clone();
@@ -987,7 +1005,7 @@ impl TorControl {
loop {
if let Err(e) = torctl(
tor_control,
tor_socks,
tor_socks.as_ref(),
&mut recv,
&mut thread_services,
&wipe_state,

View File

@@ -36,6 +36,7 @@ struct ServiceCallbackMap {
>,
get_status: BTreeMap<PackageId, Vec<CallbackHandler>>,
get_container_ip: BTreeMap<PackageId, Vec<CallbackHandler>>,
get_service_manifest: BTreeMap<PackageId, Vec<CallbackHandler>>,
}
impl ServiceCallbacks {
@@ -68,6 +69,10 @@ impl ServiceCallbacks {
v.retain(|h| h.handle.is_active() && h.seed.strong_count() > 0);
!v.is_empty()
});
this.get_service_manifest.retain(|_, v| {
v.retain(|h| h.handle.is_active() && h.seed.strong_count() > 0);
!v.is_empty()
});
})
}
@@ -250,6 +255,25 @@ impl ServiceCallbacks {
.filter(|cb| !cb.0.is_empty())
})
}
pub(super) fn add_get_service_manifest(&self, package_id: PackageId, handler: CallbackHandler) {
self.mutate(|this| {
this.get_service_manifest
.entry(package_id)
.or_default()
.push(handler)
})
}
#[must_use]
pub fn get_service_manifest(&self, package_id: &PackageId) -> Option<CallbackHandlers> {
self.mutate(|this| {
this.get_service_manifest
.remove(package_id)
.map(CallbackHandlers)
.filter(|cb| !cb.0.is_empty())
})
}
}
pub struct CallbackHandler {

View File

@@ -36,8 +36,7 @@ pub async fn restart(context: EffectContext) -> Result<(), Error> {
.as_idx_mut(id)
.or_not_found(id)?
.as_status_info_mut()
.as_desired_mut()
.map_mutate(|s| Ok(s.restart()))
.restart()
})
.await
.result?;

View File

@@ -14,7 +14,10 @@ use crate::disk::mount::filesystem::bind::{Bind, FileType};
use crate::disk::mount::filesystem::idmapped::{IdMap, IdMapped};
use crate::disk::mount::filesystem::{FileSystem, MountType};
use crate::disk::mount::util::{is_mountpoint, unmount};
use crate::s9pk::manifest::Manifest;
use crate::service::effects::callbacks::CallbackHandler;
use crate::service::effects::prelude::*;
use crate::service::rpc::CallbackId;
use crate::status::health_check::NamedHealthCheckResult;
use crate::util::{FromStrParser, VersionString};
use crate::volume::data_dir;
@@ -367,3 +370,45 @@ pub async fn check_dependencies(
}
Ok(results)
}
#[derive(Debug, Clone, Serialize, Deserialize, TS, Parser)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct GetServiceManifestParams {
pub package_id: PackageId,
#[ts(optional)]
#[arg(skip)]
pub callback: Option<CallbackId>,
}
pub async fn get_service_manifest(
context: EffectContext,
GetServiceManifestParams {
package_id,
callback,
}: GetServiceManifestParams,
) -> Result<Manifest, Error> {
let context = context.deref()?;
if let Some(callback) = callback {
let callback = callback.register(&context.seed.persistent_container);
context
.seed
.ctx
.callbacks
.add_get_service_manifest(package_id.clone(), CallbackHandler::new(&context, callback));
}
let db = context.seed.ctx.db.peek().await;
let manifest = db
.as_public()
.as_package_data()
.as_idx(&package_id)
.or_not_found(&package_id)?
.as_state_info()
.as_manifest(ManifestPreference::New)
.de()?;
Ok(manifest)
}

View File

@@ -88,6 +88,10 @@ pub fn handler<C: Context>() -> ParentHandler<C> {
"get-installed-packages",
from_fn_async(dependency::get_installed_packages).no_cli(),
)
.subcommand(
"get-service-manifest",
from_fn_async(dependency::get_service_manifest).no_cli(),
)
// health
.subcommand("set-health", from_fn_async(health::set_health).no_cli())
// subcontainer

View File

@@ -575,6 +575,17 @@ impl Service {
.await
.result?;
// Trigger manifest callbacks after successful installation
let manifest = service.seed.persistent_container.s9pk.as_manifest();
if let Some(callbacks) = ctx.callbacks.get_service_manifest(&manifest.id) {
let manifest_value =
serde_json::to_value(manifest).with_kind(ErrorKind::Serialization)?;
callbacks
.call(imbl::vector![manifest_value.into()])
.await
.log_err();
}
Ok(service)
}

View File

@@ -1,5 +1,7 @@
use std::path::Path;
use imbl::vector;
use crate::context::RpcContext;
use crate::db::model::package::{InstalledState, InstallingInfo, InstallingState, PackageState};
use crate::prelude::*;
@@ -65,6 +67,11 @@ pub async fn cleanup(ctx: &RpcContext, id: &PackageId, soft: bool) -> Result<(),
));
}
};
// Trigger manifest callbacks with null to indicate uninstall
if let Some(callbacks) = ctx.callbacks.get_service_manifest(&manifest.id) {
callbacks.call(vector![Value::Null]).await.log_err();
}
if !soft {
let path = Path::new(DATA_DIR).join(PKG_VOLUME_DIR).join(&manifest.id);
if tokio::fs::metadata(&path).await.is_ok() {

View File

@@ -53,6 +53,11 @@ impl Model<StatusInfo> {
self.as_started_mut().ser(&None)?;
Ok(())
}
pub fn restart(&mut self) -> Result<(), Error> {
self.as_desired_mut().map_mutate(|s| Ok(s.restart()))?;
self.as_health_mut().ser(&Default::default())?;
Ok(())
}
pub fn init(&mut self) -> Result<(), Error> {
self.as_started_mut().ser(&None)?;
self.as_desired_mut().map_mutate(|s| {

View File

@@ -15,6 +15,7 @@ import {
CreateTaskParams,
MountParams,
StatusInfo,
Manifest,
} from "./osBindings"
import {
PackageId,
@@ -83,6 +84,11 @@ export type Effects = {
mount(options: MountParams): Promise<string>
/** Returns a list of the ids of all installed packages */
getInstalledPackages(): Promise<string[]>
/** Returns the manifest of a service */
getServiceManifest(options: {
packageId: PackageId
callback?: () => void
}): Promise<Manifest>
// health
/** sets the result of a health check */

View File

@@ -0,0 +1,8 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { CallbackId } from "./CallbackId"
import type { PackageId } from "./PackageId"
export type GetServiceManifestParams = {
packageId: PackageId
callback?: CallbackId
}

View File

@@ -91,6 +91,7 @@ export { GetPackageParams } from "./GetPackageParams"
export { GetPackageResponseFull } from "./GetPackageResponseFull"
export { GetPackageResponse } from "./GetPackageResponse"
export { GetServiceInterfaceParams } from "./GetServiceInterfaceParams"
export { GetServiceManifestParams } from "./GetServiceManifestParams"
export { GetServicePortForwardParams } from "./GetServicePortForwardParams"
export { GetSslCertificateParams } from "./GetSslCertificateParams"
export { GetSslKeyParams } from "./GetSslKeyParams"

View File

@@ -13,6 +13,7 @@ import {
RunActionParams,
SetDataVersionParams,
SetMainStatus,
GetServiceManifestParams,
} from ".././osBindings"
import { CreateSubcontainerFsParams } from ".././osBindings"
import { DestroySubcontainerFsParams } from ".././osBindings"
@@ -64,7 +65,6 @@ describe("startosTypeValidation ", () => {
destroyFs: {} as DestroySubcontainerFsParams,
},
clearBindings: {} as ClearBindingsParams,
getInstalledPackages: undefined,
bind: {} as BindParams,
getHostInfo: {} as WithCallback<GetHostInfoParams>,
restart: undefined,
@@ -76,6 +76,8 @@ describe("startosTypeValidation ", () => {
getSslKey: {} as GetSslKeyParams,
getServiceInterface: {} as WithCallback<GetServiceInterfaceParams>,
setDependencies: {} as SetDependenciesParams,
getInstalledPackages: undefined,
getServiceManifest: {} as WithCallback<GetServiceManifestParams>,
getSystemSmtp: {} as WithCallback<GetSystemSmtpParams>,
getContainerIp: {} as WithCallback<GetContainerIpParams>,
getOsIp: undefined,

View File

@@ -46,7 +46,7 @@ import {
CheckDependencies,
checkDependencies,
} from "../../base/lib/dependencies/dependencies"
import { GetSslCertificate } from "./util"
import { GetSslCertificate, getServiceManifest } from "./util"
import { getDataVersion, setDataVersion } from "./version"
import { MaybeFn } from "../../base/lib/actions/setupActions"
import { GetInput } from "../../base/lib/actions/setupActions"
@@ -107,6 +107,7 @@ export class StartSdk<Manifest extends T.SDKManifest> {
| "getContainerIp"
| "getDataVersion"
| "setDataVersion"
| "getServiceManifest"
// prettier-ignore
type StartSdkEffectWrapper = {
@@ -441,11 +442,12 @@ export class StartSdk<Manifest extends T.SDKManifest> {
) => new ServiceInterfaceBuilder({ ...options, effects }),
getSystemSmtp: <E extends Effects>(effects: E) =>
new GetSystemSmtp(effects),
getSslCerificate: <E extends Effects>(
getSslCertificate: <E extends Effects>(
effects: E,
hostnames: string[],
algorithm?: T.Algorithm,
) => new GetSslCertificate(effects, hostnames, algorithm),
getServiceManifest,
healthCheck: {
checkPortListening,
checkWebUrl,

View File

@@ -0,0 +1,152 @@
import { Effects } from "../../../base/lib/Effects"
import { Manifest, PackageId } from "../../../base/lib/osBindings"
import { DropGenerator, DropPromise } from "../../../base/lib/util/Drop"
import { deepEqual } from "../../../base/lib/util/deepEqual"
export class GetServiceManifest<Mapped = Manifest> {
constructor(
readonly effects: Effects,
readonly packageId: PackageId,
readonly map: (manifest: Manifest | null) => Mapped,
readonly eq: (a: Mapped, b: Mapped) => boolean,
) {}
/**
* Returns the manifest of a service. Reruns the context from which it has been called if the underlying value changes
*/
async const() {
let abort = new AbortController()
const watch = this.watch(abort.signal)
const res = await watch.next()
if (this.effects.constRetry) {
watch.next().then(() => {
abort.abort()
this.effects.constRetry && this.effects.constRetry()
})
}
return res.value
}
/**
* Returns the manifest of a service. Does nothing if it changes
*/
async once() {
const manifest = await this.effects.getServiceManifest({
packageId: this.packageId,
})
return this.map(manifest)
}
private async *watchGen(abort?: AbortSignal) {
let prev = null as { value: Mapped } | null
const resolveCell = { resolve: () => {} }
this.effects.onLeaveContext(() => {
resolveCell.resolve()
})
abort?.addEventListener("abort", () => resolveCell.resolve())
while (this.effects.isInContext && !abort?.aborted) {
let callback: () => void = () => {}
const waitForNext = new Promise<void>((resolve) => {
callback = resolve
resolveCell.resolve = resolve
})
const next = this.map(
await this.effects.getServiceManifest({
packageId: this.packageId,
callback: () => callback(),
}),
)
if (!prev || !this.eq(prev.value, next)) {
prev = { value: next }
yield next
}
await waitForNext
}
return new Promise<never>((_, rej) => rej(new Error("aborted")))
}
/**
* Watches the manifest of a service. Returns an async iterator that yields whenever the value changes
*/
watch(abort?: AbortSignal): AsyncGenerator<Mapped, never, unknown> {
const ctrl = new AbortController()
abort?.addEventListener("abort", () => ctrl.abort())
return DropGenerator.of(this.watchGen(ctrl.signal), () => ctrl.abort())
}
/**
* Watches the manifest of a service. Takes a custom callback function to run whenever it changes
*/
onChange(
callback: (
value: Mapped | null,
error?: Error,
) => { cancel: boolean } | Promise<{ cancel: boolean }>,
) {
;(async () => {
const ctrl = new AbortController()
for await (const value of this.watch(ctrl.signal)) {
try {
const res = await callback(value)
if (res.cancel) {
ctrl.abort()
break
}
} catch (e) {
console.error(
"callback function threw an error @ GetServiceManifest.onChange",
e,
)
}
}
})()
.catch((e) => callback(null, e))
.catch((e) =>
console.error(
"callback function threw an error @ GetServiceManifest.onChange",
e,
),
)
}
/**
* Watches the manifest of a service. Returns when the predicate is true
*/
waitFor(pred: (value: Mapped) => boolean): Promise<Mapped> {
const ctrl = new AbortController()
return DropPromise.of(
Promise.resolve().then(async () => {
for await (const next of this.watchGen(ctrl.signal)) {
if (pred(next)) {
return next
}
}
throw new Error("context left before predicate passed")
}),
() => ctrl.abort(),
)
}
}
export function getServiceManifest(
effects: Effects,
packageId: PackageId,
): GetServiceManifest<Manifest>
export function getServiceManifest<Mapped>(
effects: Effects,
packageId: PackageId,
map: (manifest: Manifest | null) => Mapped,
eq?: (a: Mapped, b: Mapped) => boolean,
): GetServiceManifest<Mapped>
export function getServiceManifest<Mapped>(
effects: Effects,
packageId: PackageId,
map?: (manifest: Manifest | null) => Mapped,
eq?: (a: Mapped, b: Mapped) => boolean,
): GetServiceManifest<Mapped> {
return new GetServiceManifest(
effects,
packageId,
map ?? ((a) => a as Mapped),
eq ?? ((a, b) => deepEqual(a, b)),
)
}

View File

@@ -623,7 +623,10 @@ export class FileHelper<A> {
.split("\n")
.map((line) => line.trim())
.filter((line) => !line.startsWith("#") && line.includes("="))
.map((line) => line.split("=", 2)),
.map((line) => {
const pos = line.indexOf("=")
return [line.slice(0, pos), line.slice(pos + 1)]
}),
),
(data) => shape.unsafeCast(data),
transformers,

View File

@@ -1,5 +1,6 @@
export * from "../../../base/lib/util"
export { GetSslCertificate } from "./GetSslCertificate"
export { GetServiceManifest, getServiceManifest } from "./GetServiceManifest"
export { Drop } from "../../../base/lib/util/Drop"
export { Volume, Volumes } from "./Volume"

View File

@@ -1,12 +1,12 @@
{
"name": "@start9labs/start-sdk",
"version": "0.4.0-beta.46",
"version": "0.4.0-beta.47",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@start9labs/start-sdk",
"version": "0.4.0-beta.46",
"version": "0.4.0-beta.47",
"license": "MIT",
"dependencies": {
"@iarna/toml": "^3.0.0",

View File

@@ -1,6 +1,6 @@
{
"name": "@start9labs/start-sdk",
"version": "0.4.0-beta.46",
"version": "0.4.0-beta.47",
"description": "Software development kit to facilitate packaging services for StartOS",
"main": "./package/lib/index.js",
"types": "./package/lib/index.d.ts",

View File

@@ -410,7 +410,7 @@ export namespace Mock {
docsUrl: 'https://bitcoin.org',
releaseNotes: 'Even better support for Bitcoin and wallets!',
osVersion: '0.3.6',
sdkVersion: '0.4.0-beta.46',
sdkVersion: '0.4.0-beta.47',
gitHash: 'fakehash',
icon: BTC_ICON,
sourceVersion: null,
@@ -452,7 +452,7 @@ export namespace Mock {
docsUrl: 'https://bitcoinknots.org',
releaseNotes: 'Even better support for Bitcoin and wallets!',
osVersion: '0.3.6',
sdkVersion: '0.4.0-beta.46',
sdkVersion: '0.4.0-beta.47',
gitHash: 'fakehash',
icon: BTC_ICON,
sourceVersion: null,
@@ -504,7 +504,7 @@ export namespace Mock {
docsUrl: 'https://bitcoin.org',
releaseNotes: 'Even better support for Bitcoin and wallets!',
osVersion: '0.3.6',
sdkVersion: '0.4.0-beta.46',
sdkVersion: '0.4.0-beta.47',
gitHash: 'fakehash',
icon: BTC_ICON,
sourceVersion: null,
@@ -546,7 +546,7 @@ export namespace Mock {
docsUrl: 'https://bitcoinknots.org',
releaseNotes: 'Even better support for Bitcoin and wallets!',
osVersion: '0.3.6',
sdkVersion: '0.4.0-beta.46',
sdkVersion: '0.4.0-beta.47',
gitHash: 'fakehash',
icon: BTC_ICON,
sourceVersion: null,
@@ -600,7 +600,7 @@ export namespace Mock {
docsUrl: 'https://lightning.engineering/',
releaseNotes: 'Upstream release to 0.17.5',
osVersion: '0.3.6',
sdkVersion: '0.4.0-beta.46',
sdkVersion: '0.4.0-beta.47',
gitHash: 'fakehash',
icon: LND_ICON,
sourceVersion: null,
@@ -655,7 +655,7 @@ export namespace Mock {
docsUrl: 'https://lightning.engineering/',
releaseNotes: 'Upstream release to 0.17.4',
osVersion: '0.3.6',
sdkVersion: '0.4.0-beta.46',
sdkVersion: '0.4.0-beta.47',
gitHash: 'fakehash',
icon: LND_ICON,
sourceVersion: null,
@@ -714,7 +714,7 @@ export namespace Mock {
docsUrl: 'https://bitcoin.org',
releaseNotes: 'Even better support for Bitcoin and wallets!',
osVersion: '0.3.6',
sdkVersion: '0.4.0-beta.46',
sdkVersion: '0.4.0-beta.47',
gitHash: 'fakehash',
icon: BTC_ICON,
sourceVersion: null,
@@ -756,7 +756,7 @@ export namespace Mock {
docsUrl: 'https://bitcoinknots.org',
releaseNotes: 'Even better support for Bitcoin and wallets!',
osVersion: '0.3.6',
sdkVersion: '0.4.0-beta.46',
sdkVersion: '0.4.0-beta.47',
gitHash: 'fakehash',
icon: BTC_ICON,
sourceVersion: null,
@@ -808,7 +808,7 @@ export namespace Mock {
docsUrl: 'https://lightning.engineering/',
releaseNotes: 'Upstream release and minor fixes.',
osVersion: '0.3.6',
sdkVersion: '0.4.0-beta.46',
sdkVersion: '0.4.0-beta.47',
gitHash: 'fakehash',
icon: LND_ICON,
sourceVersion: null,
@@ -863,7 +863,7 @@ export namespace Mock {
marketingSite: '',
releaseNotes: 'Upstream release and minor fixes.',
osVersion: '0.3.6',
sdkVersion: '0.4.0-beta.46',
sdkVersion: '0.4.0-beta.47',
gitHash: 'fakehash',
icon: PROXY_ICON,
sourceVersion: null,