From 80461a78b02fe7f83eddebbe90b50700ed70406f Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:08:22 -0700 Subject: [PATCH] misc improvements (#2836) * misc improvements * kill proc before destroying subcontainer fs * version bump * beta.11 * use bind mount explicitly * Update sdk/base/lib/Effects.ts Co-authored-by: Dominion5254 --------- Co-authored-by: Dominion5254 --- .../src/Adapters/EffectCreator.ts | 3 + .../Systems/SystemForEmbassy/index.ts | 6 +- core/Cargo.lock | 2 +- core/startos/Cargo.toml | 2 +- core/startos/src/bins/startd.rs | 7 +- core/startos/src/context/rpc.rs | 11 +- core/startos/src/net/web_server.rs | 2 +- core/startos/src/s9pk/v2/mod.rs | 1 + core/startos/src/s9pk/v2/pack.rs | 12 +- core/startos/src/s9pk/v2/recipe.rs | 21 +++ core/startos/src/service/effects/mod.rs | 8 +- core/startos/src/service/effects/net/info.rs | 1 + .../src/service/effects/subcontainer/mod.rs | 19 +++ .../src/service/effects/subcontainer/sync.rs | 130 ++++++++++-------- core/startos/src/util/logger.rs | 33 ++--- core/startos/src/version/mod.rs | 12 +- core/startos/src/version/v0_3_6_alpha_15.rs | 39 ++++++ sdk/base/lib/Effects.ts | 2 + sdk/base/lib/osBindings/ImageMetadata.ts | 7 +- .../lib/test/startosTypeValidation.test.ts | 1 + sdk/base/lib/types.ts | 6 +- sdk/base/lib/util/getServiceInterface.ts | 8 ++ sdk/base/package-lock.json | 11 +- sdk/base/package.json | 2 +- sdk/base/tsconfig.json | 2 +- sdk/package/lib/StartSdk.ts | 41 ++++-- sdk/package/lib/health/HealthCheck.ts | 10 +- sdk/package/lib/mainFn/CommandController.ts | 24 +++- sdk/package/lib/mainFn/Daemons.ts | 6 + sdk/package/lib/mainFn/HealthDaemon.ts | 11 +- sdk/package/lib/util/SubContainer.ts | 34 ++++- sdk/package/package-lock.json | 15 +- sdk/package/package.json | 4 +- sdk/package/tsconfig.json | 2 +- web/package-lock.json | 4 +- web/package.json | 2 +- 36 files changed, 358 insertions(+), 143 deletions(-) create mode 100644 core/startos/src/s9pk/v2/recipe.rs create mode 100644 core/startos/src/version/v0_3_6_alpha_15.rs diff --git a/container-runtime/src/Adapters/EffectCreator.ts b/container-runtime/src/Adapters/EffectCreator.ts index 4bda0ed5d..725e8d22c 100644 --- a/container-runtime/src/Adapters/EffectCreator.ts +++ b/container-runtime/src/Adapters/EffectCreator.ts @@ -198,6 +198,9 @@ export function makeEffects(context: EffectContext): Effects { T.Effects["getContainerIp"] > }, + getOsIp(...[]: Parameters) { + return rpcRound("get-os-ip", {}) as ReturnType + }, getHostInfo: ((...[allOptions]: Parameters) => { const options = { ...allOptions, diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts index 9cefcde2e..380cef443 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/index.ts @@ -349,9 +349,6 @@ export class SystemForEmbassy implements System { ) { await effects.action.clearRequests({ only: ["needs-config"] }) } - await effects.setDataVersion({ - version: ExtendedVersion.parseEmver(this.manifest.version).toString(), - }) } else if (this.manifest.config) { await effects.action.request({ packageId: this.manifest.id, @@ -361,6 +358,9 @@ export class SystemForEmbassy implements System { reason: "This service must be configured before it can be run", }) } + await effects.setDataVersion({ + version: ExtendedVersion.parseEmver(this.manifest.version).toString(), + }) } async exportNetwork(effects: Effects) { for (const [id, interfaceValue] of Object.entries( diff --git a/core/Cargo.lock b/core/Cargo.lock index 15edd45d8..83c1da0e2 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -5976,7 +5976,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "start-os" -version = "0.3.6-alpha.14" +version = "0.3.6-alpha.15" dependencies = [ "aes 0.7.5", "async-acme", diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index ca7990840..dbcaf2a78 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -14,7 +14,7 @@ keywords = [ name = "start-os" readme = "README.md" repository = "https://github.com/Start9Labs/start-os" -version = "0.3.6-alpha.14" # VERSION_BUMP +version = "0.3.6-alpha.15" # VERSION_BUMP license = "MIT" [lib] diff --git a/core/startos/src/bins/startd.rs b/core/startos/src/bins/startd.rs index 4cf11ba07..01e6ac916 100644 --- a/core/startos/src/bins/startd.rs +++ b/core/startos/src/bins/startd.rs @@ -2,6 +2,7 @@ use std::cmp::max; use std::ffi::OsString; use std::net::IpAddr; use std::sync::Arc; +use std::time::Duration; use clap::Parser; use color_eyre::eyre::eyre; @@ -149,7 +150,7 @@ pub fn main(args: impl IntoIterator) { .enable_all() .build() .expect("failed to initialize runtime"); - rt.block_on(async { + let res = rt.block_on(async { let mut server = WebServer::new(Acceptor::bind_upgradable( SelfContainedNetworkInterfaceListener::bind(80), )); @@ -194,7 +195,9 @@ pub fn main(args: impl IntoIterator) { .await } } - }) + }); + rt.shutdown_timeout(Duration::from_secs(60)); + res }; match res { diff --git a/core/startos/src/context/rpc.rs b/core/startos/src/context/rpc.rs index dfaa02b01..96ca3ca63 100644 --- a/core/startos/src/context/rpc.rs +++ b/core/startos/src/context/rpc.rs @@ -1,3 +1,4 @@ +use std::backtrace; use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; @@ -484,10 +485,12 @@ impl Drop for RpcContext { fn drop(&mut self) { #[cfg(feature = "unstable")] if self.0.is_closed.load(Ordering::SeqCst) { - tracing::info!( - "RpcContext dropped. {} left.", - Arc::strong_count(&self.0) - 1 - ); + let count = Arc::strong_count(&self.0) - 1; + tracing::info!("RpcContext dropped. {} left.", count); + if count > 0 { + tracing::debug!("{}", backtrace::Backtrace::force_capture()); + tracing::debug!("{:?}", eyre!("")) + } } } } diff --git a/core/startos/src/net/web_server.rs b/core/startos/src/net/web_server.rs index 4d48316c0..005735a4a 100644 --- a/core/startos/src/net/web_server.rs +++ b/core/startos/src/net/web_server.rs @@ -267,7 +267,7 @@ impl WebServer { if !runner.is_empty() { tokio::time::timeout(Duration::from_secs(60), runner) .await - .ok(); + .log_err(); } })); Self { diff --git a/core/startos/src/s9pk/v2/mod.rs b/core/startos/src/s9pk/v2/mod.rs index 7a94c0d79..ecab202ce 100644 --- a/core/startos/src/s9pk/v2/mod.rs +++ b/core/startos/src/s9pk/v2/mod.rs @@ -27,6 +27,7 @@ pub const SIG_CONTEXT: &str = "s9pk"; pub mod compat; pub mod manifest; pub mod pack; +pub mod recipe; /** / diff --git a/core/startos/src/s9pk/v2/pack.rs b/core/startos/src/s9pk/v2/pack.rs index 9464e1ce1..eb3aaa186 100644 --- a/core/startos/src/s9pk/v2/pack.rs +++ b/core/startos/src/s9pk/v2/pack.rs @@ -26,6 +26,7 @@ use crate::s9pk::merkle_archive::source::{ into_dyn_read, ArchiveSource, DynFileSource, DynRead, FileSource, TmpSource, }; use crate::s9pk::merkle_archive::{Entry, MerkleArchive}; +use crate::s9pk::v2::recipe::DirRecipe; use crate::s9pk::v2::SIG_CONTEXT; use crate::s9pk::S9pk; use crate::util::io::{create_file, open_file, TmpDir}; @@ -363,6 +364,7 @@ pub enum ImageSource { build_args: Option>, }, DockerTag(String), + // Recipe(DirRecipe), } impl ImageSource { pub fn ingredients(&self) -> Vec { @@ -399,6 +401,8 @@ impl ImageSource { working_dir: PathBuf, #[serde(default)] user: String, + entrypoint: Option>, + cmd: Option>, } async move { match self { @@ -531,6 +535,8 @@ impl ImageSource { } else { config.user.into() }, + entrypoint: config.entrypoint, + cmd: config.cmd, }) .with_kind(ErrorKind::Serialization)? .into(), @@ -607,8 +613,8 @@ fn tar2sqfs(dest: impl AsRef) -> Result { .arg("run") .arg("-i") .arg("--rm") - .arg("-v") - .arg(format!("{}:/data:rw", directory.display())) + .arg("--mount") + .arg(format!("type=bind,src={},dst=/data", directory.display())) .arg("ghcr.io/start9labs/sdk/utils:latest") .arg("tar2sqfs") .arg("-q") @@ -625,6 +631,8 @@ pub struct ImageMetadata { pub workdir: PathBuf, #[ts(type = "string")] pub user: InternedString, + pub entrypoint: Option>, + pub cmd: Option>, } #[instrument(skip_all)] diff --git a/core/startos/src/s9pk/v2/recipe.rs b/core/startos/src/s9pk/v2/recipe.rs new file mode 100644 index 000000000..665f31253 --- /dev/null +++ b/core/startos/src/s9pk/v2/recipe.rs @@ -0,0 +1,21 @@ +use std::collections::BTreeMap; +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; +use ts_rs::TS; +use url::Url; + +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +pub struct DirRecipe(BTreeMap); + +#[derive(Debug, Clone, Deserialize, Serialize, TS)] +#[serde(rename_all = "camelCase")] +pub enum Recipe { + Make(PathBuf), + Wget { + #[ts(type = "string")] + url: Url, + checksum: String, + }, + Recipe(DirRecipe), +} diff --git a/core/startos/src/service/effects/mod.rs b/core/startos/src/service/effects/mod.rs index e9df6f9f2..48b901d07 100644 --- a/core/startos/src/service/effects/mod.rs +++ b/core/startos/src/service/effects/mod.rs @@ -1,9 +1,11 @@ +use std::net::Ipv4Addr; + use rpc_toolkit::{from_fn, from_fn_async, from_fn_blocking, Context, HandlerExt, ParentHandler}; -use crate::echo; use crate::prelude::*; use crate::service::cli::ContainerCliContext; use crate::service::effects::context::EffectContext; +use crate::{echo, HOST_IP}; mod action; pub mod callbacks; @@ -134,6 +136,10 @@ pub fn handler() -> ParentHandler { "get-container-ip", from_fn_async(net::info::get_container_ip).no_cli(), ) + .subcommand( + "get-os-ip", + from_fn(|_: C| Ok::<_, Error>(Ipv4Addr::from(HOST_IP))), + ) .subcommand( "export-service-interface", from_fn_async(net::interface::export_service_interface).no_cli(), diff --git a/core/startos/src/service/effects/net/info.rs b/core/startos/src/service/effects/net/info.rs index fe6623f44..766c0015f 100644 --- a/core/startos/src/service/effects/net/info.rs +++ b/core/startos/src/service/effects/net/info.rs @@ -1,6 +1,7 @@ use std::net::Ipv4Addr; use crate::service::effects::prelude::*; +use crate::HOST_IP; pub async fn get_container_ip(context: EffectContext) -> Result { let context = context.deref()?; diff --git a/core/startos/src/service/effects/subcontainer/mod.rs b/core/startos/src/service/effects/subcontainer/mod.rs index 943c70dbf..e7b4f73cf 100644 --- a/core/startos/src/service/effects/subcontainer/mod.rs +++ b/core/startos/src/service/effects/subcontainer/mod.rs @@ -5,6 +5,7 @@ use models::ImageId; use tokio::process::Command; use crate::disk::mount::filesystem::overlayfs::OverlayGuard; +use crate::disk::mount::guard::GenericMountGuard; use crate::rpc_continuations::Guid; use crate::service::effects::prelude::*; use crate::service::persistent_container::Subcontainer; @@ -40,6 +41,24 @@ pub async fn destroy_subcontainer_fs( .await .remove(&guid) { + #[cfg(feature = "container-runtime")] + if tokio::fs::metadata(overlay.overlay.path().join("proc/1")) + .await + .is_ok() + { + let procfs = context + .seed + .persistent_container + .lxc_container + .get() + .or_not_found("lxc container")? + .rootfs_dir() + .join("proc"); + let overlay_path = overlay.overlay.path().to_owned(); + tokio::task::spawn_blocking(move || sync::kill_init(&procfs, &overlay_path)) + .await + .with_kind(ErrorKind::Unknown)??; + } overlay.overlay.unmount(true).await?; } else { tracing::warn!("Could not find a subcontainer fs to destroy; assumming that it already is destroyed and will be skipping"); diff --git a/core/startos/src/service/effects/subcontainer/sync.rs b/core/startos/src/service/effects/subcontainer/sync.rs index 702f34bbe..f40c13543 100644 --- a/core/startos/src/service/effects/subcontainer/sync.rs +++ b/core/startos/src/service/effects/subcontainer/sync.rs @@ -20,6 +20,54 @@ const FWD_SIGNALS: &[c_int] = &[ SIGTSTP, SIGTTIN, SIGTTOU, SIGURG, SIGUSR1, SIGUSR2, SIGVTALRM, ]; +pub fn kill_init(procfs: &Path, chroot: &Path) -> Result<(), Error> { + if chroot.join("proc/1").exists() { + let ns_id = procfs::process::Process::new_with_root(chroot.join("proc/1")) + .with_ctx(|_| (ErrorKind::Filesystem, "open subcontainer procfs"))? + .namespaces() + .with_ctx(|_| (ErrorKind::Filesystem, "read subcontainer pid 1 ns"))? + .0 + .get(OsStr::new("pid")) + .or_not_found("pid namespace")? + .identifier; + for proc in procfs::process::all_processes_with_root(procfs) + .with_ctx(|_| (ErrorKind::Filesystem, "open procfs"))? + { + let proc = proc.with_ctx(|_| (ErrorKind::Filesystem, "read single process details"))?; + let pid = proc.pid(); + if proc + .namespaces() + .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("read pid {} ns", pid)))? + .0 + .get(OsStr::new("pid")) + .map_or(false, |ns| ns.identifier == ns_id) + { + let pids = proc.read::("status").with_ctx(|_| { + ( + ErrorKind::Filesystem, + lazy_format!("read pid {} NSpid", pid), + ) + })?; + if pids.0.len() == 2 && pids.0[1] == 1 { + nix::sys::signal::kill(Pid::from_raw(pid), nix::sys::signal::SIGKILL) + .with_ctx(|_| { + ( + ErrorKind::Filesystem, + lazy_format!( + "kill pid {} (determined to be pid 1 in subcontainer)", + pid + ), + ) + })?; + } + } + } + nix::mount::umount(&chroot.join("proc")) + .with_ctx(|_| (ErrorKind::Filesystem, "unmounting subcontainer procfs"))?; + } + Ok(()) +} + struct NSPid(Vec); impl procfs::FromBufRead for NSPid { fn from_buf_read(r: R) -> procfs::ProcResult { @@ -98,21 +146,27 @@ impl ExecParams { if let Some(uid) = user.as_deref().and_then(|u| u.parse::().ok()) { cmd.uid(uid); } else if let Some(user) = user { - let (uid, gid) = std::fs::read_to_string("/etc/passwd") - .with_ctx(|_| (ErrorKind::Filesystem, "read /etc/passwd"))? - .lines() - .find_map(|l| { - let mut split = l.trim().split(":"); - if user != split.next()? { - return None; - } - split.next(); // throw away x - Some((split.next()?.parse().ok()?, split.next()?.parse().ok()?)) - // uid gid - }) - .or_not_found(lazy_format!("{user} in /etc/passwd"))?; - cmd.uid(uid); - cmd.gid(gid); + let passwd = std::fs::read_to_string("/etc/passwd") + .with_ctx(|_| (ErrorKind::Filesystem, "read /etc/passwd")); + if passwd.is_err() && user == "root" { + cmd.uid(0); + cmd.gid(0); + } else { + let (uid, gid) = passwd? + .lines() + .find_map(|l| { + let mut split = l.trim().split(":"); + if user != split.next()? { + return None; + } + split.next(); // throw away x + Some((split.next()?.parse().ok()?, split.next()?.parse().ok()?)) + // uid gid + }) + .or_not_found(lazy_format!("{user} in /etc/passwd"))?; + cmd.uid(uid); + cmd.gid(gid); + } }; if let Some(workdir) = workdir { cmd.current_dir(workdir); @@ -134,51 +188,7 @@ pub fn launch( command, }: ExecParams, ) -> Result<(), Error> { - if chroot.join("proc/1").exists() { - let ns_id = procfs::process::Process::new_with_root(chroot.join("proc/1")) - .with_ctx(|_| (ErrorKind::Filesystem, "open subcontainer procfs"))? - .namespaces() - .with_ctx(|_| (ErrorKind::Filesystem, "read subcontainer pid 1 ns"))? - .0 - .get(OsStr::new("pid")) - .or_not_found("pid namespace")? - .identifier; - for proc in - procfs::process::all_processes().with_ctx(|_| (ErrorKind::Filesystem, "open procfs"))? - { - let proc = proc.with_ctx(|_| (ErrorKind::Filesystem, "read single process details"))?; - let pid = proc.pid(); - if proc - .namespaces() - .with_ctx(|_| (ErrorKind::Filesystem, lazy_format!("read pid {} ns", pid)))? - .0 - .get(OsStr::new("pid")) - .map_or(false, |ns| ns.identifier == ns_id) - { - let pids = proc.read::("status").with_ctx(|_| { - ( - ErrorKind::Filesystem, - lazy_format!("read pid {} NSpid", pid), - ) - })?; - if pids.0.len() == 2 && pids.0[1] == 1 { - nix::sys::signal::kill(Pid::from_raw(pid), nix::sys::signal::SIGKILL) - .with_ctx(|_| { - ( - ErrorKind::Filesystem, - lazy_format!( - "kill pid {} (determined to be pid 1 in subcontainer)", - pid - ), - ) - })?; - } - } - } - nix::mount::umount(&chroot.join("proc")) - .with_ctx(|_| (ErrorKind::Filesystem, "unmounting subcontainer procfs"))?; - } - + kill_init(Path::new("/proc"), &chroot)?; if (std::io::stdin().is_terminal() && std::io::stdout().is_terminal() && std::io::stderr().is_terminal()) diff --git a/core/startos/src/util/logger.rs b/core/startos/src/util/logger.rs index 816721d5b..5fc9fb98e 100644 --- a/core/startos/src/util/logger.rs +++ b/core/startos/src/util/logger.rs @@ -61,30 +61,31 @@ impl StartOSLogger { use tracing_subscriber::prelude::*; use tracing_subscriber::{fmt, EnvFilter}; - let filter_layer = EnvFilter::builder() - .with_default_directive( - format!("{}=info", std::module_path!().split("::").next().unwrap()) - .parse() - .unwrap(), - ) - .from_env_lossy(); - #[cfg(feature = "unstable")] - let filter_layer = filter_layer - .add_directive("tokio=trace".parse().unwrap()) - .add_directive("runtime=trace".parse().unwrap()); + let filter_layer = || { + EnvFilter::builder() + .with_default_directive( + format!("{}=info", std::module_path!().split("::").next().unwrap()) + .parse() + .unwrap(), + ) + .from_env_lossy() + }; + let fmt_layer = fmt::layer() .with_writer(logfile) .with_line_number(true) .with_file(true) - .with_target(true); + .with_target(true) + .with_filter(filter_layer()); - let sub = tracing_subscriber::registry() - .with(filter_layer) - .with(fmt_layer) - .with(ErrorLayer::default()); + let sub = tracing_subscriber::registry(); #[cfg(feature = "unstable")] let sub = sub.with(console_subscriber::spawn()); + #[cfg(not(feature = "unstable"))] + let sub = sub.with(filter_layer()); + + let sub = sub.with(fmt_layer).with(ErrorLayer::default()); sub } diff --git a/core/startos/src/version/mod.rs b/core/startos/src/version/mod.rs index 750a5ee07..eeed45ee5 100644 --- a/core/startos/src/version/mod.rs +++ b/core/startos/src/version/mod.rs @@ -34,8 +34,9 @@ mod v0_3_6_alpha_11; mod v0_3_6_alpha_12; mod v0_3_6_alpha_13; mod v0_3_6_alpha_14; +mod v0_3_6_alpha_15; -pub type Current = v0_3_6_alpha_14::Version; // VERSION_BUMP +pub type Current = v0_3_6_alpha_15::Version; // VERSION_BUMP impl Current { #[instrument(skip(self, db))] @@ -131,7 +132,8 @@ enum Version { V0_3_6_alpha_11(Wrapper), V0_3_6_alpha_12(Wrapper), V0_3_6_alpha_13(Wrapper), - V0_3_6_alpha_14(Wrapper), // VERSION_BUMP + V0_3_6_alpha_14(Wrapper), + V0_3_6_alpha_15(Wrapper), // VERSION_BUMP Other(exver::Version), } @@ -169,7 +171,8 @@ impl Version { Self::V0_3_6_alpha_11(v) => DynVersion(Box::new(v.0)), Self::V0_3_6_alpha_12(v) => DynVersion(Box::new(v.0)), Self::V0_3_6_alpha_13(v) => DynVersion(Box::new(v.0)), - Self::V0_3_6_alpha_14(v) => DynVersion(Box::new(v.0)), // VERSION_BUMP + Self::V0_3_6_alpha_14(v) => DynVersion(Box::new(v.0)), + Self::V0_3_6_alpha_15(v) => DynVersion(Box::new(v.0)), // VERSION_BUMP Self::Other(v) => { return Err(Error::new( eyre!("unknown version {v}"), @@ -199,7 +202,8 @@ impl Version { Version::V0_3_6_alpha_11(Wrapper(x)) => x.semver(), Version::V0_3_6_alpha_12(Wrapper(x)) => x.semver(), Version::V0_3_6_alpha_13(Wrapper(x)) => x.semver(), - Version::V0_3_6_alpha_14(Wrapper(x)) => x.semver(), // VERSION_BUMP + Version::V0_3_6_alpha_14(Wrapper(x)) => x.semver(), + Version::V0_3_6_alpha_15(Wrapper(x)) => x.semver(), // VERSION_BUMP Version::Other(x) => x.clone(), } } diff --git a/core/startos/src/version/v0_3_6_alpha_15.rs b/core/startos/src/version/v0_3_6_alpha_15.rs new file mode 100644 index 000000000..eef6c5993 --- /dev/null +++ b/core/startos/src/version/v0_3_6_alpha_15.rs @@ -0,0 +1,39 @@ +use std::collections::BTreeMap; + +use exver::{PreReleaseSegment, VersionRange}; +use imbl_value::json; + +use super::v0_3_5::V0_3_0_COMPAT; +use super::{v0_3_6_alpha_14, VersionT}; +use crate::prelude::*; + +lazy_static::lazy_static! { + static ref V0_3_6_alpha_15: exver::Version = exver::Version::new( + [0, 3, 6], + [PreReleaseSegment::String("alpha".into()), 15.into()] + ); +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct Version; + +impl VersionT for Version { + type Previous = v0_3_6_alpha_14::Version; + type PreUpRes = (); + + async fn pre_up(self) -> Result { + Ok(()) + } + fn semver(self) -> exver::Version { + V0_3_6_alpha_15.clone() + } + fn compat(self) -> &'static VersionRange { + &V0_3_0_COMPAT + } + fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> { + Ok(()) + } + fn down(self, _db: &mut Value) -> Result<(), Error> { + Ok(()) + } +} diff --git a/sdk/base/lib/Effects.ts b/sdk/base/lib/Effects.ts index dcb03af4e..135a2942c 100644 --- a/sdk/base/lib/Effects.ts +++ b/sdk/base/lib/Effects.ts @@ -131,6 +131,8 @@ export type Effects = { }): Promise /** Returns the IP address of the container */ getContainerIp(): Promise + /** Returns the IP address of StartOS */ + getOsIp(): Promise // interface /** Creates an interface bound to a specific host and port to show to the user */ exportServiceInterface(options: ExportServiceInterfaceParams): Promise diff --git a/sdk/base/lib/osBindings/ImageMetadata.ts b/sdk/base/lib/osBindings/ImageMetadata.ts index b50f7a084..2222d56e9 100644 --- a/sdk/base/lib/osBindings/ImageMetadata.ts +++ b/sdk/base/lib/osBindings/ImageMetadata.ts @@ -1,3 +1,8 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type ImageMetadata = { workdir: string; user: string } +export type ImageMetadata = { + workdir: string + user: string + entrypoint: Array | null + cmd: Array | null +} diff --git a/sdk/base/lib/test/startosTypeValidation.test.ts b/sdk/base/lib/test/startosTypeValidation.test.ts index 2de7b43a4..dfccb6506 100644 --- a/sdk/base/lib/test/startosTypeValidation.test.ts +++ b/sdk/base/lib/test/startosTypeValidation.test.ts @@ -79,6 +79,7 @@ describe("startosTypeValidation ", () => { }, getSystemSmtp: {} as WithCallback, getContainerIp: undefined, + getOsIp: undefined, getServicePortForward: {} as GetServicePortForwardParams, clearServiceInterfaces: {} as ClearServiceInterfacesParams, exportServiceInterface: {} as ExportServiceInterfaceParams, diff --git a/sdk/base/lib/types.ts b/sdk/base/lib/types.ts index 85a8c4404..056f986c2 100644 --- a/sdk/base/lib/types.ts +++ b/sdk/base/lib/types.ts @@ -127,7 +127,11 @@ export type SmtpValue = { password: string | null | undefined } -export type CommandType = string | [string, ...string[]] +export class UseEntrypoint { + constructor(readonly overridCmd?: string[]) {} +} + +export type CommandType = string | [string, ...string[]] | UseEntrypoint export type DaemonReturned = { wait(): Promise diff --git a/sdk/base/lib/util/getServiceInterface.ts b/sdk/base/lib/util/getServiceInterface.ts index 44c7c6fc8..bb57b6c3b 100644 --- a/sdk/base/lib/util/getServiceInterface.ts +++ b/sdk/base/lib/util/getServiceInterface.ts @@ -17,6 +17,7 @@ export const getHostname = (url: string): Hostname | null => { export type Filled = { hostnames: HostnameInfo[] + publicHostnames: HostnameInfo[] onionHostnames: HostnameInfo[] localHostnames: HostnameInfo[] ipHostnames: HostnameInfo[] @@ -25,6 +26,7 @@ export type Filled = { nonIpHostnames: HostnameInfo[] urls: UrlString[] + publicUrls: UrlString[] onionUrls: UrlString[] localUrls: UrlString[] ipUrls: UrlString[] @@ -105,6 +107,9 @@ export const filledAddress = ( return { ...addressInfo, hostnames, + get publicHostnames() { + return hostnames.filter((h) => h.kind === "onion" || h.public) + }, get onionHostnames() { return hostnames.filter((h) => h.kind === "onion") }, @@ -141,6 +146,9 @@ export const filledAddress = ( get urls() { return this.hostnames.flatMap(toUrl) }, + get publicUrls() { + return this.publicHostnames.flatMap(toUrl) + }, get onionUrls() { return this.onionHostnames.flatMap(toUrl) }, diff --git a/sdk/base/package-lock.json b/sdk/base/package-lock.json index 4d5625489..eadbd049b 100644 --- a/sdk/base/package-lock.json +++ b/sdk/base/package-lock.json @@ -28,7 +28,7 @@ "ts-node": "^10.9.1", "ts-pegjs": "^4.2.1", "tsx": "^4.7.1", - "typescript": "^5.0.4" + "typescript": "^5.7.3" } }, "node_modules/@ampproject/remapping": { @@ -4417,16 +4417,17 @@ } }, "node_modules/typescript": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", - "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.3.tgz", + "integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=12.20" + "node": ">=14.17" } }, "node_modules/update-browserslist-db": { diff --git a/sdk/base/package.json b/sdk/base/package.json index 6eae719a7..7a162c218 100644 --- a/sdk/base/package.json +++ b/sdk/base/package.json @@ -47,6 +47,6 @@ "ts-node": "^10.9.1", "ts-pegjs": "^4.2.1", "tsx": "^4.7.1", - "typescript": "^5.0.4" + "typescript": "^5.7.3" } } diff --git a/sdk/base/tsconfig.json b/sdk/base/tsconfig.json index cd73f3164..e73c13689 100644 --- a/sdk/base/tsconfig.json +++ b/sdk/base/tsconfig.json @@ -12,7 +12,7 @@ "skipLibCheck": true, "module": "commonjs", "outDir": "../baseDist", - "target": "es2018" + "target": "es2021" }, "include": ["lib/**/*"], "exclude": ["lib/**/*.spec.ts", "lib/**/*.gen.ts", "list", "node_modules"] diff --git a/sdk/package/lib/StartSdk.ts b/sdk/package/lib/StartSdk.ts index 3c6a05abd..5dfd797d1 100644 --- a/sdk/package/lib/StartSdk.ts +++ b/sdk/package/lib/StartSdk.ts @@ -26,7 +26,7 @@ import { import * as patterns from "../../base/lib/util/patterns" import { BackupSync, Backups } from "./backup/Backups" import { smtpInputSpec } from "../../base/lib/actions/input/inputSpecConstants" -import { Daemons } from "./mainFn/Daemons" +import { CommandController, Daemons } from "./mainFn/Daemons" import { healthCheck, HealthCheckParams } from "./health/HealthCheck" import { checkPortListening } from "./health/checkFns/checkPortListening" import { checkWebUrl, runHealthScript } from "./health/checkFns" @@ -71,6 +71,7 @@ import { GetInput } from "../../base/lib/actions/setupActions" import { Run } from "../../base/lib/actions/setupActions" import * as actions from "../../base/lib/actions" import { setupInit } from "./inits/setupInit" +import * as fs from "node:fs/promises" export const SDKVersion = testTypeVersion("0.3.6") @@ -124,6 +125,7 @@ export class StartSdk { effects.getServicePortForward(...args), clearBindings: (effects, ...args) => effects.clearBindings(...args), getContainerIp: (effects, ...args) => effects.getContainerIp(...args), + getOsIp: (effects, ...args) => effects.getOsIp(...args), getSslKey: (effects, ...args) => effects.getSslKey(...args), setDataVersion: (effects, ...args) => effects.setDataVersion(...args), getDataVersion: (effects, ...args) => effects.getDataVersion(...args), @@ -219,6 +221,8 @@ export class StartSdk { of: (effects: Effects, id: string) => new MultiHost({ id, effects }), }, nullIfEmpty, + useEntrypoint: (overrideCmd?: string[]) => + new T.UseEntrypoint(overrideCmd), runCommand: async ( effects: Effects, image: { @@ -234,13 +238,7 @@ export class StartSdk { */ name?: string, ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> => { - return runCommand( - effects, - image, - command, - options, - name || (Array.isArray(command) ? command.join(" ") : command), - ) + return runCommand(effects, image, command, options, name) }, /** * @description Use this class to create an Action. By convention, each Action should receive its own file. @@ -1081,18 +1079,37 @@ export class StartSdk { export async function runCommand( effects: Effects, image: { imageId: keyof Manifest["images"] & T.ImageId; sharedRun?: boolean }, - command: string | [string, ...string[]], + command: T.CommandType, options: CommandOptions & { mounts?: { path: string; options: MountOptions }[] }, - name: string, + name?: string, ): Promise<{ stdout: string | Buffer; stderr: string | Buffer }> { - const commands = splitCommand(command) + let commands: string[] + if (command instanceof T.UseEntrypoint) { + const imageMeta: T.ImageMetadata = await fs + .readFile(`/media/startos/images/${image.imageId}.json`, { + encoding: "utf8", + }) + .catch(() => "{}") + .then(JSON.parse) + commands = imageMeta.entrypoint ?? [] + commands.concat(...(command.overridCmd ?? imageMeta.cmd ?? [])) + } else commands = splitCommand(command) return SubContainer.with( effects, image, options.mounts || [], - name, + name || + commands + .map((c) => { + if (c.includes(" ")) { + return `"${c.replace(/"/g, `\"`)}"` + } else { + return c + } + }) + .join(" "), (subcontainer) => subcontainer.exec(commands), ) } diff --git a/sdk/package/lib/health/HealthCheck.ts b/sdk/package/lib/health/HealthCheck.ts index 50aab97f0..d921100a0 100644 --- a/sdk/package/lib/health/HealthCheck.ts +++ b/sdk/package/lib/health/HealthCheck.ts @@ -11,14 +11,17 @@ export type HealthCheckParams = { id: HealthCheckId name: string trigger?: Trigger + gracePeriod?: number fn(): Promise | HealthCheckResult onFirstSuccess?: () => unknown | Promise } export function healthCheck(o: HealthCheckParams) { new Promise(async () => { + const start = performance.now() let currentValue: TriggerInput = {} const getCurrentValue = () => currentValue + const gracePeriod = o.gracePeriod ?? 5000 const trigger = (o.trigger ?? defaultTrigger)(getCurrentValue) const triggerFirstSuccess = once(() => Promise.resolve( @@ -33,7 +36,9 @@ export function healthCheck(o: HealthCheckParams) { res = await trigger.next() ) { try { - const { result, message } = await o.fn() + let { result, message } = await o.fn() + if (result === "failure" && performance.now() - start <= gracePeriod) + result = "starting" await o.effects.setHealth({ name: o.name, id: o.id, @@ -48,7 +53,8 @@ export function healthCheck(o: HealthCheckParams) { await o.effects.setHealth({ name: o.name, id: o.id, - result: "failure", + result: + performance.now() - start <= gracePeriod ? "starting" : "failure", message: asMessage(e) || "", }) currentValue.lastResult = "failure" diff --git a/sdk/package/lib/mainFn/CommandController.ts b/sdk/package/lib/mainFn/CommandController.ts index a7375b369..d4ea171f0 100644 --- a/sdk/package/lib/mainFn/CommandController.ts +++ b/sdk/package/lib/mainFn/CommandController.ts @@ -9,6 +9,7 @@ import { } from "../util/SubContainer" import { splitCommand } from "../util" import * as cp from "child_process" +import * as fs from "node:fs/promises" export class CommandController { private constructor( @@ -45,7 +46,17 @@ export class CommandController { onStderr?: (chunk: Buffer | string | any) => void }, ) => { - const commands = splitCommand(command) + let commands: string[] + if (command instanceof T.UseEntrypoint) { + const imageMeta: T.ImageMetadata = await fs + .readFile(`/media/startos/images/${subcontainer.imageId}.json`, { + encoding: "utf8", + }) + .catch(() => "{}") + .then(JSON.parse) + commands = imageMeta.entrypoint ?? [] + commands.concat(...(command.overridCmd ?? imageMeta.cmd ?? [])) + } else commands = splitCommand(command) const subc = subcontainer instanceof SubContainer ? subcontainer @@ -55,10 +66,15 @@ export class CommandController { subcontainer, options?.subcontainerName || commands.join(" "), ) - for (let mount of options.mounts || []) { - await subc.mount(mount.options, mount.path) + try { + for (let mount of options.mounts || []) { + await subc.mount(mount.options, mount.path) + } + return subc + } catch (e) { + await subc.destroy() + throw e } - return subc })() try { diff --git a/sdk/package/lib/mainFn/Daemons.ts b/sdk/package/lib/mainFn/Daemons.ts index 8d0e6297a..e75ebec93 100644 --- a/sdk/package/lib/mainFn/Daemons.ts +++ b/sdk/package/lib/mainFn/Daemons.ts @@ -38,6 +38,12 @@ export type Ready = { fn: ( spawnable: ExecSpawnable, ) => Promise | HealthCheckResult + /** + * A duration in milliseconds to treat a failing health check as "starting" + * + * defaults to 5000 + */ + gracePeriod?: number trigger?: Trigger } diff --git a/sdk/package/lib/mainFn/HealthDaemon.ts b/sdk/package/lib/mainFn/HealthDaemon.ts index ac459f08c..3a1c1000f 100644 --- a/sdk/package/lib/mainFn/HealthDaemon.ts +++ b/sdk/package/lib/mainFn/HealthDaemon.ts @@ -25,6 +25,7 @@ export class HealthDaemon { private _health: HealthCheckResult = { result: "starting", message: null } private healthWatchers: Array<() => unknown> = [] private running = false + private started?: number private resolveReady: (() => void) | undefined private readyPromise: Promise constructor( @@ -75,6 +76,7 @@ export class HealthDaemon { if (newStatus) { ;(await this.daemon).start() + this.started = performance.now() this.setupHealthCheck() } else { ;(await this.daemon).stop() @@ -146,14 +148,21 @@ export class HealthDaemon { this._health = health this.healthWatchers.forEach((watcher) => watcher()) const display = this.ready.display - const result = health.result if (!display) { return } + let result = health.result + if ( + result === "failure" && + this.started && + performance.now() - this.started <= (this.ready.gracePeriod ?? 5000) + ) + result = "starting" await this.effects.setHealth({ ...health, id: this.id, name: display, + result, } as SetHealth) } diff --git a/sdk/package/lib/util/SubContainer.ts b/sdk/package/lib/util/SubContainer.ts index 77b31fec8..a8262efc5 100644 --- a/sdk/package/lib/util/SubContainer.ts +++ b/sdk/package/lib/util/SubContainer.ts @@ -46,6 +46,15 @@ export interface ExecSpawnable { * @see {@link ExecSpawnable} */ export class SubContainer implements ExecSpawnable { + private static finalizationEffects: { effects?: T.Effects } = {} + private static registry = new FinalizationRegistry((guid: string) => { + if (this.finalizationEffects.effects) { + this.finalizationEffects.effects.subcontainer + .destroyFs({ guid }) + .catch((e) => console.error("failed to cleanup SubContainer", guid, e)) + } + }) + private leader: cp.ChildProcess private leaderExited: boolean = false private waitProc: () => Promise @@ -55,6 +64,8 @@ export class SubContainer implements ExecSpawnable { readonly rootfs: string, readonly guid: T.Guid, ) { + if (!SubContainer.finalizationEffects.effects) + SubContainer.finalizationEffects.effects = effects this.leaderExited = false this.leader = cp.spawn("start-cli", ["subcontainer", "launch", rootfs], { killSignal: "SIGKILL", @@ -94,6 +105,8 @@ export class SubContainer implements ExecSpawnable { imageId, name, }) + const res = new SubContainer(effects, imageId, rootfs, guid) + SubContainer.registry.register(res, guid, res) const shared = ["dev", "sys"] if (!!sharedRun) { @@ -111,7 +124,7 @@ export class SubContainer implements ExecSpawnable { await execFile("mount", ["--rbind", from, to]) } - return new SubContainer(effects, imageId, rootfs, guid) + return res } static async with( @@ -202,6 +215,7 @@ export class SubContainer implements ExecSpawnable { const guid = this.guid await this.killLeader() await this.effects.subcontainer.destroyFs({ guid }) + SubContainer.registry.unregister(this) return null } } @@ -224,8 +238,9 @@ export class SubContainer implements ExecSpawnable { .catch(() => "{}") .then(JSON.parse) let extra: string[] = [] + let user = imageMeta.user || "root" if (options?.user) { - extra.push(`--user=${options.user}`) + user = options.user delete options.user } let workdir = imageMeta.workdir || "/" @@ -239,6 +254,7 @@ export class SubContainer implements ExecSpawnable { "subcontainer", "exec", `--env=/media/startos/images/${this.imageId}.env`, + `--user=${user}`, `--workdir=${workdir}`, ...extra, this.rootfs, @@ -294,15 +310,16 @@ export class SubContainer implements ExecSpawnable { options?: CommandOptions, ): Promise { await this.waitProc() - const imageMeta: any = await fs + const imageMeta: T.ImageMetadata = await fs .readFile(`/media/startos/images/${this.imageId}.json`, { encoding: "utf8", }) .catch(() => "{}") .then(JSON.parse) let extra: string[] = [] + let user = imageMeta.user || "root" if (options?.user) { - extra.push(`--user=${options.user}`) + user = options.user delete options.user } let workdir = imageMeta.workdir || "/" @@ -318,6 +335,7 @@ export class SubContainer implements ExecSpawnable { "subcontainer", "launch", `--env=/media/startos/images/${this.imageId}.env`, + `--user=${user}`, `--workdir=${workdir}`, ...extra, this.rootfs, @@ -336,15 +354,16 @@ export class SubContainer implements ExecSpawnable { options: CommandOptions & StdioOptions = { stdio: "inherit" }, ): Promise { await this.waitProc() - const imageMeta: any = await fs + const imageMeta: T.ImageMetadata = await fs .readFile(`/media/startos/images/${this.imageId}.json`, { encoding: "utf8", }) .catch(() => "{}") .then(JSON.parse) let extra: string[] = [] - if (options.user) { - extra.push(`--user=${options.user}`) + let user = imageMeta.user || "root" + if (options?.user) { + user = options.user delete options.user } let workdir = imageMeta.workdir || "/" @@ -358,6 +377,7 @@ export class SubContainer implements ExecSpawnable { "subcontainer", "exec", `--env=/media/startos/images/${this.imageId}.env`, + `--user=${user}`, `--workdir=${workdir}`, ...extra, this.rootfs, diff --git a/sdk/package/package-lock.json b/sdk/package/package-lock.json index 72a4a12a2..6e8dbbc03 100644 --- a/sdk/package/package-lock.json +++ b/sdk/package/package-lock.json @@ -1,12 +1,12 @@ { "name": "@start9labs/start-sdk", - "version": "0.3.6-beta.9", + "version": "0.3.6-beta.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@start9labs/start-sdk", - "version": "0.3.6-beta.9", + "version": "0.3.6-beta.11", "license": "MIT", "dependencies": { "@iarna/toml": "^2.2.5", @@ -28,7 +28,7 @@ "ts-node": "^10.9.1", "ts-pegjs": "^4.2.1", "tsx": "^4.7.1", - "typescript": "^5.0.4" + "typescript": "^5.7.3" } }, "../base": { @@ -4438,16 +4438,17 @@ } }, "node_modules/typescript": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", - "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.3.tgz", + "integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=12.20" + "node": ">=14.17" } }, "node_modules/update-browserslist-db": { diff --git a/sdk/package/package.json b/sdk/package/package.json index 429164006..05af0b123 100644 --- a/sdk/package/package.json +++ b/sdk/package/package.json @@ -1,6 +1,6 @@ { "name": "@start9labs/start-sdk", - "version": "0.3.6-beta.9", + "version": "0.3.6-beta.11", "description": "Software development kit to facilitate packaging services for StartOS", "main": "./package/lib/index.js", "types": "./package/lib/index.d.ts", @@ -55,6 +55,6 @@ "ts-node": "^10.9.1", "ts-pegjs": "^4.2.1", "tsx": "^4.7.1", - "typescript": "^5.0.4" + "typescript": "^5.7.3" } } diff --git a/sdk/package/tsconfig.json b/sdk/package/tsconfig.json index 7b4d4f7d8..20f7f60f7 100644 --- a/sdk/package/tsconfig.json +++ b/sdk/package/tsconfig.json @@ -12,7 +12,7 @@ "skipLibCheck": true, "module": "commonjs", "outDir": "../dist", - "target": "es2018" + "target": "es2021" }, "include": ["lib/**/*", "../base/lib/util/Hostname.ts"], "exclude": ["lib/**/*.spec.ts", "lib/**/*.gen.ts", "list", "node_modules"] diff --git a/web/package-lock.json b/web/package-lock.json index 8c19bb05b..8cea33725 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,12 +1,12 @@ { "name": "startos-ui", - "version": "0.3.6-alpha.14", + "version": "0.3.6-alpha.15", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "startos-ui", - "version": "0.3.6-alpha.14", + "version": "0.3.6-alpha.15", "license": "MIT", "dependencies": { "@angular/animations": "^14.1.0", diff --git a/web/package.json b/web/package.json index 128df447b..4808a3945 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "startos-ui", - "version": "0.3.6-alpha.14", + "version": "0.3.6-alpha.15", "author": "Start9 Labs, Inc", "homepage": "https://start9.com/", "license": "MIT",