From e08bbb142a1b5f439e77aac2fc1e56e419e5b8dc Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 5 Feb 2026 13:03:51 -0700 Subject: [PATCH] add documentation for ai agents --- .gitignore | 3 +- CLAUDE.md | 190 ++++++++++++++++++++ agents/TODO.md | 9 + agents/core-rust-patterns.md | 249 +++++++++++++++++++++++++++ agents/rpc-toolkit.md | 226 ++++++++++++++++++++++++ agents/s9pk-structure.md | 122 +++++++++++++ build/lib/scripts/chroot-and-upgrade | 2 +- container-runtime/RPCSpec.md | 107 +++++++++--- web/package.json | 4 +- 9 files changed, 884 insertions(+), 28 deletions(-) create mode 100644 CLAUDE.md create mode 100644 agents/TODO.md create mode 100644 agents/core-rust-patterns.md create mode 100644 agents/rpc-toolkit.md create mode 100644 agents/s9pk-structure.md diff --git a/.gitignore b/.gitignore index 227782023..51e25ae95 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,5 @@ secrets.db /compiled.tar /compiled-*.tar /build/lib/firmware -tmp \ No newline at end of file +tmp +web/.i18n-checked diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..9e654b2ee --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,190 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +StartOS is an open-source Linux distribution for running personal servers. It manages discovery, installation, network configuration, backups, and health monitoring of self-hosted services. + +**Tech Stack:** +- Backend: Rust (async/Tokio, Axum web framework) +- Frontend: Angular 20 + TypeScript + TaigaUI +- Container runtime: Node.js/TypeScript with LXC +- Database/State: Patch-DB (git submodule) - storage layer with reactive frontend sync +- API: JSON-RPC via rpc-toolkit (see `agents/rpc-toolkit.md`) +- Auth: Password + session cookie, public/private key signatures, local authcookie (see `core/src/middleware/auth/`) + +## Build Commands + +```bash +# Set platform (x86_64, aarch64, riscv64, raspberrypi, x86_64-nonfree, aarch64-nonfree) +# Can export or set inline. Most recent platform is remembered if not specified. +export PLATFORM=x86_64 +# or: PLATFORM=x86_64 make iso + +# Development mode - sets dev environment, prevents rebuilds on git hash changes +. ./devmode.sh + +# Full ISO build +make iso + +# Faster development iterations (same network) +make update-startbox REMOTE=start9@ # Binary + UI only (fastest, no container runtime) +make update-deb REMOTE=start9@ # Full Debian package +make update REMOTE=start9@ # OTA-style update + +# Remote device updates (different network, uses magic-wormhole) +make wormhole # Send startbox binary +make wormhole-deb # Send Debian package +make wormhole-squashfs # Send squashfs image + +# Build specific components +make all # All Rust binaries +make uis # All web UIs +make ui # Main UI only +make deb # Debian package +``` + +## Testing + +```bash +make test # All tests +make test-core # Rust tests (via ./core/run-tests.sh) +make test-sdk # SDK tests +make test-container-runtime # Container runtime tests + +# Run specific Rust test +cd core && cargo test --features=test +``` + +## Code Formatting + +```bash +make format # Rust (requires nightly) +cd web && npm run check # TypeScript type checking +cd web && npm run format # TypeScript/HTML/SCSS formatting (prettier) +``` + +## Architecture + +### Core (`/core`) +The Rust backend daemon. Main binaries: +- `startbox` - Main daemon (runs as `startd`) +- `start-cli` - CLI interface +- `start-container` - Runs inside LXC containers; communicates with host and manages subcontainers +- `registrybox` - Registry daemon +- `tunnelbox` - VPN/tunnel daemon + +**Key modules:** +- `src/context/` - Context types (RpcContext, CliContext, InitContext, DiagnosticContext) +- `src/service/` - Service lifecycle management with actor pattern (`service_actor.rs`) +- `src/db/model/` - Patch-DB models (`public.rs` synced to frontend, `private.rs` backend-only) +- `src/net/` - Networking (DNS, ACME, WiFi, Tor via Arti, WireGuard) +- `src/s9pk/` - S9PK package format (merkle archive) +- `src/registry/` - Package registry management + +**RPC Pattern:** See `agents/rpc-toolkit.md` + +### Web (`/web`) +Angular projects sharing common code: +- `projects/ui/` - Main admin interface +- `projects/setup-wizard/` - Initial setup +- `projects/start-tunnel/` - VPN management UI +- `projects/shared/` - Common library (API clients, components) +- `projects/marketplace/` - Service discovery + +**Development:** +```bash +cd web +npm ci +npm run start:ui # Dev server with mocks +npm run build:ui # Production build +npm run check # Type check all projects +``` + +### Container Runtime (`/container-runtime`) +Node.js runtime that manages service containers via RPC. See `RPCSpec.md` for protocol. + +**Container Architecture:** +``` +LXC Container (uniform base for all services) +└── systemd + └── container-runtime.service + └── Loads /usr/lib/startos/package/index.js (from s9pk javascript.squashfs) + └── Package JS launches subcontainers (from images in s9pk) +``` + +The container runtime communicates with the host via JSON-RPC over Unix socket. Package JavaScript must export functions conforming to the `ABI` type defined in `sdk/base/lib/types.ts`. + +**`/media/startos/` directory (mounted by host into container):** + +| Path | Description | +|------|-------------| +| `volumes//` | Package data volumes (id-mapped, persistent) | +| `assets/` | Read-only assets from s9pk `assets.squashfs` | +| `images//` | Container images (squashfs, used for subcontainers) | +| `images/.env` | Environment variables for image | +| `images/.json` | Image metadata | +| `backup/` | Backup mount point (mounted during backup operations) | +| `rpc/service.sock` | RPC socket (container runtime listens here) | +| `rpc/host.sock` | Host RPC socket (for effects callbacks to host) | + +**S9PK Structure:** See `agents/s9pk-structure.md` (TODO: create this doc) + +### SDK (`/sdk`) +TypeScript SDK for packaging services (`@start9labs/start-sdk`). + +- `base/` - Core types, ABI definitions, effects interface (`@start9labs/start-sdk-base`) +- `package/` - Full SDK for package developers, re-exports base + +### Patch-DB (`/patch-db`) +Git submodule providing diff-based state synchronization. Changes to `db/model/public.rs` automatically sync to the frontend. + +**Key patterns:** +- `db.peek().await` - Get a read-only snapshot of the database state +- `db.mutate(|db| { ... }).await` - Apply mutations atomically, returns `MutateResult` +- `#[derive(HasModel)]` - Derive macro for types stored in the database, generates typed accessors + +**Generated accessor types** (from `HasModel` derive): +- `as_field()` - Immutable reference: `&Model` +- `as_field_mut()` - Mutable reference: `&mut Model` +- `into_field()` - Owned value: `Model` + +**`Model` APIs** (from `db/prelude.rs`): +- `.de()` - Deserialize to `T` +- `.ser(&value)` - Serialize from `T` +- `.mutate(|v| ...)` - Deserialize, mutate, reserialize +- For maps: `.keys()`, `.as_idx(&key)`, `.as_idx_mut(&key)`, `.insert()`, `.remove()`, `.contains_key()` + +## Environment Variables + +- `PLATFORM` - Target platform +- `ENVIRONMENT` - Feature flags (comma-separated): + - `dev` - Enables password SSH before setup, other developer conveniences + - `unstable` - Adds debugging with performance penalty, enables throwing in non-prod scenarios + - `console` - Enables tokio-console for async debugging +- `PROFILE` - Build profile (`release`, `dev`) +- `GIT_BRANCH_AS_HASH` - Use git branch as version hash + +## TypeScript Bindings + +Rust types export to TypeScript via ts-rs: +```bash +make ts-bindings # Generates core/bindings/, copies to sdk/base/lib/osBindings/ +``` + +## Multi-Platform Support + +Platforms: `x86_64`, `x86_64-nonfree`, `aarch64`, `aarch64-nonfree`, `riscv64`, `raspberrypi` + +The `-nonfree` variants include non-free firmware and drivers. Some platforms like `raspberrypi` also include non-free components by necessity. + +The build system uses cross-compilation with zig for musl targets. Platform-specific code checks `PLATFORM` constant. + +## Supplementary Documentation + +The `agents/` directory contains detailed documentation and task tracking for AI assistants: + +- `TODO.md` - Pending tasks for AI agents (check this first, remove items when completed) +- `rpc-toolkit.md` - JSON-RPC patterns and handler configuration +- `core-rust-patterns.md` - Common utilities and patterns for Rust code in `/core` (guard pattern, mount guards, etc.) diff --git a/agents/TODO.md b/agents/TODO.md new file mode 100644 index 000000000..70124aa74 --- /dev/null +++ b/agents/TODO.md @@ -0,0 +1,9 @@ +# AI Agent TODOs + +Pending tasks for AI agents. Remove items when completed. + +## Unreviewed CLAUDE.md Sections + +- [ ] Architecture - Web (`/web`) - @MattDHill + + diff --git a/agents/core-rust-patterns.md b/agents/core-rust-patterns.md new file mode 100644 index 000000000..fc7058255 --- /dev/null +++ b/agents/core-rust-patterns.md @@ -0,0 +1,249 @@ +# Utilities & Patterns + +This document covers common utilities and patterns used throughout the StartOS codebase. + +## Util Module (`core/src/util/`) + +The `util` module contains reusable utilities. Key submodules: + +| Module | Purpose | +|--------|---------| +| `actor/` | Actor pattern implementation for concurrent state management | +| `collections/` | Custom collection types | +| `crypto.rs` | Cryptographic utilities (encryption, hashing) | +| `future.rs` | Future/async utilities | +| `io.rs` | File I/O helpers (create_file, canonicalize, etc.) | +| `iter.rs` | Iterator extensions | +| `net.rs` | Network utilities | +| `rpc.rs` | RPC helpers | +| `rpc_client.rs` | RPC client utilities | +| `serde.rs` | Serialization helpers (Base64, display/fromstr, etc.) | +| `sync.rs` | Synchronization primitives (SyncMutex, etc.) | + +## Command Invocation (`Invoke` trait) + +The `Invoke` trait provides a clean way to run external commands with error handling: + +```rust +use crate::util::Invoke; + +// Simple invocation +tokio::process::Command::new("ls") + .arg("-la") + .invoke(ErrorKind::Filesystem) + .await?; + +// With timeout +tokio::process::Command::new("slow-command") + .timeout(Some(Duration::from_secs(30))) + .invoke(ErrorKind::Timeout) + .await?; + +// With input +let mut input = Cursor::new(b"input data"); +tokio::process::Command::new("cat") + .input(Some(&mut input)) + .invoke(ErrorKind::Filesystem) + .await?; + +// Piped commands +tokio::process::Command::new("cat") + .arg("file.txt") + .pipe(&mut tokio::process::Command::new("grep").arg("pattern")) + .invoke(ErrorKind::Filesystem) + .await?; +``` + +## Guard Pattern + +Guards ensure cleanup happens when they go out of scope. + +### `GeneralGuard` / `GeneralBoxedGuard` + +For arbitrary cleanup actions: + +```rust +use crate::util::GeneralGuard; + +let guard = GeneralGuard::new(|| { + println!("Cleanup runs on drop"); +}); + +// Do work... + +// Explicit drop with action +guard.drop(); + +// Or skip the action +// guard.drop_without_action(); +``` + +### `FileLock` + +File-based locking with automatic unlock: + +```rust +use crate::util::FileLock; + +let lock = FileLock::new("/path/to/lockfile", true).await?; // blocking=true +// Lock held until dropped or explicitly unlocked +lock.unlock().await?; +``` + +## Mount Guard Pattern (`core/src/disk/mount/guard.rs`) + +RAII guards for filesystem mounts. Ensures filesystems are unmounted when guards are dropped. + +### `MountGuard` + +Basic mount guard: + +```rust +use crate::disk::mount::guard::MountGuard; +use crate::disk::mount::filesystem::{MountType, ReadOnly}; + +let guard = MountGuard::mount(&filesystem, "/mnt/target", ReadOnly).await?; + +// Use the mounted filesystem at guard.path() +do_something(guard.path()).await?; + +// Explicit unmount (or auto-unmounts on drop) +guard.unmount(false).await?; // false = don't delete mountpoint +``` + +### `TmpMountGuard` + +Reference-counted temporary mount (mounts to `/media/startos/tmp/`): + +```rust +use crate::disk::mount::guard::TmpMountGuard; +use crate::disk::mount::filesystem::ReadOnly; + +// Multiple clones share the same mount +let guard1 = TmpMountGuard::mount(&filesystem, ReadOnly).await?; +let guard2 = guard1.clone(); + +// Mount stays alive while any guard exists +// Auto-unmounts when last guard is dropped +``` + +### `GenericMountGuard` trait + +All mount guards implement this trait: + +```rust +pub trait GenericMountGuard: std::fmt::Debug + Send + Sync + 'static { + fn path(&self) -> &Path; + fn unmount(self) -> impl Future> + Send; +} +``` + +### `SubPath` + +Wraps a mount guard to point to a subdirectory: + +```rust +use crate::disk::mount::guard::SubPath; + +let mount = TmpMountGuard::mount(&filesystem, ReadOnly).await?; +let subdir = SubPath::new(mount, "data/subdir"); + +// subdir.path() returns the full path including subdirectory +``` + +## FileSystem Implementations (`core/src/disk/mount/filesystem/`) + +Various filesystem types that can be mounted: + +| Type | Description | +|------|-------------| +| `bind.rs` | Bind mounts | +| `block_dev.rs` | Block device mounts | +| `cifs.rs` | CIFS/SMB network shares | +| `ecryptfs.rs` | Encrypted filesystem | +| `efivarfs.rs` | EFI variables | +| `httpdirfs.rs` | HTTP directory as filesystem | +| `idmapped.rs` | ID-mapped mounts | +| `label.rs` | Mount by label | +| `loop_dev.rs` | Loop device mounts | +| `overlayfs.rs` | Overlay filesystem | + +## Other Useful Utilities + +### `Apply` / `ApplyRef` traits + +Fluent method chaining: + +```rust +use crate::util::Apply; + +let result = some_value + .apply(|v| transform(v)) + .apply(|v| another_transform(v)); +``` + +### `Container` + +Async-safe optional container: + +```rust +use crate::util::Container; + +let container = Container::new(None); +container.set(value).await; +let taken = container.take().await; +``` + +### `HashWriter` + +Write data while computing hash: + +```rust +use crate::util::HashWriter; +use sha2::Sha256; + +let writer = HashWriter::new(Sha256::new(), file); +// Write data... +let (hasher, file) = writer.finish(); +let hash = hasher.finalize(); +``` + +### `Never` type + +Uninhabited type for impossible cases: + +```rust +use crate::util::Never; + +fn impossible() -> Never { + // This function can never return +} + +let never: Never = impossible(); +never.absurd::() // Can convert to any type +``` + +### `MaybeOwned<'a, T>` + +Either borrowed or owned data: + +```rust +use crate::util::MaybeOwned; + +fn accept_either(data: MaybeOwned<'_, String>) { + // Use &*data to access the value +} + +accept_either(MaybeOwned::from(&existing_string)); +accept_either(MaybeOwned::from(owned_string)); +``` + +### `new_guid()` + +Generate a random GUID: + +```rust +use crate::util::new_guid; + +let guid = new_guid(); // Returns InternedString +``` diff --git a/agents/rpc-toolkit.md b/agents/rpc-toolkit.md new file mode 100644 index 000000000..933c345b6 --- /dev/null +++ b/agents/rpc-toolkit.md @@ -0,0 +1,226 @@ +# rpc-toolkit + +StartOS uses [rpc-toolkit](https://github.com/Start9Labs/rpc-toolkit) for its JSON-RPC API. This document covers the patterns used in this codebase. + +## Overview + +The API is JSON-RPC (not REST). All endpoints are RPC methods organized in a hierarchical command structure. + +## Handler Functions + +There are four types of handler functions, chosen based on the function's characteristics: + +### `from_fn_async` - Async handlers +For standard async functions. Most handlers use this. + +```rust +pub async fn my_handler(ctx: RpcContext, params: MyParams) -> Result { + // Can use .await +} + +from_fn_async(my_handler) +``` + +### `from_fn_async_local` - Non-thread-safe async handlers +For async functions that are not `Send` (cannot be safely moved between threads). Use when working with non-thread-safe types. + +```rust +pub async fn cli_download(ctx: CliContext, params: Params) -> Result<(), Error> { + // Non-Send async operations +} + +from_fn_async_local(cli_download) +``` + +### `from_fn_blocking` - Sync blocking handlers +For synchronous functions that perform blocking I/O or long computations. + +```rust +pub fn query_dns(ctx: RpcContext, params: DnsParams) -> Result { + // Blocking operations (file I/O, DNS lookup, etc.) +} + +from_fn_blocking(query_dns) +``` + +### `from_fn` - Sync non-blocking handlers +For pure functions or quick synchronous operations with no I/O. + +```rust +pub fn echo(ctx: RpcContext, params: EchoParams) -> Result { + Ok(params.message) +} + +from_fn(echo) +``` + +## ParentHandler + +Groups related RPC methods into a hierarchy: + +```rust +use rpc_toolkit::{Context, HandlerExt, ParentHandler, from_fn_async}; + +pub fn my_api() -> ParentHandler { + ParentHandler::new() + .subcommand("list", from_fn_async(list_handler).with_call_remote::()) + .subcommand("create", from_fn_async(create_handler).with_call_remote::()) +} +``` + +## Handler Extensions + +Chain methods to configure handler behavior. + +**Ordering rules:** +1. `with_about()` must come AFTER other CLI modifiers (`no_display()`, `with_custom_display_fn()`, etc.) +2. `with_call_remote()` must be the LAST adapter in the chain + +| Method | Purpose | +|--------|---------| +| `.with_metadata("key", Value)` | Attach metadata for middleware | +| `.no_cli()` | RPC-only, not available via CLI | +| `.no_display()` | No CLI output | +| `.with_display_serializable()` | Default JSON/YAML output for CLI | +| `.with_custom_display_fn(\|_, res\| ...)` | Custom CLI output formatting | +| `.with_about("about.description")` | Add help text (i18n key) - **after CLI modifiers** | +| `.with_call_remote::()` | Enable CLI to call remotely - **must be last** | + +### Correct ordering example: +```rust +from_fn_async(my_handler) + .with_metadata("sync_db", Value::Bool(true)) // metadata early + .no_display() // CLI modifier + .with_about("about.my-handler") // after CLI modifiers + .with_call_remote::() // always last +``` + +## Metadata by Middleware + +Metadata tags are processed by different middleware. Group them logically: + +### Auth Middleware (`middleware/auth/mod.rs`) + +| Metadata | Default | Description | +|----------|---------|-------------| +| `authenticated` | `true` | Whether endpoint requires authentication. Set to `false` for public endpoints. | + +### Session Auth Middleware (`middleware/auth/session.rs`) + +| Metadata | Default | Description | +|----------|---------|-------------| +| `login` | `false` | Special handling for login endpoints (rate limiting, cookie setting) | +| `get_session` | `false` | Inject session ID into params as `__Auth_session` | + +### Signature Auth Middleware (`middleware/auth/signature.rs`) + +| Metadata | Default | Description | +|----------|---------|-------------| +| `get_signer` | `false` | Inject signer public key into params as `__Auth_signer` | + +### Registry Auth (extends Signature Auth) + +| Metadata | Default | Description | +|----------|---------|-------------| +| `admin` | `false` | Require admin privileges (signer must be in admin list) | +| `get_device_info` | `false` | Inject device info header for hardware filtering | + +### Database Middleware (`middleware/db.rs`) + +| Metadata | Default | Description | +|----------|---------|-------------| +| `sync_db` | `false` | Sync database after mutation, add `X-Patch-Sequence` header | + +## Context Types + +Different contexts for different execution environments: + +- `RpcContext` - Web/RPC requests with full service access +- `CliContext` - CLI operations, calls remote RPC +- `InitContext` - During system initialization +- `DiagnosticContext` - Diagnostic/recovery mode +- `RegistryContext` - Registry daemon context +- `EffectContext` - Service effects context (container-to-host calls) + +## Parameter Structs + +Parameters use derive macros for JSON-RPC, CLI parsing, and TypeScript generation: + +```rust +#[derive(Deserialize, Serialize, Parser, TS)] +#[serde(rename_all = "camelCase")] // JSON-RPC uses camelCase +#[command(rename_all = "kebab-case")] // CLI uses kebab-case +#[ts(export)] // Generate TypeScript types +pub struct MyParams { + pub package_id: PackageId, +} +``` + +### Middleware Injection + +Auth middleware can inject values into params using special field names: + +```rust +#[derive(Deserialize, Serialize, Parser, TS)] +pub struct MyParams { + #[ts(skip)] + #[serde(rename = "__Auth_session")] // Injected by session auth + session: InternedString, + + #[ts(skip)] + #[serde(rename = "__Auth_signer")] // Injected by signature auth + signer: AnyVerifyingKey, + + #[ts(skip)] + #[serde(rename = "__Auth_userAgent")] // Injected during login + user_agent: Option, +} +``` + +## Common Patterns + +### Adding a New RPC Endpoint + +1. Define params struct with `Deserialize, Serialize, Parser, TS` +2. Choose handler type based on sync/async and thread-safety +3. Write handler function taking `(Context, Params) -> Result` +4. Add to parent handler with appropriate extensions (display modifiers before `with_about`) +5. TypeScript types auto-generated via `make ts-bindings` + +### Public (Unauthenticated) Endpoint + +```rust +from_fn_async(get_info) + .with_metadata("authenticated", Value::Bool(false)) + .with_display_serializable() + .with_about("about.get-info") + .with_call_remote::() // last +``` + +### Mutating Endpoint with DB Sync + +```rust +from_fn_async(update_config) + .with_metadata("sync_db", Value::Bool(true)) + .no_display() + .with_about("about.update-config") + .with_call_remote::() // last +``` + +### Session-Aware Endpoint + +```rust +from_fn_async(logout) + .with_metadata("get_session", Value::Bool(true)) + .no_display() + .with_about("about.logout") + .with_call_remote::() // last +``` + +## File Locations + +- Handler definitions: Throughout `core/src/` modules +- Main API tree: `core/src/lib.rs` (`main_api()`, `server()`, `package()`) +- Auth middleware: `core/src/middleware/auth/` +- DB middleware: `core/src/middleware/db.rs` +- Context types: `core/src/context/` diff --git a/agents/s9pk-structure.md b/agents/s9pk-structure.md new file mode 100644 index 000000000..81499336a --- /dev/null +++ b/agents/s9pk-structure.md @@ -0,0 +1,122 @@ +# S9PK Package Format + +S9PK is the package format for StartOS services. Version 2 uses a merkle archive structure for efficient downloading and cryptographic verification. + +## File Format + +S9PK files begin with a 3-byte header: `0x3b 0x3b 0x02` (magic bytes + version 2). + +The archive is cryptographically signed using Ed25519 with prehashed content (SHA-512 over blake3 merkle root hash). + +## Archive Structure + +``` +/ +├── manifest.json # Package metadata (required) +├── icon. # Package icon - any image/* format (required) +├── LICENSE.md # License text (required) +├── dependencies/ # Dependency metadata (optional) +│ └── / +│ ├── metadata.json # DependencyMetadata +│ └── icon. # Dependency icon +├── javascript.squashfs # Package JavaScript code (required) +├── assets.squashfs # Static assets (optional, legacy: assets/ directory) +└── images/ # Container images by architecture + └── / # e.g., x86_64, aarch64, riscv64 + ├── .squashfs # Container filesystem + ├── .json # Image metadata + └── .env # Environment variables +``` + +## Components + +### manifest.json + +The package manifest contains all metadata: + +| Field | Type | Description | +|-------|------|-------------| +| `id` | string | Package identifier (e.g., `bitcoind`) | +| `title` | string | Display name | +| `version` | string | Extended version string | +| `satisfies` | string[] | Version ranges this version satisfies | +| `releaseNotes` | string/object | Release notes (localized) | +| `canMigrateTo` | string | Version range for forward migration | +| `canMigrateFrom` | string | Version range for backward migration | +| `license` | string | License type | +| `wrapperRepo` | string | StartOS wrapper repository URL | +| `upstreamRepo` | string | Upstream project URL | +| `supportSite` | string | Support site URL | +| `marketingSite` | string | Marketing site URL | +| `donationUrl` | string? | Optional donation URL | +| `docsUrl` | string? | Optional documentation URL | +| `description` | object | Short and long descriptions (localized) | +| `images` | object | Image configurations by image ID | +| `volumes` | string[] | Volume IDs for persistent data | +| `alerts` | object | User alerts for lifecycle events | +| `dependencies` | object | Package dependencies | +| `hardwareRequirements` | object | Hardware requirements (arch, RAM, devices) | +| `hardwareAcceleration` | boolean | Whether package uses hardware acceleration | +| `gitHash` | string? | Git commit hash | +| `osVersion` | string | Minimum StartOS version | +| `sdkVersion` | string? | SDK version used to build | + +### javascript.squashfs + +Contains the package JavaScript that implements the `ABI` interface from `@start9labs/start-sdk-base`. This code runs in the container runtime and manages the package lifecycle. + +The squashfs is mounted at `/usr/lib/startos/package/` and the runtime loads `index.js`. + +### images/ + +Container images organized by architecture: + +- **`.squashfs`** - Container root filesystem +- **`.json`** - Image metadata (entrypoint, user, workdir, etc.) +- **`.env`** - Environment variables for the container + +Images are built from Docker/Podman and converted to squashfs. The `ImageConfig` in manifest specifies: +- `arch` - Supported architectures +- `emulateMissingAs` - Fallback architecture for emulation +- `nvidiaContainer` - Whether to enable NVIDIA container support + +### assets.squashfs + +Static assets accessible to the package, mounted read-only at `/media/startos/assets/` in the container. + +### dependencies/ + +Metadata for dependencies displayed in the UI: +- `metadata.json` - Just title for now +- `icon.` - Icon for the dependency + +## Merkle Archive + +The S9PK uses a merkle tree structure where each file and directory has a blake3 hash. This enables: + +1. **Partial downloads** - Download and verify individual files +2. **Integrity verification** - Verify any subset of the archive +3. **Efficient updates** - Only download changed portions +4. **DOS protection** - Size limits enforced before downloading content + +Files are sorted by priority for streaming (manifest first, then icon, license, dependencies, javascript, assets, images). + +## Building S9PK + +Use `start-cli s9pk pack` to build packages: + +```bash +start-cli s9pk pack -o +``` + +Images can be sourced from: +- Docker/Podman build (`--docker-build`) +- Existing Docker tag (`--docker-tag`) +- Pre-built squashfs files + +## Related Code + +- `core/src/s9pk/v2/mod.rs` - S9pk struct and serialization +- `core/src/s9pk/v2/manifest.rs` - Manifest types +- `core/src/s9pk/v2/pack.rs` - Packing logic +- `core/src/s9pk/merkle_archive/` - Merkle archive implementation diff --git a/build/lib/scripts/chroot-and-upgrade b/build/lib/scripts/chroot-and-upgrade index 792b7836d..c8e16acaf 100755 --- a/build/lib/scripts/chroot-and-upgrade +++ b/build/lib/scripts/chroot-and-upgrade @@ -111,6 +111,6 @@ if [ "$CHROOT_RES" -eq 0 ]; then reboot fi -umount -R /media/startos/next +umount /media/startos/next umount /media/startos/upper rm -rf /media/startos/upper /media/startos/next \ No newline at end of file diff --git a/container-runtime/RPCSpec.md b/container-runtime/RPCSpec.md index fd1014add..7c43467ba 100644 --- a/container-runtime/RPCSpec.md +++ b/container-runtime/RPCSpec.md @@ -1,16 +1,21 @@ -# Container RPC SERVER Specification +# Container RPC Server Specification + +The container runtime exposes a JSON-RPC server over a Unix socket at `/media/startos/rpc/service.sock`. ## Methods ### init -initialize runtime (mount `/proc`, `/sys`, `/dev`, and `/run` to each image in `/media/images`) +Initialize the runtime and system. -called after os has mounted js and images to the container +#### params -#### args - -`[]` +```ts +{ + id: string, + kind: "install" | "update" | "restore" | null, +} +``` #### response @@ -18,11 +23,16 @@ called after os has mounted js and images to the container ### exit -shutdown runtime +Shutdown runtime and optionally run exit hooks for a target version. -#### args +#### params -`[]` +```ts +{ + id: string, + target: string | null, // ExtendedVersion or VersionRange +} +``` #### response @@ -30,11 +40,11 @@ shutdown runtime ### start -run main method if not already running +Run main method if not already running. -#### args +#### params -`[]` +None #### response @@ -42,11 +52,11 @@ run main method if not already running ### stop -stop main method by sending SIGTERM to child processes, and SIGKILL after timeout +Stop main method by sending SIGTERM to child processes, and SIGKILL after timeout. -#### args +#### params -`{ timeout: millis }` +None #### response @@ -54,15 +64,16 @@ stop main method by sending SIGTERM to child processes, and SIGKILL after timeou ### execute -run a specific package procedure +Run a specific package procedure. -#### args +#### params ```ts { - procedure: JsonPath, - input: any, - timeout: millis, + id: string, // event ID + procedure: string, // JSON path (e.g., "/backup/create", "/actions/{name}/run") + input: any, + timeout: number | null, } ``` @@ -72,18 +83,64 @@ run a specific package procedure ### sandbox -run a specific package procedure in sandbox mode +Run a specific package procedure in sandbox mode. Same interface as `execute`. -#### args +UNIMPLEMENTED: this feature is planned but does not exist + +#### params ```ts { - procedure: JsonPath, - input: any, - timeout: millis, + id: string, + procedure: string, + input: any, + timeout: number | null, } ``` #### response `any` + +### callback + +Handle a callback from an effect. + +#### params + +```ts +{ + id: number, + args: any[], +} +``` + +#### response + +`null` (no response sent) + +### eval + +Evaluate a script in the runtime context. Used for debugging. + +#### params + +```ts +{ + script: string, +} +``` + +#### response + +`any` + +## Procedures + +The `execute` and `sandbox` methods route to procedures based on the `procedure` path: + +| Procedure | Description | +|-----------|-------------| +| `/backup/create` | Create a backup | +| `/actions/{name}/getInput` | Get input spec for an action | +| `/actions/{name}/run` | Run an action with input | diff --git a/web/package.json b/web/package.json index 8c1974088..49fc3a76d 100644 --- a/web/package.json +++ b/web/package.json @@ -28,7 +28,9 @@ "start:ui": "npm run-script build-config && ng serve --project ui --host 0.0.0.0", "start:tunnel": "ng serve --project start-tunnel --host 0.0.0.0", "start:ui:proxy": "npm run-script build-config && ng serve --project ui --host 0.0.0.0 --proxy-config proxy.conf.json", - "build-config": "node build-config.js" + "build-config": "node build-config.js", + "format": "prettier --write projects/", + "format:check": "prettier --check projects/" }, "dependencies": { "@angular/animations": "^20.3.0",