Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 12:11:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

feature: pack s9pk (#2642)

Browse Source 
* TODO: images

* wip

* pack s9pk images

* include path in packsource error

* debug info

* add cmd as context to invoke

* filehelper bugfix

* fix file helper

* fix exposeForDependents

* misc fixes

* force image removal

* fix filtering

* fix deadlock

* fix api

* chore: Up the version of the package.json

* always allow concurrency within same call stack

* Update core/startos/src/s9pk/merkle_archive/expected.rs

Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>

---------

Co-authored-by: J H <dragondef@gmail.com>
Co-authored-by: Jade <2364004+Blu-J@users.noreply.github.com>
This commit is contained in:
Aiden McClelland
2024-06-12 11:46:59 -06:00
committed by GitHub
parent 5aefb707fa
commit 3f380fa0da
84 changed files with 2552 additions and 2108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/startos/src/action.rs
View File

@@ -8,6 +8,7 @@ use ts_rs::TS;
use crate::config::Config;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::util::serde::{display_serializable, StdinDeserializable, WithIoFormat};
#[derive(Debug, Serialize, Deserialize)]
@@ -77,6 +78,7 @@ pub async fn action(
.as_ref()
.or_not_found(lazy_format!("Manager for {}", package_id))?
.action(
Guid::new(),
action_id,
input.map(|c| to_value(&c)).transpose()?.unwrap_or_default(),
)

1
core/startos/src/auth.rs
View File

@@ -178,6 +178,7 @@ pub fn check_password_against_db(db: &DatabaseModel, password: &str) -> Result<(
#[derive(Deserialize, Serialize, Parser, TS)]
#[serde(rename_all = "camelCase")]
#[command(rename_all = "kebab-case")]
#[ts(export)]
pub struct LoginParams {
password: Option<PasswordType>,
#[ts(skip)]

1
core/startos/src/backup/restore.rs
View File

@@ -149,7 +149,6 @@ async fn restore_packages(
S9pk::open(
backup_dir.path().join(&id).with_extension("s9pk"),
Some(&id),
true,
)
.await?,
Some(backup_dir),

5
core/startos/src/config/mod.rs
View File

@@ -16,6 +16,7 @@ use ts_rs::TS;
use crate::context::{CliContext, RpcContext};
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::util::serde::{HandlerExtSerde, StdinDeserializable};
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
@@ -156,7 +157,7 @@ pub async fn get(ctx: RpcContext, _: Empty, id: PackageId) -> Result<ConfigRes,
.await
.as_ref()
.or_not_found(lazy_format!("Manager for {id}"))?
.get_config()
.get_config(Guid::new())
.await
}
@@ -218,7 +219,7 @@ pub async fn set_impl(
ErrorKind::Unknown,
)
})?
.configure(configure_context)
.configure(Guid::new(), configure_context)
.await?;
Ok(())
}

21
core/startos/src/context/config.rs
View File

@@ -93,26 +93,28 @@ impl ClientConfig {
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct ServerConfig {
#[arg(short = 'c', long = "config")]
#[arg(short, long)]
pub config: Option<PathBuf>,
#[arg(long = "ethernet-interface")]
#[arg(long)]
pub ethernet_interface: Option<String>,
#[arg(skip)]
pub os_partitions: Option<OsPartitionInfo>,
#[arg(long = "bind-rpc")]
#[arg(long)]
pub bind_rpc: Option<SocketAddr>,
#[arg(long = "tor-control")]
#[arg(long)]
pub tor_control: Option<SocketAddr>,
#[arg(long = "tor-socks")]
#[arg(long)]
pub tor_socks: Option<SocketAddr>,
#[arg(long = "dns-bind")]
#[arg(long)]
pub dns_bind: Option<Vec<SocketAddr>>,
#[arg(long = "revision-cache-size")]
#[arg(long)]
pub revision_cache_size: Option<usize>,
#[arg(short = 'd', long = "datadir")]
#[arg(short, long)]
pub datadir: Option<PathBuf>,
#[arg(long = "disable-encryption")]
#[arg(long)]
pub disable_encryption: Option<bool>,
#[arg(long)]
pub multi_arch_s9pks: Option<bool>,
}
impl ContextConfig for ServerConfig {
fn next(&mut self) -> Option<PathBuf> {
@@ -131,6 +133,7 @@ impl ContextConfig for ServerConfig {
.or(other.revision_cache_size);
self.datadir = self.datadir.take().or(other.datadir);
self.disable_encryption = self.disable_encryption.take().or(other.disable_encryption);
self.multi_arch_s9pks = self.multi_arch_s9pks.take().or(other.multi_arch_s9pks);
}
}

6
core/startos/src/context/rpc.rs
View File

@@ -43,6 +43,7 @@ pub struct RpcContextSeed {
pub db: TypedPatchDb<Database>,
pub account: RwLock<AccountInfo>,
pub net_controller: Arc<NetController>,
pub s9pk_arch: Option<&'static str>,
pub services: ServiceMap,
pub metrics_cache: RwLock<Option<crate::system::Metrics>>,
pub shutdown: broadcast::Sender<Option<Shutdown>>,
@@ -152,6 +153,11 @@ impl RpcContext {
db,
account: RwLock::new(account),
net_controller,
s9pk_arch: if config.multi_arch_s9pks.unwrap_or(false) {
None
} else {
Some(crate::ARCH)
},
services,
metrics_cache,
shutdown,

7
core/startos/src/control.rs
View File

@@ -7,6 +7,7 @@ use ts_rs::TS;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::Error;
#[derive(Deserialize, Serialize, Parser, TS)]
@@ -23,7 +24,7 @@ pub async fn start(ctx: RpcContext, ControlParams { id }: ControlParams) -> Resu
.await
.as_ref()
.or_not_found(lazy_format!("Manager for {id}"))?
.start()
.start(Guid::new())
.await?;
Ok(())
@@ -36,7 +37,7 @@ pub async fn stop(ctx: RpcContext, ControlParams { id }: ControlParams) -> Resul
.await
.as_ref()
.ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
.stop()
.stop(Guid::new())
.await?;
Ok(())
@@ -48,7 +49,7 @@ pub async fn restart(ctx: RpcContext, ControlParams { id }: ControlParams) -> Re
.await
.as_ref()
.ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
.restart()
.restart(Guid::new())
.await?;
Ok(())

13
core/startos/src/dependencies.rs
View File

@@ -13,6 +13,7 @@ use crate::config::{Config, ConfigSpec, ConfigureContext};
use crate::context::RpcContext;
use crate::db::model::package::CurrentDependencies;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::Error;
pub fn dependency<C: Context>() -> ParentHandler<C> {
@@ -86,7 +87,7 @@ pub async fn configure_impl(
ErrorKind::Unknown,
)
})?
.configure(configure_context)
.configure(Guid::new(), configure_context)
.await?;
Ok(())
}
@@ -103,14 +104,15 @@ pub async fn configure_logic(
ctx: RpcContext,
(dependent_id, dependency_id): (PackageId, PackageId),
) -> Result<ConfigDryRes, Error> {
let procedure_id = Guid::new();
let dependency_guard = ctx.services.get(&dependency_id).await;
let dependency = dependency_guard.as_ref().or_not_found(&dependency_id)?;
let dependent_guard = ctx.services.get(&dependent_id).await;
let dependent = dependent_guard.as_ref().or_not_found(&dependent_id)?;
let config_res = dependency.get_config().await?;
let config_res = dependency.get_config(procedure_id.clone()).await?;
let diff = Value::Object(
dependent
.dependency_config(dependency_id, config_res.config.clone())
.dependency_config(procedure_id, dependency_id, config_res.config.clone())
.await?
.unwrap_or_default(),
);
@@ -129,6 +131,7 @@ pub async fn compute_dependency_config_errs(
id: &PackageId,
current_dependencies: &mut CurrentDependencies,
) -> Result<(), Error> {
let procedure_id = Guid::new();
let service_guard = ctx.services.get(id).await;
let service = service_guard.as_ref().or_not_found(id)?;
for (dep_id, dep_info) in current_dependencies.0.iter_mut() {
@@ -137,10 +140,10 @@ pub async fn compute_dependency_config_errs(
continue;
};
let dep_config = dependency.get_config().await?.config;
let dep_config = dependency.get_config(procedure_id.clone()).await?.config;
dep_info.config_satisfied = service
.dependency_config(dep_id.clone(), dep_config)
.dependency_config(procedure_id.clone(), dep_id.clone(), dep_config)
.await?
.is_none();
}

1
core/startos/src/disk/mount/backup.rs
View File

@@ -178,7 +178,6 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
Ok(())
}
}
#[async_trait::async_trait]
impl<G: GenericMountGuard> GenericMountGuard for BackupMountGuard<G> {
fn path(&self) -> &Path {
if let Some(guard) = &self.encrypted_guard {

23
core/startos/src/disk/mount/filesystem/overlayfs.rs
View File

@@ -6,8 +6,8 @@ use digest::generic_array::GenericArray;
use digest::{Digest, OutputSizeUser};
use sha2::Sha256;
use crate::disk::mount::filesystem::{FileSystem, ReadOnly, ReadWrite};
use crate::disk::mount::guard::{GenericMountGuard, MountGuard, TmpMountGuard};
use crate::disk::mount::filesystem::{FileSystem, ReadWrite};
use crate::disk::mount::guard::{GenericMountGuard, MountGuard};
use crate::prelude::*;
use crate::util::io::TmpDir;
@@ -94,17 +94,13 @@ impl<
}
#[derive(Debug)]
pub struct OverlayGuard {
lower: Option<TmpMountGuard>,
pub struct OverlayGuard<G: GenericMountGuard> {
lower: Option<G>,
upper: Option<TmpDir>,
inner_guard: MountGuard,
}
impl OverlayGuard {
pub async fn mount(
base: &impl FileSystem,
mountpoint: impl AsRef<Path>,
) -> Result<Self, Error> {
let lower = TmpMountGuard::mount(base, ReadOnly).await?;
impl<G: GenericMountGuard> OverlayGuard<G> {
pub async fn mount(lower: G, mountpoint: impl AsRef<Path>) -> Result<Self, Error> {
let upper = TmpDir::new().await?;
let inner_guard = MountGuard::mount(
&OverlayFs::new(
@@ -140,16 +136,15 @@ impl OverlayGuard {
}
}
}
#[async_trait::async_trait]
impl GenericMountGuard for OverlayGuard {
impl<G: GenericMountGuard> GenericMountGuard for OverlayGuard<G> {
fn path(&self) -> &Path {
self.inner_guard.path()
}
async fn unmount(mut self) -> Result<(), Error> {
async fn unmount(self) -> Result<(), Error> {
self.unmount(false).await
}
}
impl Drop for OverlayGuard {
impl<G: GenericMountGuard> Drop for OverlayGuard<G> {
fn drop(&mut self) {
let lower = self.lower.take();
let upper = self.upper.take();

19
core/startos/src/disk/mount/guard.rs
View File

@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Weak};
use futures::Future;
use lazy_static::lazy_static;
use models::ResultExt;
use tokio::sync::Mutex;
@@ -14,23 +15,20 @@ use crate::Error;
pub const TMP_MOUNTPOINT: &'static str = "/media/startos/tmp";
#[async_trait::async_trait]
pub trait GenericMountGuard: std::fmt::Debug + Send + Sync + 'static {
fn path(&self) -> &Path;
async fn unmount(mut self) -> Result<(), Error>;
fn unmount(self) -> impl Future<Output = Result<(), Error>> + Send;
}
#[async_trait::async_trait]
impl GenericMountGuard for Never {
fn path(&self) -> &Path {
match *self {}
}
async fn unmount(mut self) -> Result<(), Error> {
async fn unmount(self) -> Result<(), Error> {
match self {}
}
}
#[async_trait::async_trait]
impl<T> GenericMountGuard for Arc<T>
where
T: GenericMountGuard,
@@ -38,7 +36,7 @@ where
fn path(&self) -> &Path {
(&**self).path()
}
async fn unmount(mut self) -> Result<(), Error> {
async fn unmount(self) -> Result<(), Error> {
if let Ok(guard) = Arc::try_unwrap(self) {
guard.unmount().await?;
}
@@ -102,12 +100,11 @@ impl Drop for MountGuard {
}
}
}
#[async_trait::async_trait]
impl GenericMountGuard for MountGuard {
fn path(&self) -> &Path {
&self.mountpoint
}
async fn unmount(mut self) -> Result<(), Error> {
async fn unmount(self) -> Result<(), Error> {
MountGuard::unmount(self, false).await
}
}
@@ -165,12 +162,11 @@ impl TmpMountGuard {
std::mem::replace(self, unmounted)
}
}
#[async_trait::async_trait]
impl GenericMountGuard for TmpMountGuard {
fn path(&self) -> &Path {
self.guard.path()
}
async fn unmount(mut self) -> Result<(), Error> {
async fn unmount(self) -> Result<(), Error> {
self.guard.unmount().await
}
}
@@ -187,12 +183,11 @@ impl<G: GenericMountGuard> SubPath<G> {
Self { guard, path }
}
}
#[async_trait::async_trait]
impl<G: GenericMountGuard> GenericMountGuard for SubPath<G> {
fn path(&self) -> &Path {
self.path.as_path()
}
async fn unmount(mut self) -> Result<(), Error> {
async fn unmount(self) -> Result<(), Error> {
self.guard.unmount().await
}
}

2
core/startos/src/init.rs
View File

@@ -242,7 +242,7 @@ pub async fn init(cfg: &ServerConfig) -> Result<InitResult, Error> {
let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok()
|| &*server_info.version < &emver::Version::new(0, 3, 2, 0)
|| (*ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0));
|| (ARCH == "x86_64" && &*server_info.version < &emver::Version::new(0, 3, 4, 0));
let log_dir = cfg.datadir().join("main/logs");
if tokio::fs::metadata(&log_dir).await.is_err() {

2
core/startos/src/install/mod.rs
View File

@@ -152,7 +152,6 @@ pub async fn install(
.await?,
),
None, // TODO
true,
)
.await?;
@@ -262,7 +261,6 @@ pub async fn sideload(ctx: RpcContext) -> Result<SideloadResponse, Error> {
if let Err(e) = async {
let s9pk = S9pk::deserialize(
&file, None, // TODO
true,
)
.await?;
let _ = id_send.send(s9pk.as_manifest().id.clone());

6
core/startos/src/lib.rs
View File

@@ -4,12 +4,8 @@ pub const CAP_1_KiB: usize = 1024;
pub const CAP_1_MiB: usize = CAP_1_KiB * CAP_1_KiB;
pub const CAP_10_MiB: usize = 10 * CAP_1_MiB;
pub const HOST_IP: [u8; 4] = [172, 18, 0, 1];
pub const TARGET: &str = current_platform::CURRENT_PLATFORM;
pub use std::env::consts::ARCH;
lazy_static::lazy_static! {
pub static ref ARCH: &'static str = {
let (arch, _) = TARGET.split_once("-").unwrap();
arch
};
pub static ref PLATFORM: String = {
if let Ok(platform) = std::fs::read_to_string("/usr/lib/startos/PLATFORM.txt") {
platform

20
core/startos/src/lxc/mod.rs
View File

@@ -29,7 +29,7 @@ use crate::disk::mount::filesystem::bind::Bind;
use crate::disk::mount::filesystem::block_dev::BlockDev;
use crate::disk::mount::filesystem::idmapped::IdMapped;
use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
use crate::disk::mount::filesystem::{MountType, ReadWrite};
use crate::disk::mount::filesystem::{MountType, ReadOnly, ReadWrite};
use crate::disk::mount::guard::{GenericMountGuard, MountGuard, TmpMountGuard};
use crate::disk::mount::util::unmount;
use crate::prelude::*;
@@ -153,7 +153,7 @@ impl LxcManager {
pub struct LxcContainer {
manager: Weak<LxcManager>,
rootfs: OverlayGuard,
rootfs: OverlayGuard<TmpMountGuard>,
pub guid: Arc<ContainerId>,
rpc_bind: TmpMountGuard,
log_mount: Option<MountGuard>,
@@ -184,12 +184,16 @@ impl LxcContainer {
.invoke(ErrorKind::Filesystem)
.await?;
let rootfs = OverlayGuard::mount(
&IdMapped::new(
BlockDev::new("/usr/lib/startos/container-runtime/rootfs.squashfs"),
0,
100000,
65536,
),
TmpMountGuard::mount(
&IdMapped::new(
BlockDev::new("/usr/lib/startos/container-runtime/rootfs.squashfs"),
0,
100000,
65536,
),
ReadOnly,
)
.await?,
&rootfs_dir,
)
.await?;

2
core/startos/src/os_install/gpt.rs
View File

@@ -87,7 +87,7 @@ pub async fn partition(disk: &DiskInfo, overwrite: bool) -> Result<OsPartitionIn
gpt.add_partition(
"root",
15 * 1024 * 1024 * 1024,
match *crate::ARCH {
match crate::ARCH {
"x86_64" => gpt::partition_types::LINUX_ROOT_X64,
"aarch64" => gpt::partition_types::LINUX_ROOT_ARM_64,
_ => gpt::partition_types::LINUX_FS,

2
core/startos/src/os_install/mod.rs
View File

@@ -366,7 +366,7 @@ pub async fn execute<C: Context>(
if tokio::fs::metadata("/sys/firmware/efi").await.is_err() {
install.arg("--target=i386-pc");
} else {
match *ARCH {
match ARCH {
"x86_64" => install.arg("--target=x86_64-efi"),
"aarch64" => install.arg("--target=arm64-efi"),
_ => &mut install,

2
core/startos/src/registry/device_info.rs
View File

@@ -134,7 +134,7 @@ pub struct HardwareInfo {
impl From<&RpcContext> for HardwareInfo {
fn from(value: &RpcContext) -> Self {
Self {
arch: InternedString::intern(&**crate::ARCH),
arch: InternedString::intern(crate::ARCH),
ram: value.hardware.ram,
devices: value
.hardware

4
core/startos/src/registry/package/add.rs
View File

@@ -53,7 +53,6 @@ pub async fn add_package(
let s9pk = S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
Some(&commitment),
false,
)
.await?;
@@ -109,7 +108,7 @@ pub async fn cli_add_package(
..
}: HandlerArgs<CliContext, CliAddPackageParams>,
) -> Result<(), Error> {
let s9pk = S9pk::open(&file, None, false).await?;
let s9pk = S9pk::open(&file, None).await?;
let mut progress = FullProgressTracker::new();
let progress_handle = progress.handle();
@@ -143,7 +142,6 @@ pub async fn cli_add_package(
let mut src = S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
Some(&commitment),
false,
)
.await?;
src.serialize(&mut TrackingIO::new(0, tokio::io::sink()), true)

5
core/startos/src/rpc_continuations.rs
View File

@@ -39,6 +39,11 @@ impl Guid {
Some(Guid(InternedString::intern(r)))
}
}
impl Default for Guid {
fn default() -> Self {
Self::new()
}
}
impl AsRef<str> for Guid {
fn as_ref(&self) -> &str {
self.0.as_ref()

5
core/startos/src/s9pk/merkle_archive/directory_contents.rs
View File

@@ -211,7 +211,10 @@ impl<S: FileSource + Clone> DirectoryContents<S> {
if !filter(path) {
if v.hash.is_none() {
return Err(Error::new(
eyre!("cannot filter out unhashed file, run `update_hashes` first"),
eyre!(
"cannot filter out unhashed file {}, run `update_hashes` first",
path.display()
),
ErrorKind::InvalidRequest,
));
}

103
core/startos/src/s9pk/merkle_archive/expected.rs Normal file
View File

@@ -0,0 +1,103 @@
use std::ffi::OsStr;
use std::path::Path;
use crate::prelude::*;
use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
use crate::s9pk::merkle_archive::source::FileSource;
use crate::s9pk::merkle_archive::Entry;
/// An object for tracking the files expected to be in an s9pk
pub struct Expected<'a, T> {
keep: DirectoryContents<()>,
dir: &'a DirectoryContents<T>,
}
impl<'a, T> Expected<'a, T> {
pub fn new(dir: &'a DirectoryContents<T>,) -> Self {
Self {
keep: DirectoryContents::new(),
dir
}
}
}
impl<'a, T: Clone> Expected<'a, T> {
pub fn check_file(&mut self, path: impl AsRef<Path>) -> Result<(), Error> {
if self
.dir
.get_path(path.as_ref())
.and_then(|e| e.as_file())
.is_some()
{
self.keep.insert_path(path, Entry::file(()))?;
Ok(())
} else {
Err(Error::new(
eyre!("file {} missing from archive", path.as_ref().display()),
ErrorKind::ParseS9pk,
))
}
}
pub fn check_stem(
&mut self,
path: impl AsRef<Path>,
mut valid_extension: impl FnMut(Option<&OsStr>) -> bool,
) -> Result<(), Error> {
let (dir, stem) = if let Some(parent) = path.as_ref().parent().filter(|p| *p != Path::new("")) {
(
self.dir
.get_path(parent)
.and_then(|e| e.as_directory())
.ok_or_else(|| {
Error::new(
eyre!("directory {} missing from archive", parent.display()),
ErrorKind::ParseS9pk,
)
})?,
path.as_ref().strip_prefix(parent).unwrap(),
)
} else {
(self.dir, path.as_ref())
};
let name = dir
.with_stem(&stem.as_os_str().to_string_lossy())
.filter(|(_, e)| e.as_file().is_some())
.try_fold(
Err(Error::new(
eyre!(
"file {} with valid extension missing from archive",
path.as_ref().display()
),
ErrorKind::ParseS9pk,
)),
|acc, (name, _)|
if valid_extension(Path::new(&*name).extension()) {
match acc {
Ok(_) => Err(Error::new(
eyre!(
"more than one file matching {} with valid extension in archive",
path.as_ref().display()
),
ErrorKind::ParseS9pk,
)),
Err(_) => Ok(Ok(name))
}
} else {
Ok(acc)
}
)??;
self.keep
.insert_path(path.as_ref().with_file_name(name), Entry::file(()))?;
Ok(())
}
pub fn into_filter(self) -> Filter {
Filter(self.keep)
}
}
pub struct Filter(DirectoryContents<()>);
impl Filter {
pub fn keep_checked<T: FileSource + Clone>(&self, dir: &mut DirectoryContents<T>) -> Result<(), Error> {
dir.filter(|path| self.0.get_path(path).is_some())
}
}

4
core/startos/src/s9pk/merkle_archive/mod.rs
View File

@@ -19,6 +19,7 @@ use crate::util::serde::Base64;
use crate::CAP_1_MiB;
pub mod directory_contents;
pub mod expected;
pub mod file_contents;
pub mod hash;
pub mod sink;
@@ -217,6 +218,9 @@ impl<S> Entry<S> {
pub fn file(source: S) -> Self {
Self::new(EntryContents::File(FileContents::new(source)))
}
pub fn directory(directory: DirectoryContents<S>) -> Self {
Self::new(EntryContents::Directory(directory))
}
pub fn hash(&self) -> Option<(Hash, u64)> {
self.hash
}

5
core/startos/src/s9pk/merkle_archive/source/mod.rs
View File

@@ -280,3 +280,8 @@ impl<S: ArchiveSource> FileSource for Section<S> {
self.source.copy_to(self.position, self.size, w).await
}
}
pub type DynRead = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
pub fn into_dyn_read<R: AsyncRead + Unpin + Send + Sync + 'static>(r: R) -> DynRead {
Box::new(r)
}

2
core/startos/src/s9pk/merkle_archive/source/multi_cursor_file.rs
View File

@@ -97,7 +97,7 @@ impl ArchiveSource for MultiCursorFile {
.ok()
.map(|m| m.len())
}
async fn fetch_all(&self) -> Result<impl AsyncRead + Unpin + Send, Error> {
async fn fetch_all(&self) -> Result<impl AsyncRead + Unpin + Send + 'static, Error> {
use tokio::io::AsyncSeekExt;
let mut file = self.cursor().await?;

130
core/startos/src/s9pk/rpc.rs
View File

@@ -1,32 +1,26 @@
use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::path::PathBuf;
use clap::Parser;
use itertools::Itertools;
use models::ImageId;
use rpc_toolkit::{from_fn_async, Empty, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use tokio::fs::File;
use tokio::process::Command;
use ts_rs::TS;
use crate::context::CliContext;
use crate::prelude::*;
use crate::s9pk::manifest::Manifest;
use crate::s9pk::merkle_archive::source::DynFileSource;
use crate::s9pk::merkle_archive::Entry;
use crate::s9pk::v2::compat::CONTAINER_TOOL;
use crate::s9pk::v2::pack::ImageConfig;
use crate::s9pk::v2::SIG_CONTEXT;
use crate::s9pk::S9pk;
use crate::util::io::TmpDir;
use crate::util::serde::{apply_expr, HandlerExtSerde};
use crate::util::Invoke;
pub const SKIP_ENV: &[&str] = &["TERM", "container", "HOME", "HOSTNAME"];
pub fn s9pk() -> ParentHandler<CliContext> {
ParentHandler::new()
.subcommand("pack", from_fn_async(super::v2::pack::pack).no_display())
.subcommand("edit", edit())
.subcommand("inspect", inspect())
}
@@ -77,117 +71,21 @@ fn inspect() -> ParentHandler<CliContext, S9pkPath> {
#[derive(Deserialize, Serialize, Parser, TS)]
struct AddImageParams {
id: ImageId,
image: String,
arches: Option<Vec<String>>,
#[command(flatten)]
config: ImageConfig,
}
async fn add_image(
ctx: CliContext,
AddImageParams { id, image, arches }: AddImageParams,
AddImageParams { id, config }: AddImageParams,
S9pkPath { s9pk: s9pk_path }: S9pkPath,
) -> Result<(), Error> {
let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?, false)
let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?)
.await?
.into_dyn();
let arches: BTreeSet<_> = arches
.unwrap_or_else(|| vec!["x86_64".to_owned(), "aarch64".to_owned()])
.into_iter()
.collect();
s9pk.as_manifest_mut().images.insert(id, config);
let tmpdir = TmpDir::new().await?;
for arch in arches {
let sqfs_path = tmpdir.join(format!("image.{arch}.squashfs"));
let docker_platform = if arch == "x86_64" {
"--platform=linux/amd64".to_owned()
} else if arch == "aarch64" {
"--platform=linux/arm64".to_owned()
} else {
format!("--platform=linux/{arch}")
};
let env = String::from_utf8(
Command::new(CONTAINER_TOOL)
.arg("run")
.arg("--rm")
.arg(&docker_platform)
.arg("--entrypoint")
.arg("env")
.arg(&image)
.invoke(ErrorKind::Docker)
.await?,
)?
.lines()
.filter(|l| {
l.trim()
.split_once("=")
.map_or(false, |(v, _)| !SKIP_ENV.contains(&v))
})
.join("\n")
+ "\n";
let workdir = Path::new(
String::from_utf8(
Command::new(CONTAINER_TOOL)
.arg("run")
.arg(&docker_platform)
.arg("--rm")
.arg("--entrypoint")
.arg("pwd")
.arg(&image)
.invoke(ErrorKind::Docker)
.await?,
)?
.trim(),
)
.to_owned();
let container_id = String::from_utf8(
Command::new(CONTAINER_TOOL)
.arg("create")
.arg(&docker_platform)
.arg(&image)
.invoke(ErrorKind::Docker)
.await?,
)?;
Command::new("bash")
.arg("-c")
.arg(format!(
"{CONTAINER_TOOL} export {container_id} | mksquashfs - {sqfs} -tar",
container_id = container_id.trim(),
sqfs = sqfs_path.display()
))
.invoke(ErrorKind::Docker)
.await?;
Command::new(CONTAINER_TOOL)
.arg("rm")
.arg(container_id.trim())
.invoke(ErrorKind::Docker)
.await?;
let archive = s9pk.as_archive_mut();
archive.set_signer(ctx.developer_key()?.clone(), SIG_CONTEXT);
archive.contents_mut().insert_path(
Path::new("images")
.join(&arch)
.join(&id)
.with_extension("squashfs"),
Entry::file(DynFileSource::new(sqfs_path)),
)?;
archive.contents_mut().insert_path(
Path::new("images")
.join(&arch)
.join(&id)
.with_extension("env"),
Entry::file(DynFileSource::new(Arc::<[u8]>::from(Vec::from(env)))),
)?;
archive.contents_mut().insert_path(
Path::new("images")
.join(&arch)
.join(&id)
.with_extension("json"),
Entry::file(DynFileSource::new(Arc::<[u8]>::from(
serde_json::to_vec(&serde_json::json!({
"workdir": workdir
}))
.with_kind(ErrorKind::Serialization)?,
))),
)?;
}
s9pk.as_manifest_mut().images.insert(id);
s9pk.load_images(&tmpdir).await?;
s9pk.validate_and_filter(None)?;
let tmp_path = s9pk_path.with_extension("s9pk.tmp");
let mut tmp_file = File::create(&tmp_path).await?;
s9pk.serialize(&mut tmp_file, true).await?;
@@ -206,7 +104,7 @@ async fn edit_manifest(
EditManifestParams { expression }: EditManifestParams,
S9pkPath { s9pk: s9pk_path }: S9pkPath,
) -> Result<Manifest, Error> {
let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?, false).await?;
let mut s9pk = S9pk::from_file(super::load(&ctx, &s9pk_path).await?).await?;
let old = serde_json::to_value(s9pk.as_manifest()).with_kind(ErrorKind::Serialization)?;
*s9pk.as_manifest_mut() = serde_json::from_value(apply_expr(old.into(), &expression)?.into())
.with_kind(ErrorKind::Serialization)?;
@@ -227,7 +125,7 @@ async fn file_tree(
_: Empty,
S9pkPath { s9pk }: S9pkPath,
) -> Result<Vec<PathBuf>, Error> {
let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?, false).await?;
let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?).await?;
Ok(s9pk.as_archive().contents().file_paths(""))
}
@@ -244,7 +142,7 @@ async fn cat(
) -> Result<(), Error> {
use crate::s9pk::merkle_archive::source::FileSource;
let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?, false).await?;
let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?).await?;
tokio::io::copy(
&mut s9pk
.as_archive()
@@ -266,6 +164,6 @@ async fn inspect_manifest(
_: Empty,
S9pkPath { s9pk }: S9pkPath,
) -> Result<Manifest, Error> {
let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?, false).await?;
let s9pk = S9pk::from_file(super::load(&ctx, &s9pk).await?).await?;
Ok(s9pk.as_manifest().clone())
}

67
core/startos/src/s9pk/v2/compat.rs
View File

@@ -1,6 +1,5 @@
use std::collections::{BTreeMap, BTreeSet};
use std::io::Cursor;
use std::path::{Path, PathBuf};
use std::collections::BTreeMap;
use std::path::Path;
use std::sync::Arc;
use itertools::Itertools;
@@ -14,49 +13,18 @@ use crate::prelude::*;
use crate::s9pk::manifest::Manifest;
use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
use crate::s9pk::merkle_archive::source::{FileSource, Section};
use crate::s9pk::merkle_archive::source::Section;
use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
use crate::s9pk::rpc::SKIP_ENV;
use crate::s9pk::v1::manifest::{Manifest as ManifestV1, PackageProcedure};
use crate::s9pk::v1::reader::S9pkReader;
use crate::s9pk::v2::pack::{PackSource, CONTAINER_TOOL};
use crate::s9pk::v2::{S9pk, SIG_CONTEXT};
use crate::util::io::TmpDir;
use crate::util::Invoke;
pub const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x01];
#[cfg(not(feature = "docker"))]
pub const CONTAINER_TOOL: &str = "podman";
#[cfg(feature = "docker")]
pub const CONTAINER_TOOL: &str = "docker";
type DynRead = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
fn into_dyn_read<R: AsyncRead + Unpin + Send + Sync + 'static>(r: R) -> DynRead {
Box::new(r)
}
#[derive(Clone)]
enum CompatSource {
Buffered(Arc<[u8]>),
File(PathBuf),
}
impl FileSource for CompatSource {
type Reader = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
async fn size(&self) -> Result<u64, Error> {
match self {
Self::Buffered(a) => Ok(a.len() as u64),
Self::File(f) => Ok(tokio::fs::metadata(f).await?.len()),
}
}
async fn reader(&self) -> Result<Self::Reader, Error> {
match self {
Self::Buffered(a) => Ok(into_dyn_read(Cursor::new(a.clone()))),
Self::File(f) => Ok(into_dyn_read(File::open(f).await?)),
}
}
}
impl S9pk<Section<MultiCursorFile>> {
#[instrument(skip_all)]
pub async fn from_v1<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
@@ -66,7 +34,7 @@ impl S9pk<Section<MultiCursorFile>> {
) -> Result<Self, Error> {
let scratch_dir = TmpDir::new().await?;
let mut archive = DirectoryContents::<CompatSource>::new();
let mut archive = DirectoryContents::<PackSource>::new();
// manifest.json
let manifest_raw = reader.manifest().await?;
@@ -88,21 +56,21 @@ impl S9pk<Section<MultiCursorFile>> {
let license: Arc<[u8]> = reader.license().await?.to_vec().await?.into();
archive.insert_path(
"LICENSE.md",
Entry::file(CompatSource::Buffered(license.into())),
Entry::file(PackSource::Buffered(license.into())),
)?;
// instructions.md
let instructions: Arc<[u8]> = reader.instructions().await?.to_vec().await?.into();
archive.insert_path(
"instructions.md",
Entry::file(CompatSource::Buffered(instructions.into())),
Entry::file(PackSource::Buffered(instructions.into())),
)?;
// icon.md
let icon: Arc<[u8]> = reader.icon().await?.to_vec().await?.into();
archive.insert_path(
format!("icon.{}", manifest.assets.icon_type()),
Entry::file(CompatSource::Buffered(icon.into())),
Entry::file(PackSource::Buffered(icon.into())),
)?;
// images
@@ -122,7 +90,9 @@ impl S9pk<Section<MultiCursorFile>> {
.invoke(ErrorKind::Docker)
.await?;
for (image, system) in &images {
new_manifest.images.insert(image.clone());
let mut image_config = new_manifest.images.remove(image).unwrap_or_default();
image_config.arch.insert(arch.as_str().into());
new_manifest.images.insert(image.clone(), image_config);
let sqfs_path = images_dir.join(image).with_extension("squashfs");
let image_name = if *system {
format!("start9/{}:latest", image)
@@ -190,21 +160,21 @@ impl S9pk<Section<MultiCursorFile>> {
.join(&arch)
.join(&image)
.with_extension("squashfs"),
Entry::file(CompatSource::File(sqfs_path)),
Entry::file(PackSource::File(sqfs_path)),
)?;
archive.insert_path(
Path::new("images")
.join(&arch)
.join(&image)
.with_extension("env"),
Entry::file(CompatSource::Buffered(Vec::from(env).into())),
Entry::file(PackSource::Buffered(Vec::from(env).into())),
)?;
archive.insert_path(
Path::new("images")
.join(&arch)
.join(&image)
.with_extension("json"),
Entry::file(CompatSource::Buffered(
Entry::file(PackSource::Buffered(
serde_json::to_vec(&serde_json::json!({
"workdir": workdir
}))
@@ -240,7 +210,7 @@ impl S9pk<Section<MultiCursorFile>> {
.await?;
archive.insert_path(
Path::new("assets").join(&asset_id),
Entry::file(CompatSource::File(sqfs_path)),
Entry::file(PackSource::File(sqfs_path)),
)?;
}
@@ -267,12 +237,12 @@ impl S9pk<Section<MultiCursorFile>> {
.await?;
archive.insert_path(
Path::new("javascript.squashfs"),
Entry::file(CompatSource::File(sqfs_path)),
Entry::file(PackSource::File(sqfs_path)),
)?;
archive.insert_path(
"manifest.json",
Entry::file(CompatSource::Buffered(
Entry::file(PackSource::Buffered(
serde_json::to_vec::<Manifest>(&new_manifest)
.with_kind(ErrorKind::Serialization)?
.into(),
@@ -289,7 +259,6 @@ impl S9pk<Section<MultiCursorFile>> {
Ok(S9pk::deserialize(
&MultiCursorFile::from(File::open(destination.as_ref()).await?),
None,
false,
)
.await?)
}
@@ -310,7 +279,7 @@ impl From<ManifestV1> for Manifest {
marketing_site: value.marketing_site.unwrap_or_else(|| default_url.clone()),
donation_url: value.donation_url,
description: value.description,
images: BTreeSet::new(),
images: BTreeMap::new(),
assets: value
.volumes
.iter()

85
core/startos/src/s9pk/v2/manifest.rs
View File

@@ -1,10 +1,11 @@
use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use color_eyre::eyre::eyre;
use helpers::const_true;
use imbl_value::InternedString;
pub use models::PackageId;
use models::{ImageId, VolumeId};
use models::{mime, ImageId, VolumeId};
use serde::{Deserialize, Serialize};
use ts_rs::TS;
use url::Url;
@@ -12,6 +13,9 @@ use url::Url;
use crate::dependencies::Dependencies;
use crate::prelude::*;
use crate::s9pk::git_hash::GitHash;
use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
use crate::s9pk::merkle_archive::expected::{Expected, Filter};
use crate::s9pk::v2::pack::ImageConfig;
use crate::util::serde::Regex;
use crate::util::VersionString;
use crate::version::{Current, VersionT};
@@ -42,7 +46,7 @@ pub struct Manifest {
#[ts(type = "string | null")]
pub donation_url: Option<Url>,
pub description: Description,
pub images: BTreeSet<ImageId>,
pub images: BTreeMap<ImageId, ImageConfig>,
pub assets: BTreeSet<VolumeId>, // TODO: AssetsId
pub volumes: BTreeSet<VolumeId>,
#[serde(default)]
@@ -59,6 +63,83 @@ pub struct Manifest {
#[serde(default = "const_true")]
pub has_config: bool,
}
impl Manifest {
pub fn validate_for<'a, T: Clone>(
&self,
arch: Option<&str>,
archive: &'a DirectoryContents<T>,
) -> Result<Filter, Error> {
let mut expected = Expected::new(archive);
expected.check_file("manifest.json")?;
expected.check_stem("icon", |ext| {
ext.and_then(|e| e.to_str())
.and_then(mime)
.map_or(false, |mime| mime.starts_with("image/"))
})?;
expected.check_file("LICENSE.md")?;
expected.check_file("instructions.md")?;
expected.check_file("javascript.squashfs")?;
for assets in &self.assets {
expected.check_file(Path::new("assets").join(assets).with_extension("squashfs"))?;
}
for (image_id, config) in &self.images {
let mut check_arch = |arch: &str| {
let mut arch = arch;
if let Err(e) = expected.check_file(
Path::new("images")
.join(arch)
.join(image_id)
.with_extension("squashfs"),
) {
if let Some(emulate_as) = &config.emulate_missing_as {
expected.check_file(
Path::new("images")
.join(arch)
.join(image_id)
.with_extension("squashfs"),
)?;
arch = &**emulate_as;
} else {
return Err(e);
}
}
expected.check_file(
Path::new("images")
.join(arch)
.join(image_id)
.with_extension("json"),
)?;
expected.check_file(
Path::new("images")
.join(arch)
.join(image_id)
.with_extension("env"),
)?;
Ok(())
};
if let Some(arch) = arch {
check_arch(arch)?;
} else if let Some(arches) = &self.hardware_requirements.arch {
for arch in arches {
check_arch(arch)?;
}
} else if let Some(arch) = config.emulate_missing_as.as_deref() {
if !config.arch.contains(arch) {
return Err(Error::new(
eyre!("`emulateMissingAs` must match an included `arch`"),
ErrorKind::ParseS9pk,
));
}
for arch in &config.arch {
check_arch(&arch)?;
}
} else {
return Err(Error::new(eyre!("`emulateMissingAs` required for all images if no `arch` specified in `hardwareRequirements`"), ErrorKind::ParseS9pk));
}
}
Ok(expected.into_filter())
}
}
#[derive(Clone, Debug, Default, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]

76
core/startos/src/s9pk/v2/mod.rs
View File

@@ -14,7 +14,8 @@ use crate::s9pk::merkle_archive::sink::Sink;
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
use crate::s9pk::merkle_archive::source::{ArchiveSource, DynFileSource, FileSource, Section};
use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
use crate::ARCH;
use crate::s9pk::v2::pack::{ImageSource, PackSource};
use crate::util::io::TmpDir;
const MAGIC_AND_VERSION: &[u8] = &[0x3b, 0x3b, 0x02];
@@ -22,6 +23,7 @@ pub const SIG_CONTEXT: &str = "s9pk";
pub mod compat;
pub mod manifest;
pub mod pack;
/**
/
@@ -34,10 +36,14 @@ pub mod manifest;
│ └── <id>.squashfs (xN)
└── images
└── <arch>
├── <id>.json (xN)
├── <id>.env (xN)
└── <id>.squashfs (xN)
*/
// this sorts the s9pk to optimize such that the parts that are used first appear earlier in the s9pk
// this is useful for manipulating an s9pk while partially downloaded on a source that does not support
// random access
fn priority(s: &str) -> Option<usize> {
match s {
"manifest.json" => Some(0),
@@ -51,26 +57,6 @@ fn priority(s: &str) -> Option<usize> {
}
}
fn filter(p: &Path) -> bool {
match p.iter().count() {
1 if p.file_name() == Some(OsStr::new("manifest.json")) => true,
1 if p.file_stem() == Some(OsStr::new("icon")) => true,
1 if p.file_name() == Some(OsStr::new("LICENSE.md")) => true,
1 if p.file_name() == Some(OsStr::new("instructions.md")) => true,
1 if p.file_name() == Some(OsStr::new("javascript.squashfs")) => true,
1 if p.file_name() == Some(OsStr::new("assets")) => true,
1 if p.file_name() == Some(OsStr::new("images")) => true,
2 if p.parent() == Some(Path::new("assets")) => {
p.extension().map_or(false, |ext| ext == "squashfs")
}
2 if p.parent() == Some(Path::new("images")) => p.file_name() == Some(OsStr::new(&*ARCH)),
3 if p.parent() == Some(&*Path::new("images").join(&*ARCH)) => p
.extension()
.map_or(false, |ext| ext == "squashfs" || ext == "env"),
_ => false,
}
}
#[derive(Clone)]
pub struct S9pk<S = Section<MultiCursorFile>> {
pub manifest: Manifest,
@@ -108,6 +94,11 @@ impl<S: FileSource + Clone> S9pk<S> {
})
}
pub fn validate_and_filter(&mut self, arch: Option<&str>) -> Result<(), Error> {
let filter = self.manifest.validate_for(arch, self.archive.contents())?;
filter.keep_checked(self.archive.contents_mut())
}
pub async fn icon(&self) -> Result<(InternedString, FileContents<S>), Error> {
let mut best_icon = None;
for (path, icon) in self
@@ -174,12 +165,37 @@ impl<S: FileSource + Clone> S9pk<S> {
}
}
impl<S: From<PackSource> + FileSource + Clone> S9pk<S> {
pub async fn load_images(&mut self, tmpdir: &TmpDir) -> Result<(), Error> {
let id = &self.manifest.id;
let version = &self.manifest.version;
for (image_id, image_config) in &mut self.manifest.images {
self.manifest_dirty = true;
for arch in &image_config.arch {
image_config
.source
.load(
tmpdir,
id,
version,
image_id,
arch,
self.archive.contents_mut(),
)
.await?;
}
image_config.source = ImageSource::Packed;
}
Ok(())
}
}
impl<S: ArchiveSource + Clone> S9pk<Section<S>> {
#[instrument(skip_all)]
pub async fn deserialize(
source: &S,
commitment: Option<&MerkleArchiveCommitment>,
apply_filter: bool,
) -> Result<Self, Error> {
use tokio::io::AsyncReadExt;
@@ -201,10 +217,6 @@ impl<S: ArchiveSource + Clone> S9pk<Section<S>> {
let mut archive =
MerkleArchive::deserialize(source, SIG_CONTEXT, &mut header, commitment).await?;
if apply_filter {
archive.filter(filter)?;
}
archive.sort_by(|a, b| match (priority(a), priority(b)) {
(Some(a), Some(b)) => a.cmp(&b),
(Some(_), None) => std::cmp::Ordering::Less,
@@ -216,15 +228,11 @@ impl<S: ArchiveSource + Clone> S9pk<Section<S>> {
}
}
impl S9pk {
pub async fn from_file(file: File, apply_filter: bool) -> Result<Self, Error> {
Self::deserialize(&MultiCursorFile::from(file), None, apply_filter).await
pub async fn from_file(file: File) -> Result<Self, Error> {
Self::deserialize(&MultiCursorFile::from(file), None).await
}
pub async fn open(
path: impl AsRef<Path>,
id: Option<&PackageId>,
apply_filter: bool,
) -> Result<Self, Error> {
let res = Self::from_file(tokio::fs::File::open(path).await?, apply_filter).await?;
pub async fn open(path: impl AsRef<Path>, id: Option<&PackageId>) -> Result<Self, Error> {
let res = Self::from_file(tokio::fs::File::open(path).await?).await?;
if let Some(id) = id {
ensure_code!(
&res.as_manifest().id == id,

536
core/startos/src/s9pk/v2/pack.rs Normal file
View File

@@ -0,0 +1,536 @@
use std::collections::BTreeSet;
use std::ffi::OsStr;
use std::io::Cursor;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use clap::Parser;
use futures::future::{ready, BoxFuture};
use futures::{FutureExt, TryStreamExt};
use imbl_value::InternedString;
use models::{ImageId, PackageId, VersionString};
use serde::{Deserialize, Serialize};
use tokio::fs::File;
use tokio::io::AsyncRead;
use tokio::process::Command;
use tokio::sync::OnceCell;
use tokio_stream::wrappers::ReadDirStream;
use ts_rs::TS;
use crate::context::CliContext;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::s9pk::merkle_archive::directory_contents::DirectoryContents;
use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
use crate::s9pk::merkle_archive::source::{
into_dyn_read, ArchiveSource, DynFileSource, FileSource,
};
use crate::s9pk::merkle_archive::{Entry, MerkleArchive};
use crate::s9pk::v2::SIG_CONTEXT;
use crate::s9pk::S9pk;
use crate::util::io::TmpDir;
use crate::util::Invoke;
#[cfg(not(feature = "docker"))]
pub const CONTAINER_TOOL: &str = "podman";
#[cfg(feature = "docker")]
pub const CONTAINER_TOOL: &str = "docker";
pub struct SqfsDir {
path: PathBuf,
tmpdir: Arc<TmpDir>,
sqfs: OnceCell<MultiCursorFile>,
}
impl SqfsDir {
pub fn new(path: PathBuf, tmpdir: Arc<TmpDir>) -> Self {
Self {
path,
tmpdir,
sqfs: OnceCell::new(),
}
}
async fn file(&self) -> Result<&MultiCursorFile, Error> {
self.sqfs
.get_or_try_init(|| async move {
let guid = Guid::new();
let path = self.tmpdir.join(guid.as_ref()).with_extension("squashfs");
let mut cmd = Command::new("mksquashfs");
if self.path.extension().and_then(|s| s.to_str()) == Some("tar") {
cmd.arg("-tar");
}
cmd.arg(&self.path)
.arg(&path)
.invoke(ErrorKind::Filesystem)
.await?;
Ok(MultiCursorFile::from(
File::open(&path)
.await
.with_ctx(|_| (ErrorKind::Filesystem, path.display()))?,
))
})
.await
}
}
#[derive(Clone)]
pub enum PackSource {
Buffered(Arc<[u8]>),
File(PathBuf),
Squashfs(Arc<SqfsDir>),
}
impl FileSource for PackSource {
type Reader = Box<dyn AsyncRead + Unpin + Send + Sync + 'static>;
async fn size(&self) -> Result<u64, Error> {
match self {
Self::Buffered(a) => Ok(a.len() as u64),
Self::File(f) => Ok(tokio::fs::metadata(f)
.await
.with_ctx(|_| (ErrorKind::Filesystem, f.display()))?
.len()),
Self::Squashfs(dir) => dir
.file()
.await
.with_ctx(|_| (ErrorKind::Filesystem, dir.path.display()))?
.size()
.await
.or_not_found("file metadata"),
}
}
async fn reader(&self) -> Result<Self::Reader, Error> {
match self {
Self::Buffered(a) => Ok(into_dyn_read(Cursor::new(a.clone()))),
Self::File(f) => Ok(into_dyn_read(
File::open(f)
.await
.with_ctx(|_| (ErrorKind::Filesystem, f.display()))?,
)),
Self::Squashfs(dir) => dir.file().await?.fetch_all().await.map(into_dyn_read),
}
}
}
impl From<PackSource> for DynFileSource {
fn from(value: PackSource) -> Self {
DynFileSource::new(value)
}
}
#[derive(Deserialize, Serialize, Parser)]
pub struct PackParams {
pub path: Option<PathBuf>,
#[arg(short = 'o', long = "output")]
pub output: Option<PathBuf>,
#[arg(long = "javascript")]
pub javascript: Option<PathBuf>,
#[arg(long = "icon")]
pub icon: Option<PathBuf>,
#[arg(long = "license")]
pub license: Option<PathBuf>,
#[arg(long = "instructions")]
pub instructions: Option<PathBuf>,
#[arg(long = "assets")]
pub assets: Option<PathBuf>,
}
impl PackParams {
fn path(&self) -> &Path {
self.path.as_deref().unwrap_or(Path::new("."))
}
fn output(&self, id: &PackageId) -> PathBuf {
self.output
.as_ref()
.cloned()
.unwrap_or_else(|| self.path().join(id).with_extension("s9pk"))
}
fn javascript(&self) -> PathBuf {
self.javascript
.as_ref()
.cloned()
.unwrap_or_else(|| self.path().join("javascript"))
}
async fn icon(&self) -> Result<PathBuf, Error> {
if let Some(icon) = &self.icon {
Ok(icon.clone())
} else {
ReadDirStream::new(tokio::fs::read_dir(self.path()).await?).try_filter(|x| ready(x.path().file_stem() == Some(OsStr::new("icon")))).map_err(Error::from).try_fold(Err(Error::new(eyre!("icon not found"), ErrorKind::NotFound)), |acc, x| async move { match acc {
Ok(_) => Err(Error::new(eyre!("multiple icons found in working directory, please specify which to use with `--icon`"), ErrorKind::InvalidRequest)),
Err(e) => Ok({
let path = x.path();
if path.file_stem().and_then(|s| s.to_str()) == Some("icon") {
Ok(path)
} else {
Err(e)
}
})
}}).await?
}
}
fn license(&self) -> PathBuf {
self.license
.as_ref()
.cloned()
.unwrap_or_else(|| self.path().join("LICENSE.md"))
}
fn instructions(&self) -> PathBuf {
self.instructions
.as_ref()
.cloned()
.unwrap_or_else(|| self.path().join("instructions.md"))
}
fn assets(&self) -> PathBuf {
self.assets
.as_ref()
.cloned()
.unwrap_or_else(|| self.path().join("assets"))
}
}
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct ImageConfig {
pub source: ImageSource,
#[ts(type = "string[]")]
pub arch: BTreeSet<InternedString>,
#[ts(type = "string | null")]
pub emulate_missing_as: Option<InternedString>,
}
impl Default for ImageConfig {
fn default() -> Self {
Self {
source: ImageSource::Packed,
arch: BTreeSet::new(),
emulate_missing_as: None,
}
}
}
#[derive(Parser)]
struct CliImageConfig {
#[arg(long, conflicts_with("docker-tag"))]
docker_build: bool,
#[arg(long, requires("docker-build"))]
dockerfile: Option<PathBuf>,
#[arg(long, requires("docker-build"))]
workdir: Option<PathBuf>,
#[arg(long, conflicts_with_all(["dockerfile", "workdir"]))]
docker_tag: Option<String>,
#[arg(long)]
arch: Vec<InternedString>,
#[arg(long)]
emulate_missing_as: Option<InternedString>,
}
impl TryFrom<CliImageConfig> for ImageConfig {
type Error = clap::Error;
fn try_from(value: CliImageConfig) -> Result<Self, Self::Error> {
let res = Self {
source: if value.docker_build {
ImageSource::DockerBuild {
dockerfile: value.dockerfile,
workdir: value.workdir,
}
} else if let Some(tag) = value.docker_tag {
ImageSource::DockerTag(tag)
} else {
ImageSource::Packed
},
arch: value.arch.into_iter().collect(),
emulate_missing_as: value.emulate_missing_as,
};
res.emulate_missing_as
.as_ref()
.map(|a| {
if !res.arch.contains(a) {
Err(clap::Error::raw(
clap::error::ErrorKind::InvalidValue,
"`emulate-missing-as` must match one of the provided `arch`es",
))
} else {
Ok(())
}
})
.transpose()?;
Ok(res)
}
}
impl clap::Args for ImageConfig {
fn augment_args(cmd: clap::Command) -> clap::Command {
CliImageConfig::augment_args(cmd)
}
fn augment_args_for_update(cmd: clap::Command) -> clap::Command {
CliImageConfig::augment_args_for_update(cmd)
}
}
impl clap::FromArgMatches for ImageConfig {
fn from_arg_matches(matches: &clap::ArgMatches) -> Result<Self, clap::Error> {
Self::try_from(CliImageConfig::from_arg_matches(matches)?)
}
fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> {
*self = Self::try_from(CliImageConfig::from_arg_matches(matches)?)?;
Ok(())
}
}
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub enum ImageSource {
Packed,
#[serde(rename_all = "camelCase")]
DockerBuild {
workdir: Option<PathBuf>,
dockerfile: Option<PathBuf>,
},
DockerTag(String),
}
impl ImageSource {
#[instrument(skip_all)]
pub fn load<'a, S: From<PackSource> + FileSource + Clone>(
&'a self,
tmpdir: &'a TmpDir,
id: &'a PackageId,
version: &'a VersionString,
image_id: &'a ImageId,
arch: &'a str,
into: &'a mut DirectoryContents<S>,
) -> BoxFuture<'a, Result<(), Error>> {
#[derive(Deserialize)]
#[serde(rename_all = "PascalCase")]
struct DockerImageConfig {
env: Vec<String>,
#[serde(default)]
working_dir: PathBuf,
#[serde(default)]
user: String,
}
async move {
match self {
ImageSource::Packed => Ok(()),
ImageSource::DockerBuild {
workdir,
dockerfile,
} => {
let workdir = workdir.as_deref().unwrap_or(Path::new("."));
let dockerfile = dockerfile
.clone()
.unwrap_or_else(|| workdir.join("Dockerfile"));
let docker_platform = if arch == "x86_64" {
"--platform=linux/amd64".to_owned()
} else if arch == "aarch64" {
"--platform=linux/arm64".to_owned()
} else {
format!("--platform=linux/{arch}")
};
// docker buildx build ${path} -o type=image,name=start9/${id}
let tag = format!("start9/{id}/{image_id}:{version}");
Command::new(CONTAINER_TOOL)
.arg("build")
.arg(workdir)
.arg("-f")
.arg(dockerfile)
.arg("-t")
.arg(&tag)
.arg(&docker_platform)
.arg("-o")
.arg("type=image")
.capture(false)
.invoke(ErrorKind::Docker)
.await?;
ImageSource::DockerTag(tag.clone())
.load(tmpdir, id, version, image_id, arch, into)
.await?;
Command::new(CONTAINER_TOOL)
.arg("rmi")
.arg("-f")
.arg(&tag)
.invoke(ErrorKind::Docker)
.await?;
Ok(())
}
ImageSource::DockerTag(tag) => {
let docker_platform = if arch == "x86_64" {
"--platform=linux/amd64".to_owned()
} else if arch == "aarch64" {
"--platform=linux/arm64".to_owned()
} else {
format!("--platform=linux/{arch}")
};
let mut inspect_cmd = Command::new(CONTAINER_TOOL);
inspect_cmd
.arg("image")
.arg("inspect")
.arg("--format")
.arg("{{json .Config}}")
.arg(&tag);
let inspect_res = match inspect_cmd.invoke(ErrorKind::Docker).await {
Ok(a) => a,
Err(e)
if {
let msg = e.source.to_string();
#[cfg(feature = "docker")]
let matches = msg.contains("No such image:");
#[cfg(not(feature = "docker"))]
let matches = msg.contains(": image not known");
matches
} =>
{
Command::new(CONTAINER_TOOL)
.arg("pull")
.arg(&docker_platform)
.arg(tag)
.capture(false)
.invoke(ErrorKind::Docker)
.await?;
inspect_cmd.invoke(ErrorKind::Docker).await?
}
Err(e) => return Err(e),
};
let config = serde_json::from_slice::<DockerImageConfig>(&inspect_res)
.with_kind(ErrorKind::Deserialization)?;
let base_path = Path::new("images").join(arch).join(image_id);
into.insert_path(
base_path.with_extension("json"),
Entry::file(
PackSource::Buffered(
serde_json::to_vec(&ImageMetadata {
workdir: if config.working_dir == Path::new("") {
"/".into()
} else {
config.working_dir
},
user: if config.user.is_empty() {
"root".into()
} else {
config.user.into()
},
})
.with_kind(ErrorKind::Serialization)?
.into(),
)
.into(),
),
)?;
into.insert_path(
base_path.with_extension("env"),
Entry::file(
PackSource::Buffered(config.env.join("\n").into_bytes().into()).into(),
),
)?;
let dest = tmpdir.join(Guid::new().as_ref()).with_extension("squashfs");
let container = String::from_utf8(
Command::new(CONTAINER_TOOL)
.arg("create")
.arg(&docker_platform)
.arg(&tag)
.invoke(ErrorKind::Docker)
.await?,
)?;
Command::new(CONTAINER_TOOL)
.arg("export")
.arg(container.trim())
.pipe(Command::new("mksquashfs").arg("-").arg(&dest).arg("-tar"))
.capture(false)
.invoke(ErrorKind::Docker)
.await?;
Command::new(CONTAINER_TOOL)
.arg("rm")
.arg(container.trim())
.invoke(ErrorKind::Docker)
.await?;
into.insert_path(
base_path.with_extension("squashfs"),
Entry::file(PackSource::File(dest).into()),
)?;
Ok(())
}
}
}
.boxed()
}
}
#[derive(Debug, Clone, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct ImageMetadata {
pub workdir: PathBuf,
#[ts(type = "string")]
pub user: InternedString,
}
#[instrument(skip_all)]
pub async fn pack(ctx: CliContext, params: PackParams) -> Result<(), Error> {
let tmpdir = Arc::new(TmpDir::new().await?);
let mut files = DirectoryContents::<PackSource>::new();
let js_dir = params.javascript();
let manifest: Arc<[u8]> = Command::new("node")
.arg("-e")
.arg(format!(
"console.log(JSON.stringify(require('{}/index.js').manifest))",
js_dir.display()
))
.invoke(ErrorKind::Javascript)
.await?
.into();
files.insert(
"manifest.json".into(),
Entry::file(PackSource::Buffered(manifest.clone())),
);
let icon = params.icon().await?;
let icon_ext = icon
.extension()
.or_not_found("icon file extension")?
.to_string_lossy();
files.insert(
InternedString::from_display(&lazy_format!("icon.{}", icon_ext)),
Entry::file(PackSource::File(icon)),
);
files.insert(
"LICENSE.md".into(),
Entry::file(PackSource::File(params.license())),
);
files.insert(
"instructions.md".into(),
Entry::file(PackSource::File(params.instructions())),
);
files.insert(
"javascript.squashfs".into(),
Entry::file(PackSource::Squashfs(Arc::new(SqfsDir::new(
js_dir,
tmpdir.clone(),
)))),
);
let mut s9pk = S9pk::new(
MerkleArchive::new(files, ctx.developer_key()?.clone(), SIG_CONTEXT),
None,
)
.await?;
let assets_dir = params.assets();
for assets in s9pk.as_manifest().assets.clone() {
s9pk.as_archive_mut().contents_mut().insert_path(
Path::new("assets").join(&assets).with_extension("squashfs"),
Entry::file(PackSource::Squashfs(Arc::new(SqfsDir::new(
assets_dir.join(&assets),
tmpdir.clone(),
)))),
)?;
}
s9pk.load_images(&*tmpdir).await?;
s9pk.validate_and_filter(None)?;
s9pk.serialize(
&mut File::create(params.output(&s9pk.as_manifest().id)).await?,
false,
)
.await?;
drop(s9pk);
tmpdir.gc().await?;
Ok(())
}

27
core/startos/src/service/action.rs
View File

@@ -4,6 +4,7 @@ use models::{ActionId, ProcedureName};
use crate::action::ActionResult;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::config::GetConfig;
use crate::service::dependencies::DependencyConfig;
use crate::service::{Service, ServiceActor};
@@ -23,13 +24,18 @@ impl Handler<Action> for ServiceActor {
}
async fn handle(
&mut self,
Action { id, input }: Action,
id: Guid,
Action {
id: action_id,
input,
}: Action,
_: &BackgroundJobQueue,
) -> Self::Response {
let container = &self.0.persistent_container;
container
.execute::<ActionResult>(
ProcedureName::RunAction(id),
id,
ProcedureName::RunAction(action_id),
input,
Some(Duration::from_secs(30)),
)
@@ -39,7 +45,20 @@ impl Handler<Action> for ServiceActor {
}
impl Service {
pub async fn action(&self, id: ActionId, input: Value) -> Result<ActionResult, Error> {
self.actor.send(Action { id, input }).await?
pub async fn action(
&self,
id: Guid,
action_id: ActionId,
input: Value,
) -> Result<ActionResult, Error> {
self.actor
.send(
id,
Action {
id: action_id,
input,
},
)
.await?
}
}

15
core/startos/src/service/config.rs
View File

@@ -5,6 +5,7 @@ use models::ProcedureName;
use crate::config::action::ConfigRes;
use crate::config::ConfigureContext;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::dependencies::DependencyConfig;
use crate::service::{Service, ServiceActor};
use crate::util::actor::background::BackgroundJobQueue;
@@ -19,6 +20,7 @@ impl Handler<Configure> for ServiceActor {
}
async fn handle(
&mut self,
id: Guid,
Configure(ConfigureContext { timeout, config }): Configure,
_: &BackgroundJobQueue,
) -> Self::Response {
@@ -26,7 +28,7 @@ impl Handler<Configure> for ServiceActor {
let package_id = &self.0.id;
container
.execute::<NoOutput>(ProcedureName::SetConfig, to_value(&config)?, timeout)
.execute::<NoOutput>(id, ProcedureName::SetConfig, to_value(&config)?, timeout)
.await
.with_kind(ErrorKind::ConfigRulesViolation)?;
self.0
@@ -52,10 +54,11 @@ impl Handler<GetConfig> for ServiceActor {
fn conflicts_with(_: &GetConfig) -> ConflictBuilder<Self> {
ConflictBuilder::nothing().except::<Configure>()
}
async fn handle(&mut self, _: GetConfig, _: &BackgroundJobQueue) -> Self::Response {
async fn handle(&mut self, id: Guid, _: GetConfig, _: &BackgroundJobQueue) -> Self::Response {
let container = &self.0.persistent_container;
container
.execute::<ConfigRes>(
id,
ProcedureName::GetConfig,
Value::Null,
Some(Duration::from_secs(30)), // TODO timeout
@@ -66,10 +69,10 @@ impl Handler<GetConfig> for ServiceActor {
}
impl Service {
pub async fn configure(&self, ctx: ConfigureContext) -> Result<(), Error> {
self.actor.send(Configure(ctx)).await?
pub async fn configure(&self, id: Guid, ctx: ConfigureContext) -> Result<(), Error> {
self.actor.send(id, Configure(ctx)).await?
}
pub async fn get_config(&self) -> Result<ConfigRes, Error> {
self.actor.send(GetConfig).await?
pub async fn get_config(&self, id: Guid) -> Result<ConfigRes, Error> {
self.actor.send(id, GetConfig).await?
}
}

13
core/startos/src/service/control.rs
View File

@@ -1,4 +1,5 @@
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::config::GetConfig;
use crate::service::dependencies::DependencyConfig;
use crate::service::start_stop::StartStop;
@@ -15,7 +16,7 @@ impl Handler<Start> for ServiceActor {
.except::<GetConfig>()
.except::<DependencyConfig>()
}
async fn handle(&mut self, _: Start, _: &BackgroundJobQueue) -> Self::Response {
async fn handle(&mut self, _: Guid, _: Start, _: &BackgroundJobQueue) -> Self::Response {
self.0.persistent_container.state.send_modify(|x| {
x.desired_state = StartStop::Start;
});
@@ -23,8 +24,8 @@ impl Handler<Start> for ServiceActor {
}
}
impl Service {
pub async fn start(&self) -> Result<(), Error> {
self.actor.send(Start).await
pub async fn start(&self, id: Guid) -> Result<(), Error> {
self.actor.send(id, Start).await
}
}
@@ -36,7 +37,7 @@ impl Handler<Stop> for ServiceActor {
.except::<GetConfig>()
.except::<DependencyConfig>()
}
async fn handle(&mut self, _: Stop, _: &BackgroundJobQueue) -> Self::Response {
async fn handle(&mut self, _: Guid, _: Stop, _: &BackgroundJobQueue) -> Self::Response {
let mut transition_state = None;
self.0.persistent_container.state.send_modify(|x| {
x.desired_state = StartStop::Stop;
@@ -51,7 +52,7 @@ impl Handler<Stop> for ServiceActor {
}
}
impl Service {
pub async fn stop(&self) -> Result<(), Error> {
self.actor.send(Stop).await
pub async fn stop(&self, id: Guid) -> Result<(), Error> {
self.actor.send(id, Stop).await
}
}

67
core/startos/src/service/dependencies.rs
View File

@@ -4,35 +4,28 @@ use imbl_value::json;
use models::{PackageId, ProcedureName};
use crate::prelude::*;
use crate::service::{Service, ServiceActor};
use crate::rpc_continuations::Guid;
use crate::service::{Service, ServiceActor, ServiceActorSeed};
use crate::util::actor::background::BackgroundJobQueue;
use crate::util::actor::{ConflictBuilder, Handler};
use crate::Config;
pub(super) struct DependencyConfig {
dependency_id: PackageId,
remote_config: Option<Config>,
}
impl Handler<DependencyConfig> for ServiceActor {
type Response = Result<Option<Config>, Error>;
fn conflicts_with(_: &DependencyConfig) -> ConflictBuilder<Self> {
ConflictBuilder::nothing()
}
async fn handle(
&mut self,
DependencyConfig {
dependency_id,
remote_config,
}: DependencyConfig,
_: &BackgroundJobQueue,
) -> Self::Response {
let container = &self.0.persistent_container;
impl ServiceActorSeed {
async fn dependency_config(
&self,
id: Guid,
dependency_id: PackageId,
remote_config: Option<Config>,
) -> Result<Option<Config>, Error> {
let container = &self.persistent_container;
container
.sanboxed::<Option<Config>>(
id.clone(),
ProcedureName::UpdateDependency(dependency_id.clone()),
json!({
"queryResults": container
.execute::<Value>(
id,
ProcedureName::QueryDependency(dependency_id),
Value::Null,
Some(Duration::from_secs(30)),
@@ -49,17 +42,45 @@ impl Handler<DependencyConfig> for ServiceActor {
}
}
pub(super) struct DependencyConfig {
dependency_id: PackageId,
remote_config: Option<Config>,
}
impl Handler<DependencyConfig> for ServiceActor {
type Response = Result<Option<Config>, Error>;
fn conflicts_with(_: &DependencyConfig) -> ConflictBuilder<Self> {
ConflictBuilder::nothing()
}
async fn handle(
&mut self,
id: Guid,
DependencyConfig {
dependency_id,
remote_config,
}: DependencyConfig,
_: &BackgroundJobQueue,
) -> Self::Response {
self.0
.dependency_config(id, dependency_id, remote_config)
.await
}
}
impl Service {
pub async fn dependency_config(
&self,
id: Guid,
dependency_id: PackageId,
remote_config: Option<Config>,
) -> Result<Option<Config>, Error> {
self.actor
.send(DependencyConfig {
dependency_id,
remote_config,
})
.send(
id,
DependencyConfig {
dependency_id,
remote_config,
},
)
.await?
}
}

179
core/startos/src/service/mod.rs
View File

@@ -1,4 +1,5 @@
use std::sync::Arc;
use std::ops::Deref;
use std::sync::{Arc, Weak};
use std::time::Duration;
use chrono::{DateTime, Utc};
@@ -68,13 +69,87 @@ pub enum LoadDisposition {
Undo,
}
pub struct ServiceRef(Arc<Service>);
impl ServiceRef {
pub fn weak(&self) -> Weak<Service> {
Arc::downgrade(&self.0)
}
pub async fn uninstall(
self,
target_version: Option<models::VersionString>,
) -> Result<(), Error> {
self.seed
.persistent_container
.execute(
Guid::new(),
ProcedureName::Uninit,
to_value(&target_version)?,
None,
) // TODO timeout
.await?;
let id = self.seed.persistent_container.s9pk.as_manifest().id.clone();
let ctx = self.seed.ctx.clone();
self.shutdown().await?;
if target_version.is_none() {
ctx.db
.mutate(|d| d.as_public_mut().as_package_data_mut().remove(&id))
.await?;
}
Ok(())
}
pub async fn shutdown(self) -> Result<(), Error> {
if let Some((hdl, shutdown)) = self.seed.persistent_container.rpc_server.send_replace(None)
{
self.seed
.persistent_container
.rpc_client
.request(rpc::Exit, Empty {})
.await?;
shutdown.shutdown();
hdl.await.with_kind(ErrorKind::Cancelled)?;
}
let service = Arc::try_unwrap(self.0).map_err(|_| {
Error::new(
eyre!("ServiceActor held somewhere after actor shutdown"),
ErrorKind::Unknown,
)
})?;
service
.actor
.shutdown(crate::util::actor::PendingMessageStrategy::FinishAll { timeout: None }) // TODO timeout
.await;
Arc::try_unwrap(service.seed)
.map_err(|_| {
Error::new(
eyre!("ServiceActorSeed held somewhere after actor shutdown"),
ErrorKind::Unknown,
)
})?
.persistent_container
.exit()
.await?;
Ok(())
}
}
impl Deref for ServiceRef {
type Target = Service;
fn deref(&self) -> &Self::Target {
&*self.0
}
}
impl From<Service> for ServiceRef {
fn from(value: Service) -> Self {
Self(Arc::new(value))
}
}
pub struct Service {
actor: ConcurrentActor<ServiceActor>,
seed: Arc<ServiceActorSeed>,
}
impl Service {
#[instrument(skip_all)]
async fn new(ctx: RpcContext, s9pk: S9pk, start: StartStop) -> Result<Self, Error> {
async fn new(ctx: RpcContext, s9pk: S9pk, start: StartStop) -> Result<ServiceRef, Error> {
let id = s9pk.as_manifest().id.clone();
let persistent_container = PersistentContainer::new(
&ctx, s9pk,
@@ -89,13 +164,17 @@ impl Service {
ctx,
synchronized: Arc::new(Notify::new()),
});
seed.persistent_container
.init(Arc::downgrade(&seed))
.await?;
Ok(Self {
let service: ServiceRef = Self {
actor: ConcurrentActor::new(ServiceActor(seed.clone())),
seed,
})
}
.into();
service
.seed
.persistent_container
.init(service.weak())
.await?;
Ok(service)
}
#[instrument(skip_all)]
@@ -103,7 +182,7 @@ impl Service {
ctx: &RpcContext,
id: &PackageId,
disposition: LoadDisposition,
) -> Result<Option<Self>, Error> {
) -> Result<Option<ServiceRef>, Error> {
let handle_installed = {
let ctx = ctx.clone();
move |s9pk: S9pk, i: Model<PackageDataEntry>| async move {
@@ -137,7 +216,7 @@ impl Service {
match entry.as_state_info().as_match() {
PackageStateMatchModelRef::Installing(_) => {
if disposition == LoadDisposition::Retry {
if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id), true).await.map_err(|e| {
if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id)).await.map_err(|e| {
tracing::error!("Error opening s9pk for install: {e}");
tracing::debug!("{e:?}")
}) {
@@ -170,7 +249,7 @@ impl Service {
&& progress == &Progress::Complete(true)
})
{
if let Ok(s9pk) = S9pk::open(&s9pk_path, Some(id), true).await.map_err(|e| {
if let Ok(s9pk) = S9pk::open(&s9pk_path, Some(id)).await.map_err(|e| {
tracing::error!("Error opening s9pk for update: {e}");
tracing::debug!("{e:?}")
}) {
@@ -189,7 +268,7 @@ impl Service {
}
}
}
let s9pk = S9pk::open(s9pk_path, Some(id), true).await?;
let s9pk = S9pk::open(s9pk_path, Some(id)).await?;
ctx.db
.mutate({
|db| {
@@ -214,7 +293,7 @@ impl Service {
handle_installed(s9pk, entry).await
}
PackageStateMatchModelRef::Removing(_) | PackageStateMatchModelRef::Restoring(_) => {
if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id), true).await.map_err(|e| {
if let Ok(s9pk) = S9pk::open(s9pk_path, Some(id)).await.map_err(|e| {
tracing::error!("Error opening s9pk for removal: {e}");
tracing::debug!("{e:?}")
}) {
@@ -225,7 +304,7 @@ impl Service {
tracing::debug!("{e:?}")
})
{
match service.uninstall(None).await {
match ServiceRef::from(service).uninstall(None).await {
Err(e) => {
tracing::error!("Error uninstalling service: {e}");
tracing::debug!("{e:?}")
@@ -242,7 +321,7 @@ impl Service {
Ok(None)
}
PackageStateMatchModelRef::Installed(_) => {
handle_installed(S9pk::open(s9pk_path, Some(id), true).await?, entry).await
handle_installed(S9pk::open(s9pk_path, Some(id)).await?, entry).await
}
PackageStateMatchModelRef::Error(e) => Err(Error::new(
eyre!("Failed to parse PackageDataEntry, found {e:?}"),
@@ -257,7 +336,7 @@ impl Service {
s9pk: S9pk,
src_version: Option<models::VersionString>,
progress: Option<InstallProgressHandles>,
) -> Result<Self, Error> {
) -> Result<ServiceRef, Error> {
let manifest = s9pk.as_manifest().clone();
let developer_key = s9pk.as_archive().signer();
let icon = s9pk.icon_data_url().await?;
@@ -265,7 +344,12 @@ impl Service {
service
.seed
.persistent_container
.execute(ProcedureName::Init, to_value(&src_version)?, None) // TODO timeout
.execute(
Guid::new(),
ProcedureName::Init,
to_value(&src_version)?,
None,
) // TODO timeout
.await
.with_kind(ErrorKind::MigrationFailed)?; // TODO: handle cancellation
if let Some(mut progress) = progress {
@@ -301,61 +385,21 @@ impl Service {
s9pk: S9pk,
backup_source: impl GenericMountGuard,
progress: Option<InstallProgressHandles>,
) -> Result<Self, Error> {
) -> Result<ServiceRef, Error> {
let service = Service::install(ctx.clone(), s9pk, None, progress).await?;
service
.actor
.send(transition::restore::Restore {
path: backup_source.path().to_path_buf(),
})
.send(
Guid::new(),
transition::restore::Restore {
path: backup_source.path().to_path_buf(),
},
)
.await??;
Ok(service)
}
pub async fn shutdown(self) -> Result<(), Error> {
self.actor
.shutdown(crate::util::actor::PendingMessageStrategy::FinishAll { timeout: None }) // TODO timeout
.await;
if let Some((hdl, shutdown)) = self.seed.persistent_container.rpc_server.send_replace(None)
{
self.seed
.persistent_container
.rpc_client
.request(rpc::Exit, Empty {})
.await?;
shutdown.shutdown();
hdl.await.with_kind(ErrorKind::Cancelled)?;
}
Arc::try_unwrap(self.seed)
.map_err(|_| {
Error::new(
eyre!("ServiceActorSeed held somewhere after actor shutdown"),
ErrorKind::Unknown,
)
})?
.persistent_container
.exit()
.await?;
Ok(())
}
pub async fn uninstall(self, target_version: Option<models::VersionString>) -> Result<(), Error> {
self.seed
.persistent_container
.execute(ProcedureName::Uninit, to_value(&target_version)?, None) // TODO timeout
.await?;
let id = self.seed.persistent_container.s9pk.as_manifest().id.clone();
let ctx = self.seed.ctx.clone();
self.shutdown().await?;
if target_version.is_none() {
ctx.db
.mutate(|d| d.as_public_mut().as_package_data_mut().remove(&id))
.await?;
}
Ok(())
}
#[instrument(skip_all)]
pub async fn backup(&self, guard: impl GenericMountGuard) -> Result<(), Error> {
let id = &self.seed.id;
@@ -368,9 +412,12 @@ impl Service {
.await?;
drop(file);
self.actor
.send(transition::backup::Backup {
path: guard.path().to_path_buf(),
})
.send(
Guid::new(),
transition::backup::Backup {
path: guard.path().to_path_buf(),
},
)
.await??;
Ok(())
}

97
core/startos/src/service/persistent_container.rs
View File

@@ -6,8 +6,7 @@ use std::time::Duration;
use futures::future::ready;
use futures::{Future, FutureExt};
use helpers::NonDetachingJoinHandle;
use imbl_value::InternedString;
use models::{ProcedureName, VolumeId};
use models::{ImageId, ProcedureName, VolumeId};
use rpc_toolkit::{Empty, Server, ShutdownHandle};
use serde::de::DeserializeOwned;
use tokio::fs::File;
@@ -24,14 +23,15 @@ use crate::disk::mount::filesystem::idmapped::IdMapped;
use crate::disk::mount::filesystem::loop_dev::LoopDev;
use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
use crate::disk::mount::filesystem::{MountType, ReadOnly};
use crate::disk::mount::guard::MountGuard;
use crate::disk::mount::guard::{GenericMountGuard, MountGuard};
use crate::lxc::{LxcConfig, LxcContainer, HOST_RPC_SERVER_SOCKET};
use crate::net::net_controller::NetService;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::s9pk::merkle_archive::source::FileSource;
use crate::s9pk::S9pk;
use crate::service::start_stop::StartStop;
use crate::service::{rpc, RunningStatus};
use crate::service::{rpc, RunningStatus, Service};
use crate::util::rpc_client::UnixRpcClient;
use crate::util::Invoke;
use crate::volume::{asset_dir, data_dir};
@@ -89,7 +89,8 @@ pub struct PersistentContainer {
js_mount: MountGuard,
volumes: BTreeMap<VolumeId, MountGuard>,
assets: BTreeMap<VolumeId, MountGuard>,
pub(super) overlays: Arc<Mutex<BTreeMap<InternedString, OverlayGuard>>>,
pub(super) images: BTreeMap<ImageId, Arc<MountGuard>>,
pub(super) overlays: Arc<Mutex<BTreeMap<Guid, OverlayGuard<Arc<MountGuard>>>>>,
pub(super) state: Arc<watch::Sender<ServiceState>>,
pub(super) net_service: Mutex<NetService>,
destroyed: bool,
@@ -178,14 +179,62 @@ impl PersistentContainer {
.await?,
);
}
let mut images = BTreeMap::new();
let image_path = lxc_container.rootfs_dir().join("media/startos/images");
tokio::fs::create_dir_all(&image_path).await?;
for image in &s9pk.as_manifest().images {
for (image, config) in &s9pk.as_manifest().images {
let mut arch = ARCH;
let mut sqfs_path = Path::new("images")
.join(arch)
.join(image)
.with_extension("squashfs");
if !s9pk
.as_archive()
.contents()
.get_path(&sqfs_path)
.and_then(|e| e.as_file())
.is_some()
{
arch = if let Some(arch) = config.emulate_missing_as.as_deref() {
arch
} else {
continue;
};
sqfs_path = Path::new("images")
.join(arch)
.join(image)
.with_extension("squashfs");
}
let sqfs = s9pk
.as_archive()
.contents()
.get_path(&sqfs_path)
.and_then(|e| e.as_file())
.or_not_found(sqfs_path.display())?;
let mountpoint = image_path.join(image);
tokio::fs::create_dir_all(&mountpoint).await?;
Command::new("chown")
.arg("100000:100000")
.arg(&mountpoint)
.invoke(ErrorKind::Filesystem)
.await?;
images.insert(
image.clone(),
Arc::new(
MountGuard::mount(
&IdMapped::new(LoopDev::from(&**sqfs), 0, 100000, 65536),
&mountpoint,
ReadOnly,
)
.await?,
),
);
let env_filename = Path::new(image.as_ref()).with_extension("env");
if let Some(env) = s9pk
.as_archive()
.contents()
.get_path(Path::new("images").join(*ARCH).join(&env_filename))
.get_path(Path::new("images").join(arch).join(&env_filename))
.and_then(|e| e.as_file())
{
env.copy(&mut File::create(image_path.join(&env_filename)).await?)
@@ -195,7 +244,7 @@ impl PersistentContainer {
if let Some(json) = s9pk
.as_archive()
.contents()
.get_path(Path::new("images").join(*ARCH).join(&json_filename))
.get_path(Path::new("images").join(arch).join(&json_filename))
.and_then(|e| e.as_file())
{
json.copy(&mut File::create(image_path.join(&json_filename)).await?)
@@ -215,6 +264,7 @@ impl PersistentContainer {
js_mount,
volumes,
assets,
images,
overlays: Arc::new(Mutex::new(BTreeMap::new())),
state: Arc::new(watch::channel(ServiceState::new(start)).0),
net_service: Mutex::new(net_service),
@@ -257,7 +307,7 @@ impl PersistentContainer {
}
#[instrument(skip_all)]
pub async fn init(&self, seed: Weak<ServiceActorSeed>) -> Result<(), Error> {
pub async fn init(&self, seed: Weak<Service>) -> Result<(), Error> {
let socket_server_context = EffectContext::new(seed);
let server = Server::new(
move || ready(Ok(socket_server_context.clone())),
@@ -330,6 +380,7 @@ impl PersistentContainer {
let js_mount = self.js_mount.take();
let volumes = std::mem::take(&mut self.volumes);
let assets = std::mem::take(&mut self.assets);
let images = std::mem::take(&mut self.images);
let overlays = self.overlays.clone();
let lxc_container = self.lxc_container.take();
self.destroyed = true;
@@ -352,6 +403,9 @@ impl PersistentContainer {
for (_, overlay) in std::mem::take(&mut *overlays.lock().await) {
errs.handle(overlay.unmount(true).await);
}
for (_, images) in images {
errs.handle(images.unmount().await);
}
errs.handle(js_mount.unmount(true).await);
if let Some(lxc_container) = lxc_container {
errs.handle(lxc_container.exit().await);
@@ -378,6 +432,7 @@ impl PersistentContainer {
#[instrument(skip_all)]
pub async fn start(&self) -> Result<(), Error> {
self.execute(
Guid::new(),
ProcedureName::StartMain,
Value::Null,
Some(Duration::from_secs(5)), // TODO
@@ -389,7 +444,7 @@ impl PersistentContainer {
#[instrument(skip_all)]
pub async fn stop(&self) -> Result<Duration, Error> {
let timeout: Option<crate::util::serde::Duration> = self
.execute(ProcedureName::StopMain, Value::Null, None)
.execute(Guid::new(), ProcedureName::StopMain, Value::Null, None)
.await?;
Ok(timeout.map(|a| *a).unwrap_or(Duration::from_secs(30)))
}
@@ -397,6 +452,7 @@ impl PersistentContainer {
#[instrument(skip_all)]
pub async fn execute<O>(
&self,
id: Guid,
name: ProcedureName,
input: Value,
timeout: Option<Duration>,
@@ -404,7 +460,7 @@ impl PersistentContainer {
where
O: DeserializeOwned,
{
self._execute(name, input, timeout)
self._execute(id, name, input, timeout)
.await
.and_then(from_value)
}
@@ -412,6 +468,7 @@ impl PersistentContainer {
#[instrument(skip_all)]
pub async fn sanboxed<O>(
&self,
id: Guid,
name: ProcedureName,
input: Value,
timeout: Option<Duration>,
@@ -419,7 +476,7 @@ impl PersistentContainer {
where
O: DeserializeOwned,
{
self._sandboxed(name, input, timeout)
self._sandboxed(id, name, input, timeout)
.await
.and_then(from_value)
}
@@ -427,13 +484,15 @@ impl PersistentContainer {
#[instrument(skip_all)]
async fn _execute(
&self,
id: Guid,
name: ProcedureName,
input: Value,
timeout: Option<Duration>,
) -> Result<Value, Error> {
let fut = self
.rpc_client
.request(rpc::Execute, rpc::ExecuteParams::new(name, input, timeout));
let fut = self.rpc_client.request(
rpc::Execute,
rpc::ExecuteParams::new(id, name, input, timeout),
);
Ok(if let Some(timeout) = timeout {
tokio::time::timeout(timeout, fut)
@@ -447,13 +506,15 @@ impl PersistentContainer {
#[instrument(skip_all)]
async fn _sandboxed(
&self,
id: Guid,
name: ProcedureName,
input: Value,
timeout: Option<Duration>,
) -> Result<Value, Error> {
let fut = self
.rpc_client
.request(rpc::Sandbox, rpc::ExecuteParams::new(name, input, timeout));
let fut = self.rpc_client.request(
rpc::Sandbox,
rpc::ExecuteParams::new(id, name, input, timeout),
);
Ok(if let Some(timeout) = timeout {
tokio::time::timeout(timeout, fut)

2
core/startos/src/service/properties.rs
View File

@@ -3,6 +3,7 @@ use std::time::Duration;
use models::ProcedureName;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::Service;
impl Service {
@@ -11,6 +12,7 @@ impl Service {
let container = &self.seed.persistent_container;
container
.execute::<Value>(
Guid::new(),
ProcedureName::Properties,
Value::Null,
Some(Duration::from_secs(30)),

10
core/startos/src/service/rpc.rs
View File

@@ -7,6 +7,7 @@ use rpc_toolkit::Empty;
use ts_rs::TS;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
#[derive(Clone)]
pub struct Init;
@@ -46,14 +47,21 @@ impl serde::Serialize for Exit {
#[derive(Clone, serde::Deserialize, serde::Serialize, TS)]
pub struct ExecuteParams {
id: Guid,
procedure: String,
#[ts(type = "any")]
input: Value,
timeout: Option<u128>,
}
impl ExecuteParams {
pub fn new(procedure: ProcedureName, input: Value, timeout: Option<Duration>) -> Self {
pub fn new(
id: Guid,
procedure: ProcedureName,
input: Value,
timeout: Option<Duration>,
) -> Self {
Self {
id,
procedure: procedure.js_function_name(),
input,
timeout: timeout.map(|d| d.as_millis()),

326
core/startos/src/service/service_effect_handler.rs
View File

@@ -7,9 +7,9 @@ use std::str::FromStr;
use std::sync::{Arc, Weak};
use clap::builder::ValueParserFactory;
use clap::Parser;
use clap::{CommandFactory, FromArgMatches, Parser};
use emver::VersionRange;
use imbl_value::{json, InternedString};
use imbl_value::json;
use itertools::Itertools;
use models::{
ActionId, DataUrl, HealthCheckId, HostId, ImageId, PackageId, ServiceInterfaceId, VolumeId,
@@ -25,35 +25,34 @@ use crate::db::model::package::{
ActionMetadata, CurrentDependencies, CurrentDependencyInfo, CurrentDependencyKind,
ManifestPreference,
};
use crate::disk::mount::filesystem::idmapped::IdMapped;
use crate::disk::mount::filesystem::loop_dev::LoopDev;
use crate::disk::mount::filesystem::overlayfs::OverlayGuard;
use crate::echo;
use crate::net::host::address::HostAddress;
use crate::net::host::binding::{BindOptions, LanInfo};
use crate::net::host::{Host, HostKind};
use crate::net::service_interface::{AddressInfo, ServiceInterface, ServiceInterfaceType};
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::s9pk::merkle_archive::source::http::HttpSource;
use crate::s9pk::rpc::SKIP_ENV;
use crate::s9pk::S9pk;
use crate::service::cli::ContainerCliContext;
use crate::service::ServiceActorSeed;
use crate::service::Service;
use crate::status::health_check::HealthCheckResult;
use crate::status::MainStatus;
use crate::util::clap::FromStrParser;
use crate::util::{new_guid, Invoke};
use crate::{echo, ARCH};
use crate::util::Invoke;
#[derive(Clone)]
pub(super) struct EffectContext(Weak<ServiceActorSeed>);
pub(super) struct EffectContext(Weak<Service>);
impl EffectContext {
pub fn new(seed: Weak<ServiceActorSeed>) -> Self {
Self(seed)
pub fn new(service: Weak<Service>) -> Self {
Self(service)
}
}
impl Context for EffectContext {}
impl EffectContext {
fn deref(&self) -> Result<Arc<ServiceActorSeed>, Error> {
fn deref(&self) -> Result<Arc<Service>, Error> {
if let Some(seed) = Weak::upgrade(&self.0) {
Ok(seed)
} else {
@@ -66,11 +65,55 @@ impl EffectContext {
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
struct RpcData {
id: i64,
method: String,
params: Value,
#[serde(rename_all = "camelCase")]
pub struct WithProcedureId<T> {
#[serde(default)]
procedure_id: Guid,
#[serde(flatten)]
rest: T,
}
impl<T: FromArgMatches> FromArgMatches for WithProcedureId<T> {
fn from_arg_matches(matches: &clap::ArgMatches) -> Result<Self, clap::Error> {
let rest = T::from_arg_matches(matches)?;
Ok(Self {
procedure_id: matches.get_one("procedure-id").cloned().unwrap_or_default(),
rest,
})
}
fn from_arg_matches_mut(matches: &mut clap::ArgMatches) -> Result<Self, clap::Error> {
let rest = T::from_arg_matches_mut(matches)?;
Ok(Self {
procedure_id: matches.get_one("procedure-id").cloned().unwrap_or_default(),
rest,
})
}
fn update_from_arg_matches(&mut self, matches: &clap::ArgMatches) -> Result<(), clap::Error> {
self.rest.update_from_arg_matches(matches)?;
self.procedure_id = matches.get_one("procedure-id").cloned().unwrap_or_default();
Ok(())
}
fn update_from_arg_matches_mut(
&mut self,
matches: &mut clap::ArgMatches,
) -> Result<(), clap::Error> {
self.rest.update_from_arg_matches_mut(matches)?;
self.procedure_id = matches.get_one("procedure-id").cloned().unwrap_or_default();
Ok(())
}
}
impl<T: CommandFactory> CommandFactory for WithProcedureId<T> {
fn command() -> clap::Command {
T::command_for_update().arg(
clap::Arg::new("procedure-id")
.action(clap::ArgAction::Set)
.value_parser(clap::value_parser!(Guid)),
)
}
fn command_for_update() -> clap::Command {
Self::command()
}
}
pub fn service_effect_handler<C: Context>() -> ParentHandler<C> {
ParentHandler::new()
.subcommand("gitInfo", from_fn(|_: C| crate::version::git_info()))
@@ -290,6 +333,7 @@ struct MountParams {
async fn set_system_smtp(context: EffectContext, data: SetSystemSmtpParams) -> Result<(), Error> {
let context = context.deref()?;
context
.seed
.ctx
.db
.mutate(|db| {
@@ -304,6 +348,7 @@ async fn get_system_smtp(
) -> Result<String, Error> {
let context = context.deref()?;
let res = context
.seed
.ctx
.db
.peek()
@@ -323,7 +368,7 @@ async fn get_system_smtp(
}
async fn get_container_ip(context: EffectContext, _: Empty) -> Result<Ipv4Addr, Error> {
let context = context.deref()?;
let net_service = context.persistent_container.net_service.lock().await;
let net_service = context.seed.persistent_container.net_service.lock().await;
Ok(net_service.get_ip())
}
async fn get_service_port_forward(
@@ -333,14 +378,15 @@ async fn get_service_port_forward(
let internal_port = data.internal_port as u16;
let context = context.deref()?;
let net_service = context.persistent_container.net_service.lock().await;
let net_service = context.seed.persistent_container.net_service.lock().await;
net_service.get_ext_port(data.host_id, internal_port)
}
async fn clear_network_interfaces(context: EffectContext, _: Empty) -> Result<(), Error> {
let context = context.deref()?;
let package_id = context.id.clone();
let package_id = context.seed.id.clone();
context
.seed
.ctx
.db
.mutate(|db| {
@@ -369,7 +415,7 @@ async fn export_service_interface(
}: ExportServiceInterfaceParams,
) -> Result<(), Error> {
let context = context.deref()?;
let package_id = context.id.clone();
let package_id = context.seed.id.clone();
let service_interface = ServiceInterface {
id: id.clone(),
@@ -384,6 +430,7 @@ async fn export_service_interface(
let svc_interface_with_host_info = service_interface;
context
.seed
.ctx
.db
.mutate(|db| {
@@ -407,7 +454,7 @@ async fn get_primary_url(
}: GetPrimaryUrlParams,
) -> Result<Option<HostAddress>, Error> {
let context = context.deref()?;
let package_id = package_id.unwrap_or_else(|| context.id.clone());
let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
Ok(None) // TODO
}
@@ -419,9 +466,10 @@ async fn list_service_interfaces(
}: ListServiceInterfacesParams,
) -> Result<BTreeMap<ServiceInterfaceId, ServiceInterface>, Error> {
let context = context.deref()?;
let package_id = package_id.unwrap_or_else(|| context.id.clone());
let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
context
.seed
.ctx
.db
.peek()
@@ -435,9 +483,10 @@ async fn list_service_interfaces(
}
async fn remove_address(context: EffectContext, data: RemoveAddressParams) -> Result<(), Error> {
let context = context.deref()?;
let package_id = context.id.clone();
let package_id = context.seed.id.clone();
context
.seed
.ctx
.db
.mutate(|db| {
@@ -454,8 +503,9 @@ async fn remove_address(context: EffectContext, data: RemoveAddressParams) -> Re
}
async fn export_action(context: EffectContext, data: ExportActionParams) -> Result<(), Error> {
let context = context.deref()?;
let package_id = context.id.clone();
let package_id = context.seed.id.clone();
context
.seed
.ctx
.db
.mutate(|db| {
@@ -477,8 +527,9 @@ async fn export_action(context: EffectContext, data: ExportActionParams) -> Resu
}
async fn remove_action(context: EffectContext, data: RemoveActionParams) -> Result<(), Error> {
let context = context.deref()?;
let package_id = context.id.clone();
let package_id = context.seed.id.clone();
context
.seed
.ctx
.db
.mutate(|db| {
@@ -514,16 +565,16 @@ struct GetHostInfoParams {
callback: Callback,
}
async fn get_host_info(
ctx: EffectContext,
context: EffectContext,
GetHostInfoParams {
callback,
package_id,
host_id,
}: GetHostInfoParams,
) -> Result<Host, Error> {
let ctx = ctx.deref()?;
let db = ctx.ctx.db.peek().await;
let package_id = package_id.unwrap_or_else(|| ctx.id.clone());
let context = context.deref()?;
let db = context.seed.ctx.db.peek().await;
let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
db.as_public()
.as_package_data()
@@ -536,8 +587,8 @@ async fn get_host_info(
}
async fn clear_bindings(context: EffectContext, _: Empty) -> Result<(), Error> {
let ctx = context.deref()?;
let mut svc = ctx.persistent_container.net_service.lock().await;
let context = context.deref()?;
let mut svc = context.seed.persistent_container.net_service.lock().await;
svc.clear_bindings().await?;
Ok(())
}
@@ -559,8 +610,8 @@ async fn bind(context: EffectContext, bind_params: Value) -> Result<(), Error> {
internal_port,
options,
} = from_value(bind_params)?;
let ctx = context.deref()?;
let mut svc = ctx.persistent_container.net_service.lock().await;
let context = context.deref()?;
let mut svc = context.seed.persistent_container.net_service.lock().await;
svc.bind(kind, id, internal_port, options).await
}
@@ -575,16 +626,16 @@ struct GetServiceInterfaceParams {
}
async fn get_service_interface(
ctx: EffectContext,
context: EffectContext,
GetServiceInterfaceParams {
callback,
package_id,
service_interface_id,
}: GetServiceInterfaceParams,
) -> Result<ServiceInterface, Error> {
let ctx = ctx.deref()?;
let package_id = package_id.unwrap_or_else(|| ctx.id.clone());
let db = ctx.ctx.db.peek().await;
let context = context.deref()?;
let package_id = package_id.unwrap_or_else(|| context.seed.id.clone());
let db = context.seed.ctx.db.peek().await;
let interface = db
.as_public()
@@ -729,8 +780,8 @@ async fn get_store(
GetStoreParams { package_id, path }: GetStoreParams,
) -> Result<Value, Error> {
let context = context.deref()?;
let peeked = context.ctx.db.peek().await;
let package_id = package_id.unwrap_or(context.id.clone());
let peeked = context.seed.ctx.db.peek().await;
let package_id = package_id.unwrap_or(context.seed.id.clone());
let value = peeked
.as_private()
.as_package_stores()
@@ -758,8 +809,9 @@ async fn set_store(
SetStoreParams { value, path }: SetStoreParams,
) -> Result<(), Error> {
let context = context.deref()?;
let package_id = context.id.clone();
let package_id = context.seed.id.clone();
context
.seed
.ctx
.db
.mutate(|db| {
@@ -812,7 +864,7 @@ struct ParamsMaybePackageId {
async fn exists(context: EffectContext, params: ParamsPackageId) -> Result<Value, Error> {
let context = context.deref()?;
let peeked = context.ctx.db.peek().await;
let peeked = context.seed.ctx.db.peek().await;
let package = peeked
.as_public()
.as_package_data()
@@ -834,31 +886,30 @@ struct ExecuteAction {
}
async fn execute_action(
context: EffectContext,
ExecuteAction {
action_id,
input,
service_id,
}: ExecuteAction,
WithProcedureId {
procedure_id,
rest:
ExecuteAction {
service_id,
action_id,
input,
},
}: WithProcedureId<ExecuteAction>,
) -> Result<Value, Error> {
let context = context.deref()?;
let package_id = service_id.clone().unwrap_or_else(|| context.id.clone());
let service = context.ctx.services.get(&package_id).await;
let service = service.as_ref().ok_or_else(|| {
Error::new(
eyre!("Could not find package {package_id}"),
ErrorKind::Unknown,
)
})?;
let package_id = service_id
.clone()
.unwrap_or_else(|| context.seed.id.clone());
Ok(json!(service.action(action_id, input).await?))
Ok(json!(context.action(procedure_id, action_id, input).await?))
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
struct FromService {}
async fn get_configured(context: EffectContext, _: Empty) -> Result<Value, Error> {
let context = context.deref()?;
let peeked = context.ctx.db.peek().await;
let package_id = &context.id;
let peeked = context.seed.ctx.db.peek().await;
let package_id = &context.seed.id;
let package = peeked
.as_public()
.as_package_data()
@@ -872,8 +923,8 @@ async fn get_configured(context: EffectContext, _: Empty) -> Result<Value, Error
async fn stopped(context: EffectContext, params: ParamsMaybePackageId) -> Result<Value, Error> {
let context = context.deref()?;
let peeked = context.ctx.db.peek().await;
let package_id = params.package_id.unwrap_or_else(|| context.id.clone());
let peeked = context.seed.ctx.db.peek().await;
let package_id = params.package_id.unwrap_or_else(|| context.seed.id.clone());
let package = peeked
.as_public()
.as_package_data()
@@ -887,7 +938,7 @@ async fn stopped(context: EffectContext, params: ParamsMaybePackageId) -> Result
async fn running(context: EffectContext, params: ParamsPackageId) -> Result<Value, Error> {
dbg!("Starting the running {params:?}");
let context = context.deref()?;
let peeked = context.ctx.db.peek().await;
let peeked = context.seed.ctx.db.peek().await;
let package_id = params.package_id;
let package = peeked
.as_public()
@@ -900,30 +951,24 @@ async fn running(context: EffectContext, params: ParamsPackageId) -> Result<Valu
Ok(json!(matches!(package, MainStatus::Running { .. })))
}
async fn restart(context: EffectContext, _: Empty) -> Result<Value, Error> {
async fn restart(
context: EffectContext,
WithProcedureId { procedure_id, .. }: WithProcedureId<Empty>,
) -> Result<(), Error> {
let context = context.deref()?;
let service = context.ctx.services.get(&context.id).await;
let service = service.as_ref().ok_or_else(|| {
Error::new(
eyre!("Could not find package {}", context.id),
ErrorKind::Unknown,
)
})?;
service.restart().await?;
Ok(json!(()))
dbg!("here");
context.restart(procedure_id).await?;
dbg!("here");
Ok(())
}
async fn shutdown(context: EffectContext, _: Empty) -> Result<Value, Error> {
async fn shutdown(
context: EffectContext,
WithProcedureId { procedure_id, .. }: WithProcedureId<Empty>,
) -> Result<(), Error> {
let context = context.deref()?;
let service = context.ctx.services.get(&context.id).await;
let service = service.as_ref().ok_or_else(|| {
Error::new(
eyre!("Could not find package {}", context.id),
ErrorKind::Unknown,
)
})?;
service.stop().await?;
Ok(json!(()))
context.stop(procedure_id).await?;
Ok(())
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Parser, TS)]
@@ -935,8 +980,9 @@ struct SetConfigured {
}
async fn set_configured(context: EffectContext, params: SetConfigured) -> Result<Value, Error> {
let context = context.deref()?;
let package_id = &context.id;
let package_id = &context.seed.id;
context
.seed
.ctx
.db
.mutate(|db| {
@@ -989,9 +1035,9 @@ async fn set_main_status(context: EffectContext, params: SetMainStatus) -> Resul
dbg!(format!("Status for main will be is {params:?}"));
let context = context.deref()?;
match params.status {
SetMainStatusStatus::Running => context.started(),
SetMainStatusStatus::Stopped => context.stopped(),
SetMainStatusStatus::Starting => context.stopped(),
SetMainStatusStatus::Running => context.seed.started(),
SetMainStatusStatus::Stopped => context.seed.stopped(),
SetMainStatusStatus::Starting => context.seed.stopped(),
}
Ok(Value::Null)
}
@@ -1011,8 +1057,9 @@ async fn set_health(
) -> Result<Value, Error> {
let context = context.deref()?;
let package_id = &context.id;
let package_id = &context.seed.id;
context
.seed
.ctx
.db
.mutate(move |db| {
@@ -1041,17 +1088,17 @@ async fn set_health(
#[command(rename_all = "camelCase")]
#[ts(export)]
pub struct DestroyOverlayedImageParams {
#[ts(type = "string")]
guid: InternedString,
guid: Guid,
}
#[instrument(skip_all)]
pub async fn destroy_overlayed_image(
ctx: EffectContext,
context: EffectContext,
DestroyOverlayedImageParams { guid }: DestroyOverlayedImageParams,
) -> Result<(), Error> {
let ctx = ctx.deref()?;
if ctx
let context = context.deref()?;
if context
.seed
.persistent_container
.overlays
.lock()
@@ -1068,30 +1115,25 @@ pub async fn destroy_overlayed_image(
#[command(rename_all = "camelCase")]
#[ts(export)]
pub struct CreateOverlayedImageParams {
#[ts(type = "string")]
image_id: ImageId,
}
#[instrument(skip_all)]
pub async fn create_overlayed_image(
ctx: EffectContext,
context: EffectContext,
CreateOverlayedImageParams { image_id }: CreateOverlayedImageParams,
) -> Result<(PathBuf, InternedString), Error> {
let ctx = ctx.deref()?;
let path = Path::new("images")
.join(*ARCH)
.join(&image_id)
.with_extension("squashfs");
if let Some(image) = ctx
) -> Result<(PathBuf, Guid), Error> {
let context = context.deref()?;
if let Some(image) = context
.seed
.persistent_container
.s9pk
.as_archive()
.contents()
.get_path(&path)
.and_then(|e| e.as_file())
.images
.get(&image_id)
.cloned()
{
let guid = new_guid();
let rootfs_dir = ctx
let guid = Guid::new();
let rootfs_dir = context
.seed
.persistent_container
.lxc_container
.get()
@@ -1102,7 +1144,9 @@ pub async fn create_overlayed_image(
)
})?
.rootfs_dir();
let mountpoint = rootfs_dir.join("media/startos/overlays").join(&*guid);
let mountpoint = rootfs_dir
.join("media/startos/overlays")
.join(guid.as_ref());
tokio::fs::create_dir_all(&mountpoint).await?;
let container_mountpoint = Path::new("/").join(
mountpoint
@@ -1110,18 +1154,16 @@ pub async fn create_overlayed_image(
.with_kind(ErrorKind::Incoherent)?,
);
tracing::info!("Mounting overlay {guid} for {image_id}");
let guard = OverlayGuard::mount(
&IdMapped::new(LoopDev::from(&**image), 0, 100000, 65536),
&mountpoint,
)
.await?;
let guard = OverlayGuard::mount(image, &mountpoint).await?;
Command::new("chown")
.arg("100000:100000")
.arg(&mountpoint)
.invoke(ErrorKind::Filesystem)
.await?;
tracing::info!("Mounted overlay {guid} for {image_id}");
ctx.persistent_container
context
.seed
.persistent_container
.overlays
.lock()
.await
@@ -1228,13 +1270,15 @@ struct SetDependenciesParams {
}
async fn set_dependencies(
ctx: EffectContext,
SetDependenciesParams { dependencies }: SetDependenciesParams,
context: EffectContext,
WithProcedureId {
procedure_id,
rest: SetDependenciesParams { dependencies },
}: WithProcedureId<SetDependenciesParams>,
) -> Result<(), Error> {
let ctx = ctx.deref()?;
let id = &ctx.id;
let service_guard = ctx.ctx.services.get(id).await;
let service = service_guard.as_ref().or_not_found(id)?;
let context = context.deref()?;
let id = &context.seed.id;
let mut deps = BTreeMap::new();
for dependency in dependencies {
let (dep_id, kind, registry_url, version_spec) = match dependency {
@@ -1264,14 +1308,13 @@ async fn set_dependencies(
let remote_s9pk = S9pk::deserialize(
&Arc::new(
HttpSource::new(
ctx.ctx.client.clone(),
context.seed.ctx.client.clone(),
registry_url
.join(&format!("package/v2/{}.s9pk?spec={}", dep_id, version_spec))?,
)
.await?,
),
None, // TODO
true,
)
.await?;
@@ -1291,14 +1334,19 @@ async fn set_dependencies(
)
}
};
let config_satisfied = if let Some(dep_service) = &*ctx.ctx.services.get(&dep_id).await {
service
.dependency_config(dep_id.clone(), dep_service.get_config().await?.config)
.await?
.is_none()
} else {
true
};
let config_satisfied =
if let Some(dep_service) = &*context.seed.ctx.services.get(&dep_id).await {
context
.dependency_config(
procedure_id.clone(),
dep_id.clone(),
dep_service.get_config(procedure_id.clone()).await?.config,
)
.await?
.is_none()
} else {
true
};
deps.insert(
dep_id,
CurrentDependencyInfo {
@@ -1311,7 +1359,9 @@ async fn set_dependencies(
},
);
}
ctx.ctx
context
.seed
.ctx
.db
.mutate(|db| {
db.as_public_mut()
@@ -1324,10 +1374,10 @@ async fn set_dependencies(
.await
}
async fn get_dependencies(ctx: EffectContext) -> Result<Vec<DependencyRequirement>, Error> {
let ctx = ctx.deref()?;
let id = &ctx.id;
let db = ctx.ctx.db.peek().await;
async fn get_dependencies(context: EffectContext) -> Result<Vec<DependencyRequirement>, Error> {
let context = context.deref()?;
let id = &context.seed.id;
let db = context.seed.ctx.db.peek().await;
let data = db
.as_public()
.as_package_data()
@@ -1384,16 +1434,16 @@ struct CheckDependenciesResult {
}
async fn check_dependencies(
ctx: EffectContext,
context: EffectContext,
CheckDependenciesParam { package_ids }: CheckDependenciesParam,
) -> Result<Vec<CheckDependenciesResult>, Error> {
let ctx = ctx.deref()?;
let db = ctx.ctx.db.peek().await;
let context = context.deref()?;
let db = context.seed.ctx.db.peek().await;
let current_dependencies = db
.as_public()
.as_package_data()
.as_idx(&ctx.id)
.or_not_found(&ctx.id)?
.as_idx(&context.seed.id)
.or_not_found(&context.seed.id)?
.as_current_dependencies()
.de()?;
let package_ids: Vec<_> = package_ids

38
core/startos/src/service/service_map.rs
View File

@@ -25,7 +25,7 @@ use crate::progress::{
use crate::s9pk::manifest::PackageId;
use crate::s9pk::merkle_archive::source::FileSource;
use crate::s9pk::S9pk;
use crate::service::{LoadDisposition, Service};
use crate::service::{LoadDisposition, Service, ServiceRef};
use crate::status::{MainStatus, Status};
use crate::util::serde::Pem;
@@ -39,23 +39,22 @@ pub struct InstallProgressHandles {
/// This is the structure to contain all the services
#[derive(Default)]
pub struct ServiceMap(Mutex<OrdMap<PackageId, Arc<RwLock<Option<Service>>>>>);
pub struct ServiceMap(Mutex<OrdMap<PackageId, Arc<RwLock<Option<ServiceRef>>>>>);
impl ServiceMap {
async fn entry(&self, id: &PackageId) -> Arc<RwLock<Option<Service>>> {
async fn entry(&self, id: &PackageId) -> Arc<RwLock<Option<ServiceRef>>> {
let mut lock = self.0.lock().await;
dbg!(lock.keys().collect::<Vec<_>>());
lock.entry(id.clone())
.or_insert_with(|| Arc::new(RwLock::new(None)))
.clone()
}
#[instrument(skip_all)]
pub async fn get(&self, id: &PackageId) -> OwnedRwLockReadGuard<Option<Service>> {
pub async fn get(&self, id: &PackageId) -> OwnedRwLockReadGuard<Option<ServiceRef>> {
self.entry(id).await.read_owned().await
}
#[instrument(skip_all)]
pub async fn get_mut(&self, id: &PackageId) -> OwnedRwLockWriteGuard<Option<Service>> {
pub async fn get_mut(&self, id: &PackageId) -> OwnedRwLockWriteGuard<Option<ServiceRef>> {
self.entry(id).await.write_owned().await
}
@@ -83,7 +82,7 @@ impl ServiceMap {
shutdown_err = service.shutdown().await;
}
// TODO: retry on error?
*service = Service::load(ctx, id, disposition).await?;
*service = Service::load(ctx, id, disposition).await?.map(From::from);
shutdown_err?;
Ok(())
}
@@ -95,6 +94,7 @@ impl ServiceMap {
mut s9pk: S9pk<S>,
recovery_source: Option<impl GenericMountGuard>,
) -> Result<DownloadInstallFuture, Error> {
s9pk.validate_and_filter(ctx.s9pk_arch)?;
let manifest = s9pk.as_manifest().clone();
let id = manifest.id.clone();
let icon = s9pk.icon_data_url().await?;
@@ -128,7 +128,7 @@ impl ServiceMap {
);
let restoring = recovery_source.is_some();
let mut reload_guard = ServiceReloadGuard::new(ctx.clone(), id.clone(), op_name);
let mut reload_guard = ServiceRefReloadGuard::new(ctx.clone(), id.clone(), op_name);
reload_guard
.handle(ctx.db.mutate({
@@ -231,7 +231,7 @@ impl ServiceMap {
Ok(reload_guard
.handle_last(async move {
finalization_progress.start();
let s9pk = S9pk::open(&installed_path, Some(&id), true).await?;
let s9pk = S9pk::open(&installed_path, Some(&id)).await?;
let prev = if let Some(service) = service.take() {
ensure_code!(
recovery_source.is_none(),
@@ -264,7 +264,8 @@ impl ServiceMap {
progress_handle,
}),
)
.await?,
.await?
.into(),
);
} else {
*service = Some(
@@ -277,7 +278,8 @@ impl ServiceMap {
progress_handle,
}),
)
.await?,
.await?
.into(),
);
}
sync_progress_task.await.map_err(|_| {
@@ -295,7 +297,7 @@ impl ServiceMap {
pub async fn uninstall(&self, ctx: &RpcContext, id: &PackageId) -> Result<(), Error> {
let mut guard = self.get_mut(id).await;
if let Some(service) = guard.take() {
ServiceReloadGuard::new(ctx.clone(), id.clone(), "Uninstall")
ServiceRefReloadGuard::new(ctx.clone(), id.clone(), "Uninstall")
.handle_last(async move {
let res = service.uninstall(None).await;
drop(guard);
@@ -326,17 +328,17 @@ impl ServiceMap {
}
}
pub struct ServiceReloadGuard(Option<ServiceReloadInfo>);
impl Drop for ServiceReloadGuard {
pub struct ServiceRefReloadGuard(Option<ServiceRefReloadInfo>);
impl Drop for ServiceRefReloadGuard {
fn drop(&mut self) {
if let Some(info) = self.0.take() {
tokio::spawn(info.reload(None));
}
}
}
impl ServiceReloadGuard {
impl ServiceRefReloadGuard {
pub fn new(ctx: RpcContext, id: PackageId, operation: &'static str) -> Self {
Self(Some(ServiceReloadInfo { ctx, id, operation }))
Self(Some(ServiceRefReloadInfo { ctx, id, operation }))
}
pub async fn handle<T>(
@@ -365,12 +367,12 @@ impl ServiceReloadGuard {
}
}
struct ServiceReloadInfo {
struct ServiceRefReloadInfo {
ctx: RpcContext,
id: PackageId,
operation: &'static str,
}
impl ServiceReloadInfo {
impl ServiceRefReloadInfo {
async fn reload(self, error: Option<Error>) -> Result<(), Error> {
self.ctx
.services

10
core/startos/src/service/transition/backup.rs
View File

@@ -6,6 +6,7 @@ use models::ProcedureName;
use super::TempDesiredRestore;
use crate::disk::mount::filesystem::ReadWrite;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::config::GetConfig;
use crate::service::dependencies::DependencyConfig;
use crate::service::transition::{TransitionKind, TransitionState};
@@ -24,7 +25,12 @@ impl Handler<Backup> for ServiceActor {
.except::<GetConfig>()
.except::<DependencyConfig>()
}
async fn handle(&mut self, backup: Backup, jobs: &BackgroundJobQueue) -> Self::Response {
async fn handle(
&mut self,
id: Guid,
backup: Backup,
jobs: &BackgroundJobQueue,
) -> Self::Response {
// So Need a handle to just a single field in the state
let temp: TempDesiredRestore = TempDesiredRestore::new(&self.0.persistent_container.state);
let mut current = self.0.persistent_container.state.subscribe();
@@ -45,7 +51,7 @@ impl Handler<Backup> for ServiceActor {
.mount_backup(path, ReadWrite)
.await?;
seed.persistent_container
.execute(ProcedureName::CreateBackup, Value::Null, None)
.execute(id, ProcedureName::CreateBackup, Value::Null, None)
.await?;
backup_guard.unmount(true).await?;

9
core/startos/src/service/transition/restart.rs
View File

@@ -2,6 +2,7 @@ use futures::FutureExt;
use super::TempDesiredRestore;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::config::GetConfig;
use crate::service::dependencies::DependencyConfig;
use crate::service::transition::{TransitionKind, TransitionState};
@@ -18,7 +19,8 @@ impl Handler<Restart> for ServiceActor {
.except::<GetConfig>()
.except::<DependencyConfig>()
}
async fn handle(&mut self, _: Restart, jobs: &BackgroundJobQueue) -> Self::Response {
async fn handle(&mut self, _: Guid, _: Restart, jobs: &BackgroundJobQueue) -> Self::Response {
dbg!("here");
// So Need a handle to just a single field in the state
let temp = TempDesiredRestore::new(&self.0.persistent_container.state);
let mut current = self.0.persistent_container.state.subscribe();
@@ -74,7 +76,8 @@ impl Handler<Restart> for ServiceActor {
}
impl Service {
#[instrument(skip_all)]
pub async fn restart(&self) -> Result<(), Error> {
self.actor.send(Restart).await
pub async fn restart(&self, id: Guid) -> Result<(), Error> {
dbg!("here");
self.actor.send(id, Restart).await
}
}

10
core/startos/src/service/transition/restore.rs
View File

@@ -5,6 +5,7 @@ use models::ProcedureName;
use crate::disk::mount::filesystem::ReadOnly;
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::service::transition::{TransitionKind, TransitionState};
use crate::service::ServiceActor;
use crate::util::actor::background::BackgroundJobQueue;
@@ -19,7 +20,12 @@ impl Handler<Restore> for ServiceActor {
fn conflicts_with(_: &Restore) -> ConflictBuilder<Self> {
ConflictBuilder::everything()
}
async fn handle(&mut self, restore: Restore, jobs: &BackgroundJobQueue) -> Self::Response {
async fn handle(
&mut self,
id: Guid,
restore: Restore,
jobs: &BackgroundJobQueue,
) -> Self::Response {
// So Need a handle to just a single field in the state
let path = restore.path.clone();
let seed = self.0.clone();
@@ -32,7 +38,7 @@ impl Handler<Restore> for ServiceActor {
.mount_backup(path, ReadOnly)
.await?;
seed.persistent_container
.execute(ProcedureName::RestoreBackup, Value::Null, None)
.execute(id, ProcedureName::RestoreBackup, Value::Null, None)
.await?;
backup_guard.unmount(true).await?;

39
core/startos/src/util/actor/concurrent.rs
View File

@@ -8,6 +8,7 @@ use helpers::NonDetachingJoinHandle;
use tokio::sync::{mpsc, oneshot};
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::util::actor::background::{BackgroundJobQueue, BackgroundJobRunner};
use crate::util::actor::{Actor, ConflictFn, Handler, PendingMessageStrategy, Request};
@@ -18,6 +19,7 @@ struct ConcurrentRunner<A> {
waiting: Vec<Request<A>>,
recv: mpsc::UnboundedReceiver<Request<A>>,
handlers: Vec<(
Guid,
Arc<ConflictFn<A>>,
oneshot::Sender<Box<dyn Any + Send>>,
BoxFuture<'static, Box<dyn Any + Send>>,
@@ -41,16 +43,21 @@ impl<A: Actor + Clone> Future for ConcurrentRunner<A> {
}
});
if this.shutdown.is_some() {
while let std::task::Poll::Ready(Some((msg, reply))) = this.recv.poll_recv(cx) {
if this.handlers.iter().any(|(f, _, _)| f(&*msg)) {
this.waiting.push((msg, reply));
while let std::task::Poll::Ready(Some((id, msg, reply))) = this.recv.poll_recv(cx) {
if this
.handlers
.iter()
.any(|(hid, f, _, _)| &id != hid && f(&*msg))
{
this.waiting.push((id, msg, reply));
} else {
let mut actor = this.actor.clone();
let queue = this.queue.clone();
this.handlers.push((
id.clone(),
msg.conflicts_with(),
reply,
async move { msg.handle_with(&mut actor, &queue).await }.boxed(),
async move { msg.handle_with(id, &mut actor, &queue).await }.boxed(),
))
}
}
@@ -62,29 +69,34 @@ impl<A: Actor + Clone> Future for ConcurrentRunner<A> {
.handlers
.iter_mut()
.enumerate()
.filter_map(|(i, (_, _, f))| match f.poll_unpin(cx) {
.filter_map(|(i, (_, _, _, f))| match f.poll_unpin(cx) {
std::task::Poll::Pending => None,
std::task::Poll::Ready(res) => Some((i, res)),
})
.collect::<Vec<_>>();
for (idx, res) in complete.into_iter().rev() {
#[allow(clippy::let_underscore_future)]
let (f, reply, _) = this.handlers.swap_remove(idx);
let (_, f, reply, _) = this.handlers.swap_remove(idx);
let _ = reply.send(res);
// TODO: replace with Vec::extract_if once stable
if this.shutdown.is_some() {
let mut i = 0;
while i < this.waiting.len() {
if f(&*this.waiting[i].0)
&& !this.handlers.iter().any(|(f, _, _)| f(&*this.waiting[i].0))
if f(&*this.waiting[i].1)
&& !this
.handlers
.iter()
.any(|(_, f, _, _)| f(&*this.waiting[i].1))
{
let (msg, reply) = this.waiting.remove(i);
let (id, msg, reply) = this.waiting.remove(i);
let mut actor = this.actor.clone();
let queue = this.queue.clone();
this.handlers.push((
id.clone(),
msg.conflicts_with(),
reply,
async move { msg.handle_with(&mut actor, &queue).await }.boxed(),
async move { msg.handle_with(id, &mut actor, &queue).await }
.boxed(),
));
cont = true;
} else {
@@ -137,6 +149,7 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
/// Message is guaranteed to be queued immediately
pub fn queue<M: Send + 'static>(
&self,
id: Guid,
message: M,
) -> impl Future<Output = Result<A::Response, Error>>
where
@@ -150,7 +163,7 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
}
let (reply_send, reply_recv) = oneshot::channel();
self.messenger
.send((Box::new(message), reply_send))
.send((id, Box::new(message), reply_send))
.unwrap();
futures::future::Either::Right(
reply_recv
@@ -170,11 +183,11 @@ impl<A: Actor + Clone> ConcurrentActor<A> {
)
}
pub async fn send<M: Send + 'static>(&self, message: M) -> Result<A::Response, Error>
pub async fn send<M: Send + 'static>(&self, id: Guid, message: M) -> Result<A::Response, Error>
where
A: Handler<M>,
{
self.queue(message).await
self.queue(id, message).await
}
pub async fn shutdown(self, strategy: PendingMessageStrategy) {

12
core/startos/src/util/actor/mod.rs
View File

@@ -9,6 +9,7 @@ use tokio::sync::oneshot;
#[allow(unused_imports)]
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::util::actor::background::BackgroundJobQueue;
pub mod background;
@@ -28,6 +29,7 @@ pub trait Handler<M: Any + Send>: Actor {
}
fn handle(
&mut self,
id: Guid,
msg: M,
jobs: &BackgroundJobQueue,
) -> impl Future<Output = Self::Response> + Send;
@@ -39,6 +41,7 @@ trait Message<A>: Send + Any {
fn conflicts_with(&self) -> Arc<ConflictFn<A>>;
fn handle_with<'a>(
self: Box<Self>,
id: Guid,
actor: &'a mut A,
jobs: &'a BackgroundJobQueue,
) -> BoxFuture<'a, Box<dyn Any + Send>>;
@@ -52,10 +55,11 @@ where
}
fn handle_with<'a>(
self: Box<Self>,
id: Guid,
actor: &'a mut A,
jobs: &'a BackgroundJobQueue,
) -> BoxFuture<'a, Box<dyn Any + Send>> {
async move { Box::new(actor.handle(*self, jobs).await) as Box<dyn Any + Send> }.boxed()
async move { Box::new(actor.handle(id, *self, jobs).await) as Box<dyn Any + Send> }.boxed()
}
}
impl<A: Actor> dyn Message<A> {
@@ -80,7 +84,11 @@ impl<A: Actor> dyn Message<A> {
}
}
type Request<A> = (Box<dyn Message<A>>, oneshot::Sender<Box<dyn Any + Send>>);
type Request<A> = (
Guid,
Box<dyn Message<A>>,
oneshot::Sender<Box<dyn Any + Send>>,
);
pub enum PendingMessageStrategy {
CancelAll,

7
core/startos/src/util/actor/simple.rs
View File

@@ -7,6 +7,7 @@ use tokio::sync::oneshot::error::TryRecvError;
use tokio::sync::{mpsc, oneshot};
use crate::prelude::*;
use crate::rpc_continuations::Guid;
use crate::util::actor::background::BackgroundJobQueue;
use crate::util::actor::{Actor, Handler, PendingMessageStrategy, Request};
@@ -26,9 +27,9 @@ impl<A: Actor> SimpleActor<A> {
tokio::select! {
_ = &mut runner => (),
msg = messenger_recv.recv() => match msg {
Some((msg, reply)) if shutdown_recv.try_recv() == Err(TryRecvError::Empty) => {
Some((id, msg, reply)) if shutdown_recv.try_recv() == Err(TryRecvError::Empty) => {
tokio::select! {
res = msg.handle_with(&mut actor, &queue) => { let _ = reply.send(res); },
res = msg.handle_with(id, &mut actor, &queue) => { let _ = reply.send(res); },
_ = &mut runner => (),
}
}
@@ -60,7 +61,7 @@ impl<A: Actor> SimpleActor<A> {
}
let (reply_send, reply_recv) = oneshot::channel();
self.messenger
.send((Box::new(message), reply_send))
.send((Guid::new(), Box::new(message), reply_send))
.unwrap();
futures::future::Either::Right(
reply_recv

10
core/startos/src/util/io.rs
View File

@@ -681,8 +681,6 @@ impl<S: AsyncRead + AsyncWrite> AsyncWrite for TimeoutStream<S> {
}
}
pub struct TmpFile {}
#[derive(Debug)]
pub struct TmpDir {
path: PathBuf,
@@ -707,6 +705,14 @@ impl TmpDir {
tokio::fs::remove_dir_all(&self.path).await?;
Ok(())
}
pub async fn gc(self: Arc<Self>) -> Result<(), Error> {
if let Ok(dir) = Arc::try_unwrap(self) {
dir.delete().await
} else {
Ok(())
}
}
}
impl std::ops::Deref for TmpDir {
type Target = Path;

248
core/startos/src/util/mod.rs
View File

@@ -1,4 +1,4 @@
use std::collections::BTreeMap;
use std::collections::{BTreeMap, VecDeque};
use std::future::Future;
use std::marker::PhantomData;
use std::path::{Path, PathBuf};
@@ -11,6 +11,8 @@ use std::time::Duration;
use async_trait::async_trait;
use color_eyre::eyre::{self, eyre};
use fd_lock_rs::FdLock;
use futures::future::BoxFuture;
use futures::FutureExt;
use helpers::canonicalize;
pub use helpers::NonDetachingJoinHandle;
use imbl_value::InternedString;
@@ -19,7 +21,8 @@ pub use models::VersionString;
use pin_project::pin_project;
use sha2::Digest;
use tokio::fs::File;
use tokio::sync::{Mutex, OwnedMutexGuard, RwLock};
use tokio::io::{AsyncRead, AsyncReadExt, BufReader};
use tokio::sync::{oneshot, Mutex, OwnedMutexGuard, RwLock};
use tracing::instrument;
use crate::shutdown::Shutdown;
@@ -62,11 +65,16 @@ pub trait Invoke<'a> {
where
Self: 'ext,
'ext: 'a;
fn pipe<'ext: 'a>(
&'ext mut self,
next: &'ext mut tokio::process::Command,
) -> Self::Extended<'ext>;
fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext>;
fn input<'ext: 'a, Input: tokio::io::AsyncRead + Unpin + Send>(
&'ext mut self,
input: Option<&'ext mut Input>,
) -> Self::Extended<'ext>;
fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext>;
fn invoke(
&mut self,
error_kind: crate::ErrorKind,
@@ -76,7 +84,20 @@ pub trait Invoke<'a> {
pub struct ExtendedCommand<'a> {
cmd: &'a mut tokio::process::Command,
timeout: Option<Duration>,
input: Option<&'a mut (dyn tokio::io::AsyncRead + Unpin + Send)>,
input: Option<&'a mut (dyn AsyncRead + Unpin + Send)>,
pipe: VecDeque<&'a mut tokio::process::Command>,
capture: bool,
}
impl<'a> From<&'a mut tokio::process::Command> for ExtendedCommand<'a> {
fn from(value: &'a mut tokio::process::Command) -> Self {
ExtendedCommand {
cmd: value,
timeout: None,
input: None,
pipe: VecDeque::new(),
capture: true,
}
}
}
impl<'a> std::ops::Deref for ExtendedCommand<'a> {
type Target = tokio::process::Command;
@@ -95,35 +116,38 @@ impl<'a> Invoke<'a> for tokio::process::Command {
where
Self: 'ext,
'ext: 'a;
fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext> {
ExtendedCommand {
cmd: self,
timeout,
input: None,
}
fn pipe<'ext: 'a>(
&'ext mut self,
next: &'ext mut tokio::process::Command,
) -> Self::Extended<'ext> {
let mut cmd = ExtendedCommand::from(self);
cmd.pipe.push_back(next);
cmd
}
fn input<'ext: 'a, Input: tokio::io::AsyncRead + Unpin + Send>(
fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext> {
let mut cmd = ExtendedCommand::from(self);
cmd.timeout = timeout;
cmd
}
fn input<'ext: 'a, Input: AsyncRead + Unpin + Send>(
&'ext mut self,
input: Option<&'ext mut Input>,
) -> Self::Extended<'ext> {
ExtendedCommand {
cmd: self,
timeout: None,
input: if let Some(input) = input {
Some(&mut *input)
} else {
None
},
}
let mut cmd = ExtendedCommand::from(self);
cmd.input = if let Some(input) = input {
Some(&mut *input)
} else {
None
};
cmd
}
fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext> {
let mut cmd = ExtendedCommand::from(self);
cmd.capture = capture;
cmd
}
async fn invoke(&mut self, error_kind: crate::ErrorKind) -> Result<Vec<u8>, Error> {
ExtendedCommand {
cmd: self,
timeout: None,
input: None,
}
.invoke(error_kind)
.await
ExtendedCommand::from(self).invoke(error_kind).await
}
}
@@ -132,6 +156,13 @@ impl<'a> Invoke<'a> for ExtendedCommand<'a> {
where
Self: 'ext,
'ext: 'a;
fn pipe<'ext: 'a>(
&'ext mut self,
next: &'ext mut tokio::process::Command,
) -> Self::Extended<'ext> {
self.pipe.push_back(next.kill_on_drop(true));
self
}
fn timeout<'ext: 'a>(&'ext mut self, timeout: Option<Duration>) -> Self::Extended<'ext> {
self.timeout = timeout;
self
@@ -147,39 +178,150 @@ impl<'a> Invoke<'a> for ExtendedCommand<'a> {
};
self
}
fn capture<'ext: 'a>(&'ext mut self, capture: bool) -> Self::Extended<'ext> {
self.capture = capture;
self
}
#[instrument(skip_all)]
async fn invoke(&mut self, error_kind: crate::ErrorKind) -> Result<Vec<u8>, Error> {
let cmd_str = self
.cmd
.as_std()
.get_program()
.to_string_lossy()
.into_owned();
self.cmd.kill_on_drop(true);
if self.input.is_some() {
self.cmd.stdin(Stdio::piped());
}
self.cmd.stdout(Stdio::piped());
self.cmd.stderr(Stdio::piped());
let mut child = self.cmd.spawn().with_kind(error_kind)?;
if let (Some(mut stdin), Some(input)) = (child.stdin.take(), self.input.take()) {
use tokio::io::AsyncWriteExt;
tokio::io::copy(input, &mut stdin).await?;
stdin.flush().await?;
stdin.shutdown().await?;
drop(stdin);
if self.pipe.is_empty() {
if self.capture {
self.cmd.stdout(Stdio::piped());
self.cmd.stderr(Stdio::piped());
}
let mut child = self.cmd.spawn().with_ctx(|_| (error_kind, &cmd_str))?;
if let (Some(mut stdin), Some(input)) = (child.stdin.take(), self.input.take()) {
use tokio::io::AsyncWriteExt;
tokio::io::copy(input, &mut stdin).await?;
stdin.flush().await?;
stdin.shutdown().await?;
drop(stdin);
}
let res = match self.timeout {
None => child
.wait_with_output()
.await
.with_ctx(|_| (error_kind, &cmd_str))?,
Some(t) => tokio::time::timeout(t, child.wait_with_output())
.await
.with_kind(ErrorKind::Timeout)?
.with_ctx(|_| (error_kind, &cmd_str))?,
};
crate::ensure_code!(
res.status.success(),
error_kind,
"{}",
Some(&res.stderr)
.filter(|a| !a.is_empty())
.or(Some(&res.stdout))
.filter(|a| !a.is_empty())
.and_then(|a| std::str::from_utf8(a).ok())
.unwrap_or(&format!(
"{} exited with code {}",
self.cmd.as_std().get_program().to_string_lossy(),
res.status
))
);
Ok(res.stdout)
} else {
let mut futures = Vec::<BoxFuture<'_, Result<(), Error>>>::new(); // todo: predict capacity
let mut cmds = std::mem::take(&mut self.pipe);
cmds.push_front(&mut *self.cmd);
let len = cmds.len();
let timeout = self.timeout;
let mut prev = self
.input
.take()
.map(|i| Box::new(i) as Box<dyn AsyncRead + Unpin + Send>);
for (idx, cmd) in IntoIterator::into_iter(cmds).enumerate() {
let last = idx == len - 1;
if self.capture || !last {
cmd.stdout(Stdio::piped());
}
if self.capture {
cmd.stderr(Stdio::piped());
}
if prev.is_some() {
cmd.stdin(Stdio::piped());
}
let mut child = cmd.spawn().with_kind(error_kind)?;
let input = std::mem::replace(
&mut prev,
child
.stdout
.take()
.map(|i| Box::new(BufReader::new(i)) as Box<dyn AsyncRead + Unpin + Send>),
);
futures.push(
async move {
if let (Some(mut stdin), Some(mut input)) = (child.stdin.take(), input) {
use tokio::io::AsyncWriteExt;
tokio::io::copy(&mut input, &mut stdin).await?;
stdin.flush().await?;
stdin.shutdown().await?;
drop(stdin);
}
let res = match timeout {
None => child.wait_with_output().await?,
Some(t) => tokio::time::timeout(t, child.wait_with_output())
.await
.with_kind(ErrorKind::Timeout)??,
};
crate::ensure_code!(
res.status.success(),
error_kind,
"{}",
Some(&res.stderr)
.filter(|a| !a.is_empty())
.or(Some(&res.stdout))
.filter(|a| !a.is_empty())
.and_then(|a| std::str::from_utf8(a).ok())
.unwrap_or(&format!(
"{} exited with code {}",
cmd.as_std().get_program().to_string_lossy(),
res.status
))
);
Ok(())
}
.boxed(),
);
}
let (send, recv) = oneshot::channel();
futures.push(
async move {
if let Some(mut prev) = prev {
let mut res = Vec::new();
prev.read_to_end(&mut res).await?;
send.send(res).unwrap();
} else {
send.send(Vec::new()).unwrap();
}
Ok(())
}
.boxed(),
);
futures::future::try_join_all(futures).await?;
Ok(recv.await.unwrap())
}
let res = match self.timeout {
None => child.wait_with_output().await?,
Some(t) => tokio::time::timeout(t, child.wait_with_output())
.await
.with_kind(ErrorKind::Timeout)??,
};
crate::ensure_code!(
res.status.success(),
error_kind,
"{}",
Some(&res.stderr)
.filter(|a| !a.is_empty())
.or(Some(&res.stdout))
.filter(|a| !a.is_empty())
.and_then(|a| std::str::from_utf8(a).ok())
.unwrap_or(&format!("Unknown Error ({})", res.status))
);
Ok(res.stdout)
}
}