From cdca5e1b6771e899791e0ac1509a482f03473d03 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 30 Aug 2021 13:31:51 -0600 Subject: [PATCH] address bugs --- appmgr/build-prod.sh | 10 +-- appmgr/src/bin/embassy-init.rs | 14 +++- appmgr/src/context/rpc.rs | 1 + appmgr/src/control.rs | 39 ++++++--- appmgr/src/install/cleanup.rs | 96 +++++++++++++++++----- appmgr/src/install/mod.rs | 143 ++++++++++++++++++++++++--------- appmgr/src/lib.rs | 8 +- appmgr/src/manager/mod.rs | 4 +- 8 files changed, 232 insertions(+), 83 deletions(-) diff --git a/appmgr/build-prod.sh b/appmgr/build-prod.sh index 22ce6f776..ca812c44f 100755 --- a/appmgr/build-prod.sh +++ b/appmgr/build-prod.sh @@ -8,9 +8,9 @@ if [ "$0" != "./build-prod.sh" ]; then exit 1 fi -alias 'rust-arm-builder'='docker run --rm -it -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:latest' +alias 'rust-arm64-builder'='docker run --rm -it -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:aarch64' -cd .. -rust-arm-builder sh -c "(cd appmgr && cargo build --release --features=production)" -cd appmgr -rust-arm-builder arm-linux-gnueabi-strip target/armv7-unknown-linux-gnueabihf/release/appmgr \ No newline at end of file +cd ../.. +rust-arm64-builder sh -c "(cd embassy-os/appmgr && cargo +beta build --release --features=production)" +cd embassy-os/appmgr +#rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd diff --git a/appmgr/src/bin/embassy-init.rs b/appmgr/src/bin/embassy-init.rs index 8d48cbfd4..5382daf6f 100644 --- a/appmgr/src/bin/embassy-init.rs +++ b/appmgr/src/bin/embassy-init.rs @@ -1,8 +1,18 @@ use embassy::Error; async fn inner_main() -> Result<(), Error> { - // os sync - embassy::volume::disk::mount("/dev/sda", "/mnt/embassy-os-crypt").await?; + // host setup flow if needed + + // mount disk + embassy::volume::disk::mount("/dev/sda", "/mnt/embassy-os-crypt").await?; // TODO: by uuid + + // unlock disk + + // mount /var/log/journal + + // sync ssh + + // sync wifi // hostname-set embassy::hostname::sync_hostname().await?; diff --git a/appmgr/src/context/rpc.rs b/appmgr/src/context/rpc.rs index 001c49861..45ed91fe3 100644 --- a/appmgr/src/context/rpc.rs +++ b/appmgr/src/context/rpc.rs @@ -108,6 +108,7 @@ impl RpcContext { revision_cache: RwLock::new(VecDeque::new()), metrics_cache: RwLock::new(None), }); + // TODO: handle apps in bad / transient state Ok(Self(seed)) } pub async fn package_registry_url(&self) -> Result { diff --git a/appmgr/src/control.rs b/appmgr/src/control.rs index 3279e1429..86a156752 100644 --- a/appmgr/src/control.rs +++ b/appmgr/src/control.rs @@ -1,23 +1,29 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; use indexmap::IndexMap; +use patch_db::DbHandle; use rpc_toolkit::command; use crate::context::EitherContext; +use crate::db::util::WithRevision; use crate::s9pk::manifest::PackageId; use crate::status::MainStatus; use crate::util::display_none; use crate::{Error, ResultExt}; #[command(display(display_none))] -pub async fn start(#[context] ctx: EitherContext, #[arg] id: PackageId) -> Result<(), Error> { +pub async fn start( + #[context] ctx: EitherContext, + #[arg] id: PackageId, +) -> Result, Error> { let rpc_ctx = ctx.as_rpc().unwrap(); let mut db = rpc_ctx.db.handle(); + let mut tx = db.begin().await?; let installed = crate::db::DatabaseModel::new() .package_data() .idx_model(&id) .and_then(|pkg| pkg.installed()) - .expect(&mut db) + .expect(&mut tx) .await .with_ctx(|_| { ( @@ -29,10 +35,10 @@ pub async fn start(#[context] ctx: EitherContext, #[arg] id: PackageId) -> Resul .clone() .manifest() .version() - .get(&mut db, true) + .get(&mut tx, true) .await? .to_owned(); - let mut status = installed.status().main().get_mut(&mut db).await?; + let mut status = installed.status().main().get_mut(&mut tx).await?; *status = MainStatus::Running { started: Utc::now(), @@ -45,20 +51,28 @@ pub async fn start(#[context] ctx: EitherContext, #[arg] id: PackageId) -> Resul })?, ) .await?; - status.save(&mut db).await?; + status.save(&mut tx).await?; - Ok(()) + Ok(WithRevision { + revision: tx.commit(None).await?, + response: (), + }) } #[command(display(display_none))] -pub async fn stop(#[context] ctx: EitherContext, #[arg] id: PackageId) -> Result<(), Error> { +pub async fn stop( + #[context] ctx: EitherContext, + #[arg] id: PackageId, +) -> Result, Error> { let rpc_ctx = ctx.as_rpc().unwrap(); let mut db = rpc_ctx.db.handle(); + let mut tx = db.begin().await?; + let mut status = crate::db::DatabaseModel::new() .package_data() .idx_model(&id) .and_then(|pkg| pkg.installed()) - .expect(&mut db) + .expect(&mut tx) .await .with_ctx(|_| { ( @@ -68,11 +82,14 @@ pub async fn stop(#[context] ctx: EitherContext, #[arg] id: PackageId) -> Result })? .status() .main() - .get_mut(&mut db) + .get_mut(&mut tx) .await?; *status = MainStatus::Stopping; - status.save(&mut db).await?; + status.save(&mut tx).await?; - Ok(()) + Ok(WithRevision { + revision: tx.commit(None).await?, + response: (), + }) } diff --git a/appmgr/src/install/cleanup.rs b/appmgr/src/install/cleanup.rs index b710ff383..95dfcaff0 100644 --- a/appmgr/src/install/cleanup.rs +++ b/appmgr/src/install/cleanup.rs @@ -1,16 +1,19 @@ use std::borrow::Cow; +use std::collections::HashMap; use std::path::Path; use anyhow::anyhow; +use bollard::image::ListImagesOptions; use bollard::Docker; -use patch_db::DbHandle; +use patch_db::{DbHandle, PatchDbHandle}; +use tokio::process::Command; use super::PKG_PUBLIC_DIR; use crate::context::RpcContext; use crate::db::model::{InstalledPackageDataEntry, PackageDataEntry}; use crate::dependencies::DependencyError; use crate::s9pk::manifest::{Manifest, PackageId}; -use crate::util::Version; +use crate::util::{Invoke, Version}; use crate::Error; pub async fn update_dependents<'a, Db: DbHandle, I: IntoIterator>( @@ -62,7 +65,35 @@ pub async fn update_dependents<'a, Db: DbHandle, I: IntoIterator( +pub async fn cleanup(ctx: &RpcContext, id: &PackageId, version: &Version) -> Result<(), Error> { + ctx.managers.remove(&(id.clone(), version.clone())).await; + // docker images start9/$APP_ID/*:$VERSION -q | xargs docker rmi + let images = ctx + .docker + .list_images(Some(ListImagesOptions { + all: false, + filters: { + let mut f = HashMap::new(); + f.insert( + "reference".to_owned(), + vec![format!("start9/{}/*:{}", id, version)], + ); + f + }, + digests: false, + })) + .await?; + futures::future::try_join_all(images.into_iter().map(|image| async { + let image = image; // move into future + ctx.docker.remove_image(&image.id, None, None).await + })) + .await?; + // TODO: delete public dir if not a dependency + + Ok(()) +} + +pub async fn cleanup_failed( ctx: &RpcContext, db: &mut Db, id: &PackageId, @@ -76,25 +107,21 @@ pub async fn cleanup( .get(db, true) .await? .to_owned(); - if let Some(manifest) = match &pde { - PackageDataEntry::Installing { manifest, .. } => Some(manifest), + if match &pde { + PackageDataEntry::Installing { .. } => true, PackageDataEntry::Updating { manifest, .. } => { if &manifest.version != version { - Some(manifest) + true } else { - None + false } } _ => { log::warn!("{}: Nothing to clean up!", id); - None + false } } { - ctx.managers - .remove(&(manifest.id.clone(), manifest.version.clone())) - .await; - // docker images start9/$APP_ID/*:$VERSION -q | xargs docker rmi - let public_dir_path = Path::new(PKG_PUBLIC_DIR).join(id).join(version.as_str()); + cleanup(ctx, id, version).await?; } match pde { @@ -126,14 +153,39 @@ pub async fn cleanup( _ => (), } - Ok(()) // TODO -} - -pub async fn uninstall( - ctx: &RpcContext, - db: &mut Db, - entry: InstalledPackageDataEntry, -) -> Result<(), Error> { - //TODO Ok(()) } + +pub async fn uninstall( + ctx: &RpcContext, + db: &mut PatchDbHandle, + entry: &InstalledPackageDataEntry, +) -> Result<(), Error> { + cleanup(ctx, &entry.manifest.id, &entry.manifest.version).await?; + let mut tx = db.begin().await?; + crate::db::DatabaseModel::new() + .package_data() + .remove(&mut tx, &entry.manifest.id) + .await?; + update_dependents(&mut tx, &entry.manifest.id, entry.current_dependents.keys()).await?; + tx.commit(None).await?; + Ok(()) +} + +#[tokio::test] +async fn test() { + dbg!( + Docker::connect_with_socket_defaults() + .unwrap() + .list_images(Some(ListImagesOptions { + all: false, + filters: { + let mut f = HashMap::new(); + f.insert("reference", vec!["start9/*:latest"]); + f + }, + digests: false + })) + .await + ); +} diff --git a/appmgr/src/install/mod.rs b/appmgr/src/install/mod.rs index 7ffc21b20..b88f0c3b1 100644 --- a/appmgr/src/install/mod.rs +++ b/appmgr/src/install/mod.rs @@ -26,15 +26,16 @@ use sha2::{Digest, Sha256}; use tokio::fs::{File, OpenOptions}; use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt, AsyncWrite, AsyncWriteExt}; -use self::cleanup::cleanup; -use self::progress::{InstallProgress, InstallProgressTracker}; +use self::cleanup::cleanup_failed; use crate::context::{EitherContext, ExtendedContext, RpcContext}; use crate::db::model::{ CurrentDependencyInfo, InstalledPackageDataEntry, PackageDataEntry, StaticDependencyInfo, StaticFiles, }; +use crate::db::util::WithRevision; use crate::dependencies::update_current_dependents; -use crate::install::cleanup::{uninstall, update_dependents}; +use crate::install::cleanup::{cleanup, update_dependents}; +use crate::install::progress::{InstallProgress, InstallProgressTracker}; use crate::s9pk::manifest::{Manifest, PackageId}; use crate::s9pk::reader::S9pkReader; use crate::status::{DependencyErrors, MainStatus, Status}; @@ -48,7 +49,10 @@ pub const PKG_CACHE: &'static str = "/mnt/embassy-os/cache/packages"; pub const PKG_PUBLIC_DIR: &'static str = "/mnt/embassy-os/public/package-data"; #[command(display(display_none))] -pub async fn install(#[context] ctx: EitherContext, #[arg] id: String) -> Result<(), Error> { +pub async fn install( + #[context] ctx: EitherContext, + #[arg] id: String, +) -> Result, Error> { let rpc_ctx = ctx.to_rpc().unwrap(); let (pkg_id, version_str) = if let Some(split) = id.split_once("@") { split @@ -68,14 +72,105 @@ pub async fn install(#[context] ctx: EitherContext, #[arg] id: String) -> Result )) ) .with_kind(crate::ErrorKind::Registry)?; - let man = man_res.json().await.with_kind(crate::ErrorKind::Registry)?; + let man: Manifest = man_res.json().await.with_kind(crate::ErrorKind::Registry)?; + + let progress = InstallProgress::new(s9pk.content_length()); + let static_files = StaticFiles::remote(&man.id, &man.version, man.assets.icon_type()); + let mut db_handle = rpc_ctx.db.handle(); + let mut tx = db_handle.begin().await?; + let mut pde = crate::db::DatabaseModel::new() + .package_data() + .idx_model(&man.id) + .get_mut(&mut tx) + .await?; + match pde.take() { + Some(PackageDataEntry::Installed { + installed, + manifest, + static_files, + }) => { + *pde = Some(PackageDataEntry::Updating { + install_progress: progress.clone(), + static_files, + installed, + manifest, + }) + } + None => { + *pde = Some(PackageDataEntry::Installing { + install_progress: progress.clone(), + static_files, + manifest: man.clone(), + }) + } + _ => { + return Err(Error::new( + anyhow!("Cannot install over an app in a transient state"), + crate::ErrorKind::InvalidRequest, + )) + } + } + pde.save(&mut tx).await?; + let res = tx.commit(None).await?; + drop(db_handle); + tokio::spawn(async move { if let Err(e) = download_install_s9pk(&rpc_ctx, &man, s9pk).await { log::error!("Install of {}@{} Failed: {}", man.id, man.version, e); } }); - Ok(()) + Ok(WithRevision { + revision: res, + response: (), + }) +} + +#[command(display(display_none))] +pub async fn uninstall( + #[context] ctx: EitherContext, + #[arg] id: PackageId, +) -> Result, Error> { + let mut handle = ctx.as_rpc().unwrap().db.handle(); + let mut tx = handle.begin().await?; + + let mut pde = crate::db::DatabaseModel::new() + .package_data() + .idx_model(&id) + .get_mut(&mut tx) + .await?; + let (manifest, static_files, installed) = match pde.take() { + Some(PackageDataEntry::Installed { + manifest, + static_files, + installed, + }) => (manifest, static_files, installed), + _ => { + return Err(Error::new( + anyhow!("Package is not installed."), + crate::ErrorKind::NotFound, + )); + } + }; + *pde = Some(PackageDataEntry::Removing { + manifest, + static_files, + }); + pde.save(&mut tx).await?; + let res = tx.commit(None).await?; + drop(handle); + + tokio::spawn(async move { + let rpc_ctx = ctx.as_rpc().unwrap(); + if let Err(e) = cleanup::uninstall(rpc_ctx, &mut rpc_ctx.db.handle(), &installed).await { + log::error!("Uninstall of {} Failed: {}", id, e); + } + }); + + Ok(WithRevision { + revision: res, + response: (), + }) } pub async fn download_install_s9pk( @@ -96,38 +191,6 @@ pub async fn download_install_s9pk( let res = (|| async { let progress = InstallProgress::new(s9pk.content_length()); - let static_files = StaticFiles::remote(pkg_id, version, temp_manifest.assets.icon_type()); - let mut db_handle = ctx.db.handle(); - let mut pde = pkg_data_entry.get_mut(&mut db_handle).await?; - match pde.take() { - Some(PackageDataEntry::Installed { - installed, - manifest, - static_files, - }) => { - *pde = Some(PackageDataEntry::Updating { - install_progress: progress.clone(), - static_files, - installed, - manifest, - }) - } - None => { - *pde = Some(PackageDataEntry::Installing { - install_progress: progress.clone(), - static_files, - manifest: temp_manifest.clone(), - }) - } - _ => { - return Err(Error::new( - anyhow!("Cannot install over an app in a transient state"), - crate::ErrorKind::InvalidRequest, - )) - } - } - pde.save(&mut db_handle).await?; - drop(db_handle); let progress_model = pkg_data_entry.and_then(|pde| pde.install_progress()); async fn check_cache( @@ -242,7 +305,7 @@ pub async fn download_install_s9pk( let mut handle = ctx.db.handle(); let mut tx = handle.begin().await?; - if let Err(e) = cleanup(&ctx, &mut tx, pkg_id, version).await { + if let Err(e) = cleanup_failed(&ctx, &mut tx, pkg_id, version).await { log::error!( "Failed to clean up {}@{}: {}: Adding to broken packages", pkg_id, @@ -566,7 +629,7 @@ pub async fn install_s9pk( configured &= res.configured; } if &prev.manifest.version != version { - uninstall(ctx, &mut tx, prev).await?; + cleanup(ctx, &prev.manifest.id, &prev.manifest.version).await?; } if let Some(res) = manifest .migrations diff --git a/appmgr/src/lib.rs b/appmgr/src/lib.rs index 5799240e4..11b4be81b 100644 --- a/appmgr/src/lib.rs +++ b/appmgr/src/lib.rs @@ -72,7 +72,13 @@ pub fn main_api(#[context] ctx: EitherContext) -> Result Result { Ok(ctx) } diff --git a/appmgr/src/manager/mod.rs b/appmgr/src/manager/mod.rs index 595532258..87e61a438 100644 --- a/appmgr/src/manager/mod.rs +++ b/appmgr/src/manager/mod.rs @@ -291,10 +291,10 @@ impl Manager { .unwrap(); // recv is still in scope, cannot fail } Ok(Err(e)) => { - todo!("application crashed") + log::error!("application crashed: {}: {}", e.0, e.1) } Err(e) => { - todo!("failed to start application: {}", e) + log::error!("failed to start application: {}", e) } } }