use std::collections::BTreeMap; use chrono::{DateTime, Utc}; use color_eyre::eyre::eyre; use patch_db::{DbHandle, HasModel}; use serde::{Deserialize, Serialize}; use tracing::instrument; use self::health_check::HealthCheckId; use crate::context::RpcContext; use crate::dependencies::DependencyErrors; use crate::manager::{Manager, Status as ManagerStatus}; use crate::notifications::NotificationLevel; use crate::s9pk::manifest::Manifest; use crate::status::health_check::HealthCheckResult; use crate::Error; pub mod health_check; #[instrument(skip(ctx))] pub async fn synchronize_all(ctx: &RpcContext) -> Result<(), Error> { let mut db = ctx.db.handle(); let pkg_ids = crate::db::DatabaseModel::new() .package_data() .keys(&mut db, false) .await?; for id in pkg_ids { if let Err(e) = async { let (mut status, manager) = if let Some(installed) = crate::db::DatabaseModel::new() .package_data() .idx_model(&id) .and_then(|m| m.installed()) .check(&mut db) .await? { ( installed.clone().status().get_mut(&mut db).await?, ctx.managers .get(&( id.clone(), installed .manifest() .version() .get(&mut db, true) .await? .to_owned(), )) .await .ok_or_else(|| Error::new(eyre!("No Manager"), crate::ErrorKind::Docker))?, ) } else { return Ok::<_, Error>(()); }; let res = status.main.synchronize(&manager).await?; status.save(&mut db).await?; Ok(res) } .await { tracing::error!("Error syncronizing status of {}: {}", id, e); tracing::debug!("{:?}", e); } } Ok(()) } #[derive(Clone, Debug, Deserialize, Serialize, HasModel)] #[serde(rename_all = "kebab-case")] pub struct Status { pub configured: bool, pub main: MainStatus, #[model] pub dependency_errors: DependencyErrors, } #[derive(Debug, Clone, Deserialize, Serialize, HasModel)] #[serde(tag = "status")] #[serde(rename_all = "kebab-case")] pub enum MainStatus { Stopped, Stopping, Running { started: DateTime, health: BTreeMap, }, BackingUp { started: Option>, health: BTreeMap, }, Restoring { running: bool, }, } impl MainStatus { #[instrument(skip(manager))] pub async fn synchronize(&mut self, manager: &Manager) -> Result<(), Error> { match manager.status() { ManagerStatus::Stopped => match self { MainStatus::Stopped => (), MainStatus::Stopping => { *self = MainStatus::Stopped; } MainStatus::Running { started, .. } => { *started = Utc::now(); manager.start().await?; } MainStatus::BackingUp { .. } => (), MainStatus::Restoring { .. } => (), }, ManagerStatus::Running => match self { MainStatus::Stopped | MainStatus::Stopping | MainStatus::Restoring { .. } => { manager.stop().await?; } MainStatus::Running { .. } => (), MainStatus::BackingUp { .. } => { manager.pause().await?; } }, ManagerStatus::Paused => match self { MainStatus::Stopped | MainStatus::Stopping | MainStatus::Restoring { .. } => { manager.stop().await?; } MainStatus::Running { .. } => { manager.resume().await?; } MainStatus::BackingUp { .. } => (), }, } Ok(()) } #[instrument(skip(ctx, db, manifest))] pub async fn check( &mut self, ctx: &RpcContext, db: &mut Db, manifest: &Manifest, ) -> Result<(), Error> { match self { MainStatus::Running { started, health } => { *health = manifest .health_checks .check_all( ctx, *started, &manifest.id, &manifest.version, &manifest.volumes, ) .await?; let mut should_stop = false; for (check, res) in health { match &res { health_check::HealthCheckResult::Failure { error } if manifest .health_checks .0 .get(check) .map(|hc| hc.critical) .unwrap_or_default() => { ctx.notification_manager.notify( db, Some(manifest.id.clone()), NotificationLevel::Error, String::from("Critical Health Check Failed"), format!("{} was shut down because a health check required for its operation failed\n{}", manifest.title, error), (), None, ) .await?; should_stop = true; } _ => (), } } if should_stop { *self = MainStatus::Stopping; } } _ => (), } Ok(()) } pub fn running(&self) -> bool { match self { MainStatus::Running { .. } | MainStatus::BackingUp { started: Some(_), .. } | MainStatus::Restoring { running: true } => true, _ => false, } } pub fn stop(&mut self) { match self { MainStatus::Running { .. } => { *self = MainStatus::Stopping; } MainStatus::BackingUp { started, .. } => { *started = None; } MainStatus::Restoring { running } => { *running = false; } _ => (), } } }