use new locking api

appmgr/Cargo.lock

@@ -511,20 +511,6 @@ version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403"
 
-[[package]]
-name = "crossbeam"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845"
-dependencies = [
- "cfg-if 1.0.0",
- "crossbeam-channel",
- "crossbeam-deque",
- "crossbeam-epoch",
- "crossbeam-queue",
- "crossbeam-utils",
-]
-
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.1"
@@ -535,30 +521,6 @@ dependencies = [
  "crossbeam-utils",
 ]
 
-[[package]]
-name = "crossbeam-deque"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
-dependencies = [
- "cfg-if 1.0.0",
- "crossbeam-epoch",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd"
-dependencies = [
- "cfg-if 1.0.0",
- "crossbeam-utils",
- "lazy_static",
- "memoffset",
- "scopeguard",
-]
-
 [[package]]
 name = "crossbeam-queue"
 version = "0.3.2"
@@ -1918,7 +1880,6 @@ dependencies = [
  "log",
  "nix 0.20.1",
  "patch-db-macro",
- "qutex-2",
  "serde",
  "serde_cbor 0.11.1",
  "serde_json",
@@ -2192,14 +2153,6 @@ dependencies = [
  "proc-macro2 1.0.29",
 ]
 
-[[package]]
-name = "qutex-2"
-version = "0.3.0"
-dependencies = [
- "crossbeam",
- "futures",
-]
-
 [[package]]
 name = "radium"
 version = "0.5.3"

appmgr/sqlx-data.json

@@ -344,6 +344,16 @@
       "nullable": []
     }
   },
+  "a596bdc5014ba9e7b362398abf09ec6a100923e001247a79503d1e820ffe71c3": {
+    "query": "-- Add migration script here\nCREATE TABLE IF NOT EXISTS tor\n(\n    package     TEXT NOT NULL,\n    interface   TEXT NOT NULL,\n    key         BLOB NOT NULL CHECK (length(key) = 64),\n    PRIMARY KEY (package, interface)\n);\nCREATE TABLE IF NOT EXISTS session\n(\n    id         TEXT NOT NULL PRIMARY KEY,\n    logged_in TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,\n    logged_out TIMESTAMP,\n    last_active TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,\n    user_agent TEXT,\n    metadata   TEXT NOT NULL DEFAULT 'null'\n);\nCREATE TABLE IF NOT EXISTS account\n(\n    id INTEGER PRIMARY KEY CHECK (id = 0),\n    password TEXT NOT NULL,\n    tor_key BLOB NOT NULL CHECK (length(tor_key) = 64)\n);\nCREATE TABLE IF NOT EXISTS ssh_keys\n(\n    fingerprint     TEXT NOT NULL,\n    openssh_pubkey  TEXT NOT NULL,\n    created_at      TEXT NOT NULL,\n    PRIMARY KEY (fingerprint)\n);\nCREATE TABLE IF NOT EXISTS certificates\n(\n    id INTEGER PRIMARY KEY, -- Root = 0, Int = 1, Other = 2..\n    priv_key_pem TEXT NOT NULL,\n    certificate_pem TEXT NOT NULL,\n    lookup_string TEXT UNIQUE,\n    created_at TEXT,\n    updated_at TEXT\n);\nCREATE TABLE IF NOT EXISTS notifications\n(\n    id INTEGER PRIMARY KEY,\n    package_id TEXT,\n    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,\n    code INTEGER NOT NULL,\n    level TEXT NOT NULL,\n    title TEXT NOT NULL,\n    message TEXT NOT NULL,\n    data TEXT\n);",
+    "describe": {
+      "columns": [],
+      "parameters": {
+        "Right": 0
+      },
+      "nullable": []
+    }
+  },
   "a6b0c8909a3a5d6d9156aebfb359424e6b5a1d1402e028219e21726f1ebd282e": {
     "query": "SELECT fingerprint, openssh_pubkey, created_at FROM ssh_keys",
     "describe": {

appmgr/src/config/mod.rs

@@ -14,12 +14,15 @@ use serde_json::Value;
 use crate::action::docker::DockerAction;
 use crate::config::spec::PackagePointerSpecVariant;
 use crate::context::RpcContext;
-use crate::db::model::{CurrentDependencyInfo, InstalledPackageDataEntryModel};
+use crate::db::model::{
+    CurrentDependencyInfo, InstalledPackageDataEntry, InstalledPackageDataEntryModel,
+};
 use crate::db::util::WithRevision;
 use crate::dependencies::{
     update_current_dependents, BreakageRes, DependencyError, TaggedDependencyError,
 };
 use crate::s9pk::manifest::PackageId;
+use crate::status::handle_broken_dependents;
 use crate::util::{
     display_none, display_serializable, parse_duration, parse_stdin_deserializable, IoFormat,
 };
@@ -281,7 +284,7 @@ pub fn configure<'a, Db: DbHandle>(
     async move {
         crate::db::DatabaseModel::new()
             .package_data()
-            .lock(db, patch_db::LockType::Write)
+            .lock(db, true)
             .await;
         // fetch data from db
         let pkg_model = crate::db::DatabaseModel::new()
@@ -401,77 +404,6 @@ pub fn configure<'a, Db: DbHandle>(
         let prev = old_config.map(Value::Object).unwrap_or_default();
         let next = Value::Object(config.clone());
         for (dependent, dep_info) in dependents.iter().filter(|(dep_id, _)| dep_id != &id) {
-            fn handle_broken_dependents<'a, Db: DbHandle>(
-                db: &'a mut Db,
-                id: &'a PackageId,
-                dependency: &'a PackageId,
-                model: InstalledPackageDataEntryModel,
-                error: DependencyError,
-                breakages: &'a mut IndexMap<PackageId, TaggedDependencyError>,
-            ) -> BoxFuture<'a, Result<(), Error>> {
-                async move {
-                    let mut status = model.clone().status().get_mut(db).await?;
-
-                    let old = status.dependency_errors.0.remove(id);
-                    let newly_broken = old.is_none();
-                    status.dependency_errors.0.insert(
-                        id.clone(),
-                        if let Some(old) = old {
-                            old.merge_with(error.clone())
-                        } else {
-                            error.clone()
-                        },
-                    );
-                    if newly_broken {
-                        breakages.insert(
-                            id.clone(),
-                            TaggedDependencyError {
-                                dependency: dependency.clone(),
-                                error: error.clone(),
-                            },
-                        );
-                        if status.main.running() {
-                            if model
-                                .clone()
-                                .manifest()
-                                .dependencies()
-                                .idx_model(dependency)
-                                .expect(db)
-                                .await?
-                                .get(db, true)
-                                .await?
-                                .critical
-                            {
-                                status.main.stop();
-                                let dependents = model.current_dependents().get(db, true).await?;
-                                for (dependent, _) in &*dependents {
-                                    let dependent_model = crate::db::DatabaseModel::new()
-                                        .package_data()
-                                        .idx_model(dependent)
-                                        .and_then(|pkg| pkg.installed())
-                                        .expect(db)
-                                        .await?;
-                                    handle_broken_dependents(
-                                        db,
-                                        dependent,
-                                        id,
-                                        dependent_model,
-                                        DependencyError::NotRunning,
-                                        breakages,
-                                    )
-                                    .await?;
-                                }
-                            }
-                        }
-                    }
-
-                    status.save(db).await?;
-
-                    Ok(())
-                }
-                .boxed()
-            }
-
             // check if config passes dependent check
             let dependent_model = crate::db::DatabaseModel::new()
                 .package_data()

appmgr/src/config/spec.rs

@@ -1592,7 +1592,7 @@ impl PackagePointerSpec {
                 if let Some(cfg) = config_overrides.get(&self.package_id) {
                     Ok(selector.select(*multi, &Value::Object(cfg.clone())))
                 } else {
-                    let manifest_model: OptionModel<_> = crate::db::DatabaseModel::new()
+                    let manifest_model: OptionModel<Manifest> = crate::db::DatabaseModel::new()
                         .package_data()
                         .idx_model(&self.package_id)
                         .and_then(|pde| pde.installed())

appmgr/src/db/model.rs

@@ -255,6 +255,8 @@ impl HasModel for InterfaceAddressMap {
 #[derive(Debug, Deserialize, Serialize, HasModel)]
 #[serde(rename_all = "kebab-case")]
 pub struct InterfaceAddresses {
+    #[model]
     pub tor_address: Option<String>,
+    #[model]
     pub lan_address: Option<String>,
 }

appmgr/src/install/cleanup.rs

@@ -122,7 +122,7 @@ pub async fn cleanup_failed<Db: DbHandle>(
         .await?
         .get(db, true)
         .await?
-        .to_owned();
+        .into_owned();
     if match &pde {
         PackageDataEntry::Installing { .. } => true,
         PackageDataEntry::Updating { manifest, .. } => {

appmgr/src/install/mod.rs

@@ -511,7 +511,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
     let mut sql_tx = ctx.secret_store.begin().await?;
     crate::db::DatabaseModel::new()
         .package_data()
-        .lock(&mut tx, patch_db::LockType::Write)
+        .lock(&mut tx, true)
         .await;
 
     log::info!("Install {}@{}: Creating volumes", pkg_id, version);

appmgr/src/manager/mod.rs

@@ -38,14 +38,16 @@ impl ManagerMap {
             .keys(db, true)
             .await?
         {
-            let man = if let Some(installed) = crate::db::DatabaseModel::new()
+            let man: Manifest = if let Some(manifest) = crate::db::DatabaseModel::new()
                 .package_data()
                 .idx_model(&package)
                 .and_then(|pkg| pkg.installed())
-                .check(db)
+                .map(|m| m.manifest())
+                .get(db, true)
                 .await?
+                .to_owned()
             {
-                installed.manifest().get(db, true).await?.to_owned()
+                manifest
             } else {
                 continue;
             };

appmgr/src/notifications.rs

@@ -4,7 +4,7 @@ use std::str::FromStr;
 
 use anyhow::anyhow;
 use chrono::{DateTime, Utc};
-use patch_db::{LockType, PatchDb, Revision};
+use patch_db::{PatchDb, Revision};
 use rpc_toolkit::command;
 use sqlx::SqlitePool;
 
@@ -32,7 +32,7 @@ pub async fn list(
             let model = crate::db::DatabaseModel::new()
                 .server_info()
                 .unread_notification_count();
-            model.lock(&mut handle, patch_db::LockType::Write).await;
+            model.lock(&mut handle, true).await;
             let records = sqlx::query!(
                 "SELECT id, package_id, created_at, code, level, title, message, data FROM notifications ORDER BY id DESC LIMIT ?",
                 limit

appmgr/src/properties.rs

@@ -4,8 +4,8 @@ use rpc_toolkit::command;
 use serde_json::Value;
 
 use crate::context::RpcContext;
-use crate::s9pk::manifest::PackageId;
+use crate::s9pk::manifest::{Manifest, PackageId};
-use crate::{Error, ErrorKind, ResultExt};
+use crate::{Error, ErrorKind};
 
 pub fn display_properties(response: Value, _: &ArgMatches<'_>) {
     println!("{}", response);
@@ -18,17 +18,15 @@ pub async fn properties(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Res
 
 pub async fn fetch_properties(ctx: RpcContext, id: PackageId) -> Result<Value, Error> {
     let mut db = ctx.db.handle();
-    let manifest = crate::db::DatabaseModel::new()
+    let manifest: Manifest = crate::db::DatabaseModel::new()
         .package_data()
         .idx_model(&id)
         .and_then(|p| p.installed())
-        .expect(&mut db)
+        .map(|m| m.manifest())
-        .await
-        .with_kind(ErrorKind::NotFound)?
-        .manifest()
         .get(&mut db, true)
         .await?
-        .to_owned();
+        .to_owned()
+        .ok_or_else(|| Error::new(anyhow!("{} is not installed", id), ErrorKind::NotFound))?;
     if let Some(props) = manifest.properties {
         props
             .execute::<(), Value>(
@@ -41,7 +39,7 @@ pub async fn fetch_properties(ctx: RpcContext, id: PackageId) -> Result<Value, E
                 false,
             )
             .await?
-            .map_err(|_| Error::new(anyhow!("Properties failure!"), crate::ErrorKind::Docker))
+            .map_err(|_| Error::new(anyhow!("Properties failure!"), ErrorKind::Docker))
             .and_then(|a| Ok(a))
     } else {
         Ok(Value::Null)

appmgr/src/shutdown.rs

@@ -1,6 +1,6 @@
 use std::sync::Arc;
 
-use patch_db::{LockType, PatchDbHandle};
+use patch_db::PatchDbHandle;
 use rpc_toolkit::command;
 
 use crate::context::RpcContext;
@@ -67,9 +67,7 @@ impl Shutdown {
 #[command(display(display_none))]
 pub async fn shutdown(#[context] ctx: RpcContext) -> Result<(), Error> {
     let mut db = ctx.db.handle();
-    crate::db::DatabaseModel::new()
+    crate::db::DatabaseModel::new().lock(&mut db, true).await;
-        .lock(&mut db, LockType::Write)
-        .await;
     ctx.shutdown
         .send(Some(Shutdown {
             zfs_pool: ctx.zfs_pool_name.clone(),
@@ -84,9 +82,7 @@ pub async fn shutdown(#[context] ctx: RpcContext) -> Result<(), Error> {
 #[command(display(display_none))]
 pub async fn restart(#[context] ctx: RpcContext) -> Result<(), Error> {
     let mut db = ctx.db.handle();
-    crate::db::DatabaseModel::new()
+    crate::db::DatabaseModel::new().lock(&mut db, true).await;
-        .lock(&mut db, LockType::Write)
-        .await;
     ctx.shutdown
         .send(Some(Shutdown {
             zfs_pool: ctx.zfs_pool_name.clone(),

appmgr/src/status/mod.rs

@@ -3,15 +3,16 @@ use std::sync::Arc;
 
 use anyhow::anyhow;
 use chrono::{DateTime, Utc};
-use futures::StreamExt;
+use futures::future::BoxFuture;
+use futures::{FutureExt, StreamExt};
 use indexmap::IndexMap;
-use patch_db::{DbHandle, HasModel, LockType, Map, MapModel, ModelData};
+use patch_db::{DbHandle, HasModel, Map, MapModel, ModelData};
 use serde::{Deserialize, Serialize};
 
 use self::health_check::{HealthCheckId, HealthCheckResult};
 use crate::context::RpcContext;
-use crate::db::model::CurrentDependencyInfo;
+use crate::db::model::{CurrentDependencyInfo, InstalledPackageDataEntryModel};
-use crate::dependencies::DependencyError;
+use crate::dependencies::{DependencyError, TaggedDependencyError};
 use crate::manager::{Manager, Status as ManagerStatus};
 use crate::notifications::{notify, NotificationLevel, NotificationSubtype};
 use crate::s9pk::manifest::{Manifest, PackageId};
@@ -22,37 +23,31 @@ pub mod health_check;
 
 // Assume docker for now
 pub async fn synchronize_all(ctx: &RpcContext) -> Result<(), Error> {
-    let mut pkg_ids = crate::db::DatabaseModel::new()
+    let pkg_ids = crate::db::DatabaseModel::new()
         .package_data()
         .keys(&mut ctx.db.handle(), false)
         .await?;
-    // TODO: parallelize this
+    futures::stream::iter(pkg_ids)
-    for id in pkg_ids {
+        .for_each_concurrent(None, |id| async move {
-        async fn status(ctx: &RpcContext, id: PackageId) -> Result<(), Error> {
+            async fn status(ctx: &RpcContext, id: PackageId) -> Result<(), Error> {
-            let mut db = ctx.db.handle();
+                let mut db = ctx.db.handle();
-            // TODO: DRAGONS!!
+                // TODO: DRAGONS!!
-            // this locks all of package data to solve a deadlock issue below. As of the writing of this comment, it
+                // this locks all of package data to solve a deadlock issue below. As of the writing of this comment, it
-            // hangs during the `check` operation on /package-data/<id>. There is another daemon loop somewhere that
+                // hangs during the `check` operation on /package-data/<id>. There is another daemon loop somewhere that
-            // is likely iterating through packages in a different order.
+                // is likely iterating through packages in a different order.
-            crate::db::DatabaseModel::new()
+                // crate::db::DatabaseModel::new()
-                .package_data()
+                //     .package_data()
-                .lock(&mut db, LockType::Write)
+                //     .lock(&mut db)
-                .await;
+                //     .await;
 
-            // Without the above lock, the below check operation will deadlock
+                // Without the above lock, the below check operation will deadlock
-            let model = crate::db::DatabaseModel::new()
+                let (mut status, manager) = if let Some(installed) = crate::db::DatabaseModel::new()
-                .package_data()
+                    .package_data()
-                .idx_model(&id)
+                    .idx_model(&id)
-                .check(&mut db)
+                    .and_then(|m| m.installed())
-                .await?
+                    .check(&mut db)
-                .ok_or_else(|| {
+                    .await?
-                    Error::new(
+                {
-                        anyhow!("PackageDataEntry does not exist"),
-                        crate::ErrorKind::Database,
-                    )
-                })?;
-            let (mut status, manager) =
-                if let Some(installed) = model.installed().check(&mut db).await? {
                     (
                         installed.clone().status().get_mut(&mut db).await?,
                         ctx.managers
@@ -74,42 +69,41 @@ pub async fn synchronize_all(ctx: &RpcContext) -> Result<(), Error> {
                     return Ok(());
                 };
 
-            let res = status.main.synchronize(&manager).await?;
+                let res = status.main.synchronize(&manager).await?;
 
-            status.save(&mut db).await?;
+                status.save(&mut db).await?;
 
-            Ok(res)
+                Ok(res)
-        }
+            }
-        if let Err(e) = status(ctx, id.clone()).await {
+            if let Err(e) = status(ctx, id.clone()).await {
-            log::error!("Error syncronizing status of {}: {}", id, e);
+                log::error!("Error syncronizing status of {}: {}", id, e);
-        }
+            }
-    }
+        })
+        .await;
 
     Ok(())
 }
 
 pub async fn check_all(ctx: &RpcContext) -> Result<(), Error> {
     let mut db = ctx.db.handle();
+    // TODO: DRAGONS!!
+    // this locks all of package data to solve a deadlock issue below. As of the writing of this comment, it
+    // hangs during the `check` operation on /package-data/<id>. There is another daemon loop somewhere that
+    // is likely iterating through packages in a different order.
     let pkg_ids = crate::db::DatabaseModel::new()
         .package_data()
-        .keys(&mut db, false)
+        .keys(&mut db, true)
         .await?;
     let mut status_manifest = Vec::with_capacity(pkg_ids.len());
-    let mut status_deps = Vec::with_capacity(pkg_ids.len());
+    let mut installed_deps = Vec::with_capacity(pkg_ids.len());
     for id in &pkg_ids {
-        let model = crate::db::DatabaseModel::new()
+        if let Some(installed) = crate::db::DatabaseModel::new()
             .package_data()
             .idx_model(id)
+            .and_then(|m| m.installed())
             .check(&mut db)
             .await?
-            .ok_or_else(|| {
+        {
-                Error::new(
-                    anyhow!("PackageDataEntry does not exist"),
-                    crate::ErrorKind::Database,
-                )
-            })?;
-        model.lock(&mut db, LockType::Write).await;
-        if let Some(installed) = model.installed().check(&mut db).await? {
             let listed_deps = installed
                 .clone()
                 .manifest()
@@ -125,8 +119,8 @@ pub async fn check_all(ctx: &RpcContext) -> Result<(), Error> {
                 installed.clone().status(),
                 Arc::new(installed.clone().manifest().get(&mut db, true).await?),
             ));
-            status_deps.push((
+            installed_deps.push((
-                installed.clone().status(),
+                installed.clone(),
                 Arc::new({
                     installed
                         .current_dependencies()
@@ -168,10 +162,7 @@ pub async fn check_all(ctx: &RpcContext) -> Result<(), Error> {
     .for_each_concurrent(None, move |(((status, manifest), id), ctx)| {
         let status_sender = status_sender.clone();
         async move {
-            match tokio::spawn(main_status(ctx.clone(), status, manifest, ctx.db.handle()))
+            match main_status(ctx.clone(), status, manifest, ctx.db.handle()).await {
-                .await
-                .unwrap()
-            {
                 Err(e) => {
                     log::error!("Error running main health check for {}: {}", id, e);
                     log::debug!("{:?}", e);
@@ -188,30 +179,74 @@ pub async fn check_all(ctx: &RpcContext) -> Result<(), Error> {
     }
     let statuses = Arc::new(statuses);
     async fn dependency_status<Db: DbHandle>(
+        id: &PackageId,
         statuses: Arc<HashMap<PackageId, MainStatus>>,
-        status_model: StatusModel,
+        model: InstalledPackageDataEntryModel,
         current_deps: Arc<IndexMap<PackageId, CurrentDependencyInfo>>,
         mut db: Db,
     ) -> Result<(), Error> {
-        let mut status = status_model.get_mut(&mut db).await?;
+        for (dep_id, dep_info) in &*current_deps {
-
+            if let Some(err) = match statuses.get(dep_id) {
-        status
+                Some(MainStatus::Running { ref health, .. })
-            .dependency_errors
+                | Some(MainStatus::BackingUp {
-            .update_health_based(&current_deps, &*statuses)
+                    started: Some(_),
-            .await?;
+                    ref health,
-
+                }) => {
-        status.save(&mut db).await?;
+                    let mut failures = IndexMap::new();
+                    for check in &dep_info.health_checks {
+                        let res = health
+                            .get(check)
+                            .cloned()
+                            .unwrap_or_else(|| HealthCheckResult {
+                                result: HealthCheckResultVariant::Disabled,
+                                time: Utc::now(),
+                            });
+                        if !matches!(res.result, HealthCheckResultVariant::Success) {
+                            failures.insert(check.clone(), res);
+                        }
+                    }
+                    if !failures.is_empty() {
+                        Some(DependencyError::HealthChecksFailed { failures })
+                    } else {
+                        None
+                    }
+                }
+                _ => Some(DependencyError::NotRunning),
+            } {
+                handle_broken_dependents(
+                    &mut db,
+                    id,
+                    dep_id,
+                    model.clone(),
+                    err,
+                    &mut IndexMap::new(),
+                )
+                .await?;
+            } else {
+                let mut errs = model
+                    .clone()
+                    .status()
+                    .dependency_errors()
+                    .get_mut(&mut db)
+                    .await?;
+                if matches!(
+                    errs.get(dep_id),
+                    Some(DependencyError::HealthChecksFailed { .. })
+                ) {
+                    errs.0.remove(dep_id);
+                    errs.save(&mut db).await?;
+                }
+            }
+        }
 
         Ok(())
     }
-    futures::stream::iter(status_deps.into_iter().zip(pkg_ids.clone()))
+    futures::stream::iter(installed_deps.into_iter().zip(pkg_ids.clone()))
-        .for_each_concurrent(None, |((status, deps), id)| {
+        .for_each_concurrent(None, |((installed, deps), id)| {
             let statuses = statuses.clone();
             async move {
                 if let Err(e) =
-                    tokio::spawn(dependency_status(statuses, status, deps, ctx.db.handle()))
+                    dependency_status(&id, statuses, installed, deps, ctx.db.handle()).await
-                        .await
-                        .unwrap()
                 {
                     log::error!("Error running dependency health check for {}: {}", id, e);
                     log::debug!("{:?}", e);
@@ -231,7 +266,7 @@ pub struct Status {
     pub dependency_errors: DependencyErrors,
 }
 
-#[derive(Debug, Clone, Deserialize, Serialize)]
+#[derive(Debug, Clone, Deserialize, Serialize, HasModel)]
 #[serde(tag = "status")]
 #[serde(rename_all = "kebab-case")]
 pub enum MainStatus {
@@ -405,52 +440,86 @@ impl DependencyErrors {
         }
         Ok(DependencyErrors(res))
     }
-    async fn update_health_based(
+}
-        &mut self,
+
-        dependencies: &IndexMap<PackageId, CurrentDependencyInfo>,
+pub fn handle_broken_dependents<'a, Db: DbHandle>(
-        statuses: &HashMap<PackageId, MainStatus>,
+    db: &'a mut Db,
-    ) -> Result<(), Error> {
+    id: &'a PackageId,
-        for (dep_id, dep_info) in dependencies {
+    dependency: &'a PackageId,
-            if matches!(
+    model: InstalledPackageDataEntryModel,
-                self.get(&dep_id),
+    error: DependencyError,
-                Some(&DependencyError::NotRunning)
+    breakages: &'a mut IndexMap<PackageId, TaggedDependencyError>,
-                    | Some(&DependencyError::HealthChecksFailed { .. })
+) -> BoxFuture<'a, Result<(), Error>> {
-                    | None
+    async move {
-            ) {
+        let mut status = model.clone().status().get_mut(db).await?;
-                match statuses.get(dep_id) {
+
-                    Some(MainStatus::Running { ref health, .. })
+        let old = status.dependency_errors.0.remove(id);
-                    | Some(MainStatus::BackingUp {
+        let newly_broken = old.is_none();
-                        started: Some(_),
+        status.dependency_errors.0.insert(
-                        ref health,
+            id.clone(),
-                    }) => {
+            if let Some(old) = old {
-                        let mut failures = IndexMap::new();
+                old.merge_with(error.clone())
-                        for check in &dep_info.health_checks {
+            } else {
-                            let res =
+                error.clone()
-                                health
+            },
-                                    .get(check)
+        );
-                                    .cloned()
+        if newly_broken {
-                                    .unwrap_or_else(|| HealthCheckResult {
+            breakages.insert(
-                                        result: HealthCheckResultVariant::Disabled,
+                id.clone(),
-                                        time: Utc::now(),
+                TaggedDependencyError {
-                                    });
+                    dependency: dependency.clone(),
-                            if !matches!(res.result, HealthCheckResultVariant::Success) {
+                    error: error.clone(),
-                                failures.insert(check.clone(), res);
+                },
-                            }
+            );
-                        }
+            if status.main.running() {
-                        if !failures.is_empty() {
+                if model
-                            self.0.insert(
+                    .clone()
-                                dep_id.clone(),
+                    .manifest()
-                                DependencyError::HealthChecksFailed { failures },
+                    .dependencies()
-                            );
+                    .idx_model(dependency)
-                        }
+                    .get(db, true)
-                    }
+                    .await?
-                    _ => {
+                    .into_owned()
-                        self.0.insert(dep_id.clone(), DependencyError::NotRunning);
+                    .ok_or_else(|| {
+                        Error::new(
+                            anyhow!("{} not in listed dependencies", dependency),
+                            crate::ErrorKind::Database,
+                        )
+                    })?
+                    .critical
+                {
+                    status.main.stop();
+                    let dependents = model.current_dependents().get(db, true).await?;
+                    for (dependent, _) in &*dependents {
+                        let dependent_model = crate::db::DatabaseModel::new()
+                            .package_data()
+                            .idx_model(dependent)
+                            .and_then(|pkg| pkg.installed())
+                            .check(db)
+                            .await?
+                            .ok_or_else(|| {
+                                Error::new(
+                                    anyhow!("{} is not installed", dependent),
+                                    crate::ErrorKind::NotFound,
+                                )
+                            })?;
+                        handle_broken_dependents(
+                            db,
+                            dependent,
+                            id,
+                            dependent_model,
+                            DependencyError::NotRunning,
+                            breakages,
+                        )
+                        .await?;
                     }
                 }
             }
         }
 
+        status.save(db).await?;
+
         Ok(())
     }
+    .boxed()
 }

appmgr/src/util/mod.rs

@@ -11,7 +11,7 @@ use anyhow::anyhow;
 use async_trait::async_trait;
 use clap::ArgMatches;
 use digest::Digest;
-use patch_db::HasModel;
+use patch_db::{HasModel, Model};
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use serde_json::Value;
 use tokio::fs::File;
@@ -328,7 +328,7 @@ impl<'de> serde::de::Deserialize<'de> for ValuePrimative {
     }
 }
 
-#[derive(Debug, Clone, HasModel)]
+#[derive(Debug, Clone)]
 pub struct Version {
     version: emver::Version,
     string: String,
@@ -410,6 +410,9 @@ impl Serialize for Version {
         self.string.serialize(serializer)
     }
 }
+impl HasModel for Version {
+    type Model = Model<Version>;
+}
 
 #[async_trait]
 pub trait AsyncFileExt: Sized {