refactor: remove sync daemon (#765)

* refactor: remove sync daemon

* improve code organization
This commit is contained in:
Aiden McClelland
2021-11-03 12:15:38 -06:00
parent 57092cd240
commit e4a40ac32d
11 changed files with 330 additions and 313 deletions

44
appmgr/Cargo.lock generated
View File

@@ -702,6 +702,17 @@ dependencies = [
"syn 1.0.80", "syn 1.0.80",
] ]
[[package]]
name = "derivative"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b"
dependencies = [
"proc-macro2 1.0.29",
"quote 1.0.10",
"syn 1.0.80",
]
[[package]] [[package]]
name = "derive_more" name = "derive_more"
version = "0.99.16" version = "0.99.16"
@@ -857,6 +868,7 @@ dependencies = [
"nix 0.23.0", "nix 0.23.0",
"nom 7.0.0", "nom 7.0.0",
"num", "num",
"num_enum",
"openssh-keys", "openssh-keys",
"openssl", "openssl",
"patch-db", "patch-db",
@@ -1915,6 +1927,28 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "num_enum"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9bd055fb730c4f8f4f57d45d35cd6b3f0980535b056dc7ff119cee6a66ed6f"
dependencies = [
"derivative",
"num_enum_derive",
]
[[package]]
name = "num_enum_derive"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "486ea01961c4a818096de679a8b740b26d9033146ac5291b1c98557658f8cdd9"
dependencies = [
"proc-macro-crate",
"proc-macro2 1.0.29",
"quote 1.0.10",
"syn 1.0.80",
]
[[package]] [[package]]
name = "object" name = "object"
version = "0.26.2" version = "0.26.2"
@@ -2202,6 +2236,16 @@ dependencies = [
"unicode-width", "unicode-width",
] ]
[[package]]
name = "proc-macro-crate"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebace6889caf889b4d3f76becee12e90353f2b8c7d875534a71e5742f8f6f83"
dependencies = [
"thiserror",
"toml",
]
[[package]] [[package]]
name = "proc-macro-hack" name = "proc-macro-hack"
version = "0.5.19" version = "0.5.19"

View File

@@ -77,6 +77,7 @@ log = "0.4.14"
nix = "0.23.0" nix = "0.23.0"
nom = "7.0.0" nom = "7.0.0"
num = "0.4.0" num = "0.4.0"
num_enum = "0.5.4"
openssh-keys = "0.5.0" openssh-keys = "0.5.0"
openssl = { version = "0.10.36", features = ["vendored"] } openssl = { version = "0.10.36", features = ["vendored"] }
patch-db = { version = "*", path = "../patch-db/patch-db", features = [ patch-db = { version = "*", path = "../patch-db/patch-db", features = [

View File

@@ -223,27 +223,10 @@ async fn perform_backup<Db: DbHandle>(
} else { } else {
continue; continue;
}; };
installed_model.lock(&mut db, LockType::Write).await;
let manifest = installed_model
.clone()
.manifest()
.get(&mut db, true)
.await?;
let main_status_model = installed_model.clone().status().main(); let main_status_model = installed_model.clone().status().main();
let (started, health) = match main_status_model.get(&mut db, true).await?.into_owned() { let (started, health) = match main_status_model.get(&mut db, true).await?.into_owned() {
MainStatus::Running { started, health } => (Some(started.clone()), health.clone()), MainStatus::Running { started, health } => (Some(started.clone()), health.clone()),
MainStatus::Stopped | MainStatus::Stopping => (None, Default::default()), MainStatus::Stopped | MainStatus::Stopping => (None, Default::default()),
MainStatus::Restoring { .. } => {
backup_report.insert(
package_id,
PackageBackupReport {
error: Some(
"Can't do backup because service is in a restoring state".to_owned(),
),
},
);
continue;
}
MainStatus::BackingUp { .. } => { MainStatus::BackingUp { .. } => {
backup_report.insert( backup_report.insert(
package_id, package_id,
@@ -256,16 +239,33 @@ async fn perform_backup<Db: DbHandle>(
continue; continue;
} }
}; };
let mut tx = db.begin().await?; // for lock scope
main_status_model main_status_model
.put( .put(
&mut db, &mut tx,
&MainStatus::BackingUp { &MainStatus::BackingUp {
started: started.clone(), started: started.clone(),
health: health.clone(), health: health.clone(),
}, },
) )
.await?; .await?;
tx.save().await?; // drop locks
installed_model.lock(&mut db, LockType::Write).await;
let manifest = installed_model
.clone()
.manifest()
.get(&mut db, true)
.await?;
ctx.managers
.get(&(manifest.id.clone(), manifest.version.clone()))
.await
.ok_or_else(|| {
Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest)
})?
.synchronize()
.await;
let guard = backup_guard.mount_package_backup(&package_id).await?; let guard = backup_guard.mount_package_backup(&package_id).await?;
let res = manifest let res = manifest
.backup .backup

View File

@@ -11,7 +11,6 @@ use embassy::middleware::diagnostic::diagnostic;
use embassy::net::mdns::MdnsController; use embassy::net::mdns::MdnsController;
use embassy::net::tor::tor_health_check; use embassy::net::tor::tor_health_check;
use embassy::shutdown::Shutdown; use embassy::shutdown::Shutdown;
use embassy::status::synchronize_all;
use embassy::util::{daemon, Invoke}; use embassy::util::{daemon, Invoke};
use embassy::{static_server, Error, ErrorKind, ResultExt}; use embassy::{static_server, Error, ErrorKind, ResultExt};
use futures::{FutureExt, TryFutureExt}; use futures::{FutureExt, TryFutureExt};
@@ -183,22 +182,6 @@ async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
}) })
}; };
let status_ctx = rpc_ctx.clone();
let status_daemon = daemon(
move || {
let ctx = status_ctx.clone();
async move {
if let Err(e) = synchronize_all(&ctx).await {
tracing::error!("Error in Status Sync daemon: {}", e);
tracing::debug!("{:?}", e);
} else {
tracing::trace!("Status Sync completed successfully");
}
}
},
Duration::from_millis(500),
rpc_ctx.shutdown.subscribe(),
);
let tor_health_ctx = rpc_ctx.clone(); let tor_health_ctx = rpc_ctx.clone();
let tor_client = Client::builder() let tor_client = Client::builder()
.proxy( .proxy(
@@ -239,12 +222,6 @@ async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
file_server file_server
.map_err(|e| Error::new(e, ErrorKind::Network)) .map_err(|e| Error::new(e, ErrorKind::Network))
.map_ok(|_| tracing::debug!("Static File Server Shutdown")), .map_ok(|_| tracing::debug!("Static File Server Shutdown")),
status_daemon
.map_err(|e| Error::new(
e.wrap_err("Status Sync Daemon panicked!"),
ErrorKind::Unknown
))
.map_ok(|_| tracing::debug!("Status Sync Daemon Shutdown")),
tor_health_daemon tor_health_daemon
.map_err(|e| Error::new( .map_err(|e| Error::new(
e.wrap_err("Tor Health Daemon panicked!"), e.wrap_err("Tor Health Daemon panicked!"),

View File

@@ -17,7 +17,10 @@ use crate::action::docker::DockerAction;
use crate::context::RpcContext; use crate::context::RpcContext;
use crate::db::model::CurrentDependencyInfo; use crate::db::model::CurrentDependencyInfo;
use crate::db::util::WithRevision; use crate::db::util::WithRevision;
use crate::dependencies::{BreakageRes, DependencyError, DependencyErrors, TaggedDependencyError, break_transitive, heal_all_dependents_transitive, update_current_dependents}; use crate::dependencies::{
break_transitive, heal_all_dependents_transitive, update_current_dependents, BreakageRes,
DependencyError, DependencyErrors, TaggedDependencyError,
};
use crate::install::cleanup::remove_current_dependents; use crate::install::cleanup::remove_current_dependents;
use crate::s9pk::manifest::{Manifest, PackageId}; use crate::s9pk::manifest::{Manifest, PackageId};
use crate::util::{ use crate::util::{

View File

@@ -1,4 +1,3 @@
use std::borrow::Cow;
use std::collections::VecDeque; use std::collections::VecDeque;
use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4};
use std::ops::Deref; use std::ops::Deref;

View File

@@ -1,4 +1,3 @@
use std::borrow::Cow;
use std::net::{IpAddr, SocketAddr}; use std::net::{IpAddr, SocketAddr};
use std::ops::Deref; use std::ops::Deref;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};

View File

@@ -50,20 +50,20 @@ pub async fn start(
started: Utc::now(), started: Utc::now(),
health: BTreeMap::new(), health: BTreeMap::new(),
}; };
status
.synchronize(
&*ctx
.managers
.get(&(id.clone(), version))
.await
.ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::Docker))?,
)
.await?;
status.save(&mut tx).await?; status.save(&mut tx).await?;
heal_all_dependents_transitive(&ctx, &mut tx, &id).await?; heal_all_dependents_transitive(&ctx, &mut tx, &id).await?;
let revision = tx.commit(None).await?;
ctx.managers
.get(&(id, version))
.await
.ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
.synchronize()
.await;
Ok(WithRevision { Ok(WithRevision {
revision: tx.commit(None).await?, revision,
response: (), response: (),
}) })
} }

View File

@@ -1,23 +1,26 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::convert::TryInto;
use std::future::Future; use std::future::Future;
use std::sync::atomic::AtomicUsize; use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc; use std::sync::Arc;
use std::task::Poll; use std::task::Poll;
use std::time::Duration; use std::time::Duration;
use bollard::container::StopContainerOptions; use bollard::container::StopContainerOptions;
use color_eyre::eyre::eyre; use color_eyre::eyre::eyre;
use num_enum::TryFromPrimitive;
use patch_db::DbHandle; use patch_db::DbHandle;
use sqlx::{Executor, Sqlite}; use sqlx::{Executor, Sqlite};
use tokio::sync::watch::error::RecvError; use tokio::sync::watch::error::RecvError;
use tokio::sync::watch::{channel, Receiver, Sender}; use tokio::sync::watch::{channel, Receiver, Sender};
use tokio::sync::RwLock; use tokio::sync::{Notify, RwLock};
use torut::onion::TorSecretKeyV3; use torut::onion::TorSecretKeyV3;
use tracing::instrument; use tracing::instrument;
use crate::action::docker::DockerAction; use crate::action::docker::DockerAction;
use crate::action::NoOutput; use crate::action::NoOutput;
use crate::context::RpcContext; use crate::context::RpcContext;
use crate::manager::sync::synchronizer;
use crate::net::interface::InterfaceId; use crate::net::interface::InterfaceId;
use crate::notifications::NotificationLevel; use crate::notifications::NotificationLevel;
use crate::s9pk::manifest::{Manifest, PackageId}; use crate::s9pk::manifest::{Manifest, PackageId};
@@ -25,6 +28,7 @@ use crate::util::{Container, NonDetachingJoinHandle, Version};
use crate::Error; use crate::Error;
pub mod health; pub mod health;
mod sync;
#[derive(Default)] #[derive(Default)]
pub struct ManagerMap(RwLock<BTreeMap<(PackageId, Version), Arc<Manager>>>); pub struct ManagerMap(RwLock<BTreeMap<(PackageId, Version), Arc<Manager>>>);
@@ -128,19 +132,23 @@ pub struct Manager {
thread: Container<NonDetachingJoinHandle<()>>, thread: Container<NonDetachingJoinHandle<()>>,
} }
#[derive(TryFromPrimitive)]
#[repr(usize)]
pub enum Status { pub enum Status {
Running = 0, Running = 0,
Stopped = 1, Stopped = 1,
Paused = 2, Paused = 2,
} }
struct ManagerSharedState { pub struct ManagerSharedState {
ctx: RpcContext, ctx: RpcContext,
status: AtomicUsize, status: AtomicUsize,
on_stop: Sender<OnStop>, on_stop: Sender<OnStop>,
manifest: Manifest, manifest: Manifest,
container_name: String, container_name: String,
tor_keys: BTreeMap<InterfaceId, TorSecretKeyV3>, tor_keys: BTreeMap<InterfaceId, TorSecretKeyV3>,
synchronized: Notify,
synchronize_now: Notify,
} }
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
@@ -273,7 +281,7 @@ impl Manager {
manifest: Manifest, manifest: Manifest,
tor_keys: BTreeMap<InterfaceId, TorSecretKeyV3>, tor_keys: BTreeMap<InterfaceId, TorSecretKeyV3>,
) -> Result<Self, Error> { ) -> Result<Self, Error> {
let (on_stop, mut recv) = channel(OnStop::Sleep); let (on_stop, recv) = channel(OnStop::Sleep);
let shared = Arc::new(ManagerSharedState { let shared = Arc::new(ManagerSharedState {
ctx, ctx,
status: AtomicUsize::new(Status::Stopped as usize), status: AtomicUsize::new(Status::Stopped as usize),
@@ -281,83 +289,14 @@ impl Manager {
container_name: DockerAction::container_name(&manifest.id, None), container_name: DockerAction::container_name(&manifest.id, None),
manifest, manifest,
tor_keys, tor_keys,
synchronized: Notify::new(),
synchronize_now: Notify::new(),
}); });
let thread_shared = shared.clone(); let thread_shared = shared.clone();
let thread = tokio::spawn(async move { let thread = tokio::spawn(async move {
loop { tokio::select! {
fn handle_stop_action<'a>( _ = manager_thread_loop(recv, &thread_shared) => (),
recv: &'a mut Receiver<OnStop>, _ = synchronizer(&*thread_shared) => (),
) -> (
OnStop,
Option<impl Future<Output = Result<(), RecvError>> + 'a>,
) {
let val = *recv.borrow_and_update();
match val {
OnStop::Sleep => (OnStop::Sleep, Some(recv.changed())),
a => (a, None),
}
}
let (stop_action, fut) = handle_stop_action(&mut recv);
match stop_action {
OnStop::Sleep => {
if let Some(fut) = fut {
thread_shared.status.store(
Status::Stopped as usize,
std::sync::atomic::Ordering::SeqCst,
);
fut.await.unwrap();
continue;
}
}
OnStop::Exit => {
thread_shared.status.store(
Status::Stopped as usize,
std::sync::atomic::Ordering::SeqCst,
);
break;
}
OnStop::Restart => {
thread_shared.status.store(
Status::Running as usize,
std::sync::atomic::Ordering::SeqCst,
);
}
}
match run_main(&thread_shared).await {
Ok(Ok(NoOutput)) => {
thread_shared
.on_stop
.send(OnStop::Sleep)
.map_err(|_| ())
.unwrap(); // recv is still in scope, cannot fail
}
Ok(Err(e)) => {
let res = thread_shared.ctx.notification_manager
.notify(
&mut thread_shared.ctx.db.handle(),
Some(thread_shared.manifest.id.clone()),
NotificationLevel::Warning,
String::from("Service Crashed"),
format!("The service {} has crashed with the following exit code: {}\nDetails: {}", thread_shared.manifest.id.clone(), e.0, e.1),
(),
Some(900) // 15 minutes
)
.await;
match res {
Err(e) => {
// TODO for code review: Do we return this error or just log it?
tracing::error!("Failed to issue notification: {}", e);
tracing::debug!("{:?}", e);
}
Ok(()) => {}
}
tracing::error!("service crashed: {}: {}", e.0, e.1);
}
Err(e) => {
tracing::error!("failed to start service: {}", e);
tracing::debug!("{:?}", e);
}
}
} }
}); });
Ok(Manager { Ok(Manager {
@@ -366,90 +305,6 @@ impl Manager {
}) })
} }
pub fn status(&self) -> Status {
match self.shared.status.load(std::sync::atomic::Ordering::SeqCst) {
0 => Status::Running,
1 => Status::Stopped,
2 => Status::Paused,
_ => unreachable!(),
}
}
#[instrument(skip(self))]
pub async fn stop(&self) -> Result<(), Error> {
self.shared.on_stop.send(OnStop::Sleep).map_err(|_| {
Error::new(
eyre!("Manager has already been shutdown"),
crate::ErrorKind::Docker,
)
})?;
if matches!(self.status(), Status::Paused) {
self.resume().await?;
}
match self
.shared
.ctx
.docker
.stop_container(
&self.shared.container_name,
Some(StopContainerOptions { t: 30 }),
)
.await
{
Err(bollard::errors::Error::DockerResponseNotFoundError { .. })
| Err(bollard::errors::Error::DockerResponseConflictError { .. })
| Err(bollard::errors::Error::DockerResponseNotModifiedError { .. }) => (), // Already stopped
a => a?,
};
self.shared.status.store(
Status::Stopped as usize,
std::sync::atomic::Ordering::SeqCst,
);
Ok(())
}
#[instrument(skip(self))]
pub async fn start(&self) -> Result<(), Error> {
self.shared.on_stop.send(OnStop::Restart).map_err(|_| {
Error::new(
eyre!("Manager has already been shutdown"),
crate::ErrorKind::Docker,
)
})?;
self.shared.status.store(
Status::Running as usize,
std::sync::atomic::Ordering::SeqCst,
);
Ok(())
}
#[instrument(skip(self))]
pub async fn pause(&self) -> Result<(), Error> {
self.shared
.ctx
.docker
.pause_container(&self.shared.container_name)
.await?;
self.shared
.status
.store(Status::Paused as usize, std::sync::atomic::Ordering::SeqCst);
Ok(())
}
#[instrument(skip(self))]
pub async fn resume(&self) -> Result<(), Error> {
self.shared
.ctx
.docker
.unpause_container(&self.shared.container_name)
.await?;
self.shared.status.store(
Status::Running as usize,
std::sync::atomic::Ordering::SeqCst,
);
Ok(())
}
#[instrument(skip(self))] #[instrument(skip(self))]
async fn exit(&self) -> Result<(), Error> { async fn exit(&self) -> Result<(), Error> {
let _ = self.shared.on_stop.send(OnStop::Exit); let _ = self.shared.on_stop.send(OnStop::Exit);
@@ -482,4 +337,160 @@ impl Manager {
} }
Ok(()) Ok(())
} }
pub async fn synchronize(&self) {
self.shared.synchronize_now.notify_waiters();
self.shared.synchronized.notified().await
}
}
async fn manager_thread_loop(mut recv: Receiver<OnStop>, thread_shared: &Arc<ManagerSharedState>) {
loop {
fn handle_stop_action<'a>(
recv: &'a mut Receiver<OnStop>,
) -> (
OnStop,
Option<impl Future<Output = Result<(), RecvError>> + 'a>,
) {
let val = *recv.borrow_and_update();
match val {
OnStop::Sleep => (OnStop::Sleep, Some(recv.changed())),
a => (a, None),
}
}
let (stop_action, fut) = handle_stop_action(&mut recv);
match stop_action {
OnStop::Sleep => {
if let Some(fut) = fut {
thread_shared.status.store(
Status::Stopped as usize,
std::sync::atomic::Ordering::SeqCst,
);
fut.await.unwrap();
continue;
}
}
OnStop::Exit => {
thread_shared.status.store(
Status::Stopped as usize,
std::sync::atomic::Ordering::SeqCst,
);
break;
}
OnStop::Restart => {
thread_shared.status.store(
Status::Running as usize,
std::sync::atomic::Ordering::SeqCst,
);
}
}
match run_main(&thread_shared).await {
Ok(Ok(NoOutput)) => {
thread_shared
.on_stop
.send(OnStop::Sleep)
.map_err(|_| ())
.unwrap(); // recv is still in scope, cannot fail
}
Ok(Err(e)) => {
let res = thread_shared.ctx.notification_manager
.notify(
&mut thread_shared.ctx.db.handle(),
Some(thread_shared.manifest.id.clone()),
NotificationLevel::Warning,
String::from("Service Crashed"),
format!("The service {} has crashed with the following exit code: {}\nDetails: {}", thread_shared.manifest.id.clone(), e.0, e.1),
(),
Some(900) // 15 minutes
)
.await;
match res {
Err(e) => {
tracing::error!("Failed to issue notification: {}", e);
tracing::debug!("{:?}", e);
}
Ok(()) => {}
}
tracing::error!("service crashed: {}: {}", e.0, e.1);
}
Err(e) => {
tracing::error!("failed to start service: {}", e);
tracing::debug!("{:?}", e);
}
}
}
}
#[instrument(skip(shared))]
async fn stop(shared: &ManagerSharedState) -> Result<(), Error> {
shared.on_stop.send(OnStop::Sleep).map_err(|_| {
Error::new(
eyre!("Manager has already been shutdown"),
crate::ErrorKind::Docker,
)
})?;
if matches!(
shared.status.load(Ordering::SeqCst).try_into().unwrap(),
Status::Paused
) {
resume(shared).await?;
}
match shared
.ctx
.docker
.stop_container(&shared.container_name, Some(StopContainerOptions { t: 30 }))
.await
{
Err(bollard::errors::Error::DockerResponseNotFoundError { .. })
| Err(bollard::errors::Error::DockerResponseConflictError { .. })
| Err(bollard::errors::Error::DockerResponseNotModifiedError { .. }) => (), // Already stopped
a => a?,
};
shared.status.store(
Status::Stopped as usize,
std::sync::atomic::Ordering::SeqCst,
);
Ok(())
}
#[instrument(skip(shared))]
async fn start(shared: &ManagerSharedState) -> Result<(), Error> {
shared.on_stop.send(OnStop::Restart).map_err(|_| {
Error::new(
eyre!("Manager has already been shutdown"),
crate::ErrorKind::Docker,
)
})?;
shared.status.store(
Status::Running as usize,
std::sync::atomic::Ordering::SeqCst,
);
Ok(())
}
#[instrument(skip(shared))]
async fn pause(shared: &ManagerSharedState) -> Result<(), Error> {
shared
.ctx
.docker
.pause_container(&shared.container_name)
.await?;
shared
.status
.store(Status::Paused as usize, std::sync::atomic::Ordering::SeqCst);
Ok(())
}
#[instrument(skip(shared))]
async fn resume(shared: &ManagerSharedState) -> Result<(), Error> {
shared
.ctx
.docker
.unpause_container(&shared.container_name)
.await?;
shared.status.store(
Status::Running as usize,
std::sync::atomic::Ordering::SeqCst,
);
Ok(())
} }

View File

@@ -0,0 +1,79 @@
use std::convert::TryInto;
use std::sync::atomic::Ordering;
use std::time::Duration;
use chrono::Utc;
use super::{pause, resume, start, stop, ManagerSharedState, Status};
use crate::status::MainStatus;
use crate::Error;
/// Allocates a db handle. DO NOT CALL with a db handle already in scope
async fn synchronize_once(shared: &ManagerSharedState) -> Result<(), Error> {
let mut db = shared.ctx.db.handle();
let mut status = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&shared.manifest.id)
.expect(&mut db)
.await?
.installed()
.expect(&mut db)
.await?
.status()
.main()
.get_mut(&mut db)
.await?;
match shared.status.load(Ordering::SeqCst).try_into().unwrap() {
Status::Stopped => match &mut *status {
MainStatus::Stopped => (),
MainStatus::Stopping => {
*status = MainStatus::Stopped;
}
MainStatus::Running { started, .. } => {
*started = Utc::now();
start(shared).await?;
}
MainStatus::BackingUp { .. } => (),
},
Status::Running => match *status {
MainStatus::Stopped | MainStatus::Stopping => {
stop(shared).await?;
}
MainStatus::Running { .. } => (),
MainStatus::BackingUp { .. } => {
pause(shared).await?;
}
},
Status::Paused => match *status {
MainStatus::Stopped | MainStatus::Stopping => {
stop(shared).await?;
}
MainStatus::Running { .. } => {
resume(shared).await?;
}
MainStatus::BackingUp { .. } => (),
},
}
status.save(&mut db).await?;
Ok(())
}
pub async fn synchronizer(shared: &ManagerSharedState) {
loop {
if let Err(e) = synchronize_once(shared).await {
tracing::error!(
"Synchronizer for {}@{} failed: {}",
shared.manifest.id,
shared.manifest.version,
e
);
tracing::debug!("{:?}", e);
} else {
shared.synchronized.notify_waiters();
}
tokio::select! {
_ = tokio::time::sleep(Duration::from_secs(5)) => (),
_ = shared.synchronize_now.notified() => (),
}
}
}

View File

@@ -1,7 +1,6 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use color_eyre::eyre::eyre;
use patch_db::{DbHandle, HasModel}; use patch_db::{DbHandle, HasModel};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tracing::instrument; use tracing::instrument;
@@ -9,64 +8,12 @@ use tracing::instrument;
use self::health_check::HealthCheckId; use self::health_check::HealthCheckId;
use crate::context::RpcContext; use crate::context::RpcContext;
use crate::dependencies::DependencyErrors; use crate::dependencies::DependencyErrors;
use crate::manager::{Manager, Status as ManagerStatus};
use crate::notifications::NotificationLevel; use crate::notifications::NotificationLevel;
use crate::s9pk::manifest::Manifest; use crate::s9pk::manifest::Manifest;
use crate::status::health_check::HealthCheckResult; use crate::status::health_check::HealthCheckResult;
use crate::Error; use crate::Error;
pub mod health_check; pub mod health_check;
#[instrument(skip(ctx))]
pub async fn synchronize_all(ctx: &RpcContext) -> Result<(), Error> {
let mut db = ctx.db.handle();
let pkg_ids = crate::db::DatabaseModel::new()
.package_data()
.keys(&mut db, false)
.await?;
for id in pkg_ids {
if let Err(e) = async {
let (mut status, manager) = if let Some(installed) = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&id)
.and_then(|m| m.installed())
.check(&mut db)
.await?
{
(
installed.clone().status().get_mut(&mut db).await?,
ctx.managers
.get(&(
id.clone(),
installed
.manifest()
.version()
.get(&mut db, true)
.await?
.to_owned(),
))
.await
.ok_or_else(|| Error::new(eyre!("No Manager"), crate::ErrorKind::Docker))?,
)
} else {
return Ok::<_, Error>(());
};
let res = status.main.synchronize(&manager).await?;
status.save(&mut db).await?;
Ok(res)
}
.await
{
tracing::error!("Error syncronizing status of {}: {}", id, e);
tracing::debug!("{:?}", e);
}
}
Ok(())
}
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)] #[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")] #[serde(rename_all = "kebab-case")]
pub struct Status { pub struct Status {
@@ -90,47 +37,8 @@ pub enum MainStatus {
started: Option<DateTime<Utc>>, started: Option<DateTime<Utc>>,
health: BTreeMap<HealthCheckId, HealthCheckResult>, health: BTreeMap<HealthCheckId, HealthCheckResult>,
}, },
Restoring {
running: bool,
},
} }
impl MainStatus { impl MainStatus {
#[instrument(skip(manager))]
pub async fn synchronize(&mut self, manager: &Manager) -> Result<(), Error> {
match manager.status() {
ManagerStatus::Stopped => match self {
MainStatus::Stopped => (),
MainStatus::Stopping => {
*self = MainStatus::Stopped;
}
MainStatus::Running { started, .. } => {
*started = Utc::now();
manager.start().await?;
}
MainStatus::BackingUp { .. } => (),
MainStatus::Restoring { .. } => (),
},
ManagerStatus::Running => match self {
MainStatus::Stopped | MainStatus::Stopping | MainStatus::Restoring { .. } => {
manager.stop().await?;
}
MainStatus::Running { .. } => (),
MainStatus::BackingUp { .. } => {
manager.pause().await?;
}
},
ManagerStatus::Paused => match self {
MainStatus::Stopped | MainStatus::Stopping | MainStatus::Restoring { .. } => {
manager.stop().await?;
}
MainStatus::Running { .. } => {
manager.resume().await?;
}
MainStatus::BackingUp { .. } => (),
},
}
Ok(())
}
#[instrument(skip(ctx, db, manifest))] #[instrument(skip(ctx, db, manifest))]
pub async fn check<Db: DbHandle>( pub async fn check<Db: DbHandle>(
&mut self, &mut self,
@@ -189,8 +97,7 @@ impl MainStatus {
MainStatus::Running { .. } MainStatus::Running { .. }
| MainStatus::BackingUp { | MainStatus::BackingUp {
started: Some(_), .. started: Some(_), ..
} } => true,
| MainStatus::Restoring { running: true } => true,
_ => false, _ => false,
} }
} }
@@ -202,9 +109,6 @@ impl MainStatus {
MainStatus::BackingUp { started, .. } => { MainStatus::BackingUp { started, .. } => {
*started = None; *started = None;
} }
MainStatus::Restoring { running } => {
*running = false;
}
_ => (), _ => (),
} }
} }