handle crash loops better

This commit is contained in:
Aiden McClelland
2022-02-15 12:06:44 -07:00
committed by Aiden McClelland
parent 868edc636b
commit 64e38393df

View File

@@ -495,6 +495,7 @@ async fn manager_thread_loop(mut recv: Receiver<OnStop>, thread_shared: &Arc<Man
Ok(()) => {} Ok(()) => {}
} }
tracing::error!("service crashed: {}: {}", e.0, e.1); tracing::error!("service crashed: {}: {}", e.0, e.1);
tokio::time::sleep(Duration::from_secs(15)).await;
} }
Err(e) => { Err(e) => {
tracing::error!("failed to start service: {}", e); tracing::error!("failed to start service: {}", e);
@@ -572,24 +573,16 @@ async fn start(shared: &ManagerSharedState) -> Result<(), Error> {
#[instrument(skip(shared))] #[instrument(skip(shared))]
async fn pause(shared: &ManagerSharedState) -> Result<(), Error> { async fn pause(shared: &ManagerSharedState) -> Result<(), Error> {
let mut res = Ok(()); if let Err(e) = shared
for _retry in 0..5 { .ctx
res = shared .docker
.ctx .pause_container(&shared.container_name)
.docker .await
.pause_container(&shared.container_name) {
.await; tracing::error!("failed to pause container. stopping instead. {}", e);
if !matches!( tracing::debug!("{:?}", e);
res, return stop(shared).await;
Err(bollard::errors::Error::DockerResponseServerError {
status_code: 500,
..
}),
) {
break;
}
} }
res?;
shared shared
.status .status
.store(Status::Paused as usize, std::sync::atomic::Ordering::SeqCst); .store(Status::Paused as usize, std::sync::atomic::Ordering::SeqCst);