handle crash loops better

This commit is contained in:
Aiden McClelland
2022-02-15 12:06:44 -07:00
committed by Aiden McClelland
parent 868edc636b
commit 64e38393df

View File

@@ -495,6 +495,7 @@ async fn manager_thread_loop(mut recv: Receiver<OnStop>, thread_shared: &Arc<Man
Ok(()) => {} Ok(()) => {}
} }
tracing::error!("service crashed: {}: {}", e.0, e.1); tracing::error!("service crashed: {}: {}", e.0, e.1);
tokio::time::sleep(Duration::from_secs(15)).await;
} }
Err(e) => { Err(e) => {
tracing::error!("failed to start service: {}", e); tracing::error!("failed to start service: {}", e);
@@ -572,24 +573,16 @@ async fn start(shared: &ManagerSharedState) -> Result<(), Error> {
#[instrument(skip(shared))] #[instrument(skip(shared))]
async fn pause(shared: &ManagerSharedState) -> Result<(), Error> { async fn pause(shared: &ManagerSharedState) -> Result<(), Error> {
let mut res = Ok(()); if let Err(e) = shared
for _retry in 0..5 {
res = shared
.ctx .ctx
.docker .docker
.pause_container(&shared.container_name) .pause_container(&shared.container_name)
.await; .await
if !matches!( {
res, tracing::error!("failed to pause container. stopping instead. {}", e);
Err(bollard::errors::Error::DockerResponseServerError { tracing::debug!("{:?}", e);
status_code: 500, return stop(shared).await;
..
}),
) {
break;
} }
}
res?;
shared shared
.status .status
.store(Status::Paused as usize, std::sync::atomic::Ordering::SeqCst); .store(Status::Paused as usize, std::sync::atomic::Ordering::SeqCst);