From abe3ac464d16c7c2c7360847769c71bc37170bb5 Mon Sep 17 00:00:00 2001 From: J M <2364004+Blu-J@users.noreply.github.com> Date: Tue, 11 Jan 2022 10:35:03 -0700 Subject: [PATCH] Fix/timeout errors (#1051) * wip: Working with docker timeouts * fix: Make the fixes * chore: small fixes * chore: Remove not used 143 * chore: Put back --- appmgr/Cargo.lock | 49 +++++++++++++++++---------------- appmgr/Cargo.toml | 2 +- appmgr/src/action/docker.rs | 54 ++++++++++++++++++------------------- 3 files changed, 52 insertions(+), 53 deletions(-) diff --git a/appmgr/Cargo.lock b/appmgr/Cargo.lock index e0b47b964..faeecd2f0 100644 --- a/appmgr/Cargo.lock +++ b/appmgr/Cargo.lock @@ -308,7 +308,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "thiserror", - "tokio 1.12.0", + "tokio 1.15.0", "tokio-util", "url", "winapi", @@ -905,7 +905,7 @@ dependencies = [ "stderrlog", "tar", "thiserror", - "tokio 1.12.0", + "tokio 1.15.0", "tokio-compat-02", "tokio-stream", "tokio-tar", @@ -1251,7 +1251,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.12.0", + "tokio 1.15.0", "tokio-util", "tracing", ] @@ -1381,7 +1381,7 @@ dependencies = [ "itoa", "pin-project-lite 0.2.7", "socket2", - "tokio 1.12.0", + "tokio 1.15.0", "tower-service", "tracing", "want", @@ -1396,7 +1396,7 @@ dependencies = [ "bytes 1.1.0", "hyper", "native-tls", - "tokio 1.12.0", + "tokio 1.15.0", "tokio-native-tls", ] @@ -1412,7 +1412,7 @@ dependencies = [ "hyper", "log", "sha-1", - "tokio 1.12.0", + "tokio 1.15.0", "tokio-tungstenite", ] @@ -1426,7 +1426,7 @@ dependencies = [ "hex", "hyper", "pin-project", - "tokio 1.12.0", + "tokio 1.15.0", ] [[package]] @@ -2109,7 +2109,7 @@ dependencies = [ "serde_cbor 0.11.1", "serde_json", "thiserror", - "tokio 1.12.0", + "tokio 1.15.0", "tracing", "tracing-error", ] @@ -2601,7 +2601,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "time 0.2.27", - "tokio 1.12.0", + "tokio 1.15.0", "tokio-native-tls", "tokio-socks", "url", @@ -2663,7 +2663,7 @@ dependencies = [ "serde_cbor 0.11.2", "serde_json", "thiserror", - "tokio 1.12.0", + "tokio 1.15.0", "url", "yajrc", ] @@ -3180,7 +3180,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d1bd069de53442e7a320f525a6d4deb8bb0621ac7a55f7eccbc2b58b57f43d0" dependencies = [ "once_cell", - "tokio 1.12.0", + "tokio 1.15.0", "tokio-rustls", ] @@ -3533,11 +3533,10 @@ dependencies = [ [[package]] name = "tokio" -version = "1.12.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2c2416fdedca8443ae44b4527de1ea633af61d8f7169ffa6e72c5b53d24efcc" +checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" dependencies = [ - "autocfg", "bytes 1.1.0", "libc", "memchr", @@ -3561,15 +3560,15 @@ dependencies = [ "once_cell", "pin-project-lite 0.2.7", "tokio 0.2.25", - "tokio 1.12.0", + "tokio 1.15.0", "tokio-stream", ] [[package]] name = "tokio-macros" -version = "1.4.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "154794c8f499c2619acd19e839294703e9e32e7630ef5f46ea80d4ef0fbee5eb" +checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" dependencies = [ "proc-macro2 1.0.29", "quote 1.0.10", @@ -3583,7 +3582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 1.12.0", + "tokio 1.15.0", ] [[package]] @@ -3593,7 +3592,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" dependencies = [ "rustls", - "tokio 1.12.0", + "tokio 1.15.0", "webpki", ] @@ -3606,7 +3605,7 @@ dependencies = [ "either", "futures-util", "thiserror", - "tokio 1.12.0", + "tokio 1.15.0", ] [[package]] @@ -3617,7 +3616,7 @@ checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f" dependencies = [ "futures-core", "pin-project-lite 0.2.7", - "tokio 1.12.0", + "tokio 1.15.0", "tokio-util", ] @@ -3630,7 +3629,7 @@ dependencies = [ "futures-core", "libc", "redox_syscall 0.2.10", - "tokio 1.12.0", + "tokio 1.15.0", "tokio-stream", "xattr", ] @@ -3644,7 +3643,7 @@ dependencies = [ "futures-util", "log", "pin-project", - "tokio 1.12.0", + "tokio 1.15.0", "tungstenite", ] @@ -3659,7 +3658,7 @@ dependencies = [ "futures-sink", "log", "pin-project-lite 0.2.7", - "tokio 1.12.0", + "tokio 1.15.0", ] [[package]] @@ -3688,7 +3687,7 @@ dependencies = [ "serde_derive", "sha2", "sha3", - "tokio 1.12.0", + "tokio 1.15.0", ] [[package]] diff --git a/appmgr/Cargo.toml b/appmgr/Cargo.toml index 9cd3b00e9..1aba1b729 100644 --- a/appmgr/Cargo.toml +++ b/appmgr/Cargo.toml @@ -115,7 +115,7 @@ sqlx = { version = "0.5", features = [ stderrlog = "0.5.1" tar = "0.4.37" thiserror = "1.0.29" -tokio = { version = "1.12.0", features = ["full"] } +tokio = { version = "1.15.0", features = ["full"] } tokio-compat-02 = "0.2.0" tokio-stream = { version = "0.1.7", features = ["io-util", "sync"] } tokio-tar = { git = "https://github.com/dr-bonez/tokio-tar.git" } diff --git a/appmgr/src/action/docker.rs b/appmgr/src/action/docker.rs index 205e60b95..c54fd2531 100644 --- a/appmgr/src/action/docker.rs +++ b/appmgr/src/action/docker.rs @@ -20,7 +20,7 @@ use crate::util::Version; use crate::volume::{VolumeId, Volumes}; use crate::{Error, ResultExt, HOST_IP}; -pub const NET_TLD: &'static str = "embassy"; +pub const NET_TLD: &str = "embassy"; #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] @@ -97,24 +97,7 @@ impl DockerAction { EitherFuture::Right(async move { tokio::time::sleep(timeout).await; - if let Some(id) = id { - signal::kill(Pid::from_raw(id as i32), nix::sys::signal::SIGTERM) - .with_kind(crate::ErrorKind::Docker)?; - } - - tokio::time::sleep( - self.sigterm_timeout - .map(|a| *a) - .unwrap_or(Duration::from_secs(30)), - ) - .await; - - if let Some(id) = id { - signal::kill(Pid::from_raw(id as i32), signal::SIGKILL) - .with_kind(crate::ErrorKind::Docker)?; - } - - Ok(futures::future::pending().await) + Ok(()) }) } else { EitherFuture::Left(futures::future::pending::>()) @@ -126,12 +109,29 @@ impl DockerAction { .await .with_kind(crate::ErrorKind::Docker)?; } + enum Race { + Done(T), + TimedOut, + } let res = tokio::select! { - res = handle.wait_with_output() => res.with_kind(crate::ErrorKind::Docker)?, - res = timeout_fut => res?, + res = handle.wait_with_output() => Race::Done(res.with_kind(crate::ErrorKind::Docker)?), + res = timeout_fut => { + res?; + Race::TimedOut + }, + }; + let res = match res { + Race::Done(x) => x, + Race::TimedOut => { + if let Some(id) = id { + signal::kill(Pid::from_raw(id as i32), signal::SIGKILL) + .with_kind(crate::ErrorKind::Docker)?; + } + return Ok(Err((143, "Timed out. Retrying soon...".to_owned()))); + } }; Ok(if res.status.success() || res.status.code() == Some(143) { - Ok(if let Some(format) = &self.io_format { + Ok(if let Some(format) = self.io_format { match format.from_slice(&res.stdout) { Ok(a) => a, Err(e) => { @@ -230,9 +230,9 @@ impl DockerAction { } } - pub fn uncontainer_name<'a>(name: &'a str) -> Option<(PackageId<&'a str>, Option<&'a str>)> { + pub fn uncontainer_name(name: &str) -> Option<(PackageId<&str>, Option<&str>)> { let (pre_tld, _) = name.split_once(".")?; - if pre_tld.contains("_") { + if pre_tld.contains('_') { let (pkg, name) = name.split_once("_")?; Some((Id::try_from(pkg).ok()?.into(), Some(name))) } else { @@ -240,14 +240,14 @@ impl DockerAction { } } - async fn docker_args<'a>( - &'a self, + async fn docker_args( + &self, ctx: &RpcContext, pkg_id: &PackageId, pkg_version: &Version, volumes: &Volumes, allow_inject: bool, - ) -> Vec> { + ) -> Vec> { let mut res = Vec::with_capacity( (2 * self.mounts.len()) // --mount + (2 * self.shm_size_mb.is_some() as usize) // --shm-size