Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 12:11:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Cleanup by sending a command and kill when dropped (#1945)

Browse Source 
* fix: Cleanup by sending a command and kill when dropped

* chore: Fix the loadModule run command

* fix: cleans up failed health

* refactor long-running

* chore: Fixes?"

* refactor

* run iso ci on pr

* fix debuild

* fix tests

* switch to libc kill

* kill process by parent

* fix graceful shutdown

* recurse submodules

* fix compat build

* feat: Add back in the timeout

* chore: add the missing types for the unnstable

* inherited logs

Co-authored-by: J M <Blu-J@users.noreply.github.com>

* fix deleted code

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: J M <Blu-J@users.noreply.github.com>
This commit is contained in:
J M
2022-11-18 19:19:04 -07:00
committed by Aiden McClelland
parent eec8c41e20
commit a3d1b2d671
39 changed files with 1866 additions and 1845 deletions
Download Patch File Download Diff File Expand all files Collapse all files

243
backend/src/procedure/docker.rs
View File

@@ -9,19 +9,15 @@ use async_stream::stream;
use bollard::container::RemoveContainerOptions;
use color_eyre::eyre::eyre;
use color_eyre::Report;
use embassy_container_init::{InputJsonRpc, OutputJsonRpc};
use futures::future::Either as EitherFuture;
use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt};
use helpers::NonDetachingJoinHandle;
use futures::TryStreamExt;
use helpers::{NonDetachingJoinHandle, RpcClient};
use nix::sys::signal;
use nix::unistd::Pid;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio::{
io::{AsyncBufRead, AsyncBufReadExt, BufReader},
process::Child,
sync::mpsc::UnboundedReceiver,
};
use tokio::io::{AsyncBufRead, AsyncBufReadExt, BufReader};
use tracing::instrument;
use super::ProcedureName;
@@ -70,6 +66,57 @@ pub struct DockerContainer {
#[serde(default)]
pub system: bool,
}
impl DockerContainer {
/// We created a new exec runner, where we are going to be passing the commands for it to run.
/// Idea is that we are going to send it command and get the inputs be filtered back from the manager.
/// Then we could in theory run commands without the cost of running the docker exec which is known to have
/// a dely of > 200ms which is not acceptable.
#[instrument(skip(ctx))]
pub async fn long_running_execute(
&self,
ctx: &RpcContext,
pkg_id: &PackageId,
pkg_version: &Version,
volumes: &Volumes,
) -> Result<(LongRunning, RpcClient), Error> {
let container_name = DockerProcedure::container_name(pkg_id, None);
let mut cmd = LongRunning::setup_long_running_docker_cmd(
self,
ctx,
&container_name,
volumes,
pkg_id,
pkg_version,
)
.await?;
let mut handle = cmd.spawn().with_kind(crate::ErrorKind::Docker)?;
let client =
if let (Some(stdin), Some(stdout)) = (handle.stdin.take(), handle.stdout.take()) {
RpcClient::new(stdin, stdout)
} else {
return Err(Error::new(
eyre!("No stdin/stdout handle for container init"),
crate::ErrorKind::Incoherent,
));
};
let running_output = NonDetachingJoinHandle::from(tokio::spawn(async move {
if let Err(err) = handle
.wait()
.await
.map_err(|e| eyre!("Runtime error: {e:?}"))
{
tracing::error!("{}", err);
tracing::debug!("{:?}", err);
}
}));
Ok((LongRunning { running_output }, client))
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
@@ -122,23 +169,6 @@ impl DockerProcedure {
shm_size_mb: container.shm_size_mb,
}
}
#[cfg(feature = "js_engine")]
pub fn main_docker_procedure_js(
container: &DockerContainer,
_procedure: &super::js_scripts::JsProcedure,
) -> DockerProcedure {
DockerProcedure {
image: container.image.clone(),
system: container.system,
entrypoint: "sleep".to_string(),
args: Vec::new(),
inject: false,
mounts: container.mounts.clone(),
io_format: None,
sigterm_timeout: container.sigterm_timeout,
shm_size_mb: container.shm_size_mb,
}
}
pub fn validate(
&self,
@@ -346,64 +376,6 @@ impl DockerProcedure {
)
}
/// We created a new exec runner, where we are going to be passing the commands for it to run.
/// Idea is that we are going to send it command and get the inputs be filtered back from the manager.
/// Then we could in theory run commands without the cost of running the docker exec which is known to have
/// a dely of > 200ms which is not acceptable.
#[instrument(skip(ctx, input))]
pub async fn long_running_execute<S>(
&self,
ctx: &RpcContext,
pkg_id: &PackageId,
pkg_version: &Version,
name: ProcedureName,
volumes: &Volumes,
input: S,
) -> Result<LongRunning, Error>
where
S: Stream<Item = InputJsonRpc> + Send + 'static,
{
let name = name.docker_name();
let name: Option<&str> = name.as_deref();
let container_name = Self::container_name(pkg_id, name);
let mut cmd = LongRunning::setup_long_running_docker_cmd(
self,
ctx,
&container_name,
volumes,
pkg_id,
pkg_version,
)
.await?;
let mut handle = cmd.spawn().with_kind(crate::ErrorKind::Docker)?;
let input_handle = LongRunning::spawn_input_handle(&mut handle, input)?
.map_err(|e| eyre!("Input Handle Error: {e:?}"));
let (output, output_handle) = LongRunning::spawn_output_handle(&mut handle)?;
let output_handle = output_handle.map_err(|e| eyre!("Output Handle Error: {e:?}"));
let err_handle = LongRunning::spawn_error_handle(&mut handle)?
.map_err(|e| eyre!("Err Handle Error: {e:?}"));
let running_output = NonDetachingJoinHandle::from(tokio::spawn(async move {
if let Err(err) = tokio::select!(
x = handle.wait().map_err(|e| eyre!("Runtime error: {e:?}")) => x.map(|_| ()),
x = err_handle => x.map(|_| ()),
x = output_handle => x.map(|_| ()),
x = input_handle => x.map(|_| ())
) {
tracing::debug!("{:?}", err);
tracing::error!("Join error");
}
}));
Ok(LongRunning {
output,
running_output,
})
}
#[instrument(skip(_ctx, input))]
pub async fn inject<I: Serialize, O: DeserializeOwned>(
&self,
@@ -788,13 +760,12 @@ impl<T> RingVec<T> {
/// We wanted a long running since we want to be able to have the equivelent to the docker execute without the heavy costs of 400 + ms time lag.
/// Also the long running let's us have the ability to start/ end the services quicker.
pub struct LongRunning {
pub output: UnboundedReceiver<OutputJsonRpc>,
pub running_output: NonDetachingJoinHandle<()>,
}
impl LongRunning {
async fn setup_long_running_docker_cmd(
docker: &DockerProcedure,
docker: &DockerContainer,
ctx: &RpcContext,
container_name: &str,
volumes: &Volumes,
@@ -865,7 +836,7 @@ impl LongRunning {
cmd.arg(docker.image.for_package(pkg_id, Some(pkg_version)));
}
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::inherit());
cmd.stdin(std::process::Stdio::piped());
Ok(cmd)
}
@@ -894,104 +865,6 @@ impl LongRunning {
Err(e) => Err(e)?,
}
}
fn spawn_input_handle<S>(
handle: &mut Child,
input: S,
) -> Result<NonDetachingJoinHandle<()>, Error>
where
S: Stream<Item = InputJsonRpc> + Send + 'static,
{
use tokio::io::AsyncWriteExt;
let mut stdin = handle
.stdin
.take()
.ok_or_else(|| eyre!("Can't takeout stdin"))
.with_kind(crate::ErrorKind::Docker)?;
let handle = NonDetachingJoinHandle::from(tokio::spawn(async move {
let input = input;
tokio::pin!(input);
while let Some(input) = input.next().await {
let input = match serde_json::to_string(&input) {
Ok(a) => a,
Err(e) => {
tracing::debug!("{:?}", e);
tracing::error!("Docker Input Serialization issue");
continue;
}
};
if let Err(e) = stdin.write_all(format!("{input}\n").as_bytes()).await {
tracing::debug!("{:?}", e);
tracing::error!("Docker Input issue");
return;
}
}
}));
Ok(handle)
}
fn spawn_error_handle(handle: &mut Child) -> Result<NonDetachingJoinHandle<()>, Error> {
let id = handle.id();
let mut output = tokio::io::BufReader::new(
handle
.stderr
.take()
.ok_or_else(|| eyre!("Can't takeout stderr"))
.with_kind(crate::ErrorKind::Docker)?,
)
.lines();
Ok(NonDetachingJoinHandle::from(tokio::spawn(async move {
while let Ok(Some(line)) = output.next_line().await {
tracing::debug!("{:?}", id);
tracing::error!("Error from long running container");
tracing::error!("{}", line);
}
})))
}
fn spawn_output_handle(
handle: &mut Child,
) -> Result<(UnboundedReceiver<OutputJsonRpc>, NonDetachingJoinHandle<()>), Error> {
let mut output = tokio::io::BufReader::new(
handle
.stdout
.take()
.ok_or_else(|| eyre!("Can't takeout stdout for long running"))
.with_kind(crate::ErrorKind::Docker)?,
)
.lines();
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<OutputJsonRpc>();
Ok((
receiver,
NonDetachingJoinHandle::from(tokio::spawn(async move {
loop {
let next = output.next_line().await;
let next = match next {
Ok(Some(a)) => a,
Ok(None) => {
tracing::error!("The docker pipe is closed?");
break;
}
Err(e) => {
tracing::debug!("{:?}", e);
tracing::error!("Output from docker, killing");
break;
}
};
let next = match serde_json::from_str(&next) {
Ok(a) => a,
Err(_e) => {
tracing::trace!("Could not decode output from long running binary");
continue;
}
};
if let Err(e) = sender.send(next) {
tracing::debug!("{:?}", e);
tracing::error!("Could no longer send output");
break;
}
}
})),
))
}
}
async fn buf_reader_to_lines(
reader: impl AsyncBufRead + Unpin,

129
backend/src/procedure/js_scripts.rs
View File

@@ -2,17 +2,20 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use color_eyre::eyre::eyre;
use embassy_container_init::{ProcessGroupId, SignalGroup, SignalGroupParams};
use helpers::RpcClient;
pub use js_engine::JsError;
use js_engine::{JsExecutionEnvironment, PathForVolumeId};
use models::VolumeId;
use models::{ExecCommand, TermCommand};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use models::{ErrorKind, VolumeId};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use tracing::instrument;
use super::ProcedureName;
use crate::context::RpcContext;
use crate::s9pk::manifest::PackageId;
use crate::util::Version;
use crate::util::{GeneralGuard, Version};
use crate::volume::Volumes;
use crate::Error;
@@ -42,7 +45,7 @@ impl PathForVolumeId for Volumes {
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct JsProcedure {
#[serde(default)]
@@ -54,7 +57,7 @@ impl JsProcedure {
Ok(())
}
#[instrument(skip(directory, input, exec_command, term_command))]
#[instrument(skip(directory, input, rpc_client))]
pub async fn execute<I: Serialize, O: DeserializeOwned>(
&self,
directory: &PathBuf,
@@ -64,17 +67,32 @@ impl JsProcedure {
volumes: &Volumes,
input: Option<I>,
timeout: Option<Duration>,
exec_command: ExecCommand,
term_command: TermCommand,
gid: ProcessGroupId,
rpc_client: Option<Arc<RpcClient>>,
) -> Result<Result<O, (i32, String)>, Error> {
Ok(async move {
let cleaner_client = rpc_client.clone();
let cleaner = GeneralGuard::new(move || {
tokio::spawn(async move {
if let Some(client) = cleaner_client {
client
.request(SignalGroup, SignalGroupParams { gid, signal: 9 })
.await
.map_err(|e| {
Error::new(eyre!("{}: {:?}", e.message, e.data), ErrorKind::Docker)
})
} else {
Ok(())
}
})
});
let res = async move {
let running_action = JsExecutionEnvironment::load_from_package(
directory,
pkg_id,
pkg_version,
Box::new(volumes.clone()),
exec_command,
term_command,
gid,
rpc_client,
)
.await?
.run_action(name, input, self.args.clone());
@@ -88,7 +106,9 @@ impl JsProcedure {
Ok(output)
}
.await
.map_err(|(error, message)| (error.as_code_num(), message)))
.map_err(|(error, message)| (error.as_code_num(), message));
cleaner.drop().await.unwrap()?;
Ok(res)
}
#[instrument(skip(ctx, input))]
@@ -108,12 +128,8 @@ impl JsProcedure {
pkg_id,
pkg_version,
Box::new(volumes.clone()),
Arc::new(|_, _, _, _| {
Box::pin(async { Err("Can't run commands in sandox mode".to_string()) })
}),
Arc::new(|_| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
ProcessGroupId(0),
None,
)
.await?
.read_only_effects()
@@ -193,10 +209,8 @@ async fn js_action_execute() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()
@@ -252,10 +266,8 @@ async fn js_action_execute_error() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap();
@@ -300,10 +312,8 @@ async fn js_action_fetch() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()
@@ -341,23 +351,18 @@ async fn js_test_slow() {
let timeout = Some(Duration::from_secs(10));
tracing::debug!("testing start");
tokio::select! {
a = js_action
.execute::<serde_json::Value, serde_json::Value>(
&path,
&package_id,
&package_version,
name,
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
)
=> {a
.unwrap()
.unwrap();},
a = js_action
.execute::<serde_json::Value, serde_json::Value>(
&path,
&package_id,
&package_version,
name,
&volumes,
input,
timeout,
ProcessGroupId(0),
None,
) => { a.unwrap().unwrap(); },
_ = tokio::time::sleep(Duration::from_secs(1)) => ()
}
tracing::debug!("testing end should");
@@ -404,10 +409,8 @@ async fn js_action_var_arg() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()
@@ -452,10 +455,8 @@ async fn js_action_test_rename() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()
@@ -500,10 +501,8 @@ async fn js_action_test_deep_dir() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()
@@ -547,10 +546,8 @@ async fn js_action_test_deep_dir_escape() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()
@@ -595,10 +592,8 @@ async fn js_rsync() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()

99
backend/src/procedure/mod.rs
View File

@@ -3,7 +3,8 @@ use std::time::Duration;
use color_eyre::eyre::eyre;
use patch_db::HasModel;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use tracing::instrument;
use self::docker::{DockerContainers, DockerProcedure};
@@ -82,7 +83,7 @@ impl PackageProcedure {
}
#[cfg(feature = "js_engine")]
PackageProcedure::Script(procedure) => {
let exec_command = match ctx
let (gid, rpc_client) = match ctx
.managers
.get(&(pkg_id.clone(), pkg_version.clone()))
.await
@@ -93,23 +94,16 @@ impl PackageProcedure {
ErrorKind::NotFound,
))
}
Some(x) => x,
}
.exec_command();
let term_command = match ctx
.managers
.get(&(pkg_id.clone(), pkg_version.clone()))
.await
{
None => {
return Err(Error::new(
eyre!("No manager found for {}", pkg_id),
ErrorKind::NotFound,
))
}
Some(x) => x,
}
.term_command();
Some(man) => (
if matches!(name, ProcedureName::Main) {
man.new_main_gid()
} else {
man.new_gid()
},
man.rpc_client(),
),
};
procedure
.execute(
&ctx.datadir,
@@ -119,77 +113,14 @@ impl PackageProcedure {
volumes,
input,
timeout,
exec_command,
term_command,
gid,
rpc_client,
)
.await
}
}
}
#[instrument(skip(ctx, input))]
pub async fn inject<I: Serialize, O: DeserializeOwned + 'static>(
&self,
ctx: &RpcContext,
pkg_id: &PackageId,
pkg_version: &Version,
name: ProcedureName,
volumes: &Volumes,
input: Option<I>,
timeout: Option<Duration>,
) -> Result<Result<O, (i32, String)>, Error> {
match self {
PackageProcedure::Docker(procedure) => {
procedure
.inject(ctx, pkg_id, pkg_version, name, volumes, input, timeout)
.await
}
#[cfg(feature = "js_engine")]
PackageProcedure::Script(procedure) => {
let exec_command = match ctx
.managers
.get(&(pkg_id.clone(), pkg_version.clone()))
.await
{
None => {
return Err(Error::new(
eyre!("No manager found for {}", pkg_id),
ErrorKind::NotFound,
))
}
Some(x) => x,
}
.exec_command();
let term_command = match ctx
.managers
.get(&(pkg_id.clone(), pkg_version.clone()))
.await
{
None => {
return Err(Error::new(
eyre!("No manager found for {}", pkg_id),
ErrorKind::NotFound,
))
}
Some(x) => x,
}
.term_command();
procedure
.execute(
&ctx.datadir,
pkg_id,
pkg_version,
name,
volumes,
input,
timeout,
exec_command,
term_command,
)
.await
}
}
}
#[instrument(skip(ctx, input))]
pub async fn sandboxed<I: Serialize, O: DeserializeOwned>(
&self,