Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 12:11:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Cleanup by sending a command and kill when dropped (#1945)

Browse Source 
* fix: Cleanup by sending a command and kill when dropped

* chore: Fix the loadModule run command

* fix: cleans up failed health

* refactor long-running

* chore: Fixes?"

* refactor

* run iso ci on pr

* fix debuild

* fix tests

* switch to libc kill

* kill process by parent

* fix graceful shutdown

* recurse submodules

* fix compat build

* feat: Add back in the timeout

* chore: add the missing types for the unnstable

* inherited logs

Co-authored-by: J M <Blu-J@users.noreply.github.com>

* fix deleted code

Co-authored-by: Aiden McClelland <me@drbonez.dev>
Co-authored-by: J M <Blu-J@users.noreply.github.com>
This commit is contained in:
J M
2022-11-18 19:19:04 -07:00
committed by Aiden McClelland
parent eec8c41e20
commit a3d1b2d671
39 changed files with 1866 additions and 1845 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
backend/Cargo.lock generated
View File

@@ -1281,6 +1281,10 @@ dependencies = [
"async-stream",
"color-eyre",
"futures",
"helpers",
"imbl 2.0.0",
"nix 0.25.0",
"procfs",
"serde",
"serde_json",
"tokio",
@@ -1289,6 +1293,7 @@ dependencies = [
"tracing-error 0.2.0",
"tracing-futures",
"tracing-subscriber 0.3.16",
"yajrc 0.1.0 (git+https://github.com/dr-bonez/yajrc.git?branch=develop)",
]
[[package]]
@@ -1382,6 +1387,27 @@ dependencies = [
"termcolor",
]
[[package]]
name = "errno"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
dependencies = [
"errno-dragonfly",
"libc",
"winapi",
]
[[package]]
name = "errno-dragonfly"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "event-listener"
version = "2.5.3"
@@ -1747,9 +1773,11 @@ dependencies = [
"models",
"pin-project",
"serde",
"serde_json",
"tokio",
"tokio-stream",
"tracing",
"yajrc 0.1.0 (git+https://github.com/dr-bonez/yajrc.git?branch=develop)",
]
[[package]]
@@ -2030,6 +2058,12 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "io-lifetimes"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074"
[[package]]
name = "ipnet"
version = "2.5.1"
@@ -2336,6 +2370,12 @@ dependencies = [
"cc",
]
[[package]]
name = "linux-raw-sys"
version = "0.0.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d"
[[package]]
name = "lock_api"
version = "0.4.9"
@@ -2458,7 +2498,6 @@ dependencies = [
"bollard",
"color-eyre",
"ed25519-dalek",
"embassy_container_init",
"emver",
"mbrman",
"openssl",
@@ -3110,6 +3149,21 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "procfs"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dfb6451c91904606a1abe93e83a8ec851f45827fa84273f256ade45dc095818"
dependencies = [
"bitflags",
"byteorder",
"chrono",
"flate2",
"hex",
"lazy_static",
"rustix",
]
[[package]]
name = "proptest"
version = "1.0.0"
@@ -3452,7 +3506,7 @@ dependencies = [
"thiserror",
"tokio",
"url",
"yajrc",
"yajrc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -3519,6 +3573,20 @@ dependencies = [
"semver 1.0.14",
]
[[package]]
name = "rustix"
version = "0.35.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "727a1a6d65f786ec22df8a81ca3121107f235970dc1705ed681d3e6e8b9cd5f9"
dependencies = [
"bitflags",
"errno",
"io-lifetimes",
"libc",
"linux-raw-sys",
"windows-sys 0.42.0",
]
[[package]]
name = "rustls"
version = "0.20.7"
@@ -5593,6 +5661,17 @@ dependencies = [
"thiserror",
]
[[package]]
name = "yajrc"
version = "0.1.0"
source = "git+https://github.com/dr-bonez/yajrc.git?branch=develop#72a22f7ac2197d7a5cdce4be601cf20e5280eec5"
dependencies = [
"anyhow",
"serde",
"serde_json",
"thiserror",
]
[[package]]
name = "zeroize"
version = "1.5.7"

2
backend/src/bin/embassyd.rs
View File

@@ -4,8 +4,6 @@ use std::time::Duration;
use color_eyre::eyre::eyre;
use embassy::context::{DiagnosticContext, RpcContext};
use embassy::hostname::get_current_ip;
use embassy::net::embassy_service_http_server::EmbassyServiceHTTPServer;
#[cfg(feature = "avahi")]
use embassy::net::mdns::MdnsController;

2
backend/src/context/cli.rs
View File

@@ -1,6 +1,6 @@
use std::fs::File;
use std::io::BufReader;
use std::net::{Ipv4Addr, SocketAddr};
use std::net::Ipv4Addr;
use std::path::{Path, PathBuf};
use std::sync::Arc;

4
backend/src/install/cleanup.rs
View File

@@ -1,4 +1,6 @@
use std::{collections::HashMap, path::PathBuf, sync::Arc};
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use bollard::image::ListImagesOptions;
use patch_db::{DbHandle, LockReceipt, LockTargetId, LockType, PatchDbHandle, Verifier};

2
backend/src/lib.rs
View File

@@ -1,3 +1,5 @@
#![recursion_limit = "256"]
pub const DEFAULT_MARKETPLACE: &str = "https://registry.start9.com";
pub const COMMUNITY_MARKETPLACE: &str = "https://community-registry.start9.com";
pub const BUFFER_SIZE: usize = 1024;

1587
backend/src/manager/mod.rs
View File

File diff suppressed because it is too large Load Diff

42
backend/src/manager/sync.rs
View File

@@ -1,23 +1,18 @@
use std::convert::TryInto;
use std::sync::atomic::Ordering;
use std::collections::BTreeMap;
use std::time::Duration;
use std::{collections::BTreeMap, sync::Arc};
use chrono::Utc;
use super::{pause, resume, start, stop, ManagerSharedState, PersistantContainer, Status};
use super::{pause, resume, start, stop, ManagerSharedState, Status};
use crate::status::MainStatus;
use crate::Error;
/// Allocates a db handle. DO NOT CALL with a db handle already in scope
async fn synchronize_once(
shared: &ManagerSharedState,
persistant_container: Arc<PersistantContainer>,
) -> Result<Status, Error> {
let mut db = shared.ctx.db.handle();
async fn synchronize_once(shared: &ManagerSharedState) -> Result<Status, Error> {
let mut db = shared.seed.ctx.db.handle();
let mut status = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&shared.manifest.id)
.idx_model(&shared.seed.manifest.id)
.expect(&mut db)
.await?
.installed()
@@ -27,7 +22,7 @@ async fn synchronize_once(
.main()
.get_mut(&mut db)
.await?;
let manager_status = shared.status.load(Ordering::SeqCst).try_into().unwrap();
let manager_status = *shared.status.1.borrow();
match manager_status {
Status::Stopped => match &mut *status {
MainStatus::Stopped => (),
@@ -48,16 +43,16 @@ async fn synchronize_once(
},
Status::Starting => match *status {
MainStatus::Stopped | MainStatus::Stopping | MainStatus::Restarting => {
stop(shared, persistant_container).await?;
stop(shared).await?;
}
MainStatus::Starting { .. } | MainStatus::Running { .. } => (),
MainStatus::BackingUp { .. } => {
pause(shared, persistant_container).await?;
pause(shared).await?;
}
},
Status::Running => match *status {
MainStatus::Stopped | MainStatus::Stopping | MainStatus::Restarting => {
stop(shared, persistant_container).await?;
stop(shared).await?;
}
MainStatus::Starting { .. } => {
*status = MainStatus::Running {
@@ -67,12 +62,12 @@ async fn synchronize_once(
}
MainStatus::Running { .. } => (),
MainStatus::BackingUp { .. } => {
pause(shared, persistant_container).await?;
pause(shared).await?;
}
},
Status::Paused => match *status {
MainStatus::Stopped | MainStatus::Stopping | MainStatus::Restarting => {
stop(shared, persistant_container).await?;
stop(shared).await?;
}
MainStatus::Starting { .. } | MainStatus::Running { .. } => {
resume(shared).await?;
@@ -85,21 +80,20 @@ async fn synchronize_once(
Ok(manager_status)
}
pub async fn synchronizer(
shared: &ManagerSharedState,
persistant_container: Arc<PersistantContainer>,
) {
pub async fn synchronizer(shared: &ManagerSharedState) {
let mut status_recv = shared.status.0.subscribe();
loop {
tokio::select! {
_ = tokio::time::sleep(Duration::from_secs(5)) => (),
_ = shared.synchronize_now.notified() => (),
_ = status_recv.changed() => (),
}
let status = match synchronize_once(shared, persistant_container.clone()).await {
let status = match synchronize_once(shared).await {
Err(e) => {
tracing::error!(
"Synchronizer for {}@{} failed: {}",
shared.manifest.id,
shared.manifest.version,
shared.seed.manifest.id,
shared.seed.manifest.version,
e
);
tracing::debug!("{:?}", e);
@@ -107,7 +101,7 @@ pub async fn synchronizer(
}
Ok(status) => status,
};
tracing::trace!("{} status synchronized", shared.manifest.id);
tracing::trace!("{} status synchronized", shared.seed.manifest.id);
shared.synchronized.notify_waiters();
match status {
Status::Shutdown => {

1
backend/src/net/cert_resolver.rs
View File

@@ -105,7 +105,6 @@ impl ResolvesServerCert for EmbassyCertResolver {
match hostname_raw {
Some(hostname_str) => {
let full_fqdn = match ResourceFqdn::from_str(hostname_str) {
Ok(fqdn) => fqdn,
Err(_) => {

6
backend/src/net/mod.rs
View File

@@ -7,22 +7,20 @@ use indexmap::IndexSet;
use rpc_toolkit::command;
use self::interface::InterfaceId;
use crate::net::interface::LanPortConfig;
use crate::util::serde::Port;
use crate::Error;
pub mod cert_resolver;
pub mod dns;
pub mod embassy_service_http_server;
pub mod interface;
#[cfg(feature = "avahi")]
pub mod mdns;
pub mod embassy_service_http_server;
pub mod net_controller;
pub mod net_utils;
pub mod proxy_controller;
pub mod ssl;
pub mod cert_resolver;
pub mod static_server;
pub mod tor;
pub mod vhost_controller;

4
backend/src/net/net_controller.rs
View File

@@ -11,7 +11,7 @@ use torut::onion::{OnionAddressV3, TorSecretKeyV3};
use tracing::instrument;
use crate::context::RpcContext;
use crate::hostname::{get_current_ip, get_embassyd_tor_addr, get_hostname, HostNameReceipt};
use crate::hostname::{get_embassyd_tor_addr, get_hostname, HostNameReceipt};
use crate::net::dns::DnsController;
use crate::net::interface::{Interface, TorConfig};
#[cfg(feature = "avahi")]
@@ -115,7 +115,7 @@ impl NetController {
async fn setup_embassy_http_ui_handle(rpc_ctx: RpcContext) -> Result<(), Error> {
let host_name = rpc_ctx.net_controller.proxy.get_hostname().await;
let embassy_tor_addr = get_embassyd_tor_addr(rpc_ctx.clone()).await?;
let embassy_tor_fqdn: ResourceFqdn = embassy_tor_addr.parse()?;
let host_name_fqdn: ResourceFqdn = host_name.parse()?;

2
backend/src/net/proxy_controller.rs
View File

@@ -15,7 +15,7 @@ use tokio::net::TcpStream;
use tokio::sync::Mutex;
use tracing::{error, info, instrument};
use crate::net::net_utils::{host_addr_fqdn, ResourceFqdn};
use crate::net::net_utils::ResourceFqdn;
use crate::net::ssl::SslManager;
use crate::net::vhost_controller::VHOSTController;
use crate::net::{HttpClient, HttpHandler, InterfaceMetadata, PackageNetInfo};

1
backend/src/net/static_server.rs
View File

@@ -8,7 +8,6 @@ use digest::Digest;
use futures::FutureExt;
use http::response::Builder;
use hyper::{Body, Method, Request, Response, StatusCode};
use rpc_toolkit::rpc_handler;
use tokio::fs::File;
use tokio_util::codec::{BytesCodec, FramedRead};

3
backend/src/net/tor.rs
View File

@@ -377,8 +377,7 @@ pub async fn tor_health_check(client: &Client, tor_controller: &TorController) {
.await;
if let Err(e) = result {
let mut num_attempt = 1;
tracing::error!(
"Unable to reach self over tor, we will retry now...");
tracing::error!("Unable to reach self over tor, we will retry now...");
tracing::error!("The first TOR error: {}", e);
loop {

9
backend/src/net/vhost_controller.rs
View File

@@ -3,12 +3,12 @@ use std::net::SocketAddr;
use std::sync::Arc;
use tokio_rustls::rustls::ServerConfig;
use crate::net::cert_resolver::EmbassyCertResolver;
use crate::net::embassy_service_http_server::{EmbassyServiceHTTPServer};
use crate::net::cert_resolver::EmbassyCertResolver;
use crate::net::embassy_service_http_server::EmbassyServiceHTTPServer;
use crate::net::net_utils::ResourceFqdn;
use crate::net::HttpHandler;
use crate::Error;
use crate::net::net_utils::ResourceFqdn;
pub struct VHOSTController {
pub service_servers: BTreeMap<u16, EmbassyServiceHTTPServer>,
@@ -67,7 +67,6 @@ impl VHOSTController {
None
};
let mut new_service_server =
EmbassyServiceHTTPServer::new(self.embassyd_addr.ip(), external_svc_port, ssl_cfg)
.await?;
@@ -76,7 +75,7 @@ impl VHOSTController {
.await?;
self.service_servers
.insert(external_svc_port, new_service_server);
Ok(())
}
}

243
backend/src/procedure/docker.rs
View File

@@ -9,19 +9,15 @@ use async_stream::stream;
use bollard::container::RemoveContainerOptions;
use color_eyre::eyre::eyre;
use color_eyre::Report;
use embassy_container_init::{InputJsonRpc, OutputJsonRpc};
use futures::future::Either as EitherFuture;
use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt};
use helpers::NonDetachingJoinHandle;
use futures::TryStreamExt;
use helpers::{NonDetachingJoinHandle, RpcClient};
use nix::sys::signal;
use nix::unistd::Pid;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio::{
io::{AsyncBufRead, AsyncBufReadExt, BufReader},
process::Child,
sync::mpsc::UnboundedReceiver,
};
use tokio::io::{AsyncBufRead, AsyncBufReadExt, BufReader};
use tracing::instrument;
use super::ProcedureName;
@@ -70,6 +66,57 @@ pub struct DockerContainer {
#[serde(default)]
pub system: bool,
}
impl DockerContainer {
/// We created a new exec runner, where we are going to be passing the commands for it to run.
/// Idea is that we are going to send it command and get the inputs be filtered back from the manager.
/// Then we could in theory run commands without the cost of running the docker exec which is known to have
/// a dely of > 200ms which is not acceptable.
#[instrument(skip(ctx))]
pub async fn long_running_execute(
&self,
ctx: &RpcContext,
pkg_id: &PackageId,
pkg_version: &Version,
volumes: &Volumes,
) -> Result<(LongRunning, RpcClient), Error> {
let container_name = DockerProcedure::container_name(pkg_id, None);
let mut cmd = LongRunning::setup_long_running_docker_cmd(
self,
ctx,
&container_name,
volumes,
pkg_id,
pkg_version,
)
.await?;
let mut handle = cmd.spawn().with_kind(crate::ErrorKind::Docker)?;
let client =
if let (Some(stdin), Some(stdout)) = (handle.stdin.take(), handle.stdout.take()) {
RpcClient::new(stdin, stdout)
} else {
return Err(Error::new(
eyre!("No stdin/stdout handle for container init"),
crate::ErrorKind::Incoherent,
));
};
let running_output = NonDetachingJoinHandle::from(tokio::spawn(async move {
if let Err(err) = handle
.wait()
.await
.map_err(|e| eyre!("Runtime error: {e:?}"))
{
tracing::error!("{}", err);
tracing::debug!("{:?}", err);
}
}));
Ok((LongRunning { running_output }, client))
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
@@ -122,23 +169,6 @@ impl DockerProcedure {
shm_size_mb: container.shm_size_mb,
}
}
#[cfg(feature = "js_engine")]
pub fn main_docker_procedure_js(
container: &DockerContainer,
_procedure: &super::js_scripts::JsProcedure,
) -> DockerProcedure {
DockerProcedure {
image: container.image.clone(),
system: container.system,
entrypoint: "sleep".to_string(),
args: Vec::new(),
inject: false,
mounts: container.mounts.clone(),
io_format: None,
sigterm_timeout: container.sigterm_timeout,
shm_size_mb: container.shm_size_mb,
}
}
pub fn validate(
&self,
@@ -346,64 +376,6 @@ impl DockerProcedure {
)
}
/// We created a new exec runner, where we are going to be passing the commands for it to run.
/// Idea is that we are going to send it command and get the inputs be filtered back from the manager.
/// Then we could in theory run commands without the cost of running the docker exec which is known to have
/// a dely of > 200ms which is not acceptable.
#[instrument(skip(ctx, input))]
pub async fn long_running_execute<S>(
&self,
ctx: &RpcContext,
pkg_id: &PackageId,
pkg_version: &Version,
name: ProcedureName,
volumes: &Volumes,
input: S,
) -> Result<LongRunning, Error>
where
S: Stream<Item = InputJsonRpc> + Send + 'static,
{
let name = name.docker_name();
let name: Option<&str> = name.as_deref();
let container_name = Self::container_name(pkg_id, name);
let mut cmd = LongRunning::setup_long_running_docker_cmd(
self,
ctx,
&container_name,
volumes,
pkg_id,
pkg_version,
)
.await?;
let mut handle = cmd.spawn().with_kind(crate::ErrorKind::Docker)?;
let input_handle = LongRunning::spawn_input_handle(&mut handle, input)?
.map_err(|e| eyre!("Input Handle Error: {e:?}"));
let (output, output_handle) = LongRunning::spawn_output_handle(&mut handle)?;
let output_handle = output_handle.map_err(|e| eyre!("Output Handle Error: {e:?}"));
let err_handle = LongRunning::spawn_error_handle(&mut handle)?
.map_err(|e| eyre!("Err Handle Error: {e:?}"));
let running_output = NonDetachingJoinHandle::from(tokio::spawn(async move {
if let Err(err) = tokio::select!(
x = handle.wait().map_err(|e| eyre!("Runtime error: {e:?}")) => x.map(|_| ()),
x = err_handle => x.map(|_| ()),
x = output_handle => x.map(|_| ()),
x = input_handle => x.map(|_| ())
) {
tracing::debug!("{:?}", err);
tracing::error!("Join error");
}
}));
Ok(LongRunning {
output,
running_output,
})
}
#[instrument(skip(_ctx, input))]
pub async fn inject<I: Serialize, O: DeserializeOwned>(
&self,
@@ -788,13 +760,12 @@ impl<T> RingVec<T> {
/// We wanted a long running since we want to be able to have the equivelent to the docker execute without the heavy costs of 400 + ms time lag.
/// Also the long running let's us have the ability to start/ end the services quicker.
pub struct LongRunning {
pub output: UnboundedReceiver<OutputJsonRpc>,
pub running_output: NonDetachingJoinHandle<()>,
}
impl LongRunning {
async fn setup_long_running_docker_cmd(
docker: &DockerProcedure,
docker: &DockerContainer,
ctx: &RpcContext,
container_name: &str,
volumes: &Volumes,
@@ -865,7 +836,7 @@ impl LongRunning {
cmd.arg(docker.image.for_package(pkg_id, Some(pkg_version)));
}
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::inherit());
cmd.stdin(std::process::Stdio::piped());
Ok(cmd)
}
@@ -894,104 +865,6 @@ impl LongRunning {
Err(e) => Err(e)?,
}
}
fn spawn_input_handle<S>(
handle: &mut Child,
input: S,
) -> Result<NonDetachingJoinHandle<()>, Error>
where
S: Stream<Item = InputJsonRpc> + Send + 'static,
{
use tokio::io::AsyncWriteExt;
let mut stdin = handle
.stdin
.take()
.ok_or_else(|| eyre!("Can't takeout stdin"))
.with_kind(crate::ErrorKind::Docker)?;
let handle = NonDetachingJoinHandle::from(tokio::spawn(async move {
let input = input;
tokio::pin!(input);
while let Some(input) = input.next().await {
let input = match serde_json::to_string(&input) {
Ok(a) => a,
Err(e) => {
tracing::debug!("{:?}", e);
tracing::error!("Docker Input Serialization issue");
continue;
}
};
if let Err(e) = stdin.write_all(format!("{input}\n").as_bytes()).await {
tracing::debug!("{:?}", e);
tracing::error!("Docker Input issue");
return;
}
}
}));
Ok(handle)
}
fn spawn_error_handle(handle: &mut Child) -> Result<NonDetachingJoinHandle<()>, Error> {
let id = handle.id();
let mut output = tokio::io::BufReader::new(
handle
.stderr
.take()
.ok_or_else(|| eyre!("Can't takeout stderr"))
.with_kind(crate::ErrorKind::Docker)?,
)
.lines();
Ok(NonDetachingJoinHandle::from(tokio::spawn(async move {
while let Ok(Some(line)) = output.next_line().await {
tracing::debug!("{:?}", id);
tracing::error!("Error from long running container");
tracing::error!("{}", line);
}
})))
}
fn spawn_output_handle(
handle: &mut Child,
) -> Result<(UnboundedReceiver<OutputJsonRpc>, NonDetachingJoinHandle<()>), Error> {
let mut output = tokio::io::BufReader::new(
handle
.stdout
.take()
.ok_or_else(|| eyre!("Can't takeout stdout for long running"))
.with_kind(crate::ErrorKind::Docker)?,
)
.lines();
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<OutputJsonRpc>();
Ok((
receiver,
NonDetachingJoinHandle::from(tokio::spawn(async move {
loop {
let next = output.next_line().await;
let next = match next {
Ok(Some(a)) => a,
Ok(None) => {
tracing::error!("The docker pipe is closed?");
break;
}
Err(e) => {
tracing::debug!("{:?}", e);
tracing::error!("Output from docker, killing");
break;
}
};
let next = match serde_json::from_str(&next) {
Ok(a) => a,
Err(_e) => {
tracing::trace!("Could not decode output from long running binary");
continue;
}
};
if let Err(e) = sender.send(next) {
tracing::debug!("{:?}", e);
tracing::error!("Could no longer send output");
break;
}
}
})),
))
}
}
async fn buf_reader_to_lines(
reader: impl AsyncBufRead + Unpin,

129
backend/src/procedure/js_scripts.rs
View File

@@ -2,17 +2,20 @@ use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use color_eyre::eyre::eyre;
use embassy_container_init::{ProcessGroupId, SignalGroup, SignalGroupParams};
use helpers::RpcClient;
pub use js_engine::JsError;
use js_engine::{JsExecutionEnvironment, PathForVolumeId};
use models::VolumeId;
use models::{ExecCommand, TermCommand};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use models::{ErrorKind, VolumeId};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use tracing::instrument;
use super::ProcedureName;
use crate::context::RpcContext;
use crate::s9pk::manifest::PackageId;
use crate::util::Version;
use crate::util::{GeneralGuard, Version};
use crate::volume::Volumes;
use crate::Error;
@@ -42,7 +45,7 @@ impl PathForVolumeId for Volumes {
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct JsProcedure {
#[serde(default)]
@@ -54,7 +57,7 @@ impl JsProcedure {
Ok(())
}
#[instrument(skip(directory, input, exec_command, term_command))]
#[instrument(skip(directory, input, rpc_client))]
pub async fn execute<I: Serialize, O: DeserializeOwned>(
&self,
directory: &PathBuf,
@@ -64,17 +67,32 @@ impl JsProcedure {
volumes: &Volumes,
input: Option<I>,
timeout: Option<Duration>,
exec_command: ExecCommand,
term_command: TermCommand,
gid: ProcessGroupId,
rpc_client: Option<Arc<RpcClient>>,
) -> Result<Result<O, (i32, String)>, Error> {
Ok(async move {
let cleaner_client = rpc_client.clone();
let cleaner = GeneralGuard::new(move || {
tokio::spawn(async move {
if let Some(client) = cleaner_client {
client
.request(SignalGroup, SignalGroupParams { gid, signal: 9 })
.await
.map_err(|e| {
Error::new(eyre!("{}: {:?}", e.message, e.data), ErrorKind::Docker)
})
} else {
Ok(())
}
})
});
let res = async move {
let running_action = JsExecutionEnvironment::load_from_package(
directory,
pkg_id,
pkg_version,
Box::new(volumes.clone()),
exec_command,
term_command,
gid,
rpc_client,
)
.await?
.run_action(name, input, self.args.clone());
@@ -88,7 +106,9 @@ impl JsProcedure {
Ok(output)
}
.await
.map_err(|(error, message)| (error.as_code_num(), message)))
.map_err(|(error, message)| (error.as_code_num(), message));
cleaner.drop().await.unwrap()?;
Ok(res)
}
#[instrument(skip(ctx, input))]
@@ -108,12 +128,8 @@ impl JsProcedure {
pkg_id,
pkg_version,
Box::new(volumes.clone()),
Arc::new(|_, _, _, _| {
Box::pin(async { Err("Can't run commands in sandox mode".to_string()) })
}),
Arc::new(|_| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
ProcessGroupId(0),
None,
)
.await?
.read_only_effects()
@@ -193,10 +209,8 @@ async fn js_action_execute() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()
@@ -252,10 +266,8 @@ async fn js_action_execute_error() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap();
@@ -300,10 +312,8 @@ async fn js_action_fetch() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()
@@ -341,23 +351,18 @@ async fn js_test_slow() {
let timeout = Some(Duration::from_secs(10));
tracing::debug!("testing start");
tokio::select! {
a = js_action
.execute::<serde_json::Value, serde_json::Value>(
&path,
&package_id,
&package_version,
name,
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
)
=> {a
.unwrap()
.unwrap();},
a = js_action
.execute::<serde_json::Value, serde_json::Value>(
&path,
&package_id,
&package_version,
name,
&volumes,
input,
timeout,
ProcessGroupId(0),
None,
) => { a.unwrap().unwrap(); },
_ = tokio::time::sleep(Duration::from_secs(1)) => ()
}
tracing::debug!("testing end should");
@@ -404,10 +409,8 @@ async fn js_action_var_arg() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()
@@ -452,10 +455,8 @@ async fn js_action_test_rename() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()
@@ -500,10 +501,8 @@ async fn js_action_test_deep_dir() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()
@@ -547,10 +546,8 @@ async fn js_action_test_deep_dir_escape() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()
@@ -595,10 +592,8 @@ async fn js_rsync() {
&volumes,
input,
timeout,
Arc::new(|_, _, _, _| {
Box::pin(async move { Err("Can't run commands in test".to_string()) })
}),
Arc::new(|_| Box::pin(async move { Err("Can't run commands in test".to_string()) })),
ProcessGroupId(0),
None,
)
.await
.unwrap()

99
backend/src/procedure/mod.rs
View File

@@ -3,7 +3,8 @@ use std::time::Duration;
use color_eyre::eyre::eyre;
use patch_db::HasModel;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use tracing::instrument;
use self::docker::{DockerContainers, DockerProcedure};
@@ -82,7 +83,7 @@ impl PackageProcedure {
}
#[cfg(feature = "js_engine")]
PackageProcedure::Script(procedure) => {
let exec_command = match ctx
let (gid, rpc_client) = match ctx
.managers
.get(&(pkg_id.clone(), pkg_version.clone()))
.await
@@ -93,23 +94,16 @@ impl PackageProcedure {
ErrorKind::NotFound,
))
}
Some(x) => x,
}
.exec_command();
let term_command = match ctx
.managers
.get(&(pkg_id.clone(), pkg_version.clone()))
.await
{
None => {
return Err(Error::new(
eyre!("No manager found for {}", pkg_id),
ErrorKind::NotFound,
))
}
Some(x) => x,
}
.term_command();
Some(man) => (
if matches!(name, ProcedureName::Main) {
man.new_main_gid()
} else {
man.new_gid()
},
man.rpc_client(),
),
};
procedure
.execute(
&ctx.datadir,
@@ -119,77 +113,14 @@ impl PackageProcedure {
volumes,
input,
timeout,
exec_command,
term_command,
gid,
rpc_client,
)
.await
}
}
}
#[instrument(skip(ctx, input))]
pub async fn inject<I: Serialize, O: DeserializeOwned + 'static>(
&self,
ctx: &RpcContext,
pkg_id: &PackageId,
pkg_version: &Version,
name: ProcedureName,
volumes: &Volumes,
input: Option<I>,
timeout: Option<Duration>,
) -> Result<Result<O, (i32, String)>, Error> {
match self {
PackageProcedure::Docker(procedure) => {
procedure
.inject(ctx, pkg_id, pkg_version, name, volumes, input, timeout)
.await
}
#[cfg(feature = "js_engine")]
PackageProcedure::Script(procedure) => {
let exec_command = match ctx
.managers
.get(&(pkg_id.clone(), pkg_version.clone()))
.await
{
None => {
return Err(Error::new(
eyre!("No manager found for {}", pkg_id),
ErrorKind::NotFound,
))
}
Some(x) => x,
}
.exec_command();
let term_command = match ctx
.managers
.get(&(pkg_id.clone(), pkg_version.clone()))
.await
{
None => {
return Err(Error::new(
eyre!("No manager found for {}", pkg_id),
ErrorKind::NotFound,
))
}
Some(x) => x,
}
.term_command();
procedure
.execute(
&ctx.datadir,
pkg_id,
pkg_version,
name,
volumes,
input,
timeout,
exec_command,
term_command,
)
.await
}
}
}
#[instrument(skip(ctx, input))]
pub async fn sandboxed<I: Serialize, O: DeserializeOwned>(
&self,

1
backend/src/s9pk/builder.rs
View File

@@ -1,4 +1,3 @@
use nom::combinator::success;
use sha2_old::{Digest, Sha512};
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, SeekFrom};
use tracing::instrument;

3
backend/src/s9pk/manifest.rs
View File

@@ -6,6 +6,7 @@ use patch_db::HasModel;
use serde::{Deserialize, Serialize};
use url::Url;
use super::git_hash::GitHash;
use crate::action::Actions;
use crate::backup::BackupActions;
use crate::config::action::ConfigActions;
@@ -20,8 +21,6 @@ use crate::version::{Current, VersionT};
use crate::volume::Volumes;
use crate::Error;
use super::git_hash::GitHash;
fn current_version() -> Version {
Current::new().semver().into()
}

3
backend/src/update/mod.rs
View File

@@ -24,7 +24,6 @@ use crate::sound::{
CIRCLE_OF_5THS_SHORT, UPDATE_FAILED_1, UPDATE_FAILED_2, UPDATE_FAILED_3, UPDATE_FAILED_4,
};
use crate::update::latest_information::LatestInformation;
use crate::util::Invoke;
use crate::version::{Current, VersionT};
use crate::{Error, ErrorKind, ResultExt, IS_RASPBERRY_PI};
@@ -250,7 +249,7 @@ impl EosUrl {
};
Ok(format!("{host}::{version}/{arch}/")
.parse()
.map_err(|e| Error::new(eyre!("Could not parse path"), ErrorKind::ParseUrl))?)
.map_err(|_| Error::new(eyre!("Could not parse path"), ErrorKind::ParseUrl))?)
}
}

6
backend/src/version/v0_3_2.rs
View File

@@ -2,10 +2,8 @@ use emver::VersionRange;
use super::v0_3_0::V0_3_0_COMPAT;
use super::*;
use crate::{
config::util::MergeWith,
hostname::{generate_id, sync_hostname},
};
use crate::config::util::MergeWith;
use crate::hostname::{generate_id, sync_hostname};
const V0_3_2: emver::Version = emver::Version::new(0, 3, 2, 0);