Feat/long running sockets (#2090)

* wip: Working on sockets, but can't connect?

* simplify unix socket connection

* wip: Get responses back from the server at least once.

* WIP: Get the sockets working'

* feat: Sockets can start/ stop/ config/ properites/ uninstall

* fix: Restart services

* Fix: Sockets work and can stop main and not kill client

Co-authored-by: Aiden McClelland <me@drbonez.dev>

backend/src/backup/restore.rs

@@ -1,18 +1,17 @@
+use std::collections::BTreeMap;
 use std::path::Path;
 use std::sync::atomic::Ordering;
 use std::sync::Arc;
 use std::time::Duration;
-use std::{collections::BTreeMap, pin::Pin};
 
 use clap::ArgMatches;
 use color_eyre::eyre::eyre;
-use futures::{future::BoxFuture, stream, Future};
+use futures::{future::BoxFuture, stream};
 use futures::{FutureExt, StreamExt};
 use openssl::x509::X509;
 use patch_db::{DbHandle, PatchDbHandle};
 use rpc_toolkit::command;
 use tokio::fs::File;
-use tokio::task::JoinHandle;
 use torut::onion::OnionAddressV3;
 use tracing::instrument;
 

backend/src/manager/mod.rs

@@ -9,7 +9,7 @@ use std::time::Duration;
 use bollard::container::{KillContainerOptions, StopContainerOptions};
 use color_eyre::eyre::eyre;
 use embassy_container_init::{ProcessGroupId, SignalGroupParams};
-use helpers::RpcClient;
+use helpers::UnixRpcClient;
 use nix::sys::signal::Signal;
 use patch_db::DbHandle;
 use sqlx::{Executor, Postgres};
@@ -359,7 +359,7 @@ impl Manager {
         gid
     }
 
-    pub fn rpc_client(&self) -> Option<Arc<RpcClient>> {
+    pub fn rpc_client(&self) -> Option<Arc<UnixRpcClient>> {
         self.shared
             .persistent_container
             .as_ref()
@@ -449,7 +449,7 @@ async fn manager_thread_loop(mut recv: Receiver<OnStop>, thread_shared: &Arc<Man
 
 pub struct PersistentContainer {
     _running_docker: NonDetachingJoinHandle<()>,
-    rpc_client: Receiver<Arc<RpcClient>>,
+    rpc_client: Receiver<Arc<UnixRpcClient>>,
 }
 
 impl PersistentContainer {
@@ -471,12 +471,12 @@ impl PersistentContainer {
 async fn spawn_persistent_container(
     seed: Arc<ManagerSeed>,
     container: DockerContainer,
-) -> Result<(NonDetachingJoinHandle<()>, Receiver<Arc<RpcClient>>), Error> {
+) -> Result<(NonDetachingJoinHandle<()>, Receiver<Arc<UnixRpcClient>>), Error> {
     let (send_inserter, inserter) = oneshot::channel();
     Ok((
         tokio::task::spawn(async move {
-            let mut inserter_send: Option<Sender<Arc<RpcClient>>> = None;
-            let mut send_inserter: Option<oneshot::Sender<Receiver<Arc<RpcClient>>>> = Some(send_inserter);
+            let mut inserter_send: Option<Sender<Arc<UnixRpcClient>>> = None;
+            let mut send_inserter: Option<oneshot::Sender<Receiver<Arc<UnixRpcClient>>>> = Some(send_inserter);
             loop {
                 if let Err(e) = async {
                     let interfaces = main_interfaces(&*seed)?;
@@ -518,6 +518,7 @@ async fn spawn_persistent_container(
                 } else {
                     break;
                 }
+                tokio::time::sleep(Duration::from_millis(200)).await;
             }
         })
         .into(),
@@ -528,7 +529,7 @@ async fn spawn_persistent_container(
 async fn long_running_docker(
     seed: &ManagerSeed,
     container: &DockerContainer,
-) -> Result<(LongRunning, RpcClient), Error> {
+) -> Result<(LongRunning, UnixRpcClient), Error> {
     container
         .long_running_execute(
             &seed.ctx,

backend/src/procedure/docker.rs

@@ -2,7 +2,7 @@ use std::borrow::Cow;
 use std::collections::{BTreeMap, BTreeSet, VecDeque};
 use std::ffi::{OsStr, OsString};
 use std::net::Ipv4Addr;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::time::Duration;
 
 use async_stream::stream;
@@ -11,13 +11,16 @@ use color_eyre::eyre::eyre;
 use color_eyre::Report;
 use futures::future::Either as EitherFuture;
 use futures::TryStreamExt;
-use helpers::{NonDetachingJoinHandle, RpcClient};
+use helpers::{NonDetachingJoinHandle, UnixRpcClient};
 use nix::sys::signal;
 use nix::unistd::Pid;
 use serde::de::DeserializeOwned;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use tokio::io::{AsyncBufRead, AsyncBufReadExt, BufReader};
+use tokio::{
+    io::{AsyncBufRead, AsyncBufReadExt, BufReader},
+    time::timeout,
+};
 use tracing::instrument;
 
 use super::ProcedureName;
@@ -66,6 +69,7 @@ pub struct DockerContainer {
     #[serde(default)]
     pub system: bool,
 }
+
 impl DockerContainer {
     /// We created a new exec runner, where we are going to be passing the commands for it to run.
     /// Idea is that we are going to send it command and get the inputs be filtered back from the manager.
@@ -78,9 +82,16 @@ impl DockerContainer {
         pkg_id: &PackageId,
         pkg_version: &Version,
         volumes: &Volumes,
-    ) -> Result<(LongRunning, RpcClient), Error> {
+    ) -> Result<(LongRunning, UnixRpcClient), Error> {
         let container_name = DockerProcedure::container_name(pkg_id, None);
 
+        let socket_path =
+            Path::new("/tmp/embassy/containers").join(format!("{pkg_id}_{pkg_version}"));
+        if tokio::fs::metadata(&socket_path).await.is_ok() {
+            tokio::fs::remove_dir_all(&socket_path).await?;
+        }
+        tokio::fs::create_dir_all(&socket_path).await?;
+
         let mut cmd = LongRunning::setup_long_running_docker_cmd(
             self,
             ctx,
@@ -88,20 +99,13 @@ impl DockerContainer {
             volumes,
             pkg_id,
             pkg_version,
+            &socket_path,
         )
         .await?;
 
         let mut handle = cmd.spawn().with_kind(crate::ErrorKind::Docker)?;
 
-        let client =
-            if let (Some(stdin), Some(stdout)) = (handle.stdin.take(), handle.stdout.take()) {
-                RpcClient::new(stdin, stdout)
-            } else {
-                return Err(Error::new(
-                    eyre!("No stdin/stdout handle for container init"),
-                    crate::ErrorKind::Incoherent,
-                ));
-            };
+        let client = UnixRpcClient::new(socket_path.join("rpc.sock"));
 
         let running_output = NonDetachingJoinHandle::from(tokio::spawn(async move {
             if let Err(err) = handle
@@ -114,6 +118,19 @@ impl DockerContainer {
             }
         }));
 
+        {
+            let socket = socket_path.join("rpc.sock");
+            if let Err(_err) = timeout(Duration::from_secs(1), async move {
+                while tokio::fs::metadata(&socket).await.is_err() {
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                }
+            })
+            .await
+            {
+                tracing::error!("Timed out waiting for init to create socket");
+            }
+        }
+
         Ok((LongRunning { running_output }, client))
     }
 }
@@ -771,9 +788,10 @@ impl LongRunning {
         volumes: &Volumes,
         pkg_id: &PackageId,
         pkg_version: &Version,
+        socket_path: &Path,
     ) -> Result<tokio::process::Command, Error> {
-        const INIT_EXEC: &str = "/start9/embassy_container_init";
-        const BIND_LOCATION: &str = "/usr/lib/embassy/container";
+        const INIT_EXEC: &str = "/start9/bin/embassy_container_init";
+        const BIND_LOCATION: &str = "/usr/lib/embassy/container/";
         tracing::trace!("setup_long_running_docker_cmd");
 
         LongRunning::cleanup_previous_container(ctx, container_name).await?;
@@ -799,7 +817,14 @@ impl LongRunning {
             .arg("--network=start9")
             .arg(format!("--add-host=embassy:{}", Ipv4Addr::from(HOST_IP)))
             .arg("--mount")
-            .arg(format!("type=bind,src={BIND_LOCATION},dst=/start9"))
+            .arg(format!(
+                "type=bind,src={BIND_LOCATION},dst=/start9/bin/,readonly"
+            ))
+            .arg("--mount")
+            .arg(format!(
+                "type=bind,src={input},dst=/start9/sockets/",
+                input = socket_path.display()
+            ))
             .arg("--name")
             .arg(&container_name)
             .arg(format!("--hostname={}", &container_name))

backend/src/procedure/js_scripts.rs

@@ -4,7 +4,7 @@ use std::time::Duration;
 
 use color_eyre::eyre::eyre;
 use embassy_container_init::{ProcessGroupId, SignalGroup, SignalGroupParams};
-use helpers::RpcClient;
+use helpers::UnixRpcClient;
 pub use js_engine::JsError;
 use js_engine::{JsExecutionEnvironment, PathForVolumeId};
 use models::{ErrorKind, VolumeId};
@@ -68,7 +68,7 @@ impl JsProcedure {
         input: Option<I>,
         timeout: Option<Duration>,
         gid: ProcessGroupId,
-        rpc_client: Option<Arc<RpcClient>>,
+        rpc_client: Option<Arc<UnixRpcClient>>,
     ) -> Result<Result<O, (i32, String)>, Error> {
         let cleaner_client = rpc_client.clone();
         let cleaner = GeneralGuard::new(move || {
@@ -96,7 +96,7 @@ impl JsProcedure {
             )
             .await?
             .run_action(name, input, self.args.clone());
-            let output: ErrorValue = match timeout {
+            let output: Option<ErrorValue> = match timeout {
                 Some(timeout_duration) => tokio::time::timeout(timeout_duration, running_action)
                     .await
                     .map_err(|_| (JsError::Timeout, "Timed out. Retrying soon...".to_owned()))??,
@@ -134,7 +134,7 @@ impl JsProcedure {
             .await?
             .read_only_effects()
             .run_action(name, input, self.args.clone());
-            let output: ErrorValue = match timeout {
+            let output: Option<ErrorValue> = match timeout {
                 Some(timeout_duration) => tokio::time::timeout(timeout_duration, running_action)
                     .await
                     .map_err(|_| (JsError::Timeout, "Timed out. Retrying soon...".to_owned()))??,
@@ -149,8 +149,9 @@ impl JsProcedure {
 }
 
 fn unwrap_known_error<O: DeserializeOwned>(
-    error_value: ErrorValue,
+    error_value: Option<ErrorValue>,
 ) -> Result<O, (JsError, String)> {
+    let error_value = error_value.unwrap_or_else(|| ErrorValue::Result(serde_json::Value::Null));
     match error_value {
         ErrorValue::Error(error) => Err((JsError::Javascript, error)),
         ErrorValue::ErrorCode((code, message)) => Err((JsError::Code(code), message)),

backend/src/s9pk/reader.rs

@@ -231,16 +231,6 @@ impl<R: AsyncRead + AsyncSeek + Unpin + Send + Sync> S9pkReader<R> {
             &validated_image_ids,
         )?;
 
-        #[cfg(feature = "js_engine")]
-        if man.containers.is_some()
-            || matches!(man.main, crate::procedure::PackageProcedure::Script(_))
-        {
-            return Err(Error::new(
-                eyre!("Right now we don't support the containers and the long running main"),
-                crate::ErrorKind::ValidateS9pk,
-            ));
-        }
-
         if man.containers.is_some()
             && matches!(man.main, crate::procedure::PackageProcedure::Docker(_))
         {