enabling support for wireguard and firewall (#2713)

* wip: enabling support for wireguard and firewall

* wip

* wip

* wip

* wip

* wip

* implement some things

* fix warning

* wip

* alpha.23

* misc fixes

* remove ufw since no longer required

* remove debug info

* add cli bindings

* debugging

* fixes

* individualized acme and privacy settings for domains and bindings

* sdk version bump

* migration

* misc fixes

* refactor Host::update

* debug info

* refactor webserver

* misc fixes

* misc fixes

* refactor port forwarding

* recheck interfaces every 5 min if no dbus event

* misc fixes and cleanup

* misc fixes

core/startos/src/context/config.rs

@@ -13,6 +13,7 @@ use crate::disk::OsPartitionInfo;
 use crate::init::init_postgres;
 use crate::prelude::*;
 use crate::util::serde::IoFormat;
+use crate::MAIN_DATA;
 
 pub const DEVICE_CONFIG_PATH: &str = "/media/startos/config/config.yaml"; // "/media/startos/config/config.yaml";
 pub const CONFIG_PATH: &str = "/etc/startos/config.yaml";
@@ -103,8 +104,6 @@ pub struct ServerConfig {
     #[arg(skip)]
     pub os_partitions: Option<OsPartitionInfo>,
     #[arg(long)]
-    pub bind_rpc: Option<SocketAddr>,
-    #[arg(long)]
     pub tor_control: Option<SocketAddr>,
     #[arg(long)]
     pub tor_socks: Option<SocketAddr>,
@@ -112,8 +111,6 @@ pub struct ServerConfig {
     pub dns_bind: Option<Vec<SocketAddr>>,
     #[arg(long)]
     pub revision_cache_size: Option<usize>,
-    #[arg(short, long)]
-    pub datadir: Option<PathBuf>,
     #[arg(long)]
     pub disable_encryption: Option<bool>,
     #[arg(long)]
@@ -126,7 +123,6 @@ impl ContextConfig for ServerConfig {
     fn merge_with(&mut self, other: Self) {
         self.ethernet_interface = self.ethernet_interface.take().or(other.ethernet_interface);
         self.os_partitions = self.os_partitions.take().or(other.os_partitions);
-        self.bind_rpc = self.bind_rpc.take().or(other.bind_rpc);
         self.tor_control = self.tor_control.take().or(other.tor_control);
         self.tor_socks = self.tor_socks.take().or(other.tor_socks);
         self.dns_bind = self.dns_bind.take().or(other.dns_bind);
@@ -134,7 +130,6 @@ impl ContextConfig for ServerConfig {
             .revision_cache_size
             .take()
             .or(other.revision_cache_size);
-        self.datadir = self.datadir.take().or(other.datadir);
         self.disable_encryption = self.disable_encryption.take().or(other.disable_encryption);
         self.multi_arch_s9pks = self.multi_arch_s9pks.take().or(other.multi_arch_s9pks);
     }
@@ -148,13 +143,8 @@ impl ServerConfig {
         self.load_path_rec(Some(CONFIG_PATH))?;
         Ok(self)
     }
-    pub fn datadir(&self) -> &Path {
-        self.datadir
-            .as_deref()
-            .unwrap_or_else(|| Path::new("/embassy-data"))
-    }
     pub async fn db(&self) -> Result<PatchDb, Error> {
-        let db_path = self.datadir().join("main").join("embassy.db");
+        let db_path = Path::new(MAIN_DATA).join("embassy.db");
         let db = PatchDb::open(&db_path)
             .await
             .with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
@@ -163,7 +153,7 @@ impl ServerConfig {
     }
     #[instrument(skip_all)]
     pub async fn secret_store(&self) -> Result<PgPool, Error> {
-        init_postgres(self.datadir()).await?;
+        init_postgres("/media/startos/data").await?;
         let secret_store =
             PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root"))
                 .await?;

core/startos/src/context/diagnostic.rs

@@ -1,5 +1,4 @@
 use std::ops::Deref;
-use std::path::PathBuf;
 use std::sync::Arc;
 
 use rpc_toolkit::yajrc::RpcError;
@@ -13,7 +12,6 @@ use crate::shutdown::Shutdown;
 use crate::Error;
 
 pub struct DiagnosticContextSeed {
-    pub datadir: PathBuf,
     pub shutdown: Sender<Shutdown>,
     pub error: Arc<RpcError>,
     pub disk_guid: Option<Arc<String>>,
@@ -25,7 +23,7 @@ pub struct DiagnosticContext(Arc<DiagnosticContextSeed>);
 impl DiagnosticContext {
     #[instrument(skip_all)]
     pub fn init(
-        config: &ServerConfig,
+        _config: &ServerConfig,
         disk_guid: Option<Arc<String>>,
         error: Error,
     ) -> Result<Self, Error> {
@@ -35,7 +33,6 @@ impl DiagnosticContext {
         let (shutdown, _) = tokio::sync::broadcast::channel(1);
 
         Ok(Self(Arc::new(DiagnosticContextSeed {
-            datadir: config.datadir().to_owned(),
             shutdown,
             disk_guid,
             error: Arc::new(error.into()),

core/startos/src/context/rpc.rs

@@ -2,7 +2,6 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::future::Future;
 use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
 use std::ops::Deref;
-use std::path::PathBuf;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
@@ -31,6 +30,7 @@ use crate::init::check_time_is_synchronized;
 use crate::lxc::{ContainerId, LxcContainer, LxcManager};
 use crate::net::net_controller::{NetController, PreInitNetController};
 use crate::net::utils::{find_eth_iface, find_wifi_iface};
+use crate::net::web_server::{UpgradableListener, WebServerAcceptorSetter};
 use crate::net::wifi::WpaCli;
 use crate::prelude::*;
 use crate::progress::{FullProgressTracker, PhaseProgressTrackerHandle};
@@ -47,7 +47,6 @@ pub struct RpcContextSeed {
     pub os_partitions: OsPartitionInfo,
     pub wifi_interface: Option<String>,
     pub ethernet_interface: String,
-    pub datadir: PathBuf,
     pub disk_guid: Arc<String>,
     pub ephemeral_sessions: SyncMutex<Sessions>,
     pub db: TypedPatchDb<Database>,
@@ -117,6 +116,7 @@ pub struct RpcContext(Arc<RpcContextSeed>);
 impl RpcContext {
     #[instrument(skip_all)]
     pub async fn init(
+        webserver: &WebServerAcceptorSetter<UpgradableListener>,
         config: &ServerConfig,
         disk_guid: Arc<String>,
         net_ctrl: Option<PreInitNetController>,
@@ -149,7 +149,7 @@ impl RpcContext {
                 if let Some(net_ctrl) = net_ctrl {
                     net_ctrl
                 } else {
-                    PreInitNetController::init(
+                    let net_ctrl = PreInitNetController::init(
                         db.clone(),
                         config
                             .tor_control
@@ -158,7 +158,9 @@ impl RpcContext {
                         &account.hostname,
                         account.tor_key.clone(),
                     )
-                    .await?
+                    .await?;
+                    webserver.try_upgrade(|a| net_ctrl.net_iface.upgrade_listener(a))?;
+                    net_ctrl
                 },
                 config
                     .dns_bind
@@ -210,7 +212,6 @@ impl RpcContext {
 
         let seed = Arc::new(RpcContextSeed {
             is_closed: AtomicBool::new(false),
-            datadir: config.datadir().to_path_buf(),
             os_partitions: config.os_partitions.clone().ok_or_else(|| {
                 Error::new(
                     eyre!("OS Partition Information Missing"),

core/startos/src/context/setup.rs

@@ -1,5 +1,5 @@
 use std::ops::Deref;
-use std::path::PathBuf;
+use std::path::{Path};
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -10,8 +10,6 @@ use josekit::jwk::Jwk;
 use patch_db::PatchDb;
 use rpc_toolkit::Context;
 use serde::{Deserialize, Serialize};
-use sqlx::postgres::PgConnectOptions;
-use sqlx::PgPool;
 use tokio::sync::broadcast::Sender;
 use tokio::sync::OnceCell;
 use tracing::instrument;
@@ -22,12 +20,13 @@ use crate::context::config::ServerConfig;
 use crate::context::RpcContext;
 use crate::disk::OsPartitionInfo;
 use crate::hostname::Hostname;
-use crate::init::init_postgres;
+use crate::net::web_server::{UpgradableListener, WebServer, WebServerAcceptorSetter};
 use crate::prelude::*;
 use crate::progress::FullProgressTracker;
 use crate::rpc_continuations::{Guid, RpcContinuation, RpcContinuations};
 use crate::setup::SetupProgress;
 use crate::util::net::WebSocketExt;
+use crate::MAIN_DATA;
 
 lazy_static::lazy_static! {
     pub static ref CURRENT_SECRET: Jwk = Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).unwrap_or_else(|e| {
@@ -61,6 +60,7 @@ impl TryFrom<&AccountInfo> for SetupResult {
 }
 
 pub struct SetupContextSeed {
+    pub webserver: WebServerAcceptorSetter<UpgradableListener>,
     pub config: ServerConfig,
     pub os_partitions: OsPartitionInfo,
     pub disable_encryption: bool,
@@ -68,7 +68,6 @@ pub struct SetupContextSeed {
     pub task: OnceCell<NonDetachingJoinHandle<()>>,
     pub result: OnceCell<Result<(SetupResult, RpcContext), Error>>,
     pub shutdown: Sender<()>,
-    pub datadir: PathBuf,
     pub rpc_continuations: RpcContinuations,
 }
 
@@ -76,10 +75,13 @@ pub struct SetupContextSeed {
 pub struct SetupContext(Arc<SetupContextSeed>);
 impl SetupContext {
     #[instrument(skip_all)]
-    pub fn init(config: &ServerConfig) -> Result<Self, Error> {
+    pub fn init(
+        webserver: &WebServer<UpgradableListener>,
+        config: &ServerConfig,
+    ) -> Result<Self, Error> {
         let (shutdown, _) = tokio::sync::broadcast::channel(1);
-        let datadir = config.datadir().to_owned();
         Ok(Self(Arc::new(SetupContextSeed {
+            webserver: webserver.acceptor_setter(),
             config: config.clone(),
             os_partitions: config.os_partitions.clone().ok_or_else(|| {
                 Error::new(
@@ -92,13 +94,12 @@ impl SetupContext {
             task: OnceCell::new(),
             result: OnceCell::new(),
             shutdown,
-            datadir,
             rpc_continuations: RpcContinuations::new(),
         })))
     }
     #[instrument(skip_all)]
     pub async fn db(&self) -> Result<PatchDb, Error> {
-        let db_path = self.datadir.join("main").join("embassy.db");
+        let db_path = Path::new(MAIN_DATA).join("embassy.db");
         let db = PatchDb::open(&db_path)
             .await
             .with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;