enabling support for wireguard and firewall (#2713)

* wip: enabling support for wireguard and firewall

* wip

* wip

* wip

* wip

* wip

* implement some things

* fix warning

* wip

* alpha.23

* misc fixes

* remove ufw since no longer required

* remove debug info

* add cli bindings

* debugging

* fixes

* individualized acme and privacy settings for domains and bindings

* sdk version bump

* migration

* misc fixes

* refactor Host::update

* debug info

* refactor webserver

* misc fixes

* misc fixes

* refactor port forwarding

* recheck interfaces every 5 min if no dbus event

* misc fixes and cleanup

* misc fixes

core/startos/src/version/mod.rs

@@ -7,12 +7,10 @@ use futures::future::BoxFuture;
 use futures::{Future, FutureExt};
 use imbl::Vector;
 use imbl_value::{to_value, InternedString};
-use patch_db::json_ptr::{JsonPointer, ROOT};
+use patch_db::json_ptr::{ ROOT};
 
 use crate::context::RpcContext;
-use crate::db::model::Database;
 use crate::prelude::*;
-use crate::progress::PhaseProgressTrackerHandle;
 use crate::Error;
 
 mod v0_3_5;
@@ -29,7 +27,9 @@ mod v0_3_6_alpha_7;
 mod v0_3_6_alpha_8;
 mod v0_3_6_alpha_9;
 
-pub type Current = v0_3_6_alpha_9::Version; // VERSION_BUMP
+mod v0_3_6_alpha_10;
+
+pub type Current = v0_3_6_alpha_10::Version; // VERSION_BUMP
 
 impl Current {
     #[instrument(skip(self, db))]
@@ -108,6 +108,7 @@ enum Version {
     V0_3_6_alpha_7(Wrapper<v0_3_6_alpha_7::Version>),
     V0_3_6_alpha_8(Wrapper<v0_3_6_alpha_8::Version>),
     V0_3_6_alpha_9(Wrapper<v0_3_6_alpha_9::Version>),
+    V0_3_6_alpha_10(Wrapper<v0_3_6_alpha_10::Version>),
     Other(exver::Version),
 }
 
@@ -141,6 +142,7 @@ impl Version {
             Self::V0_3_6_alpha_7(v) => DynVersion(Box::new(v.0)),
             Self::V0_3_6_alpha_8(v) => DynVersion(Box::new(v.0)),
             Self::V0_3_6_alpha_9(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_10(v) => DynVersion(Box::new(v.0)),
             Self::Other(v) => {
                 return Err(Error::new(
                     eyre!("unknown version {v}"),
@@ -166,6 +168,7 @@ impl Version {
             Version::V0_3_6_alpha_7(Wrapper(x)) => x.semver(),
             Version::V0_3_6_alpha_8(Wrapper(x)) => x.semver(),
             Version::V0_3_6_alpha_9(Wrapper(x)) => x.semver(),
+            Version::V0_3_6_alpha_10(Wrapper(x)) => x.semver(),
             Version::Other(x) => x.clone(),
         }
     }

core/startos/src/version/v0_3_6_alpha_0.rs

@@ -1,19 +1,16 @@
 use std::collections::BTreeMap;
-use std::future::Future;
 use std::path::Path;
 
 use chrono::{DateTime, Utc};
+use const_format::formatcp;
 use ed25519_dalek::SigningKey;
 use exver::{PreReleaseSegment, VersionRange};
 use imbl_value::{json, InternedString};
-use itertools::Itertools;
 use models::PackageId;
-use openssl::pkey::{PKey, Private};
+use openssl::pkey::PKey;
 use openssl::x509::X509;
-use patch_db::ModelExt;
 use sqlx::postgres::PgConnectOptions;
 use sqlx::{PgPool, Row};
-use ssh_key::Fingerprint;
 use tokio::process::Command;
 use torut::onion::TorSecretKeyV3;
 
@@ -23,15 +20,11 @@ use crate::account::AccountInfo;
 use crate::auth::Sessions;
 use crate::backup::target::cifs::CifsTargets;
 use crate::context::RpcContext;
-use crate::db::model::Database;
 use crate::disk::mount::filesystem::cifs::Cifs;
 use crate::disk::mount::util::unmount;
 use crate::hostname::Hostname;
 use crate::net::forward::AvailablePorts;
 use crate::net::keys::KeyStore;
-use crate::net::ssl::CertStore;
-use crate::net::tor;
-use crate::net::tor::OnionStore;
 use crate::notifications::{Notification, Notifications};
 use crate::prelude::*;
 use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
@@ -39,6 +32,7 @@ use crate::ssh::{SshKeys, SshPubKey};
 use crate::util::crypto::ed25519_expand_key;
 use crate::util::serde::{Pem, PemEncoding};
 use crate::util::Invoke;
+use crate::{DATA_DIR, PACKAGE_DATA};
 
 lazy_static::lazy_static! {
     static ref V0_3_6_alpha_0: exver::Version = exver::Version::new(
@@ -191,7 +185,6 @@ async fn init_postgres(datadir: impl AsRef<Path>) -> Result<PgPool, Error> {
         .run(&secret_store)
         .await
         .with_kind(crate::ErrorKind::Database)?;
-    dbg!("Init Postgres Done");
     Ok(secret_store)
 }
 
@@ -208,7 +201,7 @@ impl VersionT for Version {
         &V0_3_0_COMPAT
     }
     async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
-        let pg = init_postgres("/embassy-data").await?;
+        let pg = init_postgres(DATA_DIR).await?;
         let account = previous_account_info(&pg).await?;
 
         let ssh_keys = previous_ssh_keys(&pg).await?;
@@ -315,7 +308,6 @@ impl VersionT for Version {
             "private": private,
         });
 
-        dbg!("Should be done with the up");
         *db = next;
         Ok(())
     }
@@ -329,7 +321,7 @@ impl VersionT for Version {
     #[instrument(skip(self, ctx))]
     /// MUST be idempotent, and is run after *all* db migrations
     async fn post_up(self, ctx: &RpcContext) -> Result<(), Error> {
-        let path = Path::new("/embassy-data/package-data/archive/");
+        let path = Path::new(formatcp!("{PACKAGE_DATA}/archive/"));
         if !path.is_dir() {
             return Err(Error::new(
                 eyre!(

core/startos/src/version/v0_3_6_alpha_10.rs

@@ -0,0 +1,94 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use exver::{PreReleaseSegment, VersionRange};
+use imbl_value::InternedString;
+use serde::{Deserialize, Serialize};
+use torut::onion::OnionAddressV3;
+
+use super::v0_3_5::V0_3_0_COMPAT;
+use super::{v0_3_6_alpha_9, VersionT};
+use crate::net::host::address::DomainConfig;
+use crate::prelude::*;
+
+lazy_static::lazy_static! {
+    static ref V0_3_6_alpha_10: exver::Version = exver::Version::new(
+        [0, 3, 6],
+        [PreReleaseSegment::String("alpha".into()), 10.into()]
+    );
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)]
+#[serde(rename_all = "camelCase")]
+#[serde(tag = "kind")]
+enum HostAddress {
+    Onion { address: OnionAddressV3 },
+    Domain { address: InternedString },
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct Version;
+
+impl VersionT for Version {
+    type Previous = v0_3_6_alpha_9::Version;
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
+    }
+    fn semver(self) -> exver::Version {
+        V0_3_6_alpha_10.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        for (_, package) in db["public"]["packageData"]
+            .as_object_mut()
+            .ok_or_else(|| {
+                Error::new(
+                    eyre!("expected public.packageData to be an object"),
+                    ErrorKind::Database,
+                )
+            })?
+            .iter_mut()
+        {
+            for (_, host) in package["hosts"]
+                .as_object_mut()
+                .ok_or_else(|| {
+                    Error::new(
+                        eyre!("expected public.packageData[id].hosts to be an object"),
+                        ErrorKind::Database,
+                    )
+                })?
+                .iter_mut()
+            {
+                let mut onions = BTreeSet::new();
+                let mut domains = BTreeMap::new();
+                let addresses = from_value::<BTreeSet<HostAddress>>(host["addresses"].clone())?;
+                for address in addresses {
+                    match address {
+                        HostAddress::Onion { address } => {
+                            onions.insert(address);
+                        }
+                        HostAddress::Domain { address } => {
+                            domains.insert(
+                                address,
+                                DomainConfig {
+                                    public: true,
+                                    acme: None,
+                                },
+                            );
+                        }
+                    }
+                }
+                host["onions"] = to_value(&onions)?;
+                host["domains"] = to_value(&domains)?;
+            }
+        }
+
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
+        Ok(())
+    }
+}

core/startos/src/version/v0_3_6_alpha_6.rs

@@ -27,7 +27,7 @@ impl VersionT for Version {
     async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
         Ok(())
     }
-    fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
         Ok(())
     }
     async fn post_up<'a>(self, ctx: &'a crate::context::RpcContext) -> Result<(), Error> {

core/startos/src/version/v0_3_6_alpha_7.rs

@@ -1,5 +1,5 @@
 use exver::{PreReleaseSegment, VersionRange};
-use imbl_value::{json, InOMap};
+use imbl_value::json;
 use tokio::process::Command;
 
 use super::v0_3_5::V0_3_0_COMPAT;

core/startos/src/version/v0_3_6_alpha_8.rs

@@ -1,3 +1,5 @@
+use std::path::Path;
+
 use exver::{PreReleaseSegment, VersionRange};
 use tokio::fs::File;
 
@@ -12,6 +14,7 @@ use crate::s9pk::v2::SIG_CONTEXT;
 use crate::s9pk::S9pk;
 use crate::service::LoadDisposition;
 use crate::util::io::create_file;
+use crate::DATA_DIR;
 
 lazy_static::lazy_static! {
     static ref V0_3_6_alpha_8: exver::Version = exver::Version::new(
@@ -40,7 +43,7 @@ impl VersionT for Version {
         Ok(())
     }
     async fn post_up(self, ctx: &crate::context::RpcContext) -> Result<(), Error> {
-        let s9pk_dir = ctx.datadir.join(PKG_ARCHIVE_DIR).join("installed");
+        let s9pk_dir = Path::new(DATA_DIR).join(PKG_ARCHIVE_DIR).join("installed");
 
         if tokio::fs::metadata(&s9pk_dir).await.is_ok() {
             let mut read_dir = tokio::fs::read_dir(&s9pk_dir).await?;