Feature/remove postgres (#2570)

* wip: move postgres data to patchdb

* wip

* wip

* wip

* complete notifications and clean up warnings

* fill in user agent

* move os tor bindings to single call

core/startos/src/db/model.rs

@@ -13,30 +13,46 @@ use patch_db::json_ptr::JsonPointer;
 use patch_db::{HasModel, Value};
 use reqwest::Url;
 use serde::{Deserialize, Serialize};
-use ssh_key::public::Ed25519PublicKey;
+use torut::onion::OnionAddressV3;
 
 use crate::account::AccountInfo;
+use crate::auth::Sessions;
+use crate::backup::target::cifs::CifsTargets;
+use crate::net::forward::AvailablePorts;
+use crate::net::host::HostInfo;
+use crate::net::keys::KeyStore;
 use crate::net::utils::{get_iface_ipv4_addr, get_iface_ipv6_addr};
+use crate::notifications::Notifications;
 use crate::prelude::*;
 use crate::progress::FullProgress;
 use crate::s9pk::manifest::Manifest;
+use crate::ssh::SshKeys;
 use crate::status::Status;
 use crate::util::cpupower::Governor;
+use crate::util::serde::Pem;
 use crate::util::Version;
 use crate::version::{Current, VersionT};
 use crate::{ARCH, PLATFORM};
 
+fn get_arch() -> InternedString {
+    (*ARCH).into()
+}
+
+fn get_platform() -> InternedString {
+    (&*PLATFORM).into()
+}
+
 #[derive(Debug, Deserialize, Serialize, HasModel)]
 #[serde(rename_all = "kebab-case")]
 #[model = "Model<Self>"]
 pub struct Database {
     pub public: Public,
-    pub private: (), // TODO
+    pub private: Private,
 }
 impl Database {
-    pub fn init(account: &AccountInfo) -> Self {
+    pub fn init(account: &AccountInfo) -> Result<Self, Error> {
         let lan_address = account.hostname.lan_address().parse().unwrap();
-        Database {
+        Ok(Database {
             public: Public {
                 server_info: ServerInfo {
                     arch: get_arch(),
@@ -48,9 +64,13 @@ impl Database {
                     last_wifi_region: None,
                     eos_version_compat: Current::new().compat().clone(),
                     lan_address,
-                    tor_address: format!("https://{}", account.key.tor_address())
-                        .parse()
-                        .unwrap(),
+                    onion_address: account.tor_key.public().get_onion_address(),
+                    tor_address: format!(
+                        "https://{}",
+                        account.tor_key.public().get_onion_address()
+                    )
+                    .parse()
+                    .unwrap(),
                     ip_info: BTreeMap::new(),
                     status_info: ServerStatus {
                         backup_progress: None,
@@ -70,11 +90,9 @@ impl Database {
                         clearnet: Vec::new(),
                     },
                     password_hash: account.password.clone(),
-                    pubkey: ssh_key::PublicKey::from(Ed25519PublicKey::from(
-                        &account.key.ssh_key(),
-                    ))
-                    .to_openssh()
-                    .unwrap(),
+                    pubkey: ssh_key::PublicKey::from(&account.ssh_key)
+                        .to_openssh()
+                        .unwrap(),
                     ca_fingerprint: account
                         .root_ca_cert
                         .digest(MessageDigest::sha256())
@@ -93,11 +111,22 @@ impl Database {
                 )))
                 .unwrap(),
             },
-            private: (), // TODO
-        }
+            private: Private {
+                key_store: KeyStore::new(account)?,
+                password: account.password.clone(),
+                ssh_privkey: Pem(account.ssh_key.clone()),
+                ssh_pubkeys: SshKeys::new(),
+                available_ports: AvailablePorts::new(),
+                sessions: Sessions::new(),
+                notifications: Notifications::new(),
+                cifs: CifsTargets::new(),
+            }, // TODO
+        })
     }
 }
 
+pub type DatabaseModel = Model<Database>;
+
 #[derive(Debug, Deserialize, Serialize, HasModel)]
 #[serde(rename_all = "kebab-case")]
 #[model = "Model<Self>"]
@@ -108,14 +137,18 @@ pub struct Public {
     pub ui: Value,
 }
 
-pub type DatabaseModel = Model<Database>;
-
-fn get_arch() -> InternedString {
-    (*ARCH).into()
-}
-
-fn get_platform() -> InternedString {
-    (&*PLATFORM).into()
+#[derive(Debug, Deserialize, Serialize, HasModel)]
+#[serde(rename_all = "kebab-case")]
+#[model = "Model<Self>"]
+pub struct Private {
+    pub key_store: KeyStore,
+    pub password: String, // argon2 hash
+    pub ssh_privkey: Pem<ssh_key::PrivateKey>,
+    pub ssh_pubkeys: SshKeys,
+    pub available_ports: AvailablePorts,
+    pub sessions: Sessions,
+    pub notifications: Notifications,
+    pub cifs: CifsTargets,
 }
 
 #[derive(Debug, Deserialize, Serialize, HasModel)]
@@ -134,6 +167,8 @@ pub struct ServerInfo {
     pub last_wifi_region: Option<CountryCode>,
     pub eos_version_compat: VersionRange,
     pub lan_address: Url,
+    pub onion_address: OnionAddressV3,
+    /// for backwards compatibility
     pub tor_address: Url,
     pub ip_info: BTreeMap<String, IpInfo>,
     #[serde(default)]
@@ -229,6 +264,12 @@ pub struct AllPackageData(pub BTreeMap<PackageId, PackageDataEntry>);
 impl Map for AllPackageData {
     type Key = PackageId;
     type Value = PackageDataEntry;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Ok(key)
+    }
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(key.clone().into())
+    }
 }
 
 #[derive(Debug, Deserialize, Serialize, HasModel)]
@@ -471,6 +512,7 @@ pub struct InstalledPackageInfo {
     pub current_dependents: CurrentDependents,
     pub current_dependencies: CurrentDependencies,
     pub interface_addresses: InterfaceAddressMap,
+    pub hosts: HostInfo,
     pub store: Value,
     pub store_exposed_ui: Vec<ExposedUI>,
     pub store_exposed_dependents: Vec<JsonPointer>,
@@ -512,6 +554,12 @@ impl CurrentDependents {
 impl Map for CurrentDependents {
     type Key = PackageId;
     type Value = CurrentDependencyInfo;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Ok(key)
+    }
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(key.clone().into())
+    }
 }
 #[derive(Debug, Clone, Default, Deserialize, Serialize)]
 pub struct CurrentDependencies(pub BTreeMap<PackageId, CurrentDependencyInfo>);
@@ -529,6 +577,12 @@ impl CurrentDependencies {
 impl Map for CurrentDependencies {
     type Key = PackageId;
     type Value = CurrentDependencyInfo;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Ok(key)
+    }
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(key.clone().into())
+    }
 }
 
 #[derive(Debug, Deserialize, Serialize, HasModel)]
@@ -552,6 +606,12 @@ pub struct InterfaceAddressMap(pub BTreeMap<HostId, InterfaceAddresses>);
 impl Map for InterfaceAddressMap {
     type Key = HostId;
     type Value = InterfaceAddresses;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Ok(key)
+    }
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(key.clone().into())
+    }
 }
 
 #[derive(Debug, Deserialize, Serialize, HasModel)]

core/startos/src/db/prelude.rs

@@ -1,13 +1,15 @@
 use std::collections::BTreeMap;
 use std::marker::PhantomData;
 use std::panic::UnwindSafe;
+use std::str::FromStr;
 
+use chrono::{DateTime, Utc};
 pub use imbl_value::Value;
 use patch_db::json_ptr::ROOT;
 use patch_db::value::InternedString;
 pub use patch_db::{HasModel, PatchDb};
 use serde::de::DeserializeOwned;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 
 use crate::db::model::DatabaseModel;
 use crate::prelude::*;
@@ -92,12 +94,37 @@ impl<T: Serialize> Model<T> {
     }
 }
 
+impl<T> Serialize for Model<T> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        self.value.serialize(serializer)
+    }
+}
+
+impl<'de, T: Serialize + Deserialize<'de>> Deserialize<'de> for Model<T> {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        use serde::de::Error;
+        Self::new(&T::deserialize(deserializer)?).map_err(D::Error::custom)
+    }
+}
+
 impl<T: Serialize + DeserializeOwned> Model<T> {
     pub fn replace(&mut self, value: &T) -> Result<T, Error> {
         let orig = self.de()?;
         self.ser(value)?;
         Ok(orig)
     }
+    pub fn mutate<U>(&mut self, f: impl FnOnce(&mut T) -> Result<U, Error>) -> Result<U, Error> {
+        let mut orig = self.de()?;
+        let res = f(&mut orig)?;
+        self.ser(&orig)?;
+        Ok(res)
+    }
 }
 impl<T> Clone for Model<T> {
     fn clone(&self) -> Self {
@@ -181,20 +208,38 @@ impl<T> Model<Option<T>> {
 pub trait Map: DeserializeOwned + Serialize {
     type Key;
     type Value;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error>;
+    fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
+        Ok(InternedString::intern(Self::key_str(key)?.as_ref()))
+    }
 }
 
 impl<A, B> Map for BTreeMap<A, B>
+where
+    A: serde::Serialize + serde::de::DeserializeOwned + Ord + AsRef<str>,
+    B: serde::Serialize + serde::de::DeserializeOwned,
+{
+    type Key = A;
+    type Value = B;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        Ok(key.as_ref())
+    }
+}
+
+impl<A, B> Map for BTreeMap<JsonKey<A>, B>
 where
     A: serde::Serialize + serde::de::DeserializeOwned + Ord,
     B: serde::Serialize + serde::de::DeserializeOwned,
 {
     type Key = A;
     type Value = B;
+    fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
+        serde_json::to_string(key).with_kind(ErrorKind::Serialization)
+    }
 }
 
 impl<T: Map> Model<T>
 where
-    T::Key: AsRef<str>,
     T::Value: Serialize,
 {
     pub fn insert(&mut self, key: &T::Key, value: &T::Value) -> Result<(), Error> {
@@ -202,7 +247,7 @@ where
         let v = patch_db::value::to_value(value)?;
         match &mut self.value {
             Value::Object(o) => {
-                o.insert(InternedString::intern(key.as_ref()), v);
+                o.insert(T::key_string(key)?, v);
                 Ok(())
             }
             v => Err(patch_db::value::Error {
@@ -212,13 +257,40 @@ where
             .into()),
         }
     }
+    pub fn upsert<F, D>(&mut self, key: &T::Key, value: F) -> Result<&mut Model<T::Value>, Error>
+    where
+        F: FnOnce() -> D,
+        D: AsRef<T::Value>,
+    {
+        use serde::ser::Error;
+        match &mut self.value {
+            Value::Object(o) => {
+                use patch_db::ModelExt;
+                let s = T::key_str(key)?;
+                let exists = o.contains_key(s.as_ref());
+                let res = self.transmute_mut(|v| {
+                    use patch_db::value::index::Index;
+                    s.as_ref().index_or_insert(v)
+                });
+                if !exists {
+                    res.ser(value().as_ref())?;
+                }
+                Ok(res)
+            }
+            v => Err(patch_db::value::Error {
+                source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
+                kind: patch_db::value::ErrorKind::Serialization,
+            }
+            .into()),
+        }
+    }
     pub fn insert_model(&mut self, key: &T::Key, value: Model<T::Value>) -> Result<(), Error> {
         use patch_db::ModelExt;
         use serde::ser::Error;
         let v = value.into_value();
         match &mut self.value {
             Value::Object(o) => {
-                o.insert(InternedString::intern(key.as_ref()), v);
+                o.insert(T::key_string(key)?, v);
                 Ok(())
             }
             v => Err(patch_db::value::Error {
@@ -232,25 +304,16 @@ where
 
 impl<T: Map> Model<T>
 where
-    T::Key: DeserializeOwned + Ord + Clone,
+    T::Key: FromStr + Ord + Clone,
+    Error: From<<T::Key as FromStr>::Err>,
 {
     pub fn keys(&self) -> Result<Vec<T::Key>, Error> {
         use serde::de::Error;
-        use serde::Deserialize;
         match &self.value {
             Value::Object(o) => o
                 .keys()
                 .cloned()
-                .map(|k| {
-                    T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(k))
-                        .map_err(|e| {
-                            patch_db::value::Error {
-                                kind: patch_db::value::ErrorKind::Deserialization,
-                                source: e,
-                            }
-                            .into()
-                        })
-                })
+                .map(|k| Ok(T::Key::from_str(&*k)?))
                 .collect(),
             v => Err(patch_db::value::Error {
                 source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
@@ -263,19 +326,10 @@ where
     pub fn into_entries(self) -> Result<Vec<(T::Key, Model<T::Value>)>, Error> {
         use patch_db::ModelExt;
         use serde::de::Error;
-        use serde::Deserialize;
         match self.value {
             Value::Object(o) => o
                 .into_iter()
-                .map(|(k, v)| {
-                    Ok((
-                        T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(
-                            k,
-                        ))
-                        .with_kind(ErrorKind::Deserialization)?,
-                        Model::from_value(v),
-                    ))
-                })
+                .map(|(k, v)| Ok((T::Key::from_str(&*k)?, Model::from_value(v))))
                 .collect(),
             v => Err(patch_db::value::Error {
                 source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
@@ -287,19 +341,10 @@ where
     pub fn as_entries(&self) -> Result<Vec<(T::Key, &Model<T::Value>)>, Error> {
         use patch_db::ModelExt;
         use serde::de::Error;
-        use serde::Deserialize;
         match &self.value {
             Value::Object(o) => o
                 .iter()
-                .map(|(k, v)| {
-                    Ok((
-                        T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(
-                            k.clone(),
-                        ))
-                        .with_kind(ErrorKind::Deserialization)?,
-                        Model::value_as(v),
-                    ))
-                })
+                .map(|(k, v)| Ok((T::Key::from_str(&**k)?, Model::value_as(v))))
                 .collect(),
             v => Err(patch_db::value::Error {
                 source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
@@ -311,19 +356,10 @@ where
     pub fn as_entries_mut(&mut self) -> Result<Vec<(T::Key, &mut Model<T::Value>)>, Error> {
         use patch_db::ModelExt;
         use serde::de::Error;
-        use serde::Deserialize;
         match &mut self.value {
             Value::Object(o) => o
                 .iter_mut()
-                .map(|(k, v)| {
-                    Ok((
-                        T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(
-                            k.clone(),
-                        ))
-                        .with_kind(ErrorKind::Deserialization)?,
-                        Model::value_as_mut(v),
-                    ))
-                })
+                .map(|(k, v)| Ok((T::Key::from_str(&**k)?, Model::value_as_mut(v))))
                 .collect(),
             v => Err(patch_db::value::Error {
                 source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
@@ -333,36 +369,36 @@ where
         }
     }
 }
-impl<T: Map> Model<T>
-where
-    T::Key: AsRef<str>,
-{
+impl<T: Map> Model<T> {
     pub fn into_idx(self, key: &T::Key) -> Option<Model<T::Value>> {
         use patch_db::ModelExt;
+        let s = T::key_str(key).ok()?;
         match &self.value {
-            Value::Object(o) if o.contains_key(key.as_ref()) => Some(self.transmute(|v| {
+            Value::Object(o) if o.contains_key(s.as_ref()) => Some(self.transmute(|v| {
                 use patch_db::value::index::Index;
-                key.as_ref().index_into_owned(v).unwrap()
+                s.as_ref().index_into_owned(v).unwrap()
             })),
             _ => None,
         }
     }
     pub fn as_idx<'a>(&'a self, key: &T::Key) -> Option<&'a Model<T::Value>> {
         use patch_db::ModelExt;
+        let s = T::key_str(key).ok()?;
         match &self.value {
-            Value::Object(o) if o.contains_key(key.as_ref()) => Some(self.transmute_ref(|v| {
+            Value::Object(o) if o.contains_key(s.as_ref()) => Some(self.transmute_ref(|v| {
                 use patch_db::value::index::Index;
-                key.as_ref().index_into(v).unwrap()
+                s.as_ref().index_into(v).unwrap()
             })),
             _ => None,
         }
     }
     pub fn as_idx_mut<'a>(&'a mut self, key: &T::Key) -> Option<&'a mut Model<T::Value>> {
         use patch_db::ModelExt;
+        let s = T::key_str(key).ok()?;
         match &mut self.value {
-            Value::Object(o) if o.contains_key(key.as_ref()) => Some(self.transmute_mut(|v| {
+            Value::Object(o) if o.contains_key(s.as_ref()) => Some(self.transmute_mut(|v| {
                 use patch_db::value::index::Index;
-                key.as_ref().index_or_insert(v)
+                s.as_ref().index_or_insert(v)
             })),
             _ => None,
         }
@@ -371,7 +407,7 @@ where
         use serde::ser::Error;
         match &mut self.value {
             Value::Object(o) => {
-                let v = o.remove(key.as_ref());
+                let v = o.remove(T::key_str(key)?.as_ref());
                 Ok(v.map(patch_db::ModelExt::from_value))
             }
             v => Err(patch_db::value::Error {
@@ -382,3 +418,90 @@ where
         }
     }
 }
+
+#[repr(transparent)]
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct JsonKey<T>(pub T);
+impl<T> From<T> for JsonKey<T> {
+    fn from(value: T) -> Self {
+        Self::new(value)
+    }
+}
+impl<T> JsonKey<T> {
+    pub fn new(value: T) -> Self {
+        Self(value)
+    }
+    pub fn unwrap(self) -> T {
+        self.0
+    }
+    pub fn new_ref(value: &T) -> &Self {
+        unsafe { std::mem::transmute(value) }
+    }
+    pub fn new_mut(value: &mut T) -> &mut Self {
+        unsafe { std::mem::transmute(value) }
+    }
+}
+impl<T> std::ops::Deref for JsonKey<T> {
+    type Target = T;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl<T> std::ops::DerefMut for JsonKey<T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+impl<T: Serialize> Serialize for JsonKey<T> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        use serde::ser::Error;
+        serde_json::to_string(&self.0)
+            .map_err(S::Error::custom)?
+            .serialize(serializer)
+    }
+}
+// { "foo": "bar" } -> "{ \"foo\": \"bar\" }"
+impl<'de, T: Serialize + DeserializeOwned> Deserialize<'de> for JsonKey<T> {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        use serde::de::Error;
+        let string = String::deserialize(deserializer)?;
+        Ok(Self(
+            serde_json::from_str(&string).map_err(D::Error::custom)?,
+        ))
+    }
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct WithTimeData<T> {
+    pub created_at: DateTime<Utc>,
+    pub updated_at: DateTime<Utc>,
+    pub value: T,
+}
+impl<T> WithTimeData<T> {
+    pub fn new(value: T) -> Self {
+        let now = Utc::now();
+        Self {
+            created_at: now,
+            updated_at: now,
+            value,
+        }
+    }
+}
+impl<T> std::ops::Deref for WithTimeData<T> {
+    type Target = T;
+    fn deref(&self) -> &Self::Target {
+        &self.value
+    }
+}
+impl<T> std::ops::DerefMut for WithTimeData<T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.updated_at = Utc::now();
+        &mut self.value
+    }
+}