Refactor/networking (#2189)

* refactor networking and account

* add interfaces from manifest automatically

* use nistp256 to satisfy firefox

* use ed25519 if available

* fix ip signing

* fix SQL error

* update prettytable to fix segfault

* fix migration

* fix migration

* bump welcome-ack

* add redirect if connecting to https over http

* misc rebase fixes

* fix compression

* bump rustc version

libs/models/Cargo.toml

@@ -9,15 +9,19 @@ edition = "2021"
 bollard = "0.13.0"
 color-eyre = "0.6.1"
 ed25519-dalek = { version = "1.0.1", features = ["serde"] }
+lazy_static = "1.4"
 mbrman = "0.5.0"
 emver = { version = "0.1", git = "https://github.com/Start9Labs/emver-rs.git", features = [
   "serde",
 ] }
+internment = { version = "0.7.0", features = ["arc", "serde"] }
+ipnet = "2.7.1"
 openssl = { version = "0.10.41", features = ["vendored"] }
 patch-db = { version = "*", path = "../../patch-db/patch-db", features = [
   "trace",
 ] }
 rand = "0.8"
+regex = "1.7.1"
 rpc-toolkit = "0.2.1"
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0.82"

libs/models/src/action_id.rs

@@ -6,43 +6,34 @@ use serde::{Deserialize, Serialize};
 use crate::{Id, InvalidId};
 
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
-pub struct ActionId<S: AsRef<str> = String>(Id<S>);
+pub struct ActionId(Id);
 impl FromStr for ActionId {
     type Err = InvalidId;
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ok(ActionId(Id::try_from(s.to_owned())?))
     }
 }
-impl From<ActionId> for String {
-    fn from(value: ActionId) -> Self {
-        value.0.into()
-    }
-}
-impl<S: AsRef<str>> AsRef<ActionId<S>> for ActionId<S> {
-    fn as_ref(&self) -> &ActionId<S> {
+impl AsRef<ActionId> for ActionId {
+    fn as_ref(&self) -> &ActionId {
         self
     }
 }
-impl<S: AsRef<str>> std::fmt::Display for ActionId<S> {
+impl std::fmt::Display for ActionId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", &self.0)
     }
 }
-impl<S: AsRef<str>> AsRef<str> for ActionId<S> {
+impl AsRef<str> for ActionId {
     fn as_ref(&self) -> &str {
         self.0.as_ref()
     }
 }
-impl<S: AsRef<str>> AsRef<Path> for ActionId<S> {
+impl AsRef<Path> for ActionId {
     fn as_ref(&self) -> &Path {
         self.0.as_ref().as_ref()
     }
 }
-impl<'de, S> Deserialize<'de> for ActionId<S>
-where
-    S: AsRef<str>,
-    Id<S>: Deserialize<'de>,
-{
+impl<'de> Deserialize<'de> for ActionId {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: serde::de::Deserializer<'de>,

libs/models/src/errors.rs

@@ -246,6 +246,11 @@ impl From<std::net::AddrParseError> for Error {
         Error::new(e, ErrorKind::ParseNetAddress)
     }
 }
+impl From<ipnet::AddrParseError> for Error {
+    fn from(e: ipnet::AddrParseError) -> Self {
+        Error::new(e, ErrorKind::ParseNetAddress)
+    }
+}
 impl From<openssl::error::ErrorStack> for Error {
     fn from(e: openssl::error::ErrorStack) -> Self {
         Error::new(eyre!("{}", e), ErrorKind::OpenSsl)

libs/models/src/health_check_id.rs

@@ -4,23 +4,19 @@ use serde::{Deserialize, Deserializer, Serialize};
 
 use crate::Id;
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
-pub struct HealthCheckId<S: AsRef<str> = String>(Id<S>);
-impl<S: AsRef<str>> std::fmt::Display for HealthCheckId<S> {
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
+pub struct HealthCheckId(Id);
+impl std::fmt::Display for HealthCheckId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", &self.0)
     }
 }
-impl<S: AsRef<str>> AsRef<str> for HealthCheckId<S> {
+impl AsRef<str> for HealthCheckId {
     fn as_ref(&self) -> &str {
         self.0.as_ref()
     }
 }
-impl<'de, S> Deserialize<'de> for HealthCheckId<S>
-where
-    S: AsRef<str>,
-    Id<S>: Deserialize<'de>,
-{
+impl<'de> Deserialize<'de> for HealthCheckId {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: Deserializer<'de>,
@@ -28,7 +24,7 @@ where
         Ok(HealthCheckId(Deserialize::deserialize(deserializer)?))
     }
 }
-impl<S: AsRef<str>> AsRef<Path> for HealthCheckId<S> {
+impl AsRef<Path> for HealthCheckId {
     fn as_ref(&self) -> &Path {
         self.0.as_ref().as_ref()
     }

libs/models/src/id.rs

@@ -1,72 +1,79 @@
 use std::borrow::Borrow;
 
+use internment::ArcIntern;
+use regex::Regex;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 
-use crate::id_unchecked::IdUnchecked;
 use crate::invalid_id::InvalidId;
 
-pub const SYSTEM_ID: Id<&'static str> = Id("x_system");
+lazy_static::lazy_static! {
+    static ref ID_REGEX: Regex = Regex::new("^[a-z]+(-[a-z]+)*$").unwrap();
+    pub static ref SYSTEM_ID: Id = Id(ArcIntern::from_ref("x_system"));
+}
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
-pub struct Id<S: AsRef<str> = String>(S);
-impl<S: AsRef<str>> Id<S> {
-    pub fn try_from(value: S) -> Result<Self, InvalidId> {
-        if value
-            .as_ref()
-            .chars()
-            .all(|c| c.is_ascii_lowercase() || c == '-')
-        {
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
+pub struct Id(ArcIntern<String>);
+impl TryFrom<ArcIntern<String>> for Id {
+    type Error = InvalidId;
+    fn try_from(value: ArcIntern<String>) -> Result<Self, Self::Error> {
+        if ID_REGEX.is_match(&*value) {
             Ok(Id(value))
         } else {
             Err(InvalidId)
         }
     }
 }
-impl<'a> Id<&'a str> {
-    pub fn owned(&self) -> Id {
-        Id(self.0.to_owned())
+impl TryFrom<String> for Id {
+    type Error = InvalidId;
+    fn try_from(value: String) -> Result<Self, Self::Error> {
+        if ID_REGEX.is_match(&value) {
+            Ok(Id(ArcIntern::new(value)))
+        } else {
+            Err(InvalidId)
+        }
     }
 }
-impl From<Id> for String {
-    fn from(value: Id) -> Self {
-        value.0
+impl TryFrom<&str> for Id {
+    type Error = InvalidId;
+    fn try_from(value: &str) -> Result<Self, Self::Error> {
+        if ID_REGEX.is_match(&value) {
+            Ok(Id(ArcIntern::from_ref(value)))
+        } else {
+            Err(InvalidId)
+        }
     }
 }
-impl<S: AsRef<str>> std::ops::Deref for Id<S> {
-    type Target = S;
+impl std::ops::Deref for Id {
+    type Target = String;
     fn deref(&self) -> &Self::Target {
-        &self.0
+        &*self.0
     }
 }
-impl<S: AsRef<str>> std::fmt::Display for Id<S> {
+impl std::fmt::Display for Id {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.0.as_ref())
+        write!(f, "{}", &*self.0)
     }
 }
-impl<S: AsRef<str>> AsRef<str> for Id<S> {
+impl AsRef<str> for Id {
     fn as_ref(&self) -> &str {
-        self.0.as_ref()
+        &*self.0
     }
 }
-impl<S: AsRef<str>> Borrow<str> for Id<S> {
+impl Borrow<str> for Id {
     fn borrow(&self) -> &str {
         self.0.as_ref()
     }
 }
-impl<'de, S> Deserialize<'de> for Id<S>
-where
-    S: AsRef<str>,
-    IdUnchecked<S>: Deserialize<'de>,
-{
+impl<'de> Deserialize<'de> for Id {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: Deserializer<'de>,
     {
-        let unchecked: IdUnchecked<S> = Deserialize::deserialize(deserializer)?;
-        Id::try_from(unchecked.0).map_err(serde::de::Error::custom)
+        let unchecked: String = Deserialize::deserialize(deserializer)?;
+        Id::try_from(unchecked).map_err(serde::de::Error::custom)
     }
 }
-impl<S: AsRef<str>> Serialize for Id<S> {
+impl Serialize for Id {
     fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
     where
         Ser: Serializer,

libs/models/src/id_unchecked.rs

@@ -1,55 +0,0 @@
-use std::borrow::Cow;
-
-use serde::{Deserialize, Deserializer};
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
-pub struct IdUnchecked<S: AsRef<str>>(pub S);
-impl<'de> Deserialize<'de> for IdUnchecked<Cow<'de, str>> {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        struct Visitor;
-        impl<'de> serde::de::Visitor<'de> for Visitor {
-            type Value = IdUnchecked<Cow<'de, str>>;
-            fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
-                write!(formatter, "a valid ID")
-            }
-            fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
-            where
-                E: serde::de::Error,
-            {
-                Ok(IdUnchecked(Cow::Owned(v.to_owned())))
-            }
-            fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
-            where
-                E: serde::de::Error,
-            {
-                Ok(IdUnchecked(Cow::Owned(v)))
-            }
-            fn visit_borrowed_str<E>(self, v: &'de str) -> Result<Self::Value, E>
-            where
-                E: serde::de::Error,
-            {
-                Ok(IdUnchecked(Cow::Borrowed(v)))
-            }
-        }
-        deserializer.deserialize_any(Visitor)
-    }
-}
-impl<'de> Deserialize<'de> for IdUnchecked<String> {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        Ok(IdUnchecked(String::deserialize(deserializer)?))
-    }
-}
-impl<'de> Deserialize<'de> for IdUnchecked<&'de str> {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        Ok(IdUnchecked(<&'de str>::deserialize(deserializer)?))
-    }
-}

libs/models/src/image_id.rs

@@ -0,0 +1,38 @@
+use std::fmt::Debug;
+use std::str::FromStr;
+
+use serde::{Deserialize, Deserializer, Serialize};
+
+use crate::{Id, InvalidId, PackageId, Version};
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
+pub struct ImageId(Id);
+impl std::fmt::Display for ImageId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", &self.0)
+    }
+}
+impl ImageId {
+    pub fn for_package(&self, pkg_id: &PackageId, pkg_version: Option<&Version>) -> String {
+        format!(
+            "start9/{}/{}:{}",
+            pkg_id,
+            self.0,
+            pkg_version.map(|v| { v.as_str() }).unwrap_or("latest")
+        )
+    }
+}
+impl FromStr for ImageId {
+    type Err = InvalidId;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(ImageId(Id::try_from(s.to_owned())?))
+    }
+}
+impl<'de> Deserialize<'de> for ImageId {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        Ok(ImageId(Deserialize::deserialize(deserializer)?))
+    }
+}

libs/models/src/interface_id.rs

@@ -4,34 +4,30 @@ use serde::{Deserialize, Deserializer, Serialize};
 
 use crate::Id;
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Default)]
-pub struct InterfaceId<S: AsRef<str> = String>(Id<S>);
-impl<S: AsRef<str>> From<Id<S>> for InterfaceId<S> {
-    fn from(id: Id<S>) -> Self {
+#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
+pub struct InterfaceId(Id);
+impl From<Id> for InterfaceId {
+    fn from(id: Id) -> Self {
         Self(id)
     }
 }
-impl<S: AsRef<str>> std::fmt::Display for InterfaceId<S> {
+impl std::fmt::Display for InterfaceId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", &self.0)
     }
 }
-impl<S: AsRef<str>> std::ops::Deref for InterfaceId<S> {
-    type Target = S;
+impl std::ops::Deref for InterfaceId {
+    type Target = String;
     fn deref(&self) -> &Self::Target {
         &*self.0
     }
 }
-impl<S: AsRef<str>> AsRef<str> for InterfaceId<S> {
+impl AsRef<str> for InterfaceId {
     fn as_ref(&self) -> &str {
         self.0.as_ref()
     }
 }
-impl<'de, S> Deserialize<'de> for InterfaceId<S>
-where
-    S: AsRef<str>,
-    Id<S>: Deserialize<'de>,
-{
+impl<'de> Deserialize<'de> for InterfaceId {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: Deserializer<'de>,
@@ -39,7 +35,7 @@ where
         Ok(InterfaceId(Deserialize::deserialize(deserializer)?))
     }
 }
-impl<S: AsRef<str>> AsRef<Path> for InterfaceId<S> {
+impl AsRef<Path> for InterfaceId {
     fn as_ref(&self) -> &Path {
         self.0.as_ref().as_ref()
     }

libs/models/src/lib.rs

@@ -2,7 +2,7 @@ mod action_id;
 mod errors;
 mod health_check_id;
 mod id;
-mod id_unchecked;
+mod image_id;
 mod interface_id;
 mod invalid_id;
 mod package_id;
@@ -14,7 +14,7 @@ pub use action_id::*;
 pub use errors::*;
 pub use health_check_id::*;
 pub use id::*;
-pub use id_unchecked::*;
+pub use image_id::*;
 pub use interface_id::*;
 pub use invalid_id::*;
 pub use package_id::*;

libs/models/src/package_id.rs

@@ -6,66 +6,54 @@ use serde::{Deserialize, Serialize, Serializer};
 
 use crate::{Id, InvalidId, SYSTEM_ID};
 
-pub const SYSTEM_PACKAGE_ID: PackageId<&'static str> = PackageId(SYSTEM_ID);
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub struct PackageId<S: AsRef<str> = String>(Id<S>);
-impl<'a> PackageId<&'a str> {
-    pub fn owned(&self) -> PackageId {
-        PackageId(self.0.owned())
-    }
+lazy_static::lazy_static! {
+    pub static ref SYSTEM_PACKAGE_ID: PackageId = PackageId(SYSTEM_ID.clone());
 }
+#[derive(Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct PackageId(Id);
 impl FromStr for PackageId {
     type Err = InvalidId;
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ok(PackageId(Id::try_from(s.to_owned())?))
     }
 }
-impl From<PackageId> for String {
-    fn from(value: PackageId) -> Self {
-        value.0.into()
-    }
-}
-impl<S: AsRef<str>> From<Id<S>> for PackageId<S> {
-    fn from(id: Id<S>) -> Self {
+impl From<Id> for PackageId {
+    fn from(id: Id) -> Self {
         PackageId(id)
     }
 }
-impl<S: AsRef<str>> std::ops::Deref for PackageId<S> {
-    type Target = S;
+impl std::ops::Deref for PackageId {
+    type Target = String;
     fn deref(&self) -> &Self::Target {
         &*self.0
     }
 }
-impl<S: AsRef<str>> AsRef<PackageId<S>> for PackageId<S> {
-    fn as_ref(&self) -> &PackageId<S> {
+impl AsRef<PackageId> for PackageId {
+    fn as_ref(&self) -> &PackageId {
         self
     }
 }
-impl<S: AsRef<str>> std::fmt::Display for PackageId<S> {
+impl std::fmt::Display for PackageId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", &self.0)
     }
 }
-impl<S: AsRef<str>> AsRef<str> for PackageId<S> {
+impl AsRef<str> for PackageId {
     fn as_ref(&self) -> &str {
         self.0.as_ref()
     }
 }
-impl<S: AsRef<str>> Borrow<str> for PackageId<S> {
+impl Borrow<str> for PackageId {
     fn borrow(&self) -> &str {
         self.0.as_ref()
     }
 }
-impl<S: AsRef<str>> AsRef<Path> for PackageId<S> {
+impl AsRef<Path> for PackageId {
     fn as_ref(&self) -> &Path {
         self.0.as_ref().as_ref()
     }
 }
-impl<'de, S> Deserialize<'de> for PackageId<S>
-where
-    S: AsRef<str>,
-    Id<S>: Deserialize<'de>,
-{
+impl<'de> Deserialize<'de> for PackageId {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: serde::de::Deserializer<'de>,
@@ -73,10 +61,7 @@ where
         Ok(PackageId(Deserialize::deserialize(deserializer)?))
     }
 }
-impl<S> Serialize for PackageId<S>
-where
-    S: AsRef<str>,
-{
+impl Serialize for PackageId {
     fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
     where
         Ser: Serializer,

libs/models/src/volume_id.rs

@@ -3,14 +3,14 @@ use std::path::Path;
 
 use serde::{Deserialize, Deserializer, Serialize};
 
-use crate::{Id, IdUnchecked};
+use crate::Id;
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub enum VolumeId<S: AsRef<str> = String> {
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum VolumeId {
     Backup,
-    Custom(Id<S>),
+    Custom(Id),
 }
-impl<S: AsRef<str>> std::fmt::Display for VolumeId<S> {
+impl std::fmt::Display for VolumeId {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             VolumeId::Backup => write!(f, "BACKUP"),
@@ -18,7 +18,7 @@ impl<S: AsRef<str>> std::fmt::Display for VolumeId<S> {
         }
     }
 }
-impl<S: AsRef<str>> AsRef<str> for VolumeId<S> {
+impl AsRef<str> for VolumeId {
     fn as_ref(&self) -> &str {
         match self {
             VolumeId::Backup => "BACKUP",
@@ -26,33 +26,29 @@ impl<S: AsRef<str>> AsRef<str> for VolumeId<S> {
         }
     }
 }
-impl<S: AsRef<str>> Borrow<str> for VolumeId<S> {
+impl Borrow<str> for VolumeId {
     fn borrow(&self) -> &str {
         self.as_ref()
     }
 }
-impl<S: AsRef<str>> AsRef<Path> for VolumeId<S> {
+impl AsRef<Path> for VolumeId {
     fn as_ref(&self) -> &Path {
         AsRef::<str>::as_ref(self).as_ref()
     }
 }
-impl<'de, S> Deserialize<'de> for VolumeId<S>
-where
-    S: AsRef<str>,
-    IdUnchecked<S>: Deserialize<'de>,
-{
+impl<'de> Deserialize<'de> for VolumeId {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: Deserializer<'de>,
     {
-        let unchecked: IdUnchecked<S> = Deserialize::deserialize(deserializer)?;
-        Ok(match unchecked.0.as_ref() {
+        let unchecked: String = Deserialize::deserialize(deserializer)?;
+        Ok(match unchecked.as_ref() {
             "BACKUP" => VolumeId::Backup,
-            _ => VolumeId::Custom(Id::try_from(unchecked.0).map_err(serde::de::Error::custom)?),
+            _ => VolumeId::Custom(Id::try_from(unchecked).map_err(serde::de::Error::custom)?),
         })
     }
 }
-impl<S: AsRef<str>> Serialize for VolumeId<S> {
+impl Serialize for VolumeId {
     fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
     where
         Ser: serde::Serializer,