Merge branch 'next/minor' of github.com:Start9Labs/start-os into next/major

core/startos/src/version/mod.rs

@@ -1,10 +1,15 @@
+use std::any::Any;
 use std::cmp::Ordering;
+use std::panic::{RefUnwindSafe, UnwindSafe};
 
 use color_eyre::eyre::eyre;
 use futures::future::BoxFuture;
 use futures::{Future, FutureExt};
-use imbl_value::InternedString;
+use imbl::Vector;
+use imbl_value::{to_value, InternedString};
+use patch_db::json_ptr::{JsonPointer, ROOT};
 
+use crate::context::RpcContext;
 use crate::db::model::Database;
 use crate::prelude::*;
 use crate::progress::PhaseProgressTrackerHandle;
@@ -21,8 +26,68 @@ mod v0_3_6_alpha_4;
 mod v0_3_6_alpha_5;
 mod v0_3_6_alpha_6;
 mod v0_3_6_alpha_7;
+mod v0_3_6_alpha_8;
 
-pub type Current = v0_3_6_alpha_5::Version; // VERSION_BUMP
+pub type Current = v0_3_6_alpha_8::Version; // VERSION_BUMP
+
+impl Current {
+    #[instrument(skip(self, db))]
+    pub async fn pre_init(self, db: &PatchDb) -> Result<(), Error> {
+        let from = from_value::<Version>(
+            version_accessor(&mut db.dump(&ROOT).await.value)
+                .or_not_found("`version` in db")?
+                .clone(),
+        )?
+        .as_version_t()?;
+        match from.semver().cmp(&self.semver()) {
+            Ordering::Greater => {
+                db.apply_function(|mut db| {
+                    rollback_to_unchecked(&from, &self, &mut db)?;
+                    Ok::<_, Error>((db, ()))
+                })
+                .await?;
+            }
+            Ordering::Less => {
+                let pre_ups = PreUps::load(&from, &self).await?;
+                db.apply_function(|mut db| {
+                    migrate_from_unchecked(&from, &self, pre_ups, &mut db)?;
+                    Ok::<_, Error>((db, ()))
+                })
+                .await?;
+            }
+            Ordering::Equal => (),
+        }
+        Ok(())
+    }
+}
+
+pub async fn post_init(ctx: &RpcContext) -> Result<(), Error> {
+    let mut peek;
+    while let Some(version) = {
+        peek = ctx.db.peek().await;
+        peek.as_public()
+            .as_server_info()
+            .as_post_init_migration_todos()
+            .de()?
+            .first()
+            .cloned()
+            .map(Version::from_exver_version)
+            .as_ref()
+            .map(Version::as_version_t)
+            .transpose()?
+    } {
+        version.0.post_up(ctx).await?;
+        ctx.db
+            .mutate(|db| {
+                db.as_public_mut()
+                    .as_server_info_mut()
+                    .as_post_init_migration_todos_mut()
+                    .mutate(|m| Ok(m.remove(&version.0.semver())))
+            })
+            .await?;
+    }
+    Ok(())
+}
 
 #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
 #[serde(untagged)]
@@ -40,6 +105,7 @@ enum Version {
     V0_3_6_alpha_5(Wrapper<v0_3_6_alpha_5::Version>),
     V0_3_6_alpha_6(Wrapper<v0_3_6_alpha_6::Version>),
     V0_3_6_alpha_7(Wrapper<v0_3_6_alpha_7::Version>),
+    V0_3_6_alpha_8(Wrapper<v0_3_6_alpha_8::Version>),
     Other(exver::Version),
 }
 
@@ -52,6 +118,34 @@ impl Version {
                 Version::Other(version)
             })
     }
+    fn as_version_t(&self) -> Result<DynVersion, Error> {
+        Ok(match self {
+            Self::LT0_3_5(_) => {
+                return Err(Error::new(
+                    eyre!("cannot migrate from versions before 0.3.5"),
+                    ErrorKind::MigrationFailed,
+                ))
+            }
+            Self::V0_3_5(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_5_1(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_5_2(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_0(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_1(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_2(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_3(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_4(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_5(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_6(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_7(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_8(v) => DynVersion(Box::new(v.0)),
+            Self::Other(v) => {
+                return Err(Error::new(
+                    eyre!("unknown version {v}"),
+                    ErrorKind::MigrationFailed,
+                ))
+            }
+        })
+    }
     #[cfg(test)]
     fn as_exver(&self) -> exver::Version {
         match self {
@@ -67,115 +161,262 @@ impl Version {
             Version::V0_3_6_alpha_5(Wrapper(x)) => x.semver(),
             Version::V0_3_6_alpha_6(Wrapper(x)) => x.semver(),
             Version::V0_3_6_alpha_7(Wrapper(x)) => x.semver(),
+            Version::V0_3_6_alpha_8(Wrapper(x)) => x.semver(),
             Version::Other(x) => x.clone(),
         }
     }
 }
 
+fn version_accessor(db: &mut Value) -> Option<&mut Value> {
+    if db.get("public").is_some() {
+        db.get_mut("public")?
+            .get_mut("serverInfo")?
+            .get_mut("version")
+    } else {
+        db.get_mut("server-info")?.get_mut("version")
+    }
+}
+
+fn version_compat_accessor(db: &mut Value) -> Option<&mut Value> {
+    if db.get("public").is_some() {
+        let server_info = db.get_mut("public")?.get_mut("serverInfo")?;
+        if server_info.get("packageVersionCompat").is_some() {
+            server_info.get_mut("packageVersionCompat")
+        } else {
+            if let Some(prev) = server_info.get("eosVersionCompat").cloned() {
+                server_info
+                    .as_object_mut()?
+                    .insert("packageVersionCompat".into(), prev);
+            } else if let Some(prev) = server_info.get("versionCompat").cloned() {
+                server_info
+                    .as_object_mut()?
+                    .insert("packageVersionCompat".into(), prev);
+            }
+            server_info.get_mut("packageVersionCompat")
+        }
+    } else {
+        db.get_mut("server-info")?.get_mut("eos-version-compat")
+    }
+}
+
+fn post_init_migration_todos_accessor(db: &mut Value) -> Option<&mut Value> {
+    let server_info = if db.get("public").is_some() {
+        db.get_mut("public")?.get_mut("serverInfo")?
+    } else {
+        db.get_mut("server-info")?
+    };
+    if server_info.get("postInitMigrationTodos").is_none() {
+        server_info
+            .as_object_mut()?
+            .insert("postInitMigrationTodos".into(), Value::Array(Vector::new()));
+    }
+    server_info.get_mut("postInitMigrationTodos")
+}
+
+struct PreUps {
+    prev: Option<Box<PreUps>>,
+    value: Box<dyn Any + UnwindSafe + Send + 'static>,
+}
+impl PreUps {
+    #[instrument(skip(from, to))]
+    fn load<'a, VFrom: DynVersionT + ?Sized, VTo: DynVersionT + ?Sized>(
+        from: &'a VFrom,
+        to: &'a VTo,
+    ) -> BoxFuture<'a, Result<Self, Error>> {
+        async {
+            let previous = to.previous();
+            let prev = match from.semver().cmp(&previous.semver()) {
+                Ordering::Less => Some(Box::new(PreUps::load(from, &previous).await?)),
+                Ordering::Greater => {
+                    return Err(Error::new(
+                        eyre!(
+                            "NO PATH FROM {}, THIS IS LIKELY A MISTAKE IN THE VERSION DEFINITION",
+                            from.semver()
+                        ),
+                        crate::ErrorKind::MigrationFailed,
+                    ))
+                }
+                Ordering::Equal => None,
+            };
+            Ok(Self {
+                prev,
+                value: to.pre_up().await?,
+            })
+        }
+        .boxed()
+    }
+}
+
+fn migrate_from_unchecked<VFrom: DynVersionT + ?Sized, VTo: DynVersionT + ?Sized>(
+    from: &VFrom,
+    to: &VTo,
+    pre_ups: PreUps,
+    db: &mut Value,
+) -> Result<(), Error> {
+    let previous = to.previous();
+    match pre_ups.prev {
+        Some(prev) if from.semver() < previous.semver() => {
+            migrate_from_unchecked(from, &previous, *prev, db)?
+        }
+        _ if from.semver() > previous.semver() => {
+            return Err(Error::new(
+                eyre!(
+                    "NO PATH FROM {}, THIS IS LIKELY A MISTAKE IN THE VERSION DEFINITION",
+                    from.semver()
+                ),
+                crate::ErrorKind::MigrationFailed,
+            ));
+        }
+        _ => (),
+    };
+    to.up(db, pre_ups.value)?;
+    to.commit(db)?;
+    Ok(())
+}
+
+fn rollback_to_unchecked<VFrom: DynVersionT + ?Sized, VTo: DynVersionT + ?Sized>(
+    from: &VFrom,
+    to: &VTo,
+    db: &mut Value,
+) -> Result<(), Error> {
+    let previous = from.previous();
+    from.down(db)?;
+    previous.commit(db)?;
+    if to.semver() < previous.semver() {
+        rollback_to_unchecked(&previous, to, db)?
+    } else if to.semver() > previous.semver() {
+        return Err(Error::new(
+            eyre!(
+                "NO PATH TO {}, THIS IS LIKELY A MISTAKE IN THE VERSION DEFINITION",
+                to.semver()
+            ),
+            crate::ErrorKind::MigrationFailed,
+        ));
+    }
+    Ok(())
+}
+
 pub trait VersionT
 where
-    Self: Sized + Send + Sync,
+    Self: Default + Copy + Sized + RefUnwindSafe + Send + Sync + 'static,
 {
     type Previous: VersionT;
-    fn new() -> Self;
+    type PreUpRes: Send + UnwindSafe;
+    fn semver(self) -> exver::Version;
+    fn compat(self) -> &'static exver::VersionRange;
+    /// MUST NOT change system state. Intended for async I/O reads
+    fn pre_up(self) -> impl Future<Output = Result<Self::PreUpRes, Error>> + Send + 'static;
+    fn up(self, db: &mut Value, input: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    /// MUST be idempotent, and is run after *all* db migrations
+    fn post_up<'a>(
+        self,
+        ctx: &'a RpcContext,
+    ) -> impl Future<Output = Result<(), Error>> + Send + 'a {
+        async { Ok(()) }
+    }
+    fn down(self, db: &mut Value) -> Result<(), Error> {
+        Err(Error::new(
+            eyre!("downgrades prohibited"),
+            ErrorKind::InvalidRequest,
+        ))
+    }
+    fn commit(self, db: &mut Value) -> Result<(), Error> {
+        *version_accessor(db).or_not_found("`version` in db")? = to_value(&self.semver())?;
+        *version_compat_accessor(db).or_not_found("`versionCompat` in db")? =
+            to_value(self.compat())?;
+        post_init_migration_todos_accessor(db)
+            .or_not_found("`serverInfo` in db")?
+            .as_array_mut()
+            .ok_or_else(|| {
+                Error::new(
+                    eyre!("postInitMigrationTodos is not an array"),
+                    ErrorKind::Database,
+                )
+            })?
+            .push_back(to_value(&self.semver())?);
+        Ok(())
+    }
+}
+
+struct DynVersion(Box<dyn DynVersionT>);
+unsafe impl Send for DynVersion {}
+
+trait DynVersionT: RefUnwindSafe + Send + Sync {
+    fn previous(&self) -> DynVersion;
     fn semver(&self) -> exver::Version;
     fn compat(&self) -> &'static exver::VersionRange;
-    fn up(&self, db: &TypedPatchDb<Database>) -> impl Future<Output = Result<(), Error>> + Send;
-    fn down(&self, db: &TypedPatchDb<Database>) -> impl Future<Output = Result<(), Error>> + Send;
-    fn commit(
-        &self,
-        db: &TypedPatchDb<Database>,
-    ) -> impl Future<Output = Result<(), Error>> + Send {
-        async {
-            let semver = self.semver().into();
-            let compat = self.compat().clone();
-            db.mutate(|d| {
-                d.as_public_mut()
-                    .as_server_info_mut()
-                    .as_version_mut()
-                    .ser(&semver)?;
-                d.as_public_mut()
-                    .as_server_info_mut()
-                    .as_eos_version_compat_mut()
-                    .ser(&compat)?;
-                Ok(())
-            })
-            .await?;
-            Ok(())
-        }
+    fn pre_up(&self) -> BoxFuture<'static, Result<Box<dyn Any + UnwindSafe + Send>, Error>>;
+    fn up(&self, db: &mut Value, input: Box<dyn Any + Send>) -> Result<(), Error>;
+    fn post_up<'a>(&self, ctx: &'a RpcContext) -> BoxFuture<'a, Result<(), Error>>;
+    fn down(&self, db: &mut Value) -> Result<(), Error>;
+    fn commit(&self, db: &mut Value) -> Result<(), Error>;
+}
+impl<T> DynVersionT for T
+where
+    T: VersionT,
+{
+    fn previous(&self) -> DynVersion {
+        DynVersion(Box::new(<Self as VersionT>::Previous::default()))
     }
-    fn migrate_to<V: VersionT>(
-        &self,
-        version: &V,
-        db: &TypedPatchDb<Database>,
-        progress: &mut PhaseProgressTrackerHandle,
-    ) -> impl Future<Output = Result<(), Error>> + Send {
-        async {
-            match self.semver().cmp(&version.semver()) {
-                Ordering::Greater => self.rollback_to_unchecked(version, db, progress).await,
-                Ordering::Less => version.migrate_from_unchecked(self, db, progress).await,
-                Ordering::Equal => Ok(()),
-            }
-        }
+    fn semver(&self) -> exver::Version {
+        VersionT::semver(*self)
     }
-    fn migrate_from_unchecked<'a, V: VersionT>(
-        &'a self,
-        version: &'a V,
-        db: &'a TypedPatchDb<Database>,
-        progress: &'a mut PhaseProgressTrackerHandle,
-    ) -> BoxFuture<'a, Result<(), Error>> {
-        progress.add_total(1);
-        async {
-            let previous = Self::Previous::new();
-            if version.semver() < previous.semver() {
-                previous
-                    .migrate_from_unchecked(version, db, progress)
-                    .await?;
-            } else if version.semver() > previous.semver() {
-                return Err(Error::new(
-                    eyre!(
-                        "NO PATH FROM {}, THIS IS LIKELY A MISTAKE IN THE VERSION DEFINITION",
-                        version.semver()
-                    ),
-                    crate::ErrorKind::MigrationFailed,
-                ));
-            }
-            tracing::info!("{} -> {}", previous.semver(), self.semver(),);
-            self.up(db).await?;
-            self.commit(db).await?;
-            *progress += 1;
-            Ok(())
-        }
-        .boxed()
+    fn compat(&self) -> &'static exver::VersionRange {
+        VersionT::compat(*self)
     }
-    fn rollback_to_unchecked<'a, V: VersionT>(
-        &'a self,
-        version: &'a V,
-        db: &'a TypedPatchDb<Database>,
-        progress: &'a mut PhaseProgressTrackerHandle,
-    ) -> BoxFuture<'a, Result<(), Error>> {
-        async {
-            let previous = Self::Previous::new();
-            tracing::info!("{} -> {}", self.semver(), previous.semver(),);
-            self.down(db).await?;
-            previous.commit(db).await?;
-            *progress += 1;
-            if version.semver() < previous.semver() {
-                previous
-                    .rollback_to_unchecked(version, db, progress)
-                    .await?;
-            } else if version.semver() > previous.semver() {
-                return Err(Error::new(
-                    eyre!(
-                        "NO PATH TO {}, THIS IS LIKELY A MISTAKE IN THE VERSION DEFINITION",
-                        version.semver()
-                    ),
-                    crate::ErrorKind::MigrationFailed,
-                ));
-            }
-            Ok(())
-        }
-        .boxed()
+    fn pre_up(&self) -> BoxFuture<'static, Result<Box<dyn Any + UnwindSafe + Send>, Error>> {
+        let v = *self;
+        async move { Ok(Box::new(VersionT::pre_up(v).await?) as Box<dyn Any + UnwindSafe + Send>) }
+            .boxed()
+    }
+    fn up(&self, db: &mut Value, input: Box<dyn Any + Send>) -> Result<(), Error> {
+        VersionT::up(
+            *self,
+            db,
+            *input.downcast().map_err(|_| {
+                Error::new(
+                    eyre!("pre_up returned unexpected type"),
+                    ErrorKind::Incoherent,
+                )
+            })?,
+        )
+    }
+    fn post_up<'a>(&self, ctx: &'a RpcContext) -> BoxFuture<'a, Result<(), Error>> {
+        VersionT::post_up(*self, ctx).boxed()
+    }
+    fn down(&self, db: &mut Value) -> Result<(), Error> {
+        VersionT::down(*self, db)
+    }
+    fn commit(&self, db: &mut Value) -> Result<(), Error> {
+        VersionT::commit(*self, db)
+    }
+}
+impl DynVersionT for DynVersion {
+    fn previous(&self) -> DynVersion {
+        self.0.previous()
+    }
+    fn semver(&self) -> exver::Version {
+        self.0.semver()
+    }
+    fn compat(&self) -> &'static exver::VersionRange {
+        self.0.compat()
+    }
+    fn pre_up(&self) -> BoxFuture<'static, Result<Box<dyn Any + UnwindSafe + Send>, Error>> {
+        self.0.pre_up()
+    }
+    fn up(&self, db: &mut Value, input: Box<dyn Any + Send>) -> Result<(), Error> {
+        self.0.up(db, input)
+    }
+    fn post_up<'a>(&self, ctx: &'a RpcContext) -> BoxFuture<'a, Result<(), Error>> {
+        self.0.post_up(ctx)
+    }
+    fn down(&self, db: &mut Value) -> Result<(), Error> {
+        self.0.down(db)
+    }
+    fn commit(&self, db: &mut Value) -> Result<(), Error> {
+        self.0.commit(db)
     }
 }
 
@@ -195,7 +436,7 @@ where
 {
     fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
         let v = exver::Version::deserialize(deserializer)?;
-        let version = T::new();
+        let version = T::default();
         if v < version.semver() {
             Ok(Self(version, v))
         } else {
@@ -220,7 +461,7 @@ where
 {
     fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
         let v = exver::Version::deserialize(deserializer)?;
-        let version = T::new();
+        let version = T::default();
         if v == version.semver() {
             Ok(Wrapper(version))
         } else {
@@ -229,62 +470,6 @@ where
     }
 }
 
-pub async fn init(
-    db: &TypedPatchDb<Database>,
-    mut progress: PhaseProgressTrackerHandle,
-) -> Result<(), Error> {
-    progress.start();
-    db.mutate(|db| {
-        db.as_public_mut()
-            .as_server_info_mut()
-            .as_version_mut()
-            .map_mutate(|v| {
-                Ok(if v == exver::Version::new([0, 3, 6], []) {
-                    v0_3_6_alpha_0::Version::new().semver()
-                } else {
-                    v
-                })
-            })
-    })
-    .await?; // TODO: remove before releasing 0.3.6
-    let version = Version::from_exver_version(
-        db.peek()
-            .await
-            .as_public()
-            .as_server_info()
-            .as_version()
-            .de()?,
-    );
-
-    match version {
-        Version::LT0_3_5(_) => {
-            return Err(Error::new(
-                eyre!("Cannot migrate from pre-0.3.5. Please update to v0.3.5 first."),
-                ErrorKind::MigrationFailed,
-            ));
-        }
-        Version::V0_3_5(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_5_1(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_5_2(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_0(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_1(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_2(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_3(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_4(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_5(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_6(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_7(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::Other(_) => {
-            return Err(Error::new(
-                eyre!("Cannot downgrade"),
-                crate::ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    progress.complete();
-    Ok(())
-}
-
 pub const COMMIT_HASH: &str =
     include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../GIT_HASH.txt"));
 
@@ -315,17 +500,17 @@ mod tests {
 
     fn versions() -> impl Strategy<Value = Version> {
         prop_oneof![
-            Just(Version::V0_3_5(Wrapper(v0_3_5::Version::new()))),
-            Just(Version::V0_3_5_1(Wrapper(v0_3_5_1::Version::new()))),
-            Just(Version::V0_3_5_2(Wrapper(v0_3_5_2::Version::new()))),
+            Just(Version::V0_3_5(Wrapper(v0_3_5::Version::default()))),
+            Just(Version::V0_3_5_1(Wrapper(v0_3_5_1::Version::default()))),
+            Just(Version::V0_3_5_2(Wrapper(v0_3_5_2::Version::default()))),
             Just(Version::V0_3_6_alpha_0(Wrapper(
-                v0_3_6_alpha_0::Version::new()
+                v0_3_6_alpha_0::Version::default()
             ))),
             Just(Version::V0_3_6_alpha_1(Wrapper(
-                v0_3_6_alpha_1::Version::new()
+                v0_3_6_alpha_1::Version::default()
             ))),
             Just(Version::V0_3_6_alpha_2(Wrapper(
-                v0_3_6_alpha_2::Version::new()
+                v0_3_6_alpha_2::Version::default()
             ))),
             em_version().prop_map(Version::Other),
         ]

core/startos/src/version/v0_3_5.rs

@@ -1,7 +1,6 @@
 use exver::{ExtendedVersion, VersionRange};
 
 use super::VersionT;
-use crate::db::model::Database;
 use crate::prelude::*;
 use crate::version::Current;
 
@@ -17,7 +16,7 @@ lazy_static::lazy_static! {
         VersionRange::anchor(
             exver::LTE,
             ExtendedVersion::new(
-                Current::new().semver(),
+                Current::default().semver(),
                 exver::Version::default(),
             )
         ),
@@ -25,24 +24,26 @@ lazy_static::lazy_static! {
     static ref V0_3_5: exver::Version = exver::Version::new([0, 3, 5], []);
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = Self;
-    fn new() -> Self {
-        Version
-    }
-    fn semver(&self) -> exver::Version {
-        V0_3_5.clone()
-    }
-    fn compat(&self) -> &'static VersionRange {
-        &V0_3_0_COMPAT
-    }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn semver(self) -> exver::Version {
+        V0_3_5.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_5_1.rs

@@ -2,31 +2,32 @@ use exver::VersionRange;
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_5, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
     static ref V0_3_5_1: exver::Version = exver::Version::new([0, 3, 5, 1], []);
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_5::Version;
-    fn new() -> Self {
-        Version
-    }
-    fn semver(&self) -> exver::Version {
-        V0_3_5_1.clone()
-    }
-    fn compat(&self) -> &'static VersionRange {
-        &V0_3_0_COMPAT
-    }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn semver(self) -> exver::Version {
+        V0_3_5_1.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_5_2.rs

@@ -2,31 +2,32 @@ use exver::VersionRange;
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_5_1, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
     static ref V0_3_5_2: exver::Version = exver::Version::new([0, 3, 5, 2], []);
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_5_1::Version;
-    fn new() -> Self {
-        Version
-    }
-    fn semver(&self) -> exver::Version {
-        V0_3_5_2.clone()
-    }
-    fn compat(&self) -> &'static VersionRange {
-        &V0_3_0_COMPAT
-    }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn semver(self) -> exver::Version {
+        V0_3_5_2.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_0.rs

@@ -1,9 +1,44 @@
+use std::collections::BTreeMap;
+use std::future::Future;
+use std::path::Path;
+
+use chrono::{DateTime, Utc};
+use ed25519_dalek::SigningKey;
 use exver::{PreReleaseSegment, VersionRange};
+use imbl_value::{json, InternedString};
+use itertools::Itertools;
+use models::PackageId;
+use openssl::pkey::{PKey, Private};
+use openssl::x509::X509;
+use patch_db::ModelExt;
+use sqlx::postgres::PgConnectOptions;
+use sqlx::{PgPool, Row};
+use ssh_key::Fingerprint;
+use tokio::process::Command;
+use torut::onion::TorSecretKeyV3;
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_5_2, VersionT};
+use crate::account::AccountInfo;
+use crate::auth::Sessions;
+use crate::backup::target::cifs::CifsTargets;
+use crate::context::RpcContext;
 use crate::db::model::Database;
+use crate::disk::mount::filesystem::cifs::Cifs;
+use crate::disk::mount::util::unmount;
+use crate::hostname::Hostname;
+use crate::net::forward::AvailablePorts;
+use crate::net::keys::KeyStore;
+use crate::net::ssl::CertStore;
+use crate::net::tor;
+use crate::net::tor::OnionStore;
+use crate::notifications::{Notification, Notifications};
 use crate::prelude::*;
+use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
+use crate::ssh::{SshKeys, SshPubKey};
+use crate::util::crypto::ed25519_expand_key;
+use crate::util::serde::{Pem, PemEncoding};
+use crate::util::Invoke;
 
 lazy_static::lazy_static! {
     static ref V0_3_6_alpha_0: exver::Version = exver::Version::new(
@@ -12,24 +47,537 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[tracing::instrument(skip_all)]
+async fn init_postgres(datadir: impl AsRef<Path>) -> Result<PgPool, Error> {
+    let db_dir = datadir.as_ref().join("main/postgresql");
+    if tokio::process::Command::new("mountpoint")
+        .arg("/var/lib/postgresql")
+        .stdout(std::process::Stdio::null())
+        .stderr(std::process::Stdio::null())
+        .status()
+        .await?
+        .success()
+    {
+        unmount("/var/lib/postgresql", true).await?;
+    }
+    let exists = tokio::fs::metadata(&db_dir).await.is_ok();
+    if !exists {
+        Command::new("cp")
+            .arg("-ra")
+            .arg("/var/lib/postgresql")
+            .arg(&db_dir)
+            .invoke(crate::ErrorKind::Filesystem)
+            .await?;
+    }
+    Command::new("chown")
+        .arg("-R")
+        .arg("postgres:postgres")
+        .arg(&db_dir)
+        .invoke(crate::ErrorKind::Database)
+        .await?;
+
+    let mut pg_paths = tokio::fs::read_dir("/usr/lib/postgresql").await?;
+    let mut pg_version = None;
+    while let Some(pg_path) = pg_paths.next_entry().await? {
+        let pg_path_version = pg_path
+            .file_name()
+            .to_str()
+            .map(|v| v.parse())
+            .transpose()?
+            .unwrap_or(0);
+        if pg_path_version > pg_version.unwrap_or(0) {
+            pg_version = Some(pg_path_version)
+        }
+    }
+    let pg_version = pg_version.ok_or_else(|| {
+        Error::new(
+            eyre!("could not determine postgresql version"),
+            crate::ErrorKind::Database,
+        )
+    })?;
+
+    crate::disk::mount::util::bind(&db_dir, "/var/lib/postgresql", false).await?;
+
+    let pg_version_string = pg_version.to_string();
+    let pg_version_path = db_dir.join(&pg_version_string);
+    if exists
+    // maybe migrate
+    {
+        let incomplete_path = db_dir.join(format!("{pg_version}.migration.incomplete"));
+        if tokio::fs::metadata(&incomplete_path).await.is_ok() // previous migration was incomplete
+        && tokio::fs::metadata(&pg_version_path).await.is_ok()
+        {
+            tokio::fs::remove_dir_all(&pg_version_path).await?;
+        }
+        if tokio::fs::metadata(&pg_version_path).await.is_err()
+        // need to migrate
+        {
+            let conf_dir = Path::new("/etc/postgresql").join(pg_version.to_string());
+            let conf_dir_tmp = {
+                let mut tmp = conf_dir.clone();
+                tmp.set_extension("tmp");
+                tmp
+            };
+            if tokio::fs::metadata(&conf_dir).await.is_ok() {
+                Command::new("mv")
+                    .arg(&conf_dir)
+                    .arg(&conf_dir_tmp)
+                    .invoke(ErrorKind::Filesystem)
+                    .await?;
+            }
+            let mut old_version = pg_version;
+            while old_version > 13
+            /* oldest pg version included in startos */
+            {
+                old_version -= 1;
+                let old_datadir = db_dir.join(old_version.to_string());
+                if tokio::fs::metadata(&old_datadir).await.is_ok() {
+                    tokio::fs::File::create(&incomplete_path)
+                        .await?
+                        .sync_all()
+                        .await?;
+                    Command::new("pg_upgradecluster")
+                        .arg(old_version.to_string())
+                        .arg("main")
+                        .invoke(crate::ErrorKind::Database)
+                        .await?;
+                    break;
+                }
+            }
+            if tokio::fs::metadata(&conf_dir).await.is_ok() {
+                if tokio::fs::metadata(&conf_dir).await.is_ok() {
+                    tokio::fs::remove_dir_all(&conf_dir).await?;
+                }
+                Command::new("mv")
+                    .arg(&conf_dir_tmp)
+                    .arg(&conf_dir)
+                    .invoke(ErrorKind::Filesystem)
+                    .await?;
+            }
+            tokio::fs::remove_file(&incomplete_path).await?;
+        }
+        if tokio::fs::metadata(&incomplete_path).await.is_ok() {
+            unreachable!() // paranoia
+        }
+    }
+
+    Command::new("systemctl")
+        .arg("start")
+        .arg(format!("postgresql@{pg_version}-main.service"))
+        .invoke(crate::ErrorKind::Database)
+        .await?;
+    if !exists {
+        Command::new("sudo")
+            .arg("-u")
+            .arg("postgres")
+            .arg("createuser")
+            .arg("root")
+            .invoke(crate::ErrorKind::Database)
+            .await?;
+        Command::new("sudo")
+            .arg("-u")
+            .arg("postgres")
+            .arg("createdb")
+            .arg("secrets")
+            .arg("-O")
+            .arg("root")
+            .invoke(crate::ErrorKind::Database)
+            .await?;
+    }
+
+    let secret_store =
+        PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root")).await?;
+    sqlx::migrate!()
+        .run(&secret_store)
+        .await
+        .with_kind(crate::ErrorKind::Database)?;
+    dbg!("Init Postgres Done");
+    Ok(secret_store)
+}
+
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_5_2::Version;
-    fn new() -> Self {
-        Version
-    }
-    fn semver(&self) -> exver::Version {
+    type PreUpRes = (AccountInfo, SshKeys, CifsTargets, Notifications);
+    fn semver(self) -> exver::Version {
         V0_3_6_alpha_0.clone()
     }
-    fn compat(&self) -> &'static VersionRange {
+    fn compat(self) -> &'static VersionRange {
         &V0_3_0_COMPAT
     }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
-        Err(Error::new(eyre!("unimplemented"), ErrorKind::Unknown))
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        let pg = init_postgres("/embassy-data").await?;
+        let account = previous_account_info(&pg).await?;
+
+        let ssh_keys = previous_ssh_keys(&pg).await?;
+
+        let cifs = previous_cifs(&pg).await?;
+
+        let notifications = previous_notifications(pg).await?;
+
+        Ok((account, ssh_keys, cifs, notifications))
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn up(
+        self,
+        db: &mut Value,
+        (account, ssh_keys, cifs, notifications): Self::PreUpRes,
+    ) -> Result<(), Error> {
+        let wifi = json!({
+            "infterface": db["server-info"]["wifi"]["interface"],
+            "ssids": db["server-info"]["wifi"]["ssids"],
+            "selected": db["server-info"]["wifi"]["selected"],
+            "last_region": db["server-info"]["wifi"]["last-region"],
+        });
+
+        let ip_info = {
+            let mut ip_info = json!({});
+            let empty = Default::default();
+            for (k, v) in db["server-info"]["ip-info"].as_object().unwrap_or(&empty) {
+                let k: &str = k.as_ref();
+                ip_info[k] = json!({
+                    "ipv4Range": v["ipv4-range"],
+                    "ipv6Range": v["ipv6-range"],
+                    "ipv4": v["ipv4"],
+                    "ipv6": v["ipv6"],
+                });
+            }
+            ip_info
+        };
+
+        let status_info = json!({
+            "backupProgress": db["server-info"]["status-info"]["backup-progress"],
+            "updated": db["server-info"]["status-info"]["updated"],
+            "updateProgress": db["server-info"]["status-info"]["update-progress"],
+            "shuttingDown": db["server-info"]["status-info"]["shutting-down"],
+            "restarting": db["server-info"]["status-info"]["restarting"],
+        });
+        let server_info = {
+            let mut server_info = json!({
+                "arch": db["server-info"]["arch"],
+                "platform": db["server-info"]["platform"],
+                "id": db["server-info"]["id"],
+                "hostname": db["server-info"]["hostname"],
+                "version": db["server-info"]["version"],
+                "versionCompat": db["server-info"]["eos-version-compat"],
+                "lastBackup": db["server-info"]["last-backup"],
+                "lanAddress": db["server-info"]["lan-address"],
+            });
+
+            server_info["postInitMigrationTodos"] = json!([]);
+            let tor_address: String = from_value(db["server-info"]["tor-address"].clone())?;
+            // Maybe we do this like the Public::init does
+            server_info["torAddress"] = json!(tor_address);
+            server_info["onionAddress"] = json!(tor_address
+                .replace("https://", "")
+                .replace("http://", "")
+                .replace(".onion/", ""));
+            server_info["ipInfo"] = ip_info;
+            server_info["statusInfo"] = status_info;
+            server_info["wifi"] = wifi;
+            server_info["unreadNotificationCount"] =
+                db["server-info"]["unread-notification-count"].clone();
+            server_info["passwordHash"] = db["server-info"]["password-hash"].clone();
+
+            server_info["pubkey"] = db["server-info"]["pubkey"].clone();
+            server_info["caFingerprint"] = db["server-info"]["ca-fingerprint"].clone();
+            server_info["ntpSynced"] = db["server-info"]["ntp-synced"].clone();
+            server_info["zram"] = db["server-info"]["zram"].clone();
+            server_info["governor"] = db["server-info"]["governor"].clone();
+            // This one should always be empty, doesn't exist in the previous. And the smtp is all single word key
+            server_info["smtp"] = db["server-info"]["smtp"].clone();
+            server_info
+        };
+
+        let public = json!({
+            "serverInfo": server_info,
+            "packageData": json!({}),
+            "ui": db["ui"],
+        });
+
+        let private = {
+            let mut value = json!({});
+            value["keyStore"] = to_value(&KeyStore::new(&account)?)?;
+            value["password"] = to_value(&account.password)?;
+            value["compatS9pkKey"] = to_value(&crate::db::model::private::generate_compat_key())?;
+            value["sshPrivkey"] = to_value(Pem::new_ref(&account.ssh_key))?;
+            value["sshPubkeys"] = to_value(&ssh_keys)?;
+            value["availablePorts"] = to_value(&AvailablePorts::new())?;
+            value["sessions"] = to_value(&Sessions::new())?;
+            value["notifications"] = to_value(&notifications)?;
+            value["cifs"] = to_value(&cifs)?;
+            value["packageStores"] = json!({});
+            value
+        };
+        let next: Value = json!({
+            "public": public,
+            "private": private,
+        });
+
+        dbg!("Should be done with the up");
+        *db = next;
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
+        Err(Error::new(
+            eyre!("downgrades prohibited"),
+            ErrorKind::InvalidRequest,
+        ))
+    }
+
+    #[instrument(skip(self, ctx))]
+    /// MUST be idempotent, and is run after *all* db migrations
+    async fn post_up(self, ctx: &RpcContext) -> Result<(), Error> {
+        let path = Path::new("/embassy-data/package-data/archive/");
+        if !path.is_dir() {
+            return Err(Error::new(
+                eyre!(
+                    "expected path ({}) to be a directory",
+                    path.to_string_lossy()
+                ),
+                ErrorKind::Filesystem,
+            ));
+        }
+        // Should be the name of the package
+        let mut paths = tokio::fs::read_dir(path).await?;
+        while let Some(path) = paths.next_entry().await? {
+            let path = path.path();
+            if !path.is_dir() {
+                continue;
+            }
+            // Should be the version of the package
+            let mut paths = tokio::fs::read_dir(path).await?;
+            while let Some(path) = paths.next_entry().await? {
+                let path = path.path();
+                if !path.is_dir() {
+                    continue;
+                }
+
+                // Should be s9pk
+                let mut paths = tokio::fs::read_dir(path).await?;
+                while let Some(path) = paths.next_entry().await? {
+                    let path = path.path();
+                    if path.is_dir() {
+                        continue;
+                    }
+
+                    let package_s9pk = tokio::fs::File::open(path).await?;
+                    let file = MultiCursorFile::open(&package_s9pk).await?;
+
+                    let key = ctx.db.peek().await.into_private().into_compat_s9pk_key();
+                    ctx.services
+                        .install(
+                            ctx.clone(),
+                            || crate::s9pk::load(file.clone(), || Ok(key.de()?.0), None),
+                            None::<crate::util::Never>,
+                            None,
+                        )
+                        .await?
+                        .await?
+                        .await?;
+                }
+            }
+        }
         Ok(())
     }
 }
+
+async fn previous_notifications(pg: sqlx::Pool<sqlx::Postgres>) -> Result<Notifications, Error> {
+    let notification_cursor = sqlx::query(r#"SELECT * FROM notifications"#)
+        .fetch_all(&pg)
+        .await?;
+    let notifications = {
+        let mut notifications = Notifications::default();
+        for row in notification_cursor {
+            let package_id = serde_json::from_str::<PackageId>(
+                row.try_get("package_id")
+                    .with_ctx(|_| (ErrorKind::Database, "package_id"))?,
+            )
+            .ok();
+
+            let created_at = row
+                .try_get("created_at")
+                .with_ctx(|_| (ErrorKind::Database, "created_at"))?;
+            let code = row
+                .try_get::<i64, _>("code")
+                .with_ctx(|_| (ErrorKind::Database, "code"))? as u32;
+            let id = row
+                .try_get::<i64, _>("id")
+                .with_ctx(|_| (ErrorKind::Database, "id"))? as u32;
+            let level = serde_json::from_str(
+                row.try_get("level")
+                    .with_ctx(|_| (ErrorKind::Database, "level"))?,
+            )
+            .with_kind(ErrorKind::Database)
+            .with_ctx(|_| (ErrorKind::Database, "level: serde_json "))?;
+            let title = row
+                .try_get("title")
+                .with_ctx(|_| (ErrorKind::Database, "title"))?;
+            let message = row
+                .try_get("message")
+                .with_ctx(|_| (ErrorKind::Database, "message"))?;
+            let data = serde_json::from_str(
+                row.try_get("data")
+                    .with_ctx(|_| (ErrorKind::Database, "data"))?,
+            )
+            .unwrap_or_default();
+
+            notifications.0.insert(
+                id,
+                Notification {
+                    package_id,
+                    created_at,
+                    code,
+                    level,
+                    title,
+                    message,
+                    data,
+                },
+            );
+        }
+        notifications
+    };
+    Ok(notifications)
+}
+
+#[tracing::instrument(skip_all)]
+async fn previous_cifs(pg: &sqlx::Pool<sqlx::Postgres>) -> Result<CifsTargets, Error> {
+    let cifs = sqlx::query(r#"SELECT * FROM cifs_shares"#)
+        .fetch_all(pg)
+        .await?
+        .into_iter()
+        .map(|row| {
+            let id: i64 = row.try_get("id")?;
+            Ok::<_, Error>((
+                id,
+                Cifs {
+                    hostname: row
+                        .try_get("hostname")
+                        .with_ctx(|_| (ErrorKind::Database, "hostname"))?,
+                    path: serde_json::from_str(row.try_get("path")?)
+                        .with_kind(ErrorKind::Database)
+                        .with_ctx(|_| (ErrorKind::Database, "path"))?,
+                    username: row
+                        .try_get("username")
+                        .with_ctx(|_| (ErrorKind::Database, "username"))?,
+                    password: row
+                        .try_get("password")
+                        .with_ctx(|_| (ErrorKind::Database, "password"))?,
+                },
+            ))
+        })
+        .fold(Ok::<_, Error>(CifsTargets::default()), |cifs, data| {
+            let mut cifs = cifs?;
+            let (id, cif_value) = data?;
+            cifs.0.insert(id as u32, cif_value);
+            Ok(cifs)
+        })?;
+    Ok(cifs)
+}
+
+#[tracing::instrument(skip_all)]
+async fn previous_account_info(pg: &sqlx::Pool<sqlx::Postgres>) -> Result<AccountInfo, Error> {
+    let account_query = sqlx::query(r#"SELECT * FROM account"#)
+        .fetch_one(pg)
+        .await?;
+    let account = {
+        AccountInfo {
+            password: account_query
+                .try_get("password")
+                .with_ctx(|_| (ErrorKind::Database, "password"))?,
+            tor_key: TorSecretKeyV3::try_from(
+                if let Some(bytes) = account_query
+                    .try_get::<Option<Vec<u8>>, _>("tor_key")
+                    .with_ctx(|_| (ErrorKind::Database, "tor_key"))?
+                {
+                    <[u8; 64]>::try_from(bytes)
+                        .map_err(|e| {
+                            Error::new(
+                                eyre!("expected vec of len 64, got len {}", e.len()),
+                                ErrorKind::ParseDbField,
+                            )
+                        })
+                        .with_ctx(|_| (ErrorKind::Database, "password.u8 64"))?
+                } else {
+                    ed25519_expand_key(
+                        &<[u8; 32]>::try_from(account_query.try_get::<Vec<u8>, _>("network_key")?)
+                            .map_err(|e| {
+                                Error::new(
+                                    eyre!("expected vec of len 32, got len {}", e.len()),
+                                    ErrorKind::ParseDbField,
+                                )
+                            })
+                            .with_ctx(|_| (ErrorKind::Database, "password.u8 32"))?,
+                    )
+                },
+            )?,
+            server_id: account_query
+                .try_get("server_id")
+                .with_ctx(|_| (ErrorKind::Database, "server_id"))?,
+            hostname: Hostname(
+                account_query
+                    .try_get::<String, _>("hostname")
+                    .with_ctx(|_| (ErrorKind::Database, "hostname"))?
+                    .into(),
+            ),
+            root_ca_key: PKey::private_key_from_pem(
+                &account_query
+                    .try_get::<String, _>("root_ca_key_pem")
+                    .with_ctx(|_| (ErrorKind::Database, "root_ca_key_pem"))?
+                    .as_bytes(),
+            )
+            .with_ctx(|_| (ErrorKind::Database, "private_key_from_pem"))?,
+            root_ca_cert: X509::from_pem(
+                account_query
+                    .try_get::<String, _>("root_ca_cert_pem")
+                    .with_ctx(|_| (ErrorKind::Database, "root_ca_cert_pem"))?
+                    .as_bytes(),
+            )
+            .with_ctx(|_| (ErrorKind::Database, "X509::from_pem"))?,
+            compat_s9pk_key: SigningKey::generate(&mut rand::thread_rng()),
+            ssh_key: ssh_key::PrivateKey::random(
+                &mut rand::thread_rng(),
+                ssh_key::Algorithm::Ed25519,
+            )
+            .with_ctx(|_| (ErrorKind::Database, "X509::ssh_key::PrivateKey::random"))?,
+        }
+    };
+    Ok(account)
+}
+#[tracing::instrument(skip_all)]
+async fn previous_ssh_keys(pg: &sqlx::Pool<sqlx::Postgres>) -> Result<SshKeys, Error> {
+    let ssh_query = sqlx::query(r#"SELECT * FROM ssh_keys"#)
+        .fetch_all(pg)
+        .await?;
+    let ssh_keys: SshKeys = {
+        let keys = ssh_query.into_iter().fold(
+            Ok::<_, Error>(BTreeMap::<InternedString, WithTimeData<SshPubKey>>::new()),
+            |ssh_keys, row| {
+                let mut ssh_keys = ssh_keys?;
+                let time = row
+                    .try_get::<String, _>("created_at")
+                    .map_err(Error::from)
+                    .and_then(|x| x.parse::<DateTime<Utc>>().with_kind(ErrorKind::Database))
+                    .with_ctx(|_| (ErrorKind::Database, "openssh_pubkey::created_at"))?;
+                let value: SshPubKey = row
+                    .try_get::<String, _>("openssh_pubkey")
+                    .map_err(Error::from)
+                    .and_then(|x| x.parse().map(SshPubKey).with_kind(ErrorKind::Database))
+                    .with_ctx(|_| (ErrorKind::Database, "openssh_pubkey"))?;
+                let data = WithTimeData {
+                    created_at: time,
+                    updated_at: time,
+                    value,
+                };
+                let fingerprint = row
+                    .try_get::<String, _>("fingerprint")
+                    .with_ctx(|_| (ErrorKind::Database, "fingerprint"))?;
+                ssh_keys.insert(fingerprint.into(), data);
+                Ok(ssh_keys)
+            },
+        )?;
+        SshKeys::from(keys)
+    };
+    Ok(ssh_keys)
+}

core/startos/src/version/v0_3_6_alpha_1.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_0, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
@@ -12,24 +11,26 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_0::Version;
-    fn new() -> Self {
-        Version
-    }
-    fn semver(&self) -> exver::Version {
-        V0_3_6_alpha_1.clone()
-    }
-    fn compat(&self) -> &'static VersionRange {
-        &V0_3_0_COMPAT
-    }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn semver(self) -> exver::Version {
+        V0_3_6_alpha_1.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_2.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_1, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
@@ -12,24 +11,26 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_1::Version;
-    fn new() -> Self {
-        Version
-    }
-    fn semver(&self) -> exver::Version {
-        V0_3_6_alpha_2.clone()
-    }
-    fn compat(&self) -> &'static VersionRange {
-        &V0_3_0_COMPAT
-    }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn semver(self) -> exver::Version {
+        V0_3_6_alpha_2.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_3.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_2, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
@@ -12,24 +11,26 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_2::Version;
-    fn new() -> Self {
-        Version
-    }
-    fn semver(&self) -> exver::Version {
-        V0_3_6_alpha_3.clone()
-    }
-    fn compat(&self) -> &'static VersionRange {
-        &V0_3_0_COMPAT
-    }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn semver(self) -> exver::Version {
+        V0_3_6_alpha_3.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_4.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_3, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
@@ -12,24 +11,26 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_3::Version;
-    fn new() -> Self {
-        Version
-    }
-    fn semver(&self) -> exver::Version {
-        V0_3_6_alpha_4.clone()
-    }
-    fn compat(&self) -> &'static VersionRange {
-        &V0_3_0_COMPAT
-    }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn semver(self) -> exver::Version {
+        V0_3_6_alpha_4.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_5.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_4, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
@@ -12,24 +11,26 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_4::Version;
-    fn new() -> Self {
-        Version
-    }
-    fn semver(&self) -> exver::Version {
-        V0_3_6_alpha_5.clone()
-    }
-    fn compat(&self) -> &'static VersionRange {
-        &V0_3_0_COMPAT
-    }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn semver(self) -> exver::Version {
+        V0_3_6_alpha_5.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_6.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_5, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
@@ -12,24 +11,26 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_5::Version;
-    fn new() -> Self {
-        Version
-    }
-    fn semver(&self) -> exver::Version {
-        V0_3_6_alpha_6.clone()
-    }
-    fn compat(&self) -> &'static VersionRange {
-        &V0_3_0_COMPAT
-    }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn semver(self) -> exver::Version {
+        V0_3_6_alpha_6.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_7.rs

@@ -1,9 +1,11 @@
 use exver::{PreReleaseSegment, VersionRange};
+use imbl_value::{json, InOMap};
+use tokio::process::Command;
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_6, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
+use crate::util::Invoke;
 
 lazy_static::lazy_static! {
     static ref V0_3_6_alpha_7: exver::Version = exver::Version::new(
@@ -12,24 +14,50 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_6::Version;
-    fn new() -> Self {
-        Version
-    }
-    fn semver(&self) -> exver::Version {
-        V0_3_6_alpha_7.clone()
-    }
-    fn compat(&self) -> &'static VersionRange {
-        &V0_3_0_COMPAT
-    }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn semver(self) -> exver::Version {
+        V0_3_6_alpha_7.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        let server_info = db["public"]["serverInfo"]
+            .as_object_mut()
+            .or_not_found("public.serverInfo")?;
+        server_info.insert("ram".into(), json!(0));
+        server_info.insert("devices".into(), json!([]));
+        let package_data = db["public"]["packageData"]
+            .as_object_mut()
+            .or_not_found("public.packageData")?;
+        for (_, pde) in package_data.iter_mut() {
+            if let Some(manifest) = pde["stateInfo"].get_mut("manifest") {
+                manifest["hardwareRequirements"]["device"] = json!([]);
+            }
+        }
+        Ok(())
+    }
+    async fn post_up(self, ctx: &crate::context::RpcContext) -> Result<(), Error> {
+        Command::new("systemd-firstboot")
+            .arg("--root=/media/startos/config/overlay/")
+            .arg(format!(
+                "--hostname={}",
+                ctx.account.read().await.hostname.0
+            ))
+            .invoke(ErrorKind::ParseSysInfo)
+            .await?;
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_8.rs

@@ -0,0 +1,130 @@
+use exver::{PreReleaseSegment, VersionRange};
+use tokio::fs::File;
+
+use super::v0_3_5::V0_3_0_COMPAT;
+use super::{v0_3_6_alpha_7, VersionT};
+use crate::install::PKG_ARCHIVE_DIR;
+use crate::prelude::*;
+use crate::s9pk::manifest::{DeviceFilter, Manifest};
+use crate::s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile;
+use crate::s9pk::merkle_archive::MerkleArchive;
+use crate::s9pk::v2::SIG_CONTEXT;
+use crate::s9pk::S9pk;
+use crate::service::LoadDisposition;
+use crate::util::io::create_file;
+
+lazy_static::lazy_static! {
+    static ref V0_3_6_alpha_8: exver::Version = exver::Version::new(
+        [0, 3, 6],
+        [PreReleaseSegment::String("alpha".into()), 8.into()]
+    );
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct Version;
+
+impl VersionT for Version {
+    type Previous = v0_3_6_alpha_7::Version;
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
+    }
+    fn semver(self) -> exver::Version {
+        V0_3_6_alpha_8.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, _: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    async fn post_up(self, ctx: &crate::context::RpcContext) -> Result<(), Error> {
+        let s9pk_dir = ctx.datadir.join(PKG_ARCHIVE_DIR).join("installed");
+
+        if tokio::fs::metadata(&s9pk_dir).await.is_ok() {
+            let mut read_dir = tokio::fs::read_dir(&s9pk_dir).await?;
+            while let Some(s9pk_ent) = read_dir.next_entry().await? {
+                let s9pk_path = s9pk_ent.path();
+                let matches_s9pk = s9pk_path.extension().map(|x| x == "s9pk").unwrap_or(false);
+                if !matches_s9pk {
+                    continue;
+                }
+
+                let get_archive = async {
+                    let multi_cursor = MultiCursorFile::from(File::open(&s9pk_path).await?);
+                    Ok::<_, Error>(S9pk::archive(&multi_cursor, None).await?)
+                };
+
+                let archive: MerkleArchive<
+                    crate::s9pk::merkle_archive::source::Section<MultiCursorFile>,
+                > = match get_archive.await {
+                    Ok(a) => a,
+                    Err(e) => {
+                        tracing::error!("Error opening s9pk for install: {e}");
+                        tracing::debug!("{e:?}");
+                        continue;
+                    }
+                };
+
+                let previous_manifest: Value = serde_json::from_slice::<serde_json::Value>(
+                    &archive
+                        .contents()
+                        .get_path("manifest.json")
+                        .or_not_found("manifest.json")?
+                        .read_file_to_vec()
+                        .await?,
+                )
+                .with_kind(ErrorKind::Deserialization)?
+                .into();
+
+                let mut manifest = previous_manifest.clone();
+
+                if let Some(device) =
+                    previous_manifest["hardwareRequirements"]["device"].as_object()
+                {
+                    manifest["hardwareRequirements"]["device"] = to_value(
+                        &device
+                            .into_iter()
+                            .map(|(class, product)| {
+                                Ok::<_, Error>(DeviceFilter {
+                                    pattern_description: format!(
+                                        "a {class} device matching the expression {}",
+                                        &product
+                                    ),
+                                    class: class.clone(),
+                                    pattern: from_value(product.clone())?,
+                                })
+                            })
+                            .fold(Ok::<_, Error>(Vec::new()), |acc, value| {
+                                let mut acc = acc?;
+                                acc.push(value?);
+                                Ok(acc)
+                            })?,
+                    )?;
+                }
+
+                if previous_manifest != manifest {
+                    let tmp_path = s9pk_path.with_extension("s9pk.tmp");
+                    let mut tmp_file = create_file(&tmp_path).await?;
+                    // TODO, wouldn't this break in the later versions of the manifest that would need changes, this doesn't seem to be a good way to handle this
+                    let manifest: Manifest = from_value(manifest.clone())?;
+                    let id = manifest.id.clone();
+                    let mut s9pk: S9pk<_> = S9pk::new_with_manifest(archive, None, manifest);
+                    let s9pk_compat_key = ctx.account.read().await.compat_s9pk_key.clone();
+                    s9pk.as_archive_mut()
+                        .set_signer(s9pk_compat_key, SIG_CONTEXT);
+                    s9pk.serialize(&mut tmp_file, true).await?;
+                    tmp_file.sync_all().await?;
+                    tokio::fs::rename(&tmp_path, &s9pk_path).await?;
+                    ctx.services.load(ctx, &id, LoadDisposition::Retry).await?;
+                }
+            }
+        }
+
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
+        Ok(())
+    }
+}