036 migration (#2750)

* chore: convert to use a value, cause why not

* wip: Add the up for this going up

* wip: trait changes

* wip: Add in some more of the private transformations

* chore(wip): Adding the ssh_keys todo

* wip: Add cifs

* fix migration structure

* chore: Fix the trait for the version

* wip(feat): Notifications are in the system

* fix marker trait hell

* handle key todos

* wip: Testing the migration in a system.

* fix pubkey parser

* fix: migration works

* wip: Trying to get the migration stuff?

* fix: Can now install the packages that we wanted, yay!"

* Merge branch 'next/minor' of github.com:Start9Labs/start-os into feat/migration

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>

core/startos/src/backup/backup_bulk.rs

@@ -332,10 +332,10 @@ async fn perform_backup(
 
     let timestamp = Utc::now();
 
-    backup_guard.unencrypted_metadata.version = crate::version::Current::new().semver().into();
+    backup_guard.unencrypted_metadata.version = crate::version::Current::default().semver().into();
     backup_guard.unencrypted_metadata.hostname = ctx.account.read().await.hostname.clone();
     backup_guard.unencrypted_metadata.timestamp = timestamp.clone();
-    backup_guard.metadata.version = crate::version::Current::new().semver().into();
+    backup_guard.metadata.version = crate::version::Current::default().semver().into();
     backup_guard.metadata.timestamp = Some(timestamp);
     backup_guard.metadata.package_backups = package_backups;
 

core/startos/src/bins/container_cli.rs

@@ -8,7 +8,7 @@ use crate::util::logger::EmbassyLogger;
 use crate::version::{Current, VersionT};
 
 lazy_static::lazy_static! {
-    static ref VERSION_STRING: String = Current::new().semver().to_string();
+    static ref VERSION_STRING: String = Current::default().semver().to_string();
 }
 
 pub fn main(args: impl IntoIterator<Item = OsString>) {

core/startos/src/bins/start_cli.rs

@@ -9,7 +9,7 @@ use crate::util::logger::EmbassyLogger;
 use crate::version::{Current, VersionT};
 
 lazy_static::lazy_static! {
-    static ref VERSION_STRING: String = Current::new().semver().to_string();
+    static ref VERSION_STRING: String = Current::default().semver().to_string();
 }
 
 pub fn main(args: impl IntoIterator<Item = OsString>) {

core/startos/src/context/rpc.rs

@@ -88,6 +88,7 @@ pub struct InitRpcContextPhases {
     init_net_ctrl: PhaseProgressTrackerHandle,
     read_device_info: PhaseProgressTrackerHandle,
     cleanup_init: CleanupInitPhases,
+    // TODO: migrations
 }
 impl InitRpcContextPhases {
     pub fn new(handle: &FullProgressTracker) -> Self {
@@ -286,6 +287,7 @@ impl RpcContext {
         let res = Self(seed.clone());
         res.cleanup_and_initialize(cleanup_init).await?;
         tracing::info!("Cleaned up transient states");
+        crate::version::post_init(&res).await?;
         Ok(res)
     }
 

core/startos/src/db/model/private.rs

@@ -31,6 +31,6 @@ pub struct Private {
     pub package_stores: BTreeMap<PackageId, Value>,
 }
 
-fn generate_compat_key() -> Pem<ed25519_dalek::SigningKey> {
+pub fn generate_compat_key() -> Pem<ed25519_dalek::SigningKey> {
     Pem(ed25519_dalek::SigningKey::generate(&mut rand::thread_rng()))
 }

core/startos/src/db/model/public.rs

@@ -43,10 +43,11 @@ impl Public {
                 arch: get_arch(),
                 platform: get_platform(),
                 id: account.server_id.clone(),
-                version: Current::new().semver(),
+                version: Current::default().semver(),
                 hostname: account.hostname.no_dot_host_name(),
                 last_backup: None,
-                eos_version_compat: Current::new().compat().clone(),
+                version_compat: Current::default().compat().clone(),
+                post_init_migration_todos: BTreeSet::new(),
                 lan_address,
                 onion_address: account.tor_key.public().get_onion_address(),
                 tor_address: format!("https://{}", account.tor_key.public().get_onion_address())
@@ -112,11 +113,13 @@ pub struct ServerInfo {
     pub hostname: InternedString,
     #[ts(type = "string")]
     pub version: Version,
+    #[ts(type = "string")]
+    pub version_compat: VersionRange,
+    #[ts(type = "string[]")]
+    pub post_init_migration_todos: BTreeSet<Version>,
     #[ts(type = "string | null")]
     pub last_backup: Option<DateTime<Utc>>,
     #[ts(type = "string")]
-    pub eos_version_compat: VersionRange,
-    #[ts(type = "string")]
     pub lan_address: Url,
     #[ts(type = "string")]
     pub onion_address: OnionAddressV3,

core/startos/src/init.rs

@@ -323,7 +323,9 @@ pub async fn init(
     local_auth.complete();
 
     load_database.start();
-    let db = TypedPatchDb::<Database>::load_unchecked(cfg.db().await?);
+    let db = cfg.db().await?;
+    crate::version::Current::default().pre_init(&db).await?;
+    let db = TypedPatchDb::<Database>::load_unchecked(db);
     let peek = db.peek().await;
     load_database.complete();
     tracing::info!("Opened PatchDB");
@@ -528,8 +530,6 @@ pub async fn init(
         .await?;
     launch_service_network.complete();
 
-    crate::version::init(&db, run_migrations).await?;
-
     validate_db.start();
     db.mutate(|d| {
         let model = d.de()?;

core/startos/src/notifications.rs

@@ -256,13 +256,13 @@ impl Map for Notifications {
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Notification {
-    package_id: Option<PackageId>,
+    pub package_id: Option<PackageId>,
-    created_at: DateTime<Utc>,
+    pub created_at: DateTime<Utc>,
-    code: u32,
+    pub code: u32,
-    level: NotificationLevel,
+    pub level: NotificationLevel,
-    title: String,
+    pub title: String,
-    message: String,
+    pub message: String,
-    data: Value,
+    pub data: Value,
 }
 
 #[derive(Debug, Serialize, Deserialize)]

core/startos/src/os_install/mod.rs

@@ -150,23 +150,6 @@ pub async fn execute<C: Context>(
 
     overwrite |= disk.guid.is_none() && disk.partitions.iter().all(|p| p.guid.is_none());
 
-    if !overwrite
-        && (disk
-            .guid
-            .as_ref()
-            .map_or(false, |g| g.starts_with("EMBASSY_"))
-            || disk
-                .partitions
-                .iter()
-                .flat_map(|p| p.guid.as_ref())
-                .any(|g| g.starts_with("EMBASSY_")))
-    {
-        return Err(Error::new(
-            eyre!("installing over versions before 0.3.6 is unsupported"),
-            ErrorKind::InvalidRequest,
-        ));
-    }
-
     let part_info = partition(&mut disk, overwrite).await?;
 
     if let Some(efi) = &part_info.efi {

core/startos/src/registry/device_info.rs

@@ -108,8 +108,8 @@ pub struct OsInfo {
 impl From<&RpcContext> for OsInfo {
     fn from(_: &RpcContext) -> Self {
         Self {
-            version: crate::version::Current::new().semver(),
+            version: crate::version::Current::default().semver(),
-            compat: crate::version::Current::new().compat().clone(),
+            compat: crate::version::Current::default().compat().clone(),
             platform: InternedString::intern(&*crate::PLATFORM),
         }
     }

core/startos/src/s9pk/v2/manifest.rs

@@ -22,7 +22,7 @@ use crate::util::VersionString;
 use crate::version::{Current, VersionT};
 
 fn current_version() -> Version {
-    Current::new().semver()
+    Current::default().semver()
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize, HasModel, TS)]

core/startos/src/service/mod.rs

@@ -408,6 +408,7 @@ impl Service {
         let developer_key = s9pk.as_archive().signer();
         let icon = s9pk.icon_data_url().await?;
         let service = Self::new(ctx.clone(), s9pk, StartStop::Stop).await?;
+
         if let Some(recovery_source) = recovery_source {
             service
                 .actor
@@ -429,6 +430,7 @@ impl Service {
                     .clone(),
             );
         }
+
         let procedure_id = Guid::new();
         service
             .seed
@@ -441,6 +443,7 @@ impl Service {
             ) // TODO timeout
             .await
             .with_kind(ErrorKind::MigrationFailed)?; // TODO: handle cancellation
+
         if let Some(mut progress) = progress {
             progress.finalization_progress.complete();
             progress.progress.complete();

core/startos/src/service/service_map.rs

@@ -307,7 +307,6 @@ impl ServiceMap {
                     sync_progress_task.await.map_err(|_| {
                         Error::new(eyre!("progress sync task panicked"), ErrorKind::Unknown)
                     })??;
-
                     Ok(())
                 })
                 .boxed())

core/startos/src/ssh.rs

@@ -25,6 +25,12 @@ impl SshKeys {
         Self(BTreeMap::new())
     }
 }
+
+impl From<BTreeMap<InternedString, WithTimeData<SshPubKey>>> for SshKeys {
+    fn from(map: BTreeMap<InternedString, WithTimeData<SshPubKey>>) -> Self {
+        Self(map)
+    }
+}
 impl Map for SshKeys {
     type Key = InternedString;
     type Value = WithTimeData<SshPubKey>;
@@ -41,7 +47,7 @@ impl Map for SshKeys {
 pub struct SshPubKey(
     #[serde(serialize_with = "crate::util::serde::serialize_display")]
     #[serde(deserialize_with = "crate::util::serde::deserialize_from_str")]
-    openssh_keys::PublicKey,
+    pub openssh_keys::PublicKey,
 );
 impl ValueParserFactory for SshPubKey {
     type Parser = FromStrParser<Self>;

core/startos/src/version/mod.rs

@@ -1,10 +1,15 @@
+use std::any::Any;
 use std::cmp::Ordering;
+use std::panic::{RefUnwindSafe, UnwindSafe};
 
 use color_eyre::eyre::eyre;
 use futures::future::BoxFuture;
 use futures::{Future, FutureExt};
-use imbl_value::InternedString;
+use imbl::Vector;
+use imbl_value::{to_value, InternedString};
+use patch_db::json_ptr::{JsonPointer, ROOT};
 
+use crate::context::RpcContext;
 use crate::db::model::Database;
 use crate::prelude::*;
 use crate::progress::PhaseProgressTrackerHandle;
@@ -20,10 +25,68 @@ mod v0_3_6_alpha_3;
 mod v0_3_6_alpha_4;
 mod v0_3_6_alpha_5;
 mod v0_3_6_alpha_6;
-mod v0_3_6_alpha_7;
 
 pub type Current = v0_3_6_alpha_6::Version; // VERSION_BUMP
 
+impl Current {
+    #[instrument(skip(self, db))]
+    pub async fn pre_init(self, db: &PatchDb) -> Result<(), Error> {
+        let from = from_value::<Version>(
+            version_accessor(&mut db.dump(&ROOT).await.value)
+                .or_not_found("`version` in db")?
+                .clone(),
+        )?
+        .as_version_t()?;
+        match from.semver().cmp(&self.semver()) {
+            Ordering::Greater => {
+                db.apply_function(|mut db| {
+                    rollback_to_unchecked(&from, &self, &mut db)?;
+                    Ok::<_, Error>((db, ()))
+                })
+                .await?;
+            }
+            Ordering::Less => {
+                let pre_ups = PreUps::load(&from, &self).await?;
+                db.apply_function(|mut db| {
+                    migrate_from_unchecked(&from, &self, pre_ups, &mut db)?;
+                    Ok::<_, Error>((db, ()))
+                })
+                .await?;
+            }
+            Ordering::Equal => (),
+        }
+        Ok(())
+    }
+}
+
+pub async fn post_init(ctx: &RpcContext) -> Result<(), Error> {
+    let mut peek;
+    while let Some(version) = {
+        peek = ctx.db.peek().await;
+        peek.as_public()
+            .as_server_info()
+            .as_post_init_migration_todos()
+            .de()?
+            .first()
+            .cloned()
+            .map(Version::from_exver_version)
+            .as_ref()
+            .map(Version::as_version_t)
+            .transpose()?
+    } {
+        version.0.post_up(ctx).await?;
+        ctx.db
+            .mutate(|db| {
+                db.as_public_mut()
+                    .as_server_info_mut()
+                    .as_post_init_migration_todos_mut()
+                    .mutate(|m| Ok(m.remove(&version.0.semver())))
+            })
+            .await?;
+    }
+    Ok(())
+}
+
 #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
 #[serde(untagged)]
 #[allow(non_camel_case_types)]
@@ -39,7 +102,6 @@ enum Version {
     V0_3_6_alpha_4(Wrapper<v0_3_6_alpha_4::Version>),
     V0_3_6_alpha_5(Wrapper<v0_3_6_alpha_5::Version>),
     V0_3_6_alpha_6(Wrapper<v0_3_6_alpha_6::Version>),
-    V0_3_6_alpha_7(Wrapper<v0_3_6_alpha_7::Version>),
     Other(exver::Version),
 }
 
@@ -52,6 +114,32 @@ impl Version {
                 Version::Other(version)
             })
     }
+    fn as_version_t(&self) -> Result<DynVersion, Error> {
+        Ok(match self {
+            Self::LT0_3_5(_) => {
+                return Err(Error::new(
+                    eyre!("cannot migrate from versions before 0.3.5"),
+                    ErrorKind::MigrationFailed,
+                ))
+            }
+            Self::V0_3_5(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_5_1(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_5_2(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_0(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_1(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_2(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_3(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_4(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_5(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_6(v) => DynVersion(Box::new(v.0)),
+            Self::Other(v) => {
+                return Err(Error::new(
+                    eyre!("unknown version {v}"),
+                    ErrorKind::MigrationFailed,
+                ))
+            }
+        })
+    }
     #[cfg(test)]
     fn as_exver(&self) -> exver::Version {
         match self {
@@ -66,117 +154,247 @@ impl Version {
             Version::V0_3_6_alpha_4(Wrapper(x)) => x.semver(),
             Version::V0_3_6_alpha_5(Wrapper(x)) => x.semver(),
             Version::V0_3_6_alpha_6(Wrapper(x)) => x.semver(),
-            Version::V0_3_6_alpha_7(Wrapper(x)) => x.semver(),
             Version::Other(x) => x.clone(),
         }
     }
 }
 
-pub trait VersionT
+fn version_accessor(db: &mut Value) -> Option<&mut Value> {
-where
+    if db.get("public").is_some() {
-    Self: Sized + Send + Sync,
+        db.get_mut("public")?
-{
+            .get_mut("serverInfo")?
-    type Previous: VersionT;
+            .get_mut("version")
-    fn new() -> Self;
+    } else {
-    fn semver(&self) -> exver::Version;
+        db.get_mut("server-info")?.get_mut("version")
-    fn compat(&self) -> &'static exver::VersionRange;
+    }
-    fn up(&self, db: &TypedPatchDb<Database>) -> impl Future<Output = Result<(), Error>> + Send;
+}
-    fn down(&self, db: &TypedPatchDb<Database>) -> impl Future<Output = Result<(), Error>> + Send;
+
-    fn commit(
+fn version_compat_accessor(db: &mut Value) -> Option<&mut Value> {
-        &self,
+    if db.get("public").is_some() {
-        db: &TypedPatchDb<Database>,
+        db.get_mut("public")?
-    ) -> impl Future<Output = Result<(), Error>> + Send {
+            .get_mut("serverInfo")?
+            .get_mut("versionCompat")
+    } else {
+        db.get_mut("server-info")?.get_mut("eos-version-compat")
+    }
+}
+
+fn post_init_migration_todos_accessor(db: &mut Value) -> Option<&mut Value> {
+    let server_info = if db.get("public").is_some() {
+        db.get_mut("public")?.get_mut("serverInfo")?
+    } else {
+        db.get_mut("server-info")?
+    };
+    if server_info.get("postInitMigrationTodos").is_none() {
+        server_info
+            .as_object_mut()?
+            .insert("postInitMigrationTodos".into(), Value::Array(Vector::new()));
+    }
+    server_info.get_mut("postInitMigrationTodos")
+}
+
+struct PreUps {
+    prev: Option<Box<PreUps>>,
+    value: Box<dyn Any + UnwindSafe + Send + 'static>,
+}
+impl PreUps {
+    #[instrument(skip(from, to))]
+    fn load<'a, VFrom: DynVersionT + ?Sized, VTo: DynVersionT + ?Sized>(
+        from: &'a VFrom,
+        to: &'a VTo,
+    ) -> BoxFuture<'a, Result<Self, Error>> {
         async {
-            let semver = self.semver().into();
+            let previous = to.previous();
-            let compat = self.compat().clone();
+            let prev = match from.semver().cmp(&previous.semver()) {
-            db.mutate(|d| {
+                Ordering::Less => Some(Box::new(PreUps::load(from, &previous).await?)),
-                d.as_public_mut()
+                Ordering::Greater => {
-                    .as_server_info_mut()
-                    .as_version_mut()
-                    .ser(&semver)?;
-                d.as_public_mut()
-                    .as_server_info_mut()
-                    .as_eos_version_compat_mut()
-                    .ser(&compat)?;
-                Ok(())
-            })
-            .await?;
-            Ok(())
-        }
-    }
-    fn migrate_to<V: VersionT>(
-        &self,
-        version: &V,
-        db: &TypedPatchDb<Database>,
-        progress: &mut PhaseProgressTrackerHandle,
-    ) -> impl Future<Output = Result<(), Error>> + Send {
-        async {
-            match self.semver().cmp(&version.semver()) {
-                Ordering::Greater => self.rollback_to_unchecked(version, db, progress).await,
-                Ordering::Less => version.migrate_from_unchecked(self, db, progress).await,
-                Ordering::Equal => Ok(()),
-            }
-        }
-    }
-    fn migrate_from_unchecked<'a, V: VersionT>(
-        &'a self,
-        version: &'a V,
-        db: &'a TypedPatchDb<Database>,
-        progress: &'a mut PhaseProgressTrackerHandle,
-    ) -> BoxFuture<'a, Result<(), Error>> {
-        progress.add_total(1);
-        async {
-            let previous = Self::Previous::new();
-            if version.semver() < previous.semver() {
-                previous
-                    .migrate_from_unchecked(version, db, progress)
-                    .await?;
-            } else if version.semver() > previous.semver() {
                     return Err(Error::new(
                         eyre!(
                             "NO PATH FROM {}, THIS IS LIKELY A MISTAKE IN THE VERSION DEFINITION",
-                        version.semver()
+                            from.semver()
+                        ),
+                        crate::ErrorKind::MigrationFailed,
+                    ))
+                }
+                Ordering::Equal => None,
+            };
+            Ok(Self {
+                prev,
+                value: to.pre_up().await?,
+            })
+        }
+        .boxed()
+    }
+}
+
+fn migrate_from_unchecked<VFrom: DynVersionT + ?Sized, VTo: DynVersionT + ?Sized>(
+    from: &VFrom,
+    to: &VTo,
+    pre_ups: PreUps,
+    db: &mut Value,
+) -> Result<(), Error> {
+    let previous = to.previous();
+    match pre_ups.prev {
+        Some(prev) if from.semver() < previous.semver() => {
+            migrate_from_unchecked(from, &previous, *prev, db)?
+        }
+        _ if from.semver() > previous.semver() => {
+            return Err(Error::new(
+                eyre!(
+                    "NO PATH FROM {}, THIS IS LIKELY A MISTAKE IN THE VERSION DEFINITION",
+                    from.semver()
                 ),
                 crate::ErrorKind::MigrationFailed,
             ));
         }
-            tracing::info!("{} -> {}", previous.semver(), self.semver(),);
+        _ => (),
-            self.up(db).await?;
+    };
-            self.commit(db).await?;
+    to.up(db, pre_ups.value)?;
-            *progress += 1;
+    to.commit(db)?;
     Ok(())
 }
-        .boxed()
+
-    }
+fn rollback_to_unchecked<VFrom: DynVersionT + ?Sized, VTo: DynVersionT + ?Sized>(
-    fn rollback_to_unchecked<'a, V: VersionT>(
+    from: &VFrom,
-        &'a self,
+    to: &VTo,
-        version: &'a V,
+    db: &mut Value,
-        db: &'a TypedPatchDb<Database>,
+) -> Result<(), Error> {
-        progress: &'a mut PhaseProgressTrackerHandle,
+    let previous = from.previous();
-    ) -> BoxFuture<'a, Result<(), Error>> {
+    from.down(db)?;
-        async {
+    previous.commit(db)?;
-            let previous = Self::Previous::new();
+    if to.semver() < previous.semver() {
-            tracing::info!("{} -> {}", self.semver(), previous.semver(),);
+        rollback_to_unchecked(&previous, to, db)?
-            self.down(db).await?;
+    } else if to.semver() > previous.semver() {
-            previous.commit(db).await?;
-            *progress += 1;
-            if version.semver() < previous.semver() {
-                previous
-                    .rollback_to_unchecked(version, db, progress)
-                    .await?;
-            } else if version.semver() > previous.semver() {
         return Err(Error::new(
             eyre!(
                 "NO PATH TO {}, THIS IS LIKELY A MISTAKE IN THE VERSION DEFINITION",
-                        version.semver()
+                to.semver()
             ),
             crate::ErrorKind::MigrationFailed,
         ));
     }
     Ok(())
 }
+
+pub trait VersionT
+where
+    Self: Default + Copy + Sized + RefUnwindSafe + Send + Sync + 'static,
+{
+    type Previous: VersionT;
+    type PreUpRes: Send + UnwindSafe;
+    fn semver(self) -> exver::Version;
+    fn compat(self) -> &'static exver::VersionRange;
+    /// MUST NOT change system state. Intended for async I/O reads
+    fn pre_up(self) -> impl Future<Output = Result<Self::PreUpRes, Error>> + Send + 'static;
+    fn up(self, db: &mut Value, input: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    /// MUST be idempotent, and is run after *all* db migrations
+    fn post_up(self, ctx: &RpcContext) -> impl Future<Output = Result<(), Error>> + Send + 'static {
+        async { Ok(()) }
+    }
+    fn down(self, db: &mut Value) -> Result<(), Error> {
+        Err(Error::new(
+            eyre!("downgrades prohibited"),
+            ErrorKind::InvalidRequest,
+        ))
+    }
+    fn commit(self, db: &mut Value) -> Result<(), Error> {
+        *version_accessor(db).or_not_found("`version` in db")? = to_value(&self.semver())?;
+        *version_compat_accessor(db).or_not_found("`versionCompat` in db")? =
+            to_value(self.compat())?;
+        post_init_migration_todos_accessor(db)
+            .or_not_found("`serverInfo` in db")?
+            .as_array_mut()
+            .ok_or_else(|| {
+                Error::new(
+                    eyre!("postInitMigrationTodos is not an array"),
+                    ErrorKind::Database,
+                )
+            })?
+            .push_back(to_value(&self.semver())?);
+        Ok(())
+    }
+}
+
+struct DynVersion(Box<dyn DynVersionT>);
+unsafe impl Send for DynVersion {}
+
+trait DynVersionT: RefUnwindSafe + Send + Sync {
+    fn previous(&self) -> DynVersion;
+    fn semver(&self) -> exver::Version;
+    fn compat(&self) -> &'static exver::VersionRange;
+    fn pre_up(&self) -> BoxFuture<'static, Result<Box<dyn Any + UnwindSafe + Send>, Error>>;
+    fn up(&self, db: &mut Value, input: Box<dyn Any + Send>) -> Result<(), Error>;
+    fn post_up<'a>(&self, ctx: &'a RpcContext) -> BoxFuture<'a, Result<(), Error>>;
+    fn down(&self, db: &mut Value) -> Result<(), Error>;
+    fn commit(&self, db: &mut Value) -> Result<(), Error>;
+}
+impl<T> DynVersionT for T
+where
+    T: VersionT,
+{
+    fn previous(&self) -> DynVersion {
+        DynVersion(Box::new(<Self as VersionT>::Previous::default()))
+    }
+    fn semver(&self) -> exver::Version {
+        VersionT::semver(*self)
+    }
+    fn compat(&self) -> &'static exver::VersionRange {
+        VersionT::compat(*self)
+    }
+    fn pre_up(&self) -> BoxFuture<'static, Result<Box<dyn Any + UnwindSafe + Send>, Error>> {
+        let v = *self;
+        async move { Ok(Box::new(VersionT::pre_up(v).await?) as Box<dyn Any + UnwindSafe + Send>) }
             .boxed()
     }
+    fn up(&self, db: &mut Value, input: Box<dyn Any + Send>) -> Result<(), Error> {
+        VersionT::up(
+            *self,
+            db,
+            *input.downcast().map_err(|_| {
+                Error::new(
+                    eyre!("pre_up returned unexpected type"),
+                    ErrorKind::Incoherent,
+                )
+            })?,
+        )
+    }
+    fn post_up<'a>(&self, ctx: &'a RpcContext) -> BoxFuture<'a, Result<(), Error>> {
+        VersionT::post_up(*self, ctx).boxed()
+    }
+    fn down(&self, db: &mut Value) -> Result<(), Error> {
+        VersionT::down(*self, db)
+    }
+    fn commit(&self, db: &mut Value) -> Result<(), Error> {
+        VersionT::commit(*self, db)
+    }
+}
+impl DynVersionT for DynVersion {
+    fn previous(&self) -> DynVersion {
+        self.0.previous()
+    }
+    fn semver(&self) -> exver::Version {
+        self.0.semver()
+    }
+    fn compat(&self) -> &'static exver::VersionRange {
+        self.0.compat()
+    }
+    fn pre_up(&self) -> BoxFuture<'static, Result<Box<dyn Any + UnwindSafe + Send>, Error>> {
+        self.0.pre_up()
+    }
+    fn up(&self, db: &mut Value, input: Box<dyn Any + Send>) -> Result<(), Error> {
+        self.0.up(db, input)
+    }
+    fn post_up<'a>(&self, ctx: &'a RpcContext) -> BoxFuture<'a, Result<(), Error>> {
+        self.0.post_up(ctx)
+    }
+    fn down(&self, db: &mut Value) -> Result<(), Error> {
+        self.0.down(db)
+    }
+    fn commit(&self, db: &mut Value) -> Result<(), Error> {
+        self.0.commit(db)
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -195,7 +413,7 @@ where
 {
     fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
         let v = exver::Version::deserialize(deserializer)?;
-        let version = T::new();
+        let version = T::default();
         if v < version.semver() {
             Ok(Self(version, v))
         } else {
@@ -220,7 +438,7 @@ where
 {
     fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
         let v = exver::Version::deserialize(deserializer)?;
-        let version = T::new();
+        let version = T::default();
         if v == version.semver() {
             Ok(Wrapper(version))
         } else {
@@ -229,62 +447,6 @@ where
     }
 }
 
-pub async fn init(
-    db: &TypedPatchDb<Database>,
-    mut progress: PhaseProgressTrackerHandle,
-) -> Result<(), Error> {
-    progress.start();
-    db.mutate(|db| {
-        db.as_public_mut()
-            .as_server_info_mut()
-            .as_version_mut()
-            .map_mutate(|v| {
-                Ok(if v == exver::Version::new([0, 3, 6], []) {
-                    v0_3_6_alpha_0::Version::new().semver()
-                } else {
-                    v
-                })
-            })
-    })
-    .await?; // TODO: remove before releasing 0.3.6
-    let version = Version::from_exver_version(
-        db.peek()
-            .await
-            .as_public()
-            .as_server_info()
-            .as_version()
-            .de()?,
-    );
-
-    match version {
-        Version::LT0_3_5(_) => {
-            return Err(Error::new(
-                eyre!("Cannot migrate from pre-0.3.5. Please update to v0.3.5 first."),
-                ErrorKind::MigrationFailed,
-            ));
-        }
-        Version::V0_3_5(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_5_1(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_5_2(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_0(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_1(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_2(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_3(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_4(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_5(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_6(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::V0_3_6_alpha_7(v) => v.0.migrate_to(&Current::new(), &db, &mut progress).await?,
-        Version::Other(_) => {
-            return Err(Error::new(
-                eyre!("Cannot downgrade"),
-                crate::ErrorKind::InvalidRequest,
-            ))
-        }
-    }
-    progress.complete();
-    Ok(())
-}
-
 pub const COMMIT_HASH: &str =
     include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../GIT_HASH.txt"));
 
@@ -315,17 +477,17 @@ mod tests {
 
     fn versions() -> impl Strategy<Value = Version> {
         prop_oneof![
-            Just(Version::V0_3_5(Wrapper(v0_3_5::Version::new()))),
+            Just(Version::V0_3_5(Wrapper(v0_3_5::Version::default()))),
-            Just(Version::V0_3_5_1(Wrapper(v0_3_5_1::Version::new()))),
+            Just(Version::V0_3_5_1(Wrapper(v0_3_5_1::Version::default()))),
-            Just(Version::V0_3_5_2(Wrapper(v0_3_5_2::Version::new()))),
+            Just(Version::V0_3_5_2(Wrapper(v0_3_5_2::Version::default()))),
             Just(Version::V0_3_6_alpha_0(Wrapper(
-                v0_3_6_alpha_0::Version::new()
+                v0_3_6_alpha_0::Version::default()
             ))),
             Just(Version::V0_3_6_alpha_1(Wrapper(
-                v0_3_6_alpha_1::Version::new()
+                v0_3_6_alpha_1::Version::default()
             ))),
             Just(Version::V0_3_6_alpha_2(Wrapper(
-                v0_3_6_alpha_2::Version::new()
+                v0_3_6_alpha_2::Version::default()
             ))),
             em_version().prop_map(Version::Other),
         ]

core/startos/src/version/v0_3_5.rs

@@ -1,7 +1,6 @@
 use exver::{ExtendedVersion, VersionRange};
 
 use super::VersionT;
-use crate::db::model::Database;
 use crate::prelude::*;
 use crate::version::Current;
 
@@ -17,7 +16,7 @@ lazy_static::lazy_static! {
         VersionRange::anchor(
             exver::LTE,
             ExtendedVersion::new(
-                Current::new().semver(),
+                Current::default().semver(),
                 exver::Version::default(),
             )
         ),
@@ -25,24 +24,26 @@ lazy_static::lazy_static! {
     static ref V0_3_5: exver::Version = exver::Version::new([0, 3, 5], []);
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = Self;
-    fn new() -> Self {
+    type PreUpRes = ();
-        Version
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
     }
-    fn semver(&self) -> exver::Version {
+    fn semver(self) -> exver::Version {
         V0_3_5.clone()
     }
-    fn compat(&self) -> &'static VersionRange {
+    fn compat(self) -> &'static VersionRange {
         &V0_3_0_COMPAT
     }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_5_1.rs

@@ -2,31 +2,32 @@ use exver::VersionRange;
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_5, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
     static ref V0_3_5_1: exver::Version = exver::Version::new([0, 3, 5, 1], []);
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_5::Version;
-    fn new() -> Self {
+    type PreUpRes = ();
-        Version
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
     }
-    fn semver(&self) -> exver::Version {
+    fn semver(self) -> exver::Version {
         V0_3_5_1.clone()
     }
-    fn compat(&self) -> &'static VersionRange {
+    fn compat(self) -> &'static VersionRange {
         &V0_3_0_COMPAT
     }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_5_2.rs

@@ -2,31 +2,32 @@ use exver::VersionRange;
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_5_1, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
     static ref V0_3_5_2: exver::Version = exver::Version::new([0, 3, 5, 2], []);
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_5_1::Version;
-    fn new() -> Self {
+    type PreUpRes = ();
-        Version
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
     }
-    fn semver(&self) -> exver::Version {
+    fn semver(self) -> exver::Version {
         V0_3_5_2.clone()
     }
-    fn compat(&self) -> &'static VersionRange {
+    fn compat(self) -> &'static VersionRange {
         &V0_3_0_COMPAT
     }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_0.rs

@@ -1,9 +1,44 @@
+use std::collections::BTreeMap;
+use std::future::Future;
+use std::path::Path;
+
+use chrono::{DateTime, Utc};
+use ed25519_dalek::SigningKey;
 use exver::{PreReleaseSegment, VersionRange};
+use imbl_value::{json, InternedString};
+use itertools::Itertools;
+use models::PackageId;
+use openssl::pkey::{PKey, Private};
+use openssl::x509::X509;
+use patch_db::ModelExt;
+use sqlx::postgres::PgConnectOptions;
+use sqlx::{PgPool, Row};
+use ssh_key::Fingerprint;
+use tokio::process::Command;
+use torut::onion::TorSecretKeyV3;
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_5_2, VersionT};
 use crate::db::model::Database;
+use crate::disk::mount::filesystem::cifs::Cifs;
+use crate::disk::mount::util::unmount;
+use crate::hostname::Hostname;
+use crate::net::forward::AvailablePorts;
+use crate::net::keys::KeyStore;
+use crate::net::ssl::CertStore;
+use crate::net::tor::OnionStore;
+use crate::notifications::{Notification, Notifications};
 use crate::prelude::*;
+use crate::ssh::{SshKeys, SshPubKey};
+use crate::util::crypto::ed25519_expand_key;
+use crate::util::serde::{Pem, PemEncoding};
+use crate::util::Invoke;
+use crate::{account::AccountInfo, net::tor};
+use crate::{auth::Sessions, context::RpcContext};
+use crate::{
+    backup::target::cifs::CifsTargets,
+    s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile,
+};
 
 lazy_static::lazy_static! {
     static ref V0_3_6_alpha_0: exver::Version = exver::Version::new(
@@ -12,24 +47,540 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[tracing::instrument(skip_all)]
+async fn init_postgres(datadir: impl AsRef<Path>) -> Result<PgPool, Error> {
+    let db_dir = datadir.as_ref().join("main/postgresql");
+    if tokio::process::Command::new("mountpoint")
+        .arg("/var/lib/postgresql")
+        .stdout(std::process::Stdio::null())
+        .stderr(std::process::Stdio::null())
+        .status()
+        .await?
+        .success()
+    {
+        unmount("/var/lib/postgresql", true).await?;
+    }
+    let exists = tokio::fs::metadata(&db_dir).await.is_ok();
+    if !exists {
+        Command::new("cp")
+            .arg("-ra")
+            .arg("/var/lib/postgresql")
+            .arg(&db_dir)
+            .invoke(crate::ErrorKind::Filesystem)
+            .await?;
+    }
+    Command::new("chown")
+        .arg("-R")
+        .arg("postgres:postgres")
+        .arg(&db_dir)
+        .invoke(crate::ErrorKind::Database)
+        .await?;
+
+    let mut pg_paths = tokio::fs::read_dir("/usr/lib/postgresql").await?;
+    let mut pg_version = None;
+    while let Some(pg_path) = pg_paths.next_entry().await? {
+        let pg_path_version = pg_path
+            .file_name()
+            .to_str()
+            .map(|v| v.parse())
+            .transpose()?
+            .unwrap_or(0);
+        if pg_path_version > pg_version.unwrap_or(0) {
+            pg_version = Some(pg_path_version)
+        }
+    }
+    let pg_version = pg_version.ok_or_else(|| {
+        Error::new(
+            eyre!("could not determine postgresql version"),
+            crate::ErrorKind::Database,
+        )
+    })?;
+
+    crate::disk::mount::util::bind(&db_dir, "/var/lib/postgresql", false).await?;
+
+    let pg_version_string = pg_version.to_string();
+    let pg_version_path = db_dir.join(&pg_version_string);
+    if exists
+    // maybe migrate
+    {
+        let incomplete_path = db_dir.join(format!("{pg_version}.migration.incomplete"));
+        if tokio::fs::metadata(&incomplete_path).await.is_ok() // previous migration was incomplete
+        && tokio::fs::metadata(&pg_version_path).await.is_ok()
+        {
+            tokio::fs::remove_dir_all(&pg_version_path).await?;
+        }
+        if tokio::fs::metadata(&pg_version_path).await.is_err()
+        // need to migrate
+        {
+            let conf_dir = Path::new("/etc/postgresql").join(pg_version.to_string());
+            let conf_dir_tmp = {
+                let mut tmp = conf_dir.clone();
+                tmp.set_extension("tmp");
+                tmp
+            };
+            if tokio::fs::metadata(&conf_dir).await.is_ok() {
+                Command::new("mv")
+                    .arg(&conf_dir)
+                    .arg(&conf_dir_tmp)
+                    .invoke(ErrorKind::Filesystem)
+                    .await?;
+            }
+            let mut old_version = pg_version;
+            while old_version > 13
+            /* oldest pg version included in startos */
+            {
+                old_version -= 1;
+                let old_datadir = db_dir.join(old_version.to_string());
+                if tokio::fs::metadata(&old_datadir).await.is_ok() {
+                    tokio::fs::File::create(&incomplete_path)
+                        .await?
+                        .sync_all()
+                        .await?;
+                    Command::new("pg_upgradecluster")
+                        .arg(old_version.to_string())
+                        .arg("main")
+                        .invoke(crate::ErrorKind::Database)
+                        .await?;
+                    break;
+                }
+            }
+            if tokio::fs::metadata(&conf_dir).await.is_ok() {
+                if tokio::fs::metadata(&conf_dir).await.is_ok() {
+                    tokio::fs::remove_dir_all(&conf_dir).await?;
+                }
+                Command::new("mv")
+                    .arg(&conf_dir_tmp)
+                    .arg(&conf_dir)
+                    .invoke(ErrorKind::Filesystem)
+                    .await?;
+            }
+            tokio::fs::remove_file(&incomplete_path).await?;
+        }
+        if tokio::fs::metadata(&incomplete_path).await.is_ok() {
+            unreachable!() // paranoia
+        }
+    }
+
+    Command::new("systemctl")
+        .arg("start")
+        .arg(format!("postgresql@{pg_version}-main.service"))
+        .invoke(crate::ErrorKind::Database)
+        .await?;
+    if !exists {
+        Command::new("sudo")
+            .arg("-u")
+            .arg("postgres")
+            .arg("createuser")
+            .arg("root")
+            .invoke(crate::ErrorKind::Database)
+            .await?;
+        Command::new("sudo")
+            .arg("-u")
+            .arg("postgres")
+            .arg("createdb")
+            .arg("secrets")
+            .arg("-O")
+            .arg("root")
+            .invoke(crate::ErrorKind::Database)
+            .await?;
+    }
+
+    let secret_store =
+        PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root")).await?;
+    sqlx::migrate!()
+        .run(&secret_store)
+        .await
+        .with_kind(crate::ErrorKind::Database)?;
+    dbg!("Init Postgres Done");
+    Ok(secret_store)
+}
+
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_5_2::Version;
-    fn new() -> Self {
+    type PreUpRes = (AccountInfo, SshKeys, CifsTargets, Notifications);
-        Version
+    fn semver(self) -> exver::Version {
-    }
-    fn semver(&self) -> exver::Version {
         V0_3_6_alpha_0.clone()
     }
-    fn compat(&self) -> &'static VersionRange {
+    fn compat(self) -> &'static VersionRange {
         &V0_3_0_COMPAT
     }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
-        Err(Error::new(eyre!("unimplemented"), ErrorKind::Unknown))
+        let pg = init_postgres("/embassy-data").await?;
+        let account = previous_account_info(&pg).await?;
+
+        let ssh_keys = previous_ssh_keys(&pg).await?;
+
+        let cifs = previous_cifs(&pg).await?;
+
+        let notifications = previous_notifications(pg).await?;
+
+        Ok((account, ssh_keys, cifs, notifications))
+    }
+    fn up(
+        self,
+        db: &mut Value,
+        (account, ssh_keys, cifs, notifications): Self::PreUpRes,
+    ) -> Result<(), Error> {
+        let wifi = json!({
+            "infterface": db["server-info"]["wifi"]["interface"],
+            "ssids": db["server-info"]["wifi"]["ssids"],
+            "selected": db["server-info"]["wifi"]["selected"],
+            "last_region": db["server-info"]["wifi"]["last-region"],
+        });
+
+        let ip_info = {
+            let mut ip_info = json!({});
+            let empty = Default::default();
+            for (k, v) in db["server-info"]["ip-info"].as_object().unwrap_or(&empty) {
+                let k: &str = k.as_ref();
+                ip_info[k] = json!({
+                    "ipv4Range": v["ipv4-range"],
+                    "ipv6Range": v["ipv6-range"],
+                    "ipv4": v["ipv4"],
+                    "ipv6": v["ipv6"],
+                });
+            }
+            ip_info
+        };
+
+        let status_info = json!({
+            "backupProgress": db["server-info"]["status-info"]["backup-progress"],
+            "updated": db["server-info"]["status-info"]["updated"],
+            "updateProgress": db["server-info"]["status-info"]["update-progress"],
+            "shuttingDown": db["server-info"]["status-info"]["shutting-down"],
+            "restarting": db["server-info"]["status-info"]["restarting"],
+        });
+        let server_info = {
+            let mut server_info = json!({
+                "arch": db["server-info"]["arch"],
+                "platform": db["server-info"]["platform"],
+                "id": db["server-info"]["id"],
+                "hostname": db["server-info"]["hostname"],
+                "version": db["server-info"]["version"],
+                "versionCompat": db["server-info"]["eos-version-compat"],
+                "lastBackup": db["server-info"]["last-backup"],
+                "lanAddress": db["server-info"]["lan-address"],
+            });
+
+            server_info["postInitMigrationTodos"] = json!([]);
+            let tor_address: String = from_value(db["server-info"]["tor-address"].clone())?;
+            // Maybe we do this like the Public::init does
+            server_info["torAddress"] = json!(tor_address);
+            server_info["onionAddress"] = json!(tor_address
+                .replace("https://", "")
+                .replace("http://", "")
+                .replace(".onion/", ""));
+            server_info["ipInfo"] = ip_info;
+            server_info["statusInfo"] = status_info;
+            server_info["wifi"] = wifi;
+            server_info["unreadNotificationCount"] =
+                db["server-info"]["unread-notification-count"].clone();
+            server_info["passwordHash"] = db["server-info"]["password-hash"].clone();
+
+            server_info["pubkey"] = db["server-info"]["pubkey"].clone();
+            server_info["caFingerprint"] = db["server-info"]["ca-fingerprint"].clone();
+            server_info["ntpSynced"] = db["server-info"]["ntp-synced"].clone();
+            server_info["zram"] = db["server-info"]["zram"].clone();
+            server_info["governor"] = db["server-info"]["governor"].clone();
+            // This one should always be empty, doesn't exist in the previous. And the smtp is all single word key
+            server_info["smtp"] = db["server-info"]["smtp"].clone();
+            server_info
+        };
+
+        let public = json!({
+            "serverInfo": server_info,
+            "packageData": json!({}),
+            "ui": db["ui"],
+        });
+
+        let private = {
+            let mut value = json!({});
+            value["keyStore"] = to_value(&KeyStore::new(&account)?)?;
+            value["password"] = to_value(&account.password)?;
+            value["compatS9pkKey"] = to_value(&crate::db::model::private::generate_compat_key())?;
+            value["sshPrivkey"] = to_value(Pem::new_ref(&account.ssh_key))?;
+            value["sshPubkeys"] = to_value(&ssh_keys)?;
+            value["availablePorts"] = to_value(&AvailablePorts::new())?;
+            value["sessions"] = to_value(&Sessions::new())?;
+            value["notifications"] = to_value(&notifications)?;
+            value["cifs"] = to_value(&cifs)?;
+            value["packageStores"] = json!({});
+            value
+        };
+        let next: Value = json!({
+            "public": public,
+            "private": private,
+        });
+
+        dbg!("Should be done with the up");
+        *db = next;
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
+        Err(Error::new(
+            eyre!("downgrades prohibited"),
+            ErrorKind::InvalidRequest,
+        ))
+    }
+
+    #[instrument(skip(self, ctx))]
+    /// MUST be idempotent, and is run after *all* db migrations
+    fn post_up(self, ctx: &RpcContext) -> impl Future<Output = Result<(), Error>> + Send + 'static {
+        let ctx = ctx.clone();
+        async move {
+            let path = Path::new("/embassy-data/package-data/archive/");
+            if !path.is_dir() {
+                return Err(Error::new(
+                    eyre!(
+                        "expected path ({}) to be a directory",
+                        path.to_string_lossy()
+                    ),
+                    ErrorKind::Filesystem,
+                ));
+            }
+            // Should be the name of the package
+            let mut paths = tokio::fs::read_dir(path).await?;
+            while let Some(path) = paths.next_entry().await? {
+                let path = path.path();
+                if !path.is_dir() {
+                    continue;
+                }
+                // Should be the version of the package
+                let mut paths = tokio::fs::read_dir(path).await?;
+                while let Some(path) = paths.next_entry().await? {
+                    let path = path.path();
+                    if !path.is_dir() {
+                        continue;
+                    }
+
+                    // Should be s9pk
+                    let mut paths = tokio::fs::read_dir(path).await?;
+                    while let Some(path) = paths.next_entry().await? {
+                        let path = path.path();
+                        if path.is_dir() {
+                            continue;
+                        }
+
+                        let package_s9pk = tokio::fs::File::open(path).await?;
+                        let file = MultiCursorFile::open(&package_s9pk).await?;
+
+                        let key = ctx.db.peek().await.into_private().into_compat_s9pk_key();
+                        ctx.services
+                            .install(
+                                ctx.clone(),
+                                || crate::s9pk::load(file.clone(), || Ok(key.de()?.0), None),
+                                None::<crate::util::Never>,
+                                None,
+                            )
+                            .await?
+                            .await?
+                            .await?;
+                    }
+                }
             }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
             Ok(())
         }
     }
+}
+
+async fn previous_notifications(pg: sqlx::Pool<sqlx::Postgres>) -> Result<Notifications, Error> {
+    let notification_cursor = sqlx::query(r#"SELECT * FROM notifications"#)
+        .fetch_all(&pg)
+        .await?;
+    let notifications = {
+        let mut notifications = Notifications::default();
+        for row in notification_cursor {
+            let package_id = serde_json::from_str::<PackageId>(
+                row.try_get("package_id")
+                    .with_ctx(|_| (ErrorKind::Database, "package_id"))?,
+            )
+            .ok();
+
+            let created_at = row
+                .try_get("created_at")
+                .with_ctx(|_| (ErrorKind::Database, "created_at"))?;
+            let code = row
+                .try_get::<i64, _>("code")
+                .with_ctx(|_| (ErrorKind::Database, "code"))? as u32;
+            let id = row
+                .try_get::<i64, _>("id")
+                .with_ctx(|_| (ErrorKind::Database, "id"))? as u32;
+            let level = serde_json::from_str(
+                row.try_get("level")
+                    .with_ctx(|_| (ErrorKind::Database, "level"))?,
+            )
+            .with_kind(ErrorKind::Database)
+            .with_ctx(|_| (ErrorKind::Database, "level: serde_json "))?;
+            let title = row
+                .try_get("title")
+                .with_ctx(|_| (ErrorKind::Database, "title"))?;
+            let message = row
+                .try_get("message")
+                .with_ctx(|_| (ErrorKind::Database, "message"))?;
+            let data = serde_json::from_str(
+                row.try_get("data")
+                    .with_ctx(|_| (ErrorKind::Database, "data"))?,
+            )
+            .unwrap_or_default();
+
+            notifications.0.insert(
+                id,
+                Notification {
+                    package_id,
+                    created_at,
+                    code,
+                    level,
+                    title,
+                    message,
+                    data,
+                },
+            );
+        }
+        notifications
+    };
+    Ok(notifications)
+}
+
+#[tracing::instrument(skip_all)]
+async fn previous_cifs(pg: &sqlx::Pool<sqlx::Postgres>) -> Result<CifsTargets, Error> {
+    let cifs = sqlx::query(r#"SELECT * FROM cifs_shares"#)
+        .fetch_all(pg)
+        .await?
+        .into_iter()
+        .map(|row| {
+            let id: i64 = row.try_get("id")?;
+            Ok::<_, Error>((
+                id,
+                Cifs {
+                    hostname: row
+                        .try_get("hostname")
+                        .with_ctx(|_| (ErrorKind::Database, "hostname"))?,
+                    path: serde_json::from_str(row.try_get("path")?)
+                        .with_kind(ErrorKind::Database)
+                        .with_ctx(|_| (ErrorKind::Database, "path"))?,
+                    username: row
+                        .try_get("username")
+                        .with_ctx(|_| (ErrorKind::Database, "username"))?,
+                    password: row
+                        .try_get("password")
+                        .with_ctx(|_| (ErrorKind::Database, "password"))?,
+                },
+            ))
+        })
+        .fold(Ok::<_, Error>(CifsTargets::default()), |cifs, data| {
+            let mut cifs = cifs?;
+            let (id, cif_value) = data?;
+            cifs.0.insert(id as u32, cif_value);
+            Ok(cifs)
+        })?;
+    Ok(cifs)
+}
+
+#[tracing::instrument(skip_all)]
+async fn previous_account_info(pg: &sqlx::Pool<sqlx::Postgres>) -> Result<AccountInfo, Error> {
+    let account_query = sqlx::query(r#"SELECT * FROM account"#)
+        .fetch_one(pg)
+        .await?;
+    let account = {
+        AccountInfo {
+            password: account_query
+                .try_get("password")
+                .with_ctx(|_| (ErrorKind::Database, "password"))?,
+            tor_key: TorSecretKeyV3::try_from(
+                if let Some(bytes) = account_query
+                    .try_get::<Option<Vec<u8>>, _>("tor_key")
+                    .with_ctx(|_| (ErrorKind::Database, "tor_key"))?
+                {
+                    <[u8; 64]>::try_from(bytes)
+                        .map_err(|e| {
+                            Error::new(
+                                eyre!("expected vec of len 64, got len {}", e.len()),
+                                ErrorKind::ParseDbField,
+                            )
+                        })
+                        .with_ctx(|_| (ErrorKind::Database, "password.u8 64"))?
+                } else {
+                    ed25519_expand_key(
+                        &<[u8; 32]>::try_from(account_query.try_get::<Vec<u8>, _>("network_key")?)
+                            .map_err(|e| {
+                                Error::new(
+                                    eyre!("expected vec of len 32, got len {}", e.len()),
+                                    ErrorKind::ParseDbField,
+                                )
+                            })
+                            .with_ctx(|_| (ErrorKind::Database, "password.u8 32"))?,
+                    )
+                },
+            )?,
+            server_id: account_query
+                .try_get("server_id")
+                .with_ctx(|_| (ErrorKind::Database, "server_id"))?,
+            hostname: Hostname(
+                account_query
+                    .try_get::<String, _>("hostname")
+                    .with_ctx(|_| (ErrorKind::Database, "hostname"))?
+                    .into(),
+            ),
+            root_ca_key: PKey::private_key_from_pem(
+                &account_query
+                    .try_get::<String, _>("root_ca_key_pem")
+                    .with_ctx(|_| (ErrorKind::Database, "root_ca_key_pem"))?
+                    .as_bytes(),
+            )
+            .with_ctx(|_| (ErrorKind::Database, "private_key_from_pem"))?,
+            root_ca_cert: X509::from_pem(
+                account_query
+                    .try_get::<String, _>("root_ca_cert_pem")
+                    .with_ctx(|_| (ErrorKind::Database, "root_ca_cert_pem"))?
+                    .as_bytes(),
+            )
+            .with_ctx(|_| (ErrorKind::Database, "X509::from_pem"))?,
+            compat_s9pk_key: SigningKey::generate(&mut rand::thread_rng()),
+            ssh_key: ssh_key::PrivateKey::random(
+                &mut rand::thread_rng(),
+                ssh_key::Algorithm::Ed25519,
+            )
+            .with_ctx(|_| (ErrorKind::Database, "X509::ssh_key::PrivateKey::random"))?,
+        }
+    };
+    Ok(account)
+}
+#[tracing::instrument(skip_all)]
+async fn previous_ssh_keys(pg: &sqlx::Pool<sqlx::Postgres>) -> Result<SshKeys, Error> {
+    let ssh_query = sqlx::query(r#"SELECT * FROM ssh_keys"#)
+        .fetch_all(pg)
+        .await?;
+    let ssh_keys: SshKeys = {
+        let keys = ssh_query.into_iter().fold(
+            Ok::<_, Error>(BTreeMap::<InternedString, WithTimeData<SshPubKey>>::new()),
+            |ssh_keys, row| {
+                let mut ssh_keys = ssh_keys?;
+                let time = row
+                    .try_get::<String, _>("created_at")
+                    .map_err(Error::from)
+                    .and_then(|x| x.parse::<DateTime<Utc>>().with_kind(ErrorKind::Database))
+                    .with_ctx(|_| (ErrorKind::Database, "openssh_pubkey::created_at"))?;
+                let value: SshPubKey = row
+                    .try_get::<String, _>("openssh_pubkey")
+                    .map_err(Error::from)
+                    .and_then(|x| x.parse().map(SshPubKey).with_kind(ErrorKind::Database))
+                    .with_ctx(|_| (ErrorKind::Database, "openssh_pubkey"))?;
+                let data = WithTimeData {
+                    created_at: time,
+                    updated_at: time,
+                    value,
+                };
+                let fingerprint = row
+                    .try_get::<String, _>("fingerprint")
+                    .with_ctx(|_| (ErrorKind::Database, "fingerprint"))?;
+                ssh_keys.insert(fingerprint.into(), data);
+                Ok(ssh_keys)
+            },
+        )?;
+        SshKeys::from(keys)
+    };
+    Ok(ssh_keys)
+}

core/startos/src/version/v0_3_6_alpha_1.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_0, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
@@ -12,24 +11,26 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_0::Version;
-    fn new() -> Self {
+    type PreUpRes = ();
-        Version
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
     }
-    fn semver(&self) -> exver::Version {
+    fn semver(self) -> exver::Version {
         V0_3_6_alpha_1.clone()
     }
-    fn compat(&self) -> &'static VersionRange {
+    fn compat(self) -> &'static VersionRange {
         &V0_3_0_COMPAT
     }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_2.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_1, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
@@ -12,24 +11,26 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_1::Version;
-    fn new() -> Self {
+    type PreUpRes = ();
-        Version
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
     }
-    fn semver(&self) -> exver::Version {
+    fn semver(self) -> exver::Version {
         V0_3_6_alpha_2.clone()
     }
-    fn compat(&self) -> &'static VersionRange {
+    fn compat(self) -> &'static VersionRange {
         &V0_3_0_COMPAT
     }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_3.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_2, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
@@ -12,24 +11,26 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_2::Version;
-    fn new() -> Self {
+    type PreUpRes = ();
-        Version
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
     }
-    fn semver(&self) -> exver::Version {
+    fn semver(self) -> exver::Version {
         V0_3_6_alpha_3.clone()
     }
-    fn compat(&self) -> &'static VersionRange {
+    fn compat(self) -> &'static VersionRange {
         &V0_3_0_COMPAT
     }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_4.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_3, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
@@ -12,24 +11,26 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_3::Version;
-    fn new() -> Self {
+    type PreUpRes = ();
-        Version
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
     }
-    fn semver(&self) -> exver::Version {
+    fn semver(self) -> exver::Version {
         V0_3_6_alpha_4.clone()
     }
-    fn compat(&self) -> &'static VersionRange {
+    fn compat(self) -> &'static VersionRange {
         &V0_3_0_COMPAT
     }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_5.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_4, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
@@ -12,24 +11,26 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_4::Version;
-    fn new() -> Self {
+    type PreUpRes = ();
-        Version
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
     }
-    fn semver(&self) -> exver::Version {
+    fn semver(self) -> exver::Version {
         V0_3_6_alpha_5.clone()
     }
-    fn compat(&self) -> &'static VersionRange {
+    fn compat(self) -> &'static VersionRange {
         &V0_3_0_COMPAT
     }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_6.rs

@@ -2,7 +2,6 @@ use exver::{PreReleaseSegment, VersionRange};
 
 use super::v0_3_5::V0_3_0_COMPAT;
 use super::{v0_3_6_alpha_5, VersionT};
-use crate::db::model::Database;
 use crate::prelude::*;
 
 lazy_static::lazy_static! {
@@ -12,24 +11,26 @@ lazy_static::lazy_static! {
     );
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub struct Version;
 
 impl VersionT for Version {
     type Previous = v0_3_6_alpha_5::Version;
-    fn new() -> Self {
+    type PreUpRes = ();
-        Version
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
     }
-    fn semver(&self) -> exver::Version {
+    fn semver(self) -> exver::Version {
         V0_3_6_alpha_6.clone()
     }
-    fn compat(&self) -> &'static VersionRange {
+    fn compat(self) -> &'static VersionRange {
         &V0_3_0_COMPAT
     }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn up(self, _db: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
         Ok(())
     }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
         Ok(())
     }
 }

core/startos/src/version/v0_3_6_alpha_7.rs

@@ -1,35 +0,0 @@
-use exver::{PreReleaseSegment, VersionRange};
-
-use super::v0_3_5::V0_3_0_COMPAT;
-use super::{v0_3_6_alpha_6, VersionT};
-use crate::db::model::Database;
-use crate::prelude::*;
-
-lazy_static::lazy_static! {
-    static ref V0_3_6_alpha_7: exver::Version = exver::Version::new(
-        [0, 3, 6],
-        [PreReleaseSegment::String("alpha".into()), 7.into()]
-    );
-}
-
-#[derive(Clone, Debug)]
-pub struct Version;
-
-impl VersionT for Version {
-    type Previous = v0_3_6_alpha_6::Version;
-    fn new() -> Self {
-        Version
-    }
-    fn semver(&self) -> exver::Version {
-        V0_3_6_alpha_7.clone()
-    }
-    fn compat(&self) -> &'static VersionRange {
-        &V0_3_0_COMPAT
-    }
-    async fn up(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
-        Ok(())
-    }
-    async fn down(&self, _db: &TypedPatchDb<Database>) -> Result<(), Error> {
-        Ok(())
-    }
-}

sdk/base/lib/osBindings/ServerInfo.ts

@@ -11,8 +11,9 @@ export type ServerInfo = {
   id: string
   hostname: string
   version: string
+  versionCompat: string
+  postInitMigrationTodos: string[]
   lastBackup: string | null
-  eosVersionCompat: string
   lanAddress: string
   onionAddress: string
   /**

web/patchdb-ui-seed.json

@@ -21,5 +21,5 @@
   "ackInstructions": {},
   "theme": "Dark",
   "widgets": [],
-  "ack-welcome": "0.3.6-alpha.5"
+  "ack-welcome": "0.3.6-alpha.6"
 }

web/projects/ui/src/app/pages/server-routes/sideload/sideload.page.ts

@@ -1,12 +1,13 @@
 import { Component } from '@angular/core'
 import { isPlatform } from '@ionic/angular'
 import { ErrorService, LoadingService } from '@start9labs/shared'
-import { S9pk } from '@start9labs/start-sdk'
+import { S9pk, T } from '@start9labs/start-sdk'
 import cbor from 'cbor'
 import { ApiService } from 'src/app/services/api/embassy-api.service'
 import { ConfigService } from 'src/app/services/config.service'
 import { SideloadService } from './sideload.service'
 import { firstValueFrom } from 'rxjs'
+import mime from 'mime'
 
 interface Positions {
   [key: string]: [bigint, bigint] // [position, length]

web/projects/ui/src/app/services/api/mock-patch.ts

@@ -60,7 +60,7 @@ export const mockPatchData: DataModel = {
     // password is asdfasdf
     passwordHash:
       '$argon2d$v=19$m=1024,t=1,p=1$YXNkZmFzZGZhc2RmYXNkZg$Ceev1I901G6UwU+hY0sHrFZ56D+o+LNJ',
-    eosVersionCompat: '>=0.3.0 <=0.3.6',
+    versionCompat: '>=0.3.0 <=0.3.6',
     statusInfo: {
       backupProgress: null,
       updated: false,
@@ -82,6 +82,7 @@ export const mockPatchData: DataModel = {
       selected: null,
       lastRegion: null,
     },
+    postInitMigrationTodos: [],
   },
   packageData: {
     bitcoind: {