Feat/patch migration (#1890)

* feat: Most of the ui conversions and removing the package-recovered

* chore: Include the down process.

* feat: Add in the potential community packages.

* chore: Add in the services

* Make it so we skip rebuil.

* update version to 033 in FE

* chore: Revert to next version for patch-db

* fix: Build and check

Co-authored-by: Lucy Cifferello <12953208+elvece@users.noreply.github.com>

Makefile

@@ -8,7 +8,7 @@ COMPAT_SRC := $(shell find system-images/compat/ -not -path 'system-images/compa
 UTILS_SRC := $(shell find system-images/utils/ -not -name *.tar)
 BINFMT_SRC := $(shell find system-images/binfmt/ -not -name *.tar)
 BACKEND_SRC := $(shell find backend/src) $(shell find backend/migrations) $(shell find patch-db/*/src) backend/Cargo.toml backend/Cargo.lock
-FRONTEND_SHARED_SRC := $(shell find frontend/projects/shared) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/node_modules frontend/config.json patch-db/client/dist frontend/patchdb-ui-seed.json
+FRONTEND_SHARED_SRC := $(shell find frontend/projects/shared) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/package.json frontend/node_modules frontend/config.json patch-db/client/dist frontend/patchdb-ui-seed.json
 FRONTEND_UI_SRC := $(shell find frontend/projects/ui)
 FRONTEND_SETUP_WIZARD_SRC := $(shell find frontend/projects/setup-wizard)
 FRONTEND_DIAGNOSTIC_UI_SRC := $(shell find frontend/projects/diagnostic-ui)

backend/src/db/model.rs

@@ -28,8 +28,6 @@ pub struct Database {
     pub server_info: ServerInfo,
     #[model]
     pub package_data: AllPackageData,
-    #[model]
-    pub recovered_packages: BTreeMap<PackageId, RecoveredPackageInfo>,
     pub ui: Value,
 }
 impl Database {
@@ -68,7 +66,6 @@ impl Database {
                 password_hash,
             },
             package_data: AllPackageData::default(),
-            recovered_packages: BTreeMap::new(),
             ui: serde_json::from_str(include_str!("../../../frontend/patchdb-ui-seed.json"))
                 .unwrap(),
         }

backend/src/init.rs

@@ -14,7 +14,6 @@ use crate::db::model::ServerStatus;
 use crate::install::PKG_DOCKER_DIR;
 use crate::sound::CIRCLE_OF_5THS_SHORT;
 use crate::util::Invoke;
-use crate::version::VersionT;
 use crate::Error;
 
 pub const SYSTEM_REBUILD_PATH: &str = "/media/embassy/config/system-rebuild";
@@ -225,8 +224,7 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
     let receipts = InitReceipts::new(&mut handle).await?;
 
     let should_rebuild = tokio::fs::metadata(SYSTEM_REBUILD_PATH).await.is_ok()
-        || &*receipts.server_version.get(&mut handle).await?
+        || &*receipts.server_version.get(&mut handle).await? < &emver::Version::new(0, 3, 2, 0);
-            < &crate::version::Current::new().semver();
 
     let song = if should_rebuild {
         Some(NonDetachingJoinHandle::from(tokio::spawn(async {

backend/src/install/mod.rs

@@ -14,7 +14,7 @@ use futures::{FutureExt, StreamExt, TryStreamExt};
 use http::header::CONTENT_LENGTH;
 use http::{Request, Response, StatusCode};
 use hyper::Body;
-use patch_db::{DbHandle, LockReceipt, LockType};
+use patch_db::{DbHandle, LockType};
 use reqwest::Url;
 use rpc_toolkit::command;
 use rpc_toolkit::yajrc::RpcError;
@@ -30,7 +30,7 @@ use crate::context::{CliContext, RpcContext};
 use crate::core::rpc_continuations::{RequestGuid, RpcContinuation};
 use crate::db::model::{
     CurrentDependencies, CurrentDependencyInfo, CurrentDependents, InstalledPackageDataEntry,
-    PackageDataEntry, RecoveredPackageInfo, StaticDependencyInfo, StaticFiles,
+    PackageDataEntry, StaticDependencyInfo, StaticFiles,
 };
 use crate::dependencies::{
     add_dependent_to_current_dependents_lists, break_all_dependents_transitive,
@@ -667,43 +667,6 @@ pub async fn uninstall_impl(ctx: RpcContext, id: PackageId) -> Result<(), Error>
     Ok(())
 }
 
-#[command(
-    rename = "delete-recovered",
-    display(display_none),
-    metadata(sync_db = true)
-)]
-pub async fn delete_recovered(
-    #[context] ctx: RpcContext,
-    #[arg] id: PackageId,
-) -> Result<(), Error> {
-    let mut handle = ctx.db.handle();
-    let mut tx = handle.begin().await?;
-    let mut sql_tx = ctx.secret_store.begin().await?;
-
-    let mut recovered_packages = crate::db::DatabaseModel::new()
-        .recovered_packages()
-        .get_mut(&mut tx)
-        .await?;
-    recovered_packages.remove(&id).ok_or_else(|| {
-        Error::new(
-            eyre!("{} not in recovered-packages", id),
-            crate::ErrorKind::NotFound,
-        )
-    })?;
-    recovered_packages.save(&mut tx).await?;
-
-    let volumes = ctx.datadir.join(crate::volume::PKG_VOLUME_DIR).join(&id);
-    if tokio::fs::metadata(&volumes).await.is_ok() {
-        tokio::fs::remove_dir_all(&volumes).await?;
-    }
-    cleanup::remove_tor_keys(&mut sql_tx, &id).await?;
-
-    tx.commit().await?;
-    sql_tx.commit().await?;
-
-    Ok(())
-}
-
 pub struct DownloadInstallReceipts {
     package_receipts: crate::db::package::PackageReceipts,
     manifest_receipts: crate::db::package::ManifestReceipts,
@@ -864,8 +827,6 @@ pub async fn download_install_s9pk(
 
 pub struct InstallS9Receipts {
     config: ConfigReceipts,
-
-    recovered_packages: LockReceipt<BTreeMap<PackageId, RecoveredPackageInfo>, ()>,
 }
 
 impl InstallS9Receipts {
@@ -880,16 +841,9 @@ impl InstallS9Receipts {
         locks: &mut Vec<patch_db::LockTargetId>,
     ) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
         let config = ConfigReceipts::setup(locks);
-
-        let recovered_packages = crate::db::DatabaseModel::new()
-            .recovered_packages()
-            .make_locker(LockType::Write)
-            .add_to_keys(locks);
-
         move |skeleton_key| {
             Ok(Self {
                 config: config(skeleton_key)?,
-                recovered_packages: recovered_packages.verify(skeleton_key)?,
             })
         }
     }
@@ -1436,38 +1390,6 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
             &receipts.config.update_dependency_receipts,
         )
         .await?;
-    } else if let Some(recovered) = {
-        receipts
-            .recovered_packages
-            .get(&mut tx)
-            .await?
-            .remove(pkg_id)
-    } {
-        handle_recovered_package(
-            recovered,
-            manifest,
-            ctx,
-            pkg_id,
-            version,
-            &mut tx,
-            &receipts.config,
-        )
-        .await?;
-        add_dependent_to_current_dependents_lists(
-            &mut tx,
-            pkg_id,
-            &current_dependencies,
-            &receipts.config.current_dependents,
-        )
-        .await?;
-        update_dependency_errors_of_dependents(
-            ctx,
-            &mut tx,
-            pkg_id,
-            &current_dependents,
-            &receipts.config.update_dependency_receipts,
-        )
-        .await?;
     } else {
         add_dependent_to_current_dependents_lists(
             &mut tx,
@@ -1486,16 +1408,6 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
         .await?;
     }
 
-    let recovered_packages = {
-        let mut r = receipts.recovered_packages.get(&mut tx).await?;
-        r.remove(pkg_id);
-        r
-    };
-    receipts
-        .recovered_packages
-        .set(&mut tx, recovered_packages)
-        .await?;
-
     if let Some(installed) = pde.installed() {
         reconfigure_dependents_with_live_pointers(ctx, &mut tx, &receipts.config, installed)
             .await?;
@@ -1509,46 +1421,6 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
     Ok(())
 }
 
-#[instrument(skip(ctx, tx, receipts))]
-async fn handle_recovered_package(
-    recovered: RecoveredPackageInfo,
-    manifest: Manifest,
-    ctx: &RpcContext,
-    pkg_id: &PackageId,
-    version: &Version,
-    tx: &mut patch_db::Transaction<&mut patch_db::PatchDbHandle>,
-    receipts: &ConfigReceipts,
-) -> Result<(), Error> {
-    let configured = if let Some(migration) = manifest.migrations.from(
-        &manifest.containers,
-        ctx,
-        &recovered.version,
-        pkg_id,
-        version,
-        &manifest.volumes,
-    ) {
-        migration.await?.configured
-    } else {
-        false
-    };
-    if configured && manifest.config.is_some() {
-        crate::config::configure(
-            ctx,
-            tx,
-            pkg_id,
-            None,
-            &None,
-            false,
-            &mut BTreeMap::new(),
-            &mut BTreeMap::new(),
-            &receipts,
-        )
-        .await?;
-    }
-
-    Ok(())
-}
-
 #[instrument(skip(datadir))]
 pub fn load_images<'a, P: AsRef<Path> + 'a + Send + Sync>(
     datadir: P,

backend/src/lib.rs

@@ -1,4 +1,5 @@
 pub const DEFAULT_MARKETPLACE: &str = "https://registry.start9.com";
+pub const COMMUNITY_MARKETPLACE: &str = "https://community-registry.start9.com";
 pub const BUFFER_SIZE: usize = 1024;
 pub const HOST_IP: [u8; 4] = [172, 18, 0, 1];
 pub const TARGET: &str = current_platform::CURRENT_PLATFORM;
@@ -97,7 +98,6 @@ pub fn server() -> Result<(), RpcError> {
     install::install,
     install::sideload,
     install::uninstall,
-    install::delete_recovered,
     install::list,
     install::update::update,
     config::config,

backend/src/version/mod.rs

@@ -17,8 +17,9 @@ mod v0_3_1_1;
 mod v0_3_1_2;
 mod v0_3_2;
 mod v0_3_2_1;
+mod v0_3_3;
 
-pub type Current = v0_3_2_1::Version;
+pub type Current = v0_3_3::Version;
 
 #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
 #[serde(untagged)]
@@ -32,6 +33,7 @@ enum Version {
     V0_3_1_2(Wrapper<v0_3_1_2::Version>),
     V0_3_2(Wrapper<v0_3_2::Version>),
     V0_3_2_1(Wrapper<v0_3_2_1::Version>),
+    V0_3_3(Wrapper<v0_3_3::Version>),
     Other(emver::Version),
 }
 
@@ -56,6 +58,7 @@ impl Version {
             Version::V0_3_1_2(Wrapper(x)) => x.semver(),
             Version::V0_3_2(Wrapper(x)) => x.semver(),
             Version::V0_3_2_1(Wrapper(x)) => x.semver(),
+            Version::V0_3_3(Wrapper(x)) => x.semver(),
             Version::Other(x) => x.clone(),
         }
     }
@@ -191,6 +194,7 @@ pub async fn init<Db: DbHandle>(
         Version::V0_3_1_2(v) => v.0.migrate_to(&Current::new(), db, receipts).await?,
         Version::V0_3_2(v) => v.0.migrate_to(&Current::new(), db, receipts).await?,
         Version::V0_3_2_1(v) => v.0.migrate_to(&Current::new(), db, receipts).await?,
+        Version::V0_3_3(v) => v.0.migrate_to(&Current::new(), db, receipts).await?,
         Version::Other(_) => {
             return Err(Error::new(
                 eyre!("Cannot downgrade"),
@@ -232,6 +236,7 @@ mod tests {
             Just(Version::V0_3_1_2(Wrapper(v0_3_1_2::Version::new()))),
             Just(Version::V0_3_2(Wrapper(v0_3_2::Version::new()))),
             Just(Version::V0_3_2_1(Wrapper(v0_3_2_1::Version::new()))),
+            Just(Version::V0_3_3(Wrapper(v0_3_3::Version::new()))),
             em_version().prop_map(Version::Other),
         ]
     }

backend/src/version/v0_3_3.rs

@@ -0,0 +1,164 @@
+use async_trait::async_trait;
+use emver::VersionRange;
+use serde_json::{json, Value};
+
+use super::v0_3_0::V0_3_0_COMPAT;
+use super::*;
+use crate::{COMMUNITY_MARKETPLACE, DEFAULT_MARKETPLACE};
+
+const V0_3_3: emver::Version = emver::Version::new(0, 3, 3, 0);
+
+lazy_static::lazy_static! {
+    static ref COMMUNITY_PACKAGES: Vec<&'static str> = vec![
+        "robosats",
+        "syncthing",
+        "balanceofsatoshis",
+        "lightning-jet",
+        "mastodon",
+        "sphinx-relay",
+        "agora",
+        "lndg",
+        "synapse",
+        "thunderhub",
+    ];
+}
+
+#[derive(Clone, Debug)]
+pub struct Version;
+
+#[async_trait]
+impl VersionT for Version {
+    type Previous = v0_3_2_1::Version;
+    fn new() -> Self {
+        Version
+    }
+    fn semver(&self) -> emver::Version {
+        V0_3_3
+    }
+    fn compat(&self) -> &'static VersionRange {
+        &*V0_3_0_COMPAT
+    }
+    async fn up<Db: DbHandle>(&self, db: &mut Db) -> Result<(), Error> {
+        let mut ui = crate::db::DatabaseModel::new().ui().get_mut(db).await?;
+
+        if let Some(Value::String(selected_url)) =
+            ui["marketplace"]
+                .get("selected-id")
+                .and_then(|selected_id| {
+                    if let Value::String(selected_id) = selected_id {
+                        return Some(ui["marketplace"]["known-hosts"].get(&selected_id)?);
+                    }
+                    None
+                })
+        {
+            ui["marketplace"]["selected-url"] = json!(selected_url);
+        }
+        if let Value::Object(ref mut obj) = *ui {
+            obj.remove("pkg-order");
+            obj.remove("auto-check-updates");
+        }
+        let known_hosts = ui["marketplace"]["known-hosts"].take();
+        if let Value::Object(known_hosts) = known_hosts {
+            for (_id, value) in known_hosts {
+                if let (Value::String(name), Value::String(url)) = (&value["name"], &value["url"]) {
+                    ui["marketplace"]["known-hosts"][name] = json!(url);
+                }
+            }
+        }
+        if let Some(Value::Object(ref mut obj)) = ui.get_mut("marketplace") {
+            obj.remove("selected-id");
+        }
+        if ui["marketplace"]["selected-url"].is_null() {
+            ui["marketplace"]["selected-url"] = json!(MarketPlaceUrls::Default.url());
+        }
+        ui.save(db).await?;
+
+        for package_id in crate::db::DatabaseModel::new()
+            .package_data()
+            .keys(db, false)
+            .await?
+            .iter()
+        {
+            let id: &str = &**package_id;
+            if COMMUNITY_PACKAGES.iter().find(|x| x == &&id).is_none() {
+                continue;
+            }
+            let mut package = crate::db::DatabaseModel::new()
+                .package_data()
+                .idx_model(package_id)
+                .and_then(|x| x.installed())
+                .get_mut(db)
+                .await?;
+            if let Some(ref mut package) = *package {
+                package.marketplace_url = Some(MarketPlaceUrls::Community.url().parse().unwrap());
+            }
+            package.save(db).await?;
+        }
+
+        Ok(())
+    }
+    async fn down<Db: DbHandle>(&self, db: &mut Db) -> Result<(), Error> {
+        let mut ui = crate::db::DatabaseModel::new().ui().get_mut(db).await?;
+        let selected_url = ui["marketplace"]["selected-url"]
+            .as_str()
+            .map(|x| x.to_owned());
+        let known_hosts = ui["marketplace"]["known-hosts"].take();
+        ui["marketplace"]["known-hosts"] = json!({});
+        if let Value::Object(known_hosts) = known_hosts {
+            for (url, name) in known_hosts {
+                if let Value::String(name) = name {
+                    let id = uuid::Uuid::new_v4().to_string();
+                    if Some(&name) == selected_url.as_ref() {
+                        ui["marketplace"]["selected-id"] = Value::String(id.clone());
+                    }
+                    ui["marketplace"]["known-hosts"][id.as_str()] = json!({
+                        "name": name,
+                        "url": url
+                    });
+                }
+            }
+        }
+        ui["auto-check-updates"] = Value::Bool(true);
+        ui["pkg-order"] = json!(crate::db::DatabaseModel::new()
+            .package_data()
+            .keys(db, false)
+            .await?
+            .iter()
+            .map(|x| x.to_string())
+            .collect::<Vec<String>>());
+        if let Some(Value::Object(ref mut obj)) = ui.get_mut("marketplace") {
+            obj.remove("selected-url");
+        }
+        ui.save(db).await?;
+        Ok(())
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum MarketPlaceUrls {
+    Default,
+    Community,
+}
+
+impl MarketPlaceUrls {
+    pub fn url(&self) -> String {
+        let url_string = match self {
+            MarketPlaceUrls::Default => DEFAULT_MARKETPLACE,
+            MarketPlaceUrls::Community => COMMUNITY_MARKETPLACE,
+        };
+        format!("{url_string}/")
+    }
+}
+
+#[test]
+fn test_that_ui_includes_url() {
+    let ui: Value =
+        serde_json::from_str(include_str!("../../../frontend/patchdb-ui-seed.json")).unwrap();
+    for market_place in [MarketPlaceUrls::Default, MarketPlaceUrls::Community] {
+        let url = market_place.url();
+        assert!(
+            !ui["marketplace"]["known-hosts"][&url].is_null(),
+            "Should have a market place for {url}"
+        );
+    }
+}

frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "embassy-os",
-  "version": "0.3.2.1",
+  "version": "0.3.3",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "embassy-os",
-      "version": "0.3.2.1",
+      "version": "0.3.3",
       "dependencies": {
         "@angular/animations": "^14.1.0",
         "@angular/common": "^14.1.0",

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "embassy-os",
-  "version": "0.3.2.1",
+  "version": "0.3.3",
   "author": "Start9 Labs, Inc",
   "homepage": "https://start9.com/",
   "scripts": {

frontend/patchdb-ui-seed.json

@@ -1,6 +1,6 @@
 {
   "name": null,
-  "ack-welcome": "0.3.2.1",
+  "ack-welcome": "0.3.3",
   "marketplace": {
     "selected-url": "https://registry.start9.com/",
     "known-hosts": {

frontend/projects/ui/src/app/modals/os-welcome/os-welcome.page.html

@@ -11,11 +11,11 @@
 
 <ion-content class="ion-padding">
   <h2>This release</h2>
-  <h4>0.3.2~1</h4>
+  <h4>0.3.3</h4>
   <p class="note-padding">
     View the complete
     <a
-      href="https://github.com/Start9Labs/embassy-os/releases/tag/v0.3.2.1"
+      href="https://github.com/Start9Labs/embassy-os/releases/tag/v0.3.3"
       target="_blank"
       noreferrer
       >release notes</a
@@ -24,30 +24,12 @@
   </p>
   <h6>Highlights</h6>
   <ul class="spaced-list">
-    <li>Bugfix for LAN URLs</li>
+    <li>Kiosk mode</li>
-  </ul>
+    <li>x86_64 architecture compatibility</li>
-
+    <li>Community marketplaces</li>
-  <br />
+    <li>New update all tab</li>
-  <h2>Previous releases in this series</h2>
+    <li>Various UI/UX improvements</li>
-  <h4>0.3.2</h4>
+    <li>Various bugfixes and optimizations</li>
-  <p class="note-padding">
-    View the complete
-    <a
-      href="https://github.com/Start9Labs/embassy-os/releases/tag/v0.3.2"
-      target="_blank"
-      noreferrer
-      >release notes</a
-    >
-    for more details.
-  </p>
-  <h6>Highlights</h6>
-  <ul class="spaced-list">
-    <li>Autoscrolling for logs</li>
-    <li>Improved connectivity between browser and Embassy</li>
-    <li>Switch to Postgres for EOS database for better performance</li>
-    <li>Multiple bug fixes and under-the-hood improvements</li>
-    <li>Various UI/UX enhancements</li>
-    <li>Removal of product keys</li>
   </ul>
   <div class="ion-text-center ion-padding">
     <ion-button

frontend/projects/ui/src/app/services/api/api.fixures.ts

@@ -20,9 +20,10 @@ export module Mock {
     updated: true,
   }
   export const MarketplaceEos: RR.GetMarketplaceEOSRes = {
-    version: '0.3.2.1',
+    version: '0.3.3',
     headline: 'Our biggest release ever.',
     'release-notes': {
+      '0.3.3': 'Some **Markdown** release _notes_ for 0.3.3',
       '0.3.2.1': 'Some **Markdown** release _notes_ for 0.3.2.1',
       '0.3.2': 'Some **Markdown** release _notes_ for 0.3.2',
       '0.3.1': 'Some **Markdown** release _notes_ for 0.3.1',

frontend/projects/ui/src/app/services/api/mock-patch.ts

@@ -32,7 +32,7 @@ export const mockPatchData: DataModel = {
   },
   'server-info': {
     id: 'abcdefgh',
-    version: '0.3.2.1',
+    version: '0.3.3',
     'last-backup': new Date(new Date().valueOf() - 604800001).toISOString(),
     'lan-address': 'https://embassy-abcdefgh.local',
     'tor-address': 'http://myveryownspecialtoraddress.onion',

package-lock.json

@@ -1,6 +0,0 @@
-{
-  "name": "embassy-os",
-  "lockfileVersion": 2,
-  "requires": true,
-  "packages": {}
-}