Feature/backup fs (#2665)

* port 040 config, WIP

* update fixtures

* use taiga modal for backups too

* fix: update Taiga UI and refactor everything to work

* chore: package-lock

* fix interfaces and mocks for interfaces

* better mocks

* function to transform old spec to new

* delete unused fns

* delete unused FE config utils

* fix exports from sdk

* reorganize exports

* functions to translate config

* rename unionSelectKey and unionValueKey

* new backup fs

* update sdk types

* change types, include fuse module

* fix casing

* rework setup wiz

* rework UI

* only fuse3

* fix arm build

* misc fixes

* fix duplicate server select

* fix: fix throwing inside dialog

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>
Co-authored-by: waterplea <alexander@inkin.ru>
Co-authored-by: Matt Hill <MattDHill@users.noreply.github.com>

core/startos/src/disk/mod.rs

@@ -1,5 +1,7 @@
 use std::path::{Path, PathBuf};
 
+use itertools::Itertools;
+use lazy_format::lazy_format;
 use rpc_toolkit::{from_fn_async, CallRemoteHandler, Context, Empty, HandlerExt, ParentHandler};
 use serde::{Deserialize, Serialize};
 
@@ -102,10 +104,18 @@ fn display_disk_info(params: WithIoFormat<Empty>, args: Vec<DiskInfo>) {
                 } else {
                     "N/A"
                 },
-                &if let Some(eos) = part.start_os.as_ref() {
-                    eos.version.to_string()
-                } else {
+                &if part.start_os.is_empty() {
                     "N/A".to_owned()
+                } else if part.start_os.len() == 1 {
+                    part.start_os
+                        .first_key_value()
+                        .map(|(_, info)| info.version.to_string())
+                        .unwrap()
+                } else {
+                    part.start_os
+                        .iter()
+                        .map(|(id, info)| lazy_format!("{} ({})", info.version, id))
+                        .join(", ")
                 },
             ];
             table.add_row(row);

core/startos/src/disk/mount/backup.rs

@@ -11,9 +11,10 @@ use super::filesystem::ecryptfs::EcryptFS;
 use super::guard::{GenericMountGuard, TmpMountGuard};
 use crate::auth::check_password;
 use crate::backup::target::BackupInfo;
+use crate::disk::mount::filesystem::backupfs::BackupFS;
 use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::mount::guard::SubPath;
-use crate::disk::util::EmbassyOsRecoveryInfo;
+use crate::disk::util::StartOsRecoveryInfo;
 use crate::util::crypto::{decrypt_slice, encrypt_slice};
 use crate::util::serde::IoFormat;
 use crate::{Error, ErrorKind, ResultExt};
@@ -23,29 +24,27 @@ pub struct BackupMountGuard<G: GenericMountGuard> {
     backup_disk_mount_guard: Option<G>,
     encrypted_guard: Option<TmpMountGuard>,
     enc_key: String,
-    pub unencrypted_metadata: EmbassyOsRecoveryInfo,
+    unencrypted_metadata_path: PathBuf,
+    pub unencrypted_metadata: StartOsRecoveryInfo,
     pub metadata: BackupInfo,
 }
 impl<G: GenericMountGuard> BackupMountGuard<G> {
-    fn backup_disk_path(&self) -> &Path {
-        if let Some(guard) = &self.backup_disk_mount_guard {
-            guard.path()
-        } else {
-            unreachable!()
-        }
-    }
-
     #[instrument(skip_all)]
-    pub async fn mount(backup_disk_mount_guard: G, password: &str) -> Result<Self, Error> {
+    pub async fn mount(
+        backup_disk_mount_guard: G,
+        server_id: &str,
+        password: &str,
+    ) -> Result<Self, Error> {
         let backup_disk_path = backup_disk_mount_guard.path();
-        let unencrypted_metadata_path =
-            backup_disk_path.join("EmbassyBackups/unencrypted-metadata.cbor");
-        let mut unencrypted_metadata: EmbassyOsRecoveryInfo =
+        let backup_dir = backup_disk_path.join("StartOSBackups").join(server_id);
+        let unencrypted_metadata_path = backup_dir.join("unencrypted-metadata.json");
+        let crypt_path = backup_dir.join("crypt");
+        let mut unencrypted_metadata: StartOsRecoveryInfo =
             if tokio::fs::metadata(&unencrypted_metadata_path)
                 .await
                 .is_ok()
             {
-                IoFormat::Cbor.from_slice(
+                IoFormat::Json.from_slice(
                     &tokio::fs::read(&unencrypted_metadata_path)
                         .await
                         .with_ctx(|_| {
@@ -56,6 +55,9 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
                         })?,
                 )?
             } else {
+                if tokio::fs::metadata(&crypt_path).await.is_ok() {
+                    tokio::fs::remove_dir_all(&crypt_path).await?;
+                }
                 Default::default()
             };
         let enc_key = if let (Some(hash), Some(wrapped_key)) = (
@@ -96,7 +98,6 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
             ));
         }
 
-        let crypt_path = backup_disk_path.join("EmbassyBackups/crypt");
         if tokio::fs::metadata(&crypt_path).await.is_err() {
             tokio::fs::create_dir_all(&crypt_path).await.with_ctx(|_| {
                 (
@@ -106,11 +107,11 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
             })?;
         }
         let encrypted_guard =
-            TmpMountGuard::mount(&EcryptFS::new(&crypt_path, &enc_key), ReadWrite).await?;
+            TmpMountGuard::mount(&BackupFS::new(&crypt_path, &enc_key), ReadWrite).await?;
 
-        let metadata_path = encrypted_guard.path().join("metadata.cbor");
+        let metadata_path = encrypted_guard.path().join("metadata.json");
         let metadata: BackupInfo = if tokio::fs::metadata(&metadata_path).await.is_ok() {
-            IoFormat::Cbor.from_slice(&tokio::fs::read(&metadata_path).await.with_ctx(|_| {
+            IoFormat::Json.from_slice(&tokio::fs::read(&metadata_path).await.with_ctx(|_| {
                 (
                     crate::ErrorKind::Filesystem,
                     metadata_path.display().to_string(),
@@ -124,6 +125,7 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
             backup_disk_mount_guard: Some(backup_disk_mount_guard),
             encrypted_guard: Some(encrypted_guard),
             enc_key,
+            unencrypted_metadata_path,
             unencrypted_metadata,
             metadata,
         })
@@ -152,20 +154,17 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
 
     #[instrument(skip_all)]
     pub async fn save(&self) -> Result<(), Error> {
-        let metadata_path = self.path().join("metadata.cbor");
-        let backup_disk_path = self.backup_disk_path();
+        let metadata_path = self.path().join("metadata.json");
         let mut file = AtomicFile::new(&metadata_path, None::<PathBuf>)
             .await
             .with_kind(ErrorKind::Filesystem)?;
-        file.write_all(&IoFormat::Cbor.to_vec(&self.metadata)?)
+        file.write_all(&IoFormat::Json.to_vec(&self.metadata)?)
             .await?;
         file.save().await.with_kind(ErrorKind::Filesystem)?;
-        let unencrypted_metadata_path =
-            backup_disk_path.join("EmbassyBackups/unencrypted-metadata.cbor");
-        let mut file = AtomicFile::new(&unencrypted_metadata_path, None::<PathBuf>)
+        let mut file = AtomicFile::new(&self.unencrypted_metadata_path, None::<PathBuf>)
             .await
             .with_kind(ErrorKind::Filesystem)?;
-        file.write_all(&IoFormat::Cbor.to_vec(&self.unencrypted_metadata)?)
+        file.write_all(&IoFormat::Json.to_vec(&self.unencrypted_metadata)?)
             .await?;
         file.save().await.with_kind(ErrorKind::Filesystem)?;
         Ok(())

core/startos/src/disk/mount/filesystem/backupfs.rs

@@ -0,0 +1,55 @@
+use std::fmt::{self, Display};
+use std::os::unix::ffi::OsStrExt;
+use std::path::Path;
+
+use digest::generic_array::GenericArray;
+use digest::{Digest, OutputSizeUser};
+use sha2::Sha256;
+
+use super::FileSystem;
+use crate::prelude::*;
+
+pub struct BackupFS<DataDir: AsRef<Path>, Password: fmt::Display> {
+    data_dir: DataDir,
+    password: Password,
+}
+impl<DataDir: AsRef<Path>, Password: fmt::Display> BackupFS<DataDir, Password> {
+    pub fn new(data_dir: DataDir, password: Password) -> Self {
+        BackupFS { data_dir, password }
+    }
+}
+impl<DataDir: AsRef<Path> + Send + Sync, Password: fmt::Display + Send + Sync> FileSystem
+    for BackupFS<DataDir, Password>
+{
+    fn mount_type(&self) -> Option<impl AsRef<str>> {
+        Some("backup-fs")
+    }
+    fn mount_options(&self) -> impl IntoIterator<Item = impl Display> {
+        [
+            format!("password={}", self.password),
+            format!("file-size-padding=0.05"),
+        ]
+    }
+    async fn source(&self) -> Result<Option<impl AsRef<Path>>, Error> {
+        Ok(Some(&self.data_dir))
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        let mut sha = Sha256::new();
+        sha.update("BackupFS");
+        sha.update(
+            tokio::fs::canonicalize(self.data_dir.as_ref())
+                .await
+                .with_ctx(|_| {
+                    (
+                        crate::ErrorKind::Filesystem,
+                        self.data_dir.as_ref().display().to_string(),
+                    )
+                })?
+                .as_os_str()
+                .as_bytes(),
+        );
+        Ok(sha.finalize())
+    }
+}

core/startos/src/disk/mount/filesystem/mod.rs

@@ -1,6 +1,7 @@
 use std::ffi::OsStr;
 use std::fmt::{Display, Write};
 use std::path::Path;
+use std::time::Duration;
 
 use digest::generic_array::GenericArray;
 use digest::OutputSizeUser;
@@ -11,6 +12,7 @@ use tokio::process::Command;
 use crate::prelude::*;
 use crate::util::Invoke;
 
+pub mod backupfs;
 pub mod bind;
 pub mod block_dev;
 pub mod cifs;
@@ -71,6 +73,7 @@ pub(self) async fn default_mount_impl(
     fs.pre_mount().await?;
     tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
     Command::from(default_mount_command(fs, mountpoint, mount_type).await?)
+        .capture(false)
         .invoke(ErrorKind::Filesystem)
         .await?;
 

core/startos/src/disk/util.rs

@@ -1,6 +1,7 @@
 use std::collections::{BTreeMap, BTreeSet};
 use std::path::{Path, PathBuf};
 
+use chrono::{DateTime, Utc};
 use color_eyre::eyre::{self, eyre};
 use futures::TryStreamExt;
 use nom::bytes::complete::{tag, take_till1};
@@ -19,6 +20,7 @@ use super::mount::filesystem::ReadOnly;
 use super::mount::guard::TmpMountGuard;
 use crate::disk::mount::guard::GenericMountGuard;
 use crate::disk::OsPartitionInfo;
+use crate::hostname::Hostname;
 use crate::util::serde::IoFormat;
 use crate::util::Invoke;
 use crate::{Error, ResultExt as _};
@@ -49,15 +51,16 @@ pub struct PartitionInfo {
     pub label: Option<String>,
     pub capacity: u64,
     pub used: Option<u64>,
-    pub start_os: Option<EmbassyOsRecoveryInfo>,
+    pub start_os: BTreeMap<String, StartOsRecoveryInfo>,
     pub guid: Option<String>,
 }
 
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 #[serde(rename_all = "camelCase")]
-pub struct EmbassyOsRecoveryInfo {
+pub struct StartOsRecoveryInfo {
+    pub hostname: Hostname,
     pub version: exver::Version,
-    pub full: bool,
+    pub timestamp: DateTime<Utc>,
     pub password_hash: Option<String>,
     pub wrapped_key: Option<String>,
 }
@@ -223,29 +226,38 @@ pub async fn pvscan() -> Result<BTreeMap<PathBuf, Option<String>>, Error> {
 
 pub async fn recovery_info(
     mountpoint: impl AsRef<Path>,
-) -> Result<Option<EmbassyOsRecoveryInfo>, Error> {
-    let backup_unencrypted_metadata_path = mountpoint
-        .as_ref()
-        .join("EmbassyBackups/unencrypted-metadata.cbor");
-    if tokio::fs::metadata(&backup_unencrypted_metadata_path)
-        .await
-        .is_ok()
-    {
-        return Ok(Some(
-            IoFormat::Cbor.from_slice(
-                &tokio::fs::read(&backup_unencrypted_metadata_path)
-                    .await
-                    .with_ctx(|_| {
-                        (
-                            crate::ErrorKind::Filesystem,
-                            backup_unencrypted_metadata_path.display().to_string(),
-                        )
-                    })?,
-            )?,
-        ));
+) -> Result<BTreeMap<String, StartOsRecoveryInfo>, Error> {
+    let backup_root = mountpoint.as_ref().join("StartOSBackups");
+    let mut res = BTreeMap::new();
+    if tokio::fs::metadata(&backup_root).await.is_ok() {
+        let mut dir = tokio::fs::read_dir(&backup_root).await?;
+        while let Some(entry) = dir.next_entry().await? {
+            let server_id = entry.file_name().to_string_lossy().into_owned();
+            let backup_unencrypted_metadata_path = backup_root
+                .join(&server_id)
+                .join("unencrypted-metadata.json");
+            if tokio::fs::metadata(&backup_unencrypted_metadata_path)
+                .await
+                .is_ok()
+            {
+                res.insert(
+                    server_id,
+                    IoFormat::Json.from_slice(
+                        &tokio::fs::read(&backup_unencrypted_metadata_path)
+                            .await
+                            .with_ctx(|_| {
+                                (
+                                    crate::ErrorKind::Filesystem,
+                                    backup_unencrypted_metadata_path.display().to_string(),
+                                )
+                            })?,
+                    )?,
+                );
+            }
+        }
     }
 
-    Ok(None)
+    Ok(res)
 }
 
 #[instrument(skip_all)]
@@ -390,7 +402,7 @@ async fn disk_info(disk: PathBuf) -> DiskInfo {
 }
 
 async fn part_info(part: PathBuf) -> PartitionInfo {
-    let mut start_os = None;
+    let mut start_os = BTreeMap::new();
     let label = get_label(&part)
         .await
         .map_err(|e| tracing::warn!("Could not get label of {}: {}", part.display(), e.source))
@@ -410,14 +422,13 @@ async fn part_info(part: PathBuf) -> PartitionInfo {
                     tracing::warn!("Could not get usage of {}: {}", part.display(), e.source)
                 })
                 .ok();
-            if let Some(recovery_info) = match recovery_info(mount_guard.path()).await {
-                Ok(a) => a,
+            match recovery_info(mount_guard.path()).await {
+                Ok(a) => {
+                    start_os = a;
+                }
                 Err(e) => {
                     tracing::error!("Error fetching unencrypted backup metadata: {}", e);
-                    None
                 }
-            } {
-                start_os = Some(recovery_info)
             }
             if let Err(e) = mount_guard.unmount().await {
                 tracing::error!("Error unmounting partition {}: {}", part.display(), e);