retry file copies during migration (#1221)

* retry file copies during migration up to 10x

* fix build

* readonly mounts, and validation on v2 recovery

* banish booleans

backend/src/disk/main.rs

@@ -7,6 +7,7 @@ use tracing::instrument;
 
 use super::util::pvscan;
 use crate::disk::mount::filesystem::block_dev::mount;
+use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::mount::util::unmount;
 use crate::util::Invoke;
 use crate::{Error, ResultExt};
@@ -128,6 +129,7 @@ pub async fn create_fs<P: AsRef<Path>>(
     mount(
         Path::new("/dev/mapper").join(format!("{}_{}", guid, name)),
         datadir.as_ref().join(name),
+        ReadWrite,
     )
     .await?;
     tokio::fs::remove_file(PASSWORD_PATH)
@@ -268,6 +270,7 @@ pub async fn mount_fs<P: AsRef<Path>>(
     mount(
         Path::new("/dev/mapper").join(format!("{}_{}", guid, name)),
         datadir.as_ref().join(name),
+        ReadWrite,
     )
     .await?;
 

backend/src/disk/mount/backup.rs

@@ -9,6 +9,7 @@ use super::guard::{GenericMountGuard, TmpMountGuard};
 use super::util::{bind, unmount};
 use crate::auth::check_password;
 use crate::backup::target::BackupInfo;
+use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::util::EmbassyOsRecoveryInfo;
 use crate::middleware::encrypt::{decrypt_slice, encrypt_slice};
 use crate::s9pk::manifest::PackageId;
@@ -103,7 +104,8 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
                 )
             })?;
         }
-        let encrypted_guard = TmpMountGuard::mount(&EcryptFS::new(&crypt_path, &enc_key)).await?;
+        let encrypted_guard =
+            TmpMountGuard::mount(&EcryptFS::new(&crypt_path, &enc_key), ReadWrite).await?;
 
         let metadata_path = encrypted_guard.as_ref().join("metadata.cbor");
         let metadata: BackupInfo = if tokio::fs::metadata(&metadata_path).await.is_ok() {

backend/src/disk/mount/filesystem/block_dev.rs

@@ -7,20 +7,22 @@ use digest::Digest;
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
 
-use super::FileSystem;
+use super::{FileSystem, MountType, ReadOnly};
 use crate::util::Invoke;
 use crate::{Error, ResultExt};
 
 pub async fn mount(
     logicalname: impl AsRef<Path>,
     mountpoint: impl AsRef<Path>,
+    mount_type: MountType,
 ) -> Result<(), Error> {
     tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
-    tokio::process::Command::new("mount")
-        .arg(logicalname.as_ref())
-        .arg(mountpoint.as_ref())
-        .invoke(crate::ErrorKind::Filesystem)
-        .await?;
+    let mut cmd = tokio::process::Command::new("mount");
+    cmd.arg(logicalname.as_ref()).arg(mountpoint.as_ref());
+    if mount_type == ReadOnly {
+        cmd.arg("-o").arg("ro");
+    }
+    cmd.invoke(crate::ErrorKind::Filesystem).await?;
     Ok(())
 }
 
@@ -36,8 +38,12 @@ impl<LogicalName: AsRef<Path>> BlockDev<LogicalName> {
 }
 #[async_trait]
 impl<LogicalName: AsRef<Path> + Send + Sync> FileSystem for BlockDev<LogicalName> {
-    async fn mount<P: AsRef<Path> + Send + Sync>(&self, mountpoint: P) -> Result<(), Error> {
-        mount(self.logicalname.as_ref(), mountpoint).await
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
+        mount(self.logicalname.as_ref(), mountpoint, mount_type).await
     }
     async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
         let mut sha = Sha256::new();

backend/src/disk/mount/filesystem/cifs.rs

@@ -10,7 +10,7 @@ use sha2::Sha256;
 use tokio::process::Command;
 use tracing::instrument;
 
-use super::FileSystem;
+use super::{FileSystem, MountType, ReadOnly};
 use crate::disk::mount::guard::TmpMountGuard;
 use crate::util::Invoke;
 use crate::Error;
@@ -40,19 +40,22 @@ pub async fn mount_cifs(
     username: &str,
     password: Option<&str>,
     mountpoint: impl AsRef<Path>,
+    mount_type: MountType,
 ) -> Result<(), Error> {
     tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
     let ip: IpAddr = resolve_hostname(hostname).await?;
     let absolute_path = Path::new("/").join(path.as_ref());
-    Command::new("mount")
-        .arg("-t")
+    let mut cmd = Command::new("mount");
+    cmd.arg("-t")
         .arg("cifs")
         .env("USER", username)
         .env("PASSWD", password.unwrap_or_default())
         .arg(format!("//{}{}", ip, absolute_path.display()))
-        .arg(mountpoint.as_ref())
-        .invoke(crate::ErrorKind::Filesystem)
-        .await?;
+        .arg(mountpoint.as_ref());
+    if mount_type == ReadOnly {
+        cmd.arg("-o").arg("ro");
+    }
+    cmd.invoke(crate::ErrorKind::Filesystem).await?;
     Ok(())
 }
 
@@ -66,7 +69,7 @@ pub struct Cifs {
 }
 impl Cifs {
     pub async fn mountable(&self) -> Result<(), Error> {
-        let guard = TmpMountGuard::mount(self).await?;
+        let guard = TmpMountGuard::mount(self, ReadOnly).await?;
         guard.unmount().await?;
         Ok(())
     }
@@ -76,6 +79,7 @@ impl FileSystem for Cifs {
     async fn mount<P: AsRef<std::path::Path> + Send + Sync>(
         &self,
         mountpoint: P,
+        mount_type: MountType,
     ) -> Result<(), Error> {
         mount_cifs(
             &self.hostname,
@@ -83,6 +87,7 @@ impl FileSystem for Cifs {
             &self.username,
             self.password.as_ref().map(|p| p.as_str()),
             mountpoint,
+            mount_type,
         )
         .await
     }

backend/src/disk/mount/filesystem/ecryptfs.rs

@@ -8,7 +8,7 @@ use digest::Digest;
 use sha2::Sha256;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
-use super::FileSystem;
+use super::{FileSystem, MountType};
 use crate::{Error, ResultExt};
 
 pub async fn mount_ecryptfs<P0: AsRef<Path>, P1: AsRef<Path>>(
@@ -56,7 +56,11 @@ impl<EncryptedDir: AsRef<Path>, Key: AsRef<str>> EcryptFS<EncryptedDir, Key> {
 impl<EncryptedDir: AsRef<Path> + Send + Sync, Key: AsRef<str> + Send + Sync> FileSystem
     for EcryptFS<EncryptedDir, Key>
 {
-    async fn mount<P: AsRef<Path> + Send + Sync>(&self, mountpoint: P) -> Result<(), Error> {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        _mount_type: MountType, // ignored - inherited from parent fs
+    ) -> Result<(), Error> {
         mount_ecryptfs(self.encrypted_dir.as_ref(), mountpoint, self.key.as_ref()).await
     }
     async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {

backend/src/disk/mount/filesystem/label.rs

@@ -5,18 +5,22 @@ use digest::generic_array::GenericArray;
 use digest::Digest;
 use sha2::Sha256;
 
-use super::FileSystem;
+use super::{FileSystem, MountType, ReadOnly};
 use crate::util::Invoke;
 use crate::Error;
 
-pub async fn mount_label(label: &str, mountpoint: impl AsRef<Path>) -> Result<(), Error> {
+pub async fn mount_label(
+    label: &str,
+    mountpoint: impl AsRef<Path>,
+    mount_type: MountType,
+) -> Result<(), Error> {
     tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
-    tokio::process::Command::new("mount")
-        .arg("-L")
-        .arg(label)
-        .arg(mountpoint.as_ref())
-        .invoke(crate::ErrorKind::Filesystem)
-        .await?;
+    let mut cmd = tokio::process::Command::new("mount");
+    cmd.arg("-L").arg(label).arg(mountpoint.as_ref());
+    if mount_type == ReadOnly {
+        cmd.arg("-o").arg("ro");
+    }
+    cmd.invoke(crate::ErrorKind::Filesystem).await?;
     Ok(())
 }
 
@@ -30,8 +34,12 @@ impl<S: AsRef<str>> Label<S> {
 }
 #[async_trait]
 impl<S: AsRef<str> + Send + Sync> FileSystem for Label<S> {
-    async fn mount<P: AsRef<Path> + Send + Sync>(&self, mountpoint: P) -> Result<(), Error> {
-        mount_label(self.label.as_ref(), mountpoint).await
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
+        mount_label(self.label.as_ref(), mountpoint, mount_type).await
     }
     async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
         let mut sha = Sha256::new();

backend/src/disk/mount/filesystem/mod.rs

@@ -12,8 +12,20 @@ pub mod cifs;
 pub mod ecryptfs;
 pub mod label;
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum MountType {
+    ReadOnly,
+    ReadWrite,
+}
+
+pub use MountType::*;
+
 #[async_trait]
 pub trait FileSystem {
-    async fn mount<P: AsRef<Path> + Send + Sync>(&self, mountpoint: P) -> Result<(), Error>;
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error>;
     async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error>;
 }

backend/src/disk/mount/guard.rs

@@ -6,8 +6,9 @@ use lazy_static::lazy_static;
 use tokio::sync::Mutex;
 use tracing::instrument;
 
-use super::filesystem::FileSystem;
+use super::filesystem::{FileSystem, MountType, ReadOnly, ReadWrite};
 use super::util::unmount;
+use crate::util::Invoke;
 use crate::Error;
 
 pub const TMP_MOUNTPOINT: &'static str = "/media/embassy-os/tmp";
@@ -26,9 +27,10 @@ impl MountGuard {
     pub async fn mount(
         filesystem: &impl FileSystem,
         mountpoint: impl AsRef<Path>,
+        mount_type: MountType,
     ) -> Result<Self, Error> {
         let mountpoint = mountpoint.as_ref().to_owned();
-        filesystem.mount(&mountpoint).await?;
+        filesystem.mount(&mountpoint, mount_type).await?;
         Ok(MountGuard {
             mountpoint,
             mounted: true,
@@ -70,7 +72,8 @@ async fn tmp_mountpoint(source: &impl FileSystem) -> Result<PathBuf, Error> {
 }
 
 lazy_static! {
-    static ref TMP_MOUNTS: Mutex<BTreeMap<PathBuf, Weak<MountGuard>>> = Mutex::new(BTreeMap::new());
+    static ref TMP_MOUNTS: Mutex<BTreeMap<PathBuf, (MountType, Weak<MountGuard>)>> =
+        Mutex::new(BTreeMap::new());
 }
 
 #[derive(Debug)]
@@ -78,19 +81,31 @@ pub struct TmpMountGuard {
     guard: Arc<MountGuard>,
 }
 impl TmpMountGuard {
+    /// DRAGONS: if you try to mount something as ro and rw at the same time, the ro mount will be upgraded to rw.
     #[instrument(skip(filesystem))]
-    pub async fn mount(filesystem: &impl FileSystem) -> Result<Self, Error> {
+    pub async fn mount(filesystem: &impl FileSystem, mount_type: MountType) -> Result<Self, Error> {
         let mountpoint = tmp_mountpoint(filesystem).await?;
         let mut tmp_mounts = TMP_MOUNTS.lock().await;
         if !tmp_mounts.contains_key(&mountpoint) {
-            tmp_mounts.insert(mountpoint.clone(), Weak::new());
+            tmp_mounts.insert(mountpoint.clone(), (mount_type, Weak::new()));
         }
-        let weak_slot = tmp_mounts.get_mut(&mountpoint).unwrap();
+        let (prev_mt, weak_slot) = tmp_mounts.get_mut(&mountpoint).unwrap();
         if let Some(guard) = weak_slot.upgrade() {
+            // upgrade to rw
+            if *prev_mt == ReadOnly && mount_type == ReadWrite {
+                tokio::process::Command::new("mount")
+                    .arg("-o")
+                    .arg("remount,rw")
+                    .arg(&mountpoint)
+                    .invoke(crate::ErrorKind::Filesystem)
+                    .await?;
+                *prev_mt = ReadWrite;
+            }
             Ok(TmpMountGuard { guard })
         } else {
-            let guard = Arc::new(MountGuard::mount(filesystem, &mountpoint).await?);
+            let guard = Arc::new(MountGuard::mount(filesystem, &mountpoint, mount_type).await?);
             *weak_slot = Arc::downgrade(&guard);
+            *prev_mt = mount_type;
             Ok(TmpMountGuard { guard })
         }
     }

backend/src/disk/util.rs

@@ -17,6 +17,7 @@ use tokio::process::Command;
 use tracing::instrument;
 
 use super::mount::filesystem::block_dev::BlockDev;
+use super::mount::filesystem::ReadOnly;
 use super::mount::guard::TmpMountGuard;
 use super::quirks::{fetch_quirks, save_quirks, update_quirks};
 use crate::util::io::from_yaml_async_reader;
@@ -325,7 +326,7 @@ pub async fn list() -> Result<DiskListResponse, Error> {
                     .unwrap_or_default();
                 let mut used = None;
 
-                match TmpMountGuard::mount(&BlockDev::new(&part)).await {
+                match TmpMountGuard::mount(&BlockDev::new(&part), ReadOnly).await {
                     Err(e) => tracing::warn!("Could not collect usage information: {}", e.source),
                     Ok(mount_guard) => {
                         used = get_used(&mount_guard)