retry file copies during migration (#1221)

* retry file copies during migration up to 10x

* fix build

* readonly mounts, and validation on v2 recovery

* banish booleans

backend/src/backup/backup_bulk.rs

@@ -10,7 +10,6 @@ use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use tokio::io::AsyncWriteExt;
-use tokio::sync::oneshot::Sender;
 use torut::onion::TorSecretKeyV3;
 use tracing::instrument;
 
@@ -21,6 +20,7 @@ use crate::backup::{BackupReport, ServerBackupReport};
 use crate::context::RpcContext;
 use crate::db::util::WithRevision;
 use crate::disk::mount::backup::BackupMountGuard;
+use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::mount::guard::TmpMountGuard;
 use crate::notifications::NotificationLevel;
 use crate::s9pk::manifest::PackageId;
@@ -126,7 +126,7 @@ pub async fn backup_all(
         .load(&mut ctx.secret_store.acquire().await?)
         .await?;
     let mut backup_guard = BackupMountGuard::mount(
-        TmpMountGuard::mount(&fs).await?,
+        TmpMountGuard::mount(&fs, ReadWrite).await?,
         old_password.as_ref().unwrap_or(&password),
     )
     .await?;

backend/src/backup/restore.rs

@@ -22,6 +22,7 @@ use crate::context::{RpcContext, SetupContext};
 use crate::db::model::{PackageDataEntry, StaticFiles};
 use crate::db::util::WithRevision;
 use crate::disk::mount::backup::{BackupMountGuard, PackageBackupMountGuard};
+use crate::disk::mount::filesystem::ReadOnly;
 use crate::disk::mount::guard::TmpMountGuard;
 use crate::install::progress::InstallProgress;
 use crate::install::{download_install_s9pk, PKG_PUBLIC_DIR};
@@ -57,7 +58,7 @@ pub async fn restore_packages_rpc(
         .load(&mut ctx.secret_store.acquire().await?)
         .await?;
     let mut backup_guard = BackupMountGuard::mount(
-        TmpMountGuard::mount(&fs).await?,
+        TmpMountGuard::mount(&fs, ReadOnly).await?,
         old_password.as_ref().unwrap_or(&password),
     )
     .await?;

backend/src/backup/target/cifs.rs

@@ -9,6 +9,7 @@ use sqlx::{Executor, Sqlite};
 use super::{BackupTarget, BackupTargetId};
 use crate::context::RpcContext;
 use crate::disk::mount::filesystem::cifs::Cifs;
+use crate::disk::mount::filesystem::ReadOnly;
 use crate::disk::mount::guard::TmpMountGuard;
 use crate::disk::util::{recovery_info, EmbassyOsRecoveryInfo};
 use crate::util::display_none;
@@ -44,7 +45,7 @@ pub async fn add(
         username,
         password,
     };
-    let guard = TmpMountGuard::mount(&cifs).await?;
+    let guard = TmpMountGuard::mount(&cifs, ReadOnly).await?;
     let embassy_os = recovery_info(&guard).await?;
     guard.unmount().await?;
     let path_string = Path::new("/").join(&cifs.path).display().to_string();
@@ -92,7 +93,7 @@ pub async fn update(
         username,
         password,
     };
-    let guard = TmpMountGuard::mount(&cifs).await?;
+    let guard = TmpMountGuard::mount(&cifs, ReadOnly).await?;
     let embassy_os = recovery_info(&guard).await?;
     guard.unmount().await?;
     let path_string = Path::new("/").join(&cifs.path).display().to_string();
@@ -188,7 +189,7 @@ where
                 password: record.password,
             };
             let embassy_os = async {
-                let guard = TmpMountGuard::mount(&mount_info).await?;
+                let guard = TmpMountGuard::mount(&mount_info, ReadOnly).await?;
                 let embassy_os = recovery_info(&guard).await?;
                 guard.unmount().await?;
                 Ok::<_, Error>(embassy_os)

backend/src/backup/target/mod.rs

@@ -18,7 +18,7 @@ use crate::context::RpcContext;
 use crate::disk::mount::backup::BackupMountGuard;
 use crate::disk::mount::filesystem::block_dev::BlockDev;
 use crate::disk::mount::filesystem::cifs::Cifs;
-use crate::disk::mount::filesystem::FileSystem;
+use crate::disk::mount::filesystem::{FileSystem, MountType, ReadOnly};
 use crate::disk::mount::guard::TmpMountGuard;
 use crate::disk::util::PartitionInfo;
 use crate::s9pk::manifest::PackageId;
@@ -109,10 +109,14 @@ pub enum BackupTargetFS {
 }
 #[async_trait]
 impl FileSystem for BackupTargetFS {
-    async fn mount<P: AsRef<Path> + Send + Sync>(&self, mountpoint: P) -> Result<(), Error> {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
         match self {
-            BackupTargetFS::Disk(a) => a.mount(mountpoint).await,
-            BackupTargetFS::Cifs(a) => a.mount(mountpoint).await,
+            BackupTargetFS::Disk(a) => a.mount(mountpoint, mount_type).await,
+            BackupTargetFS::Cifs(a) => a.mount(mountpoint, mount_type).await,
         }
     }
     async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
@@ -228,6 +232,7 @@ pub async fn info(
             &target_id
                 .load(&mut ctx.secret_store.acquire().await?)
                 .await?,
+            ReadOnly,
         )
         .await?,
         &password,

backend/src/disk/main.rs

@@ -7,6 +7,7 @@ use tracing::instrument;
 
 use super::util::pvscan;
 use crate::disk::mount::filesystem::block_dev::mount;
+use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::mount::util::unmount;
 use crate::util::Invoke;
 use crate::{Error, ResultExt};
@@ -128,6 +129,7 @@ pub async fn create_fs<P: AsRef<Path>>(
     mount(
         Path::new("/dev/mapper").join(format!("{}_{}", guid, name)),
         datadir.as_ref().join(name),
+        ReadWrite,
     )
     .await?;
     tokio::fs::remove_file(PASSWORD_PATH)
@@ -268,6 +270,7 @@ pub async fn mount_fs<P: AsRef<Path>>(
     mount(
         Path::new("/dev/mapper").join(format!("{}_{}", guid, name)),
         datadir.as_ref().join(name),
+        ReadWrite,
     )
     .await?;
 

backend/src/disk/mount/backup.rs

@@ -9,6 +9,7 @@ use super::guard::{GenericMountGuard, TmpMountGuard};
 use super::util::{bind, unmount};
 use crate::auth::check_password;
 use crate::backup::target::BackupInfo;
+use crate::disk::mount::filesystem::ReadWrite;
 use crate::disk::util::EmbassyOsRecoveryInfo;
 use crate::middleware::encrypt::{decrypt_slice, encrypt_slice};
 use crate::s9pk::manifest::PackageId;
@@ -103,7 +104,8 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
                 )
             })?;
         }
-        let encrypted_guard = TmpMountGuard::mount(&EcryptFS::new(&crypt_path, &enc_key)).await?;
+        let encrypted_guard =
+            TmpMountGuard::mount(&EcryptFS::new(&crypt_path, &enc_key), ReadWrite).await?;
 
         let metadata_path = encrypted_guard.as_ref().join("metadata.cbor");
         let metadata: BackupInfo = if tokio::fs::metadata(&metadata_path).await.is_ok() {

backend/src/disk/mount/filesystem/block_dev.rs

@@ -7,20 +7,22 @@ use digest::Digest;
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
 
-use super::FileSystem;
+use super::{FileSystem, MountType, ReadOnly};
 use crate::util::Invoke;
 use crate::{Error, ResultExt};
 
 pub async fn mount(
     logicalname: impl AsRef<Path>,
     mountpoint: impl AsRef<Path>,
+    mount_type: MountType,
 ) -> Result<(), Error> {
     tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
-    tokio::process::Command::new("mount")
-        .arg(logicalname.as_ref())
-        .arg(mountpoint.as_ref())
-        .invoke(crate::ErrorKind::Filesystem)
-        .await?;
+    let mut cmd = tokio::process::Command::new("mount");
+    cmd.arg(logicalname.as_ref()).arg(mountpoint.as_ref());
+    if mount_type == ReadOnly {
+        cmd.arg("-o").arg("ro");
+    }
+    cmd.invoke(crate::ErrorKind::Filesystem).await?;
     Ok(())
 }
 
@@ -36,8 +38,12 @@ impl<LogicalName: AsRef<Path>> BlockDev<LogicalName> {
 }
 #[async_trait]
 impl<LogicalName: AsRef<Path> + Send + Sync> FileSystem for BlockDev<LogicalName> {
-    async fn mount<P: AsRef<Path> + Send + Sync>(&self, mountpoint: P) -> Result<(), Error> {
-        mount(self.logicalname.as_ref(), mountpoint).await
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
+        mount(self.logicalname.as_ref(), mountpoint, mount_type).await
     }
     async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
         let mut sha = Sha256::new();

backend/src/disk/mount/filesystem/cifs.rs

@@ -10,7 +10,7 @@ use sha2::Sha256;
 use tokio::process::Command;
 use tracing::instrument;
 
-use super::FileSystem;
+use super::{FileSystem, MountType, ReadOnly};
 use crate::disk::mount::guard::TmpMountGuard;
 use crate::util::Invoke;
 use crate::Error;
@@ -40,19 +40,22 @@ pub async fn mount_cifs(
     username: &str,
     password: Option<&str>,
     mountpoint: impl AsRef<Path>,
+    mount_type: MountType,
 ) -> Result<(), Error> {
     tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
     let ip: IpAddr = resolve_hostname(hostname).await?;
     let absolute_path = Path::new("/").join(path.as_ref());
-    Command::new("mount")
-        .arg("-t")
+    let mut cmd = Command::new("mount");
+    cmd.arg("-t")
         .arg("cifs")
         .env("USER", username)
         .env("PASSWD", password.unwrap_or_default())
         .arg(format!("//{}{}", ip, absolute_path.display()))
-        .arg(mountpoint.as_ref())
-        .invoke(crate::ErrorKind::Filesystem)
-        .await?;
+        .arg(mountpoint.as_ref());
+    if mount_type == ReadOnly {
+        cmd.arg("-o").arg("ro");
+    }
+    cmd.invoke(crate::ErrorKind::Filesystem).await?;
     Ok(())
 }
 
@@ -66,7 +69,7 @@ pub struct Cifs {
 }
 impl Cifs {
     pub async fn mountable(&self) -> Result<(), Error> {
-        let guard = TmpMountGuard::mount(self).await?;
+        let guard = TmpMountGuard::mount(self, ReadOnly).await?;
         guard.unmount().await?;
         Ok(())
     }
@@ -76,6 +79,7 @@ impl FileSystem for Cifs {
     async fn mount<P: AsRef<std::path::Path> + Send + Sync>(
         &self,
         mountpoint: P,
+        mount_type: MountType,
     ) -> Result<(), Error> {
         mount_cifs(
             &self.hostname,
@@ -83,6 +87,7 @@ impl FileSystem for Cifs {
             &self.username,
             self.password.as_ref().map(|p| p.as_str()),
             mountpoint,
+            mount_type,
         )
         .await
     }

backend/src/disk/mount/filesystem/ecryptfs.rs

@@ -8,7 +8,7 @@ use digest::Digest;
 use sha2::Sha256;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
-use super::FileSystem;
+use super::{FileSystem, MountType};
 use crate::{Error, ResultExt};
 
 pub async fn mount_ecryptfs<P0: AsRef<Path>, P1: AsRef<Path>>(
@@ -56,7 +56,11 @@ impl<EncryptedDir: AsRef<Path>, Key: AsRef<str>> EcryptFS<EncryptedDir, Key> {
 impl<EncryptedDir: AsRef<Path> + Send + Sync, Key: AsRef<str> + Send + Sync> FileSystem
     for EcryptFS<EncryptedDir, Key>
 {
-    async fn mount<P: AsRef<Path> + Send + Sync>(&self, mountpoint: P) -> Result<(), Error> {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        _mount_type: MountType, // ignored - inherited from parent fs
+    ) -> Result<(), Error> {
         mount_ecryptfs(self.encrypted_dir.as_ref(), mountpoint, self.key.as_ref()).await
     }
     async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {

backend/src/disk/mount/filesystem/label.rs

@@ -5,18 +5,22 @@ use digest::generic_array::GenericArray;
 use digest::Digest;
 use sha2::Sha256;
 
-use super::FileSystem;
+use super::{FileSystem, MountType, ReadOnly};
 use crate::util::Invoke;
 use crate::Error;
 
-pub async fn mount_label(label: &str, mountpoint: impl AsRef<Path>) -> Result<(), Error> {
+pub async fn mount_label(
+    label: &str,
+    mountpoint: impl AsRef<Path>,
+    mount_type: MountType,
+) -> Result<(), Error> {
     tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
-    tokio::process::Command::new("mount")
-        .arg("-L")
-        .arg(label)
-        .arg(mountpoint.as_ref())
-        .invoke(crate::ErrorKind::Filesystem)
-        .await?;
+    let mut cmd = tokio::process::Command::new("mount");
+    cmd.arg("-L").arg(label).arg(mountpoint.as_ref());
+    if mount_type == ReadOnly {
+        cmd.arg("-o").arg("ro");
+    }
+    cmd.invoke(crate::ErrorKind::Filesystem).await?;
     Ok(())
 }
 
@@ -30,8 +34,12 @@ impl<S: AsRef<str>> Label<S> {
 }
 #[async_trait]
 impl<S: AsRef<str> + Send + Sync> FileSystem for Label<S> {
-    async fn mount<P: AsRef<Path> + Send + Sync>(&self, mountpoint: P) -> Result<(), Error> {
-        mount_label(self.label.as_ref(), mountpoint).await
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
+        mount_label(self.label.as_ref(), mountpoint, mount_type).await
     }
     async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
         let mut sha = Sha256::new();

backend/src/disk/mount/filesystem/mod.rs

@@ -12,8 +12,20 @@ pub mod cifs;
 pub mod ecryptfs;
 pub mod label;
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum MountType {
+    ReadOnly,
+    ReadWrite,
+}
+
+pub use MountType::*;
+
 #[async_trait]
 pub trait FileSystem {
-    async fn mount<P: AsRef<Path> + Send + Sync>(&self, mountpoint: P) -> Result<(), Error>;
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error>;
     async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error>;
 }

backend/src/disk/mount/guard.rs

@@ -6,8 +6,9 @@ use lazy_static::lazy_static;
 use tokio::sync::Mutex;
 use tracing::instrument;
 
-use super::filesystem::FileSystem;
+use super::filesystem::{FileSystem, MountType, ReadOnly, ReadWrite};
 use super::util::unmount;
+use crate::util::Invoke;
 use crate::Error;
 
 pub const TMP_MOUNTPOINT: &'static str = "/media/embassy-os/tmp";
@@ -26,9 +27,10 @@ impl MountGuard {
     pub async fn mount(
         filesystem: &impl FileSystem,
         mountpoint: impl AsRef<Path>,
+        mount_type: MountType,
     ) -> Result<Self, Error> {
         let mountpoint = mountpoint.as_ref().to_owned();
-        filesystem.mount(&mountpoint).await?;
+        filesystem.mount(&mountpoint, mount_type).await?;
         Ok(MountGuard {
             mountpoint,
             mounted: true,
@@ -70,7 +72,8 @@ async fn tmp_mountpoint(source: &impl FileSystem) -> Result<PathBuf, Error> {
 }
 
 lazy_static! {
-    static ref TMP_MOUNTS: Mutex<BTreeMap<PathBuf, Weak<MountGuard>>> = Mutex::new(BTreeMap::new());
+    static ref TMP_MOUNTS: Mutex<BTreeMap<PathBuf, (MountType, Weak<MountGuard>)>> =
+        Mutex::new(BTreeMap::new());
 }
 
 #[derive(Debug)]
@@ -78,19 +81,31 @@ pub struct TmpMountGuard {
     guard: Arc<MountGuard>,
 }
 impl TmpMountGuard {
+    /// DRAGONS: if you try to mount something as ro and rw at the same time, the ro mount will be upgraded to rw.
     #[instrument(skip(filesystem))]
-    pub async fn mount(filesystem: &impl FileSystem) -> Result<Self, Error> {
+    pub async fn mount(filesystem: &impl FileSystem, mount_type: MountType) -> Result<Self, Error> {
         let mountpoint = tmp_mountpoint(filesystem).await?;
         let mut tmp_mounts = TMP_MOUNTS.lock().await;
         if !tmp_mounts.contains_key(&mountpoint) {
-            tmp_mounts.insert(mountpoint.clone(), Weak::new());
+            tmp_mounts.insert(mountpoint.clone(), (mount_type, Weak::new()));
         }
-        let weak_slot = tmp_mounts.get_mut(&mountpoint).unwrap();
+        let (prev_mt, weak_slot) = tmp_mounts.get_mut(&mountpoint).unwrap();
         if let Some(guard) = weak_slot.upgrade() {
+            // upgrade to rw
+            if *prev_mt == ReadOnly && mount_type == ReadWrite {
+                tokio::process::Command::new("mount")
+                    .arg("-o")
+                    .arg("remount,rw")
+                    .arg(&mountpoint)
+                    .invoke(crate::ErrorKind::Filesystem)
+                    .await?;
+                *prev_mt = ReadWrite;
+            }
             Ok(TmpMountGuard { guard })
         } else {
-            let guard = Arc::new(MountGuard::mount(filesystem, &mountpoint).await?);
+            let guard = Arc::new(MountGuard::mount(filesystem, &mountpoint, mount_type).await?);
             *weak_slot = Arc::downgrade(&guard);
+            *prev_mt = mount_type;
             Ok(TmpMountGuard { guard })
         }
     }

backend/src/disk/util.rs

@@ -17,6 +17,7 @@ use tokio::process::Command;
 use tracing::instrument;
 
 use super::mount::filesystem::block_dev::BlockDev;
+use super::mount::filesystem::ReadOnly;
 use super::mount::guard::TmpMountGuard;
 use super::quirks::{fetch_quirks, save_quirks, update_quirks};
 use crate::util::io::from_yaml_async_reader;
@@ -325,7 +326,7 @@ pub async fn list() -> Result<DiskListResponse, Error> {
                     .unwrap_or_default();
                 let mut used = None;
 
-                match TmpMountGuard::mount(&BlockDev::new(&part)).await {
+                match TmpMountGuard::mount(&BlockDev::new(&part), ReadOnly).await {
                     Err(e) => tracing::warn!("Could not collect usage information: {}", e.source),
                     Ok(mount_guard) => {
                         used = get_used(&mount_guard)

backend/src/setup.rs

@@ -6,6 +6,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use color_eyre::eyre::eyre;
+use digest::generic_array::GenericArray;
 use futures::future::BoxFuture;
 use futures::{FutureExt, TryFutureExt, TryStreamExt};
 use nix::unistd::{Gid, Uid};
@@ -14,6 +15,7 @@ use patch_db::LockType;
 use rpc_toolkit::command;
 use rpc_toolkit::yajrc::RpcError;
 use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
 use sqlx::{Executor, Sqlite};
 use tokio::fs::File;
 use tokio::io::AsyncWriteExt;
@@ -28,9 +30,10 @@ use crate::db::model::RecoveredPackageInfo;
 use crate::disk::main::DEFAULT_PASSWORD;
 use crate::disk::mount::filesystem::block_dev::BlockDev;
 use crate::disk::mount::filesystem::cifs::Cifs;
+use crate::disk::mount::filesystem::ReadOnly;
 use crate::disk::mount::guard::TmpMountGuard;
 use crate::disk::util::{pvscan, recovery_info, DiskListResponse, EmbassyOsRecoveryInfo};
-use crate::hostname::{get_product_key, PRODUCT_KEY_PATH};
+use crate::hostname::PRODUCT_KEY_PATH;
 use crate::id::Id;
 use crate::init::init;
 use crate::install::PKG_PUBLIC_DIR;
@@ -135,7 +138,7 @@ pub fn v2() -> Result<(), Error> {
 
 #[command(rpc_only, metadata(authenticated = false))]
 pub async fn set(#[context] ctx: SetupContext, #[arg] logicalname: PathBuf) -> Result<(), Error> {
-    let guard = TmpMountGuard::mount(&BlockDev::new(&logicalname)).await?;
+    let guard = TmpMountGuard::mount(&BlockDev::new(&logicalname), ReadOnly).await?;
     let product_key = tokio::fs::read_to_string(guard.as_ref().join("root/agent/product_key"))
         .await?
         .trim()
@@ -173,12 +176,15 @@ pub async fn verify_cifs(
     #[arg] username: String,
     #[arg] password: Option<String>,
 ) -> Result<EmbassyOsRecoveryInfo, Error> {
-    let guard = TmpMountGuard::mount(&Cifs {
-        hostname,
-        path,
-        username,
-        password,
-    })
+    let guard = TmpMountGuard::mount(
+        &Cifs {
+            hostname,
+            path,
+            username,
+            password,
+        },
+        ReadOnly,
+    )
     .await?;
     let embassy_os = recovery_info(&guard).await?;
     guard.unmount().await?;
@@ -386,7 +392,7 @@ async fn recover(
     recovery_source: BackupTargetFS,
     recovery_password: Option<String>,
 ) -> Result<(OnionAddressV3, X509, BoxFuture<'static, Result<(), Error>>), Error> {
-    let recovery_source = TmpMountGuard::mount(&recovery_source).await?;
+    let recovery_source = TmpMountGuard::mount(&recovery_source, ReadOnly).await?;
     let recovery_version = recovery_info(&recovery_source)
         .await?
         .as_ref()
@@ -413,6 +419,47 @@ async fn recover(
     Ok(res)
 }
 
+async fn shasum(
+    path: impl AsRef<Path>,
+) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
+    use tokio::io::AsyncReadExt;
+
+    let mut rdr = tokio::fs::File::open(path).await?;
+    let mut hasher = Sha256::new();
+    let mut buf = [0; 1024];
+    let mut read;
+    while {
+        read = rdr.read(&mut buf).await?;
+        read != 0
+    } {
+        hasher.update(&buf[0..read]);
+    }
+    Ok(hasher.finalize())
+}
+
+async fn validated_copy(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> Result<(), Error> {
+    let src_path = src.as_ref();
+    let dst_path = dst.as_ref();
+    tokio::fs::copy(src_path, dst_path).await.with_ctx(|_| {
+        (
+            crate::ErrorKind::Filesystem,
+            format!("cp {} -> {}", src_path.display(), dst_path.display()),
+        )
+    })?;
+    let (src_hash, dst_hash) = tokio::try_join!(shasum(src_path), shasum(dst_path))?;
+    if src_hash != dst_hash {
+        Err(Error::new(
+            eyre!(
+                "source hash does not match destination hash for {}",
+                dst_path.display()
+            ),
+            crate::ErrorKind::Filesystem,
+        ))
+    } else {
+        Ok(())
+    }
+}
+
 fn dir_copy<'a, P0: AsRef<Path> + 'a + Send + Sync, P1: AsRef<Path> + 'a + Send + Sync>(
     src: P0,
     dst: P1,
@@ -459,12 +506,14 @@ fn dir_copy<'a, P0: AsRef<Path> + 'a + Send + Sync, P1: AsRef<Path> + 'a + Send
                 let dst_path = dst_path.join(e.file_name());
                 if m.is_file() {
                     let len = m.len();
-                    tokio::fs::copy(&src_path, &dst_path).await.with_ctx(|_| {
-                        (
-                            crate::ErrorKind::Filesystem,
-                            format!("cp {} -> {}", src_path.display(), dst_path.display()),
-                        )
-                    })?;
+                    let mut cp_res = Ok(());
+                    for _ in 0..10 {
+                        cp_res = validated_copy(&src_path, &dst_path).await;
+                        if cp_res.is_ok() {
+                            break;
+                        }
+                    }
+                    cp_res?;
                     let tmp_dst_path = dst_path.clone();
                     tokio::task::spawn_blocking(move || {
                         nix::unistd::chown(

backend/src/update/mod.rs

@@ -26,7 +26,7 @@ use crate::context::RpcContext;
 use crate::db::model::UpdateProgress;
 use crate::db::util::WithRevision;
 use crate::disk::mount::filesystem::block_dev::BlockDev;
-use crate::disk::mount::filesystem::FileSystem;
+use crate::disk::mount::filesystem::{FileSystem, ReadWrite};
 use crate::disk::mount::guard::TmpMountGuard;
 use crate::disk::BOOT_RW_PATH;
 use crate::notifications::NotificationLevel;
@@ -114,12 +114,6 @@ impl WritableDrives {
     fn as_fs(&self) -> impl FileSystem {
         BlockDev::new(self.block_dev())
     }
-    fn invert(&self) -> WritableDrives {
-        match self {
-            Self::Green => Self::Blue,
-            Self::Blue => Self::Green,
-        }
-    }
 }
 
 /// This will be where we are going to be putting the new update
@@ -393,14 +387,14 @@ async fn check_download(hash_from_header: &str, file_digest: Vec<u8>) -> Result<
 }
 
 async fn copy_machine_id(new_label: NewLabel) -> Result<(), Error> {
-    let new_guard = TmpMountGuard::mount(&new_label.0.as_fs()).await?;
+    let new_guard = TmpMountGuard::mount(&new_label.0.as_fs(), ReadWrite).await?;
     tokio::fs::copy("/etc/machine-id", new_guard.as_ref().join("etc/machine-id")).await?;
     new_guard.unmount().await?;
     Ok(())
 }
 
 async fn copy_ssh_host_keys(new_label: NewLabel) -> Result<(), Error> {
-    let new_guard = TmpMountGuard::mount(&new_label.0.as_fs()).await?;
+    let new_guard = TmpMountGuard::mount(&new_label.0.as_fs(), ReadWrite).await?;
     tokio::fs::copy(
         "/etc/ssh/ssh_host_rsa_key",
         new_guard.as_ref().join("etc/ssh/ssh_host_rsa_key"),
@@ -443,7 +437,7 @@ async fn swap_boot_label(new_label: NewLabel) -> Result<(), Error> {
         .arg(new_label.0.label())
         .invoke(crate::ErrorKind::BlockDevice)
         .await?;
-    let mounted = TmpMountGuard::mount(&new_label.0.as_fs()).await?;
+    let mounted = TmpMountGuard::mount(&new_label.0.as_fs(), ReadWrite).await?;
     Command::new("sed")
         .arg("-i")
         .arg(&format!(