From c22c80d3b0c9a8ce4b6a14d65ce6cb59c1b3dbae Mon Sep 17 00:00:00 2001 From: J M <2364004+Blu-J@users.noreply.github.com> Date: Fri, 22 Jul 2022 14:08:49 -0600 Subject: [PATCH] feat: atomic writing (#1673) * feat: atomic writing * Apply suggestions from code review * clean up temp files on error Co-authored-by: Aiden McClelland --- backend/Cargo.lock | 7 +- backend/src/backup/backup_bulk.rs | 18 +++- backend/src/backup/mod.rs | 19 ++-- backend/src/context/rpc.rs | 15 ++- backend/src/disk/mount/backup.rs | 17 ++-- backend/src/disk/quirks.rs | 12 ++- backend/src/util/mod.rs | 98 +------------------- libs/Cargo.lock | 148 +++++++++++++++++++++++++++++- libs/helpers/Cargo.toml | 2 + libs/helpers/src/lib.rs | 121 ++++++++++++++++++++++++ libs/js_engine/Cargo.toml | 1 + libs/js_engine/src/lib.rs | 21 ++++- 12 files changed, 357 insertions(+), 122 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index f96fac727..b28457788 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -558,9 +558,9 @@ dependencies = [ [[package]] name = "color-eyre" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ebf286c900a6d5867aeff75cfee3192857bb7f24b547d4f0df2ed6baa812c90" +checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204" dependencies = [ "backtrace", "color-spantrace", @@ -1619,6 +1619,8 @@ dependencies = [ name = "helpers" version = "0.1.0" dependencies = [ + "color-eyre", + "futures", "models", "pin-project", "tokio", @@ -1940,6 +1942,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2 0.10.2", "swc_atoms", "swc_common", "swc_config", diff --git a/backend/src/backup/backup_bulk.rs b/backend/src/backup/backup_bulk.rs index 40100948c..8331c9d64 100644 --- a/backend/src/backup/backup_bulk.rs +++ b/backend/src/backup/backup_bulk.rs @@ -1,9 +1,11 @@ use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; use std::sync::Arc; use chrono::Utc; use clap::ArgMatches; use color_eyre::eyre::eyre; +use helpers::AtomicFile; use openssl::pkey::{PKey, Private}; use openssl::x509::X509; use patch_db::{DbHandle, LockType, PatchDbHandle, Revision}; @@ -27,10 +29,10 @@ use crate::disk::mount::guard::TmpMountGuard; use crate::notifications::NotificationLevel; use crate::s9pk::manifest::PackageId; use crate::status::MainStatus; +use crate::util::display_none; use crate::util::serde::IoFormat; -use crate::util::{display_none, AtomicFile}; use crate::version::VersionT; -use crate::Error; +use crate::{Error, ErrorKind, ResultExt}; #[derive(Debug)] pub struct OsBackup { @@ -424,7 +426,12 @@ async fn perform_backup( .await?; let (root_ca_key, root_ca_cert) = ctx.net_controller.ssl.export_root_ca().await?; - let mut os_backup_file = AtomicFile::new(backup_guard.as_ref().join("os-backup.cbor")).await?; + let mut os_backup_file = AtomicFile::new( + backup_guard.as_ref().join("os-backup.cbor"), + None::, + ) + .await + .with_kind(ErrorKind::Filesystem)?; os_backup_file .write_all( &IoFormat::Cbor.to_vec(&OsBackup { @@ -439,7 +446,10 @@ async fn perform_backup( })?, ) .await?; - os_backup_file.save().await?; + os_backup_file + .save() + .await + .with_kind(ErrorKind::Filesystem)?; let timestamp = Some(Utc::now()); diff --git a/backend/src/backup/mod.rs b/backend/src/backup/mod.rs index 5acb7ea95..2d6174999 100644 --- a/backend/src/backup/mod.rs +++ b/backend/src/backup/mod.rs @@ -1,8 +1,9 @@ use std::collections::{BTreeMap, BTreeSet}; -use std::path::Path; +use std::path::{Path, PathBuf}; use chrono::{DateTime, Utc}; use color_eyre::eyre::eyre; +use helpers::AtomicFile; use patch_db::{DbHandle, HasModel, LockType}; use rpc_toolkit::command; use serde::{Deserialize, Serialize}; @@ -20,10 +21,10 @@ use crate::net::interface::{InterfaceId, Interfaces}; use crate::procedure::{NoOutput, PackageProcedure, ProcedureName}; use crate::s9pk::manifest::PackageId; use crate::util::serde::IoFormat; -use crate::util::{AtomicFile, Version}; +use crate::util::Version; use crate::version::{Current, VersionT}; use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR}; -use crate::{Error, ResultExt}; +use crate::{Error, ErrorKind, ResultExt}; pub mod backup_bulk; pub mod restore; @@ -129,7 +130,9 @@ impl BackupActions { .join(pkg_version.as_str()) .join(format!("{}.s9pk", pkg_id)); let mut infile = File::open(&s9pk_path).await?; - let mut outfile = AtomicFile::new(&tmp_path).await?; + let mut outfile = AtomicFile::new(&tmp_path, None::) + .await + .with_kind(ErrorKind::Filesystem)?; tokio::io::copy(&mut infile, &mut *outfile) .await .with_ctx(|_| { @@ -138,17 +141,19 @@ impl BackupActions { format!("cp {} -> {}", s9pk_path.display(), tmp_path.display()), ) })?; - outfile.save().await?; + outfile.save().await.with_kind(ErrorKind::Filesystem)?; let timestamp = Utc::now(); let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor"); - let mut outfile = AtomicFile::new(&metadata_path).await?; + let mut outfile = AtomicFile::new(&metadata_path, None::) + .await + .with_kind(ErrorKind::Filesystem)?; outfile .write_all(&IoFormat::Cbor.to_vec(&BackupMetadata { timestamp, tor_keys, })?) .await?; - outfile.save().await?; + outfile.save().await.with_kind(ErrorKind::Filesystem)?; Ok(PackageBackupInfo { os_version: Current::new().semver().into(), title: pkg_title.to_owned(), diff --git a/backend/src/context/rpc.rs b/backend/src/context/rpc.rs index a4bd7a77f..aaf7cbd01 100644 --- a/backend/src/context/rpc.rs +++ b/backend/src/context/rpc.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use std::time::Duration; use bollard::Docker; +use helpers::to_tmp_path; use patch_db::json_ptr::JsonPointer; use patch_db::{DbHandle, LockReceipt, LockType, PatchDb, Revision}; use reqwest::Url; @@ -35,7 +36,7 @@ use crate::shutdown::Shutdown; use crate::status::{MainStatus, Status}; use crate::util::io::from_yaml_async_reader; use crate::util::{AsyncFileExt, Invoke}; -use crate::{Error, ResultExt}; +use crate::{volume, Error, ErrorKind, ResultExt}; #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "kebab-case")] @@ -336,6 +337,18 @@ impl RpcContext { static_files, manifest, } => { + for (volume_id, volume_info) in &*manifest.volumes { + let tmp_path = to_tmp_path(volume_info.path_for( + &self.datadir, + &package_id, + &manifest.version, + &volume_id, + )) + .with_kind(ErrorKind::Filesystem)?; + if tokio::fs::metadata(&tmp_path).await.is_ok() { + tokio::fs::remove_dir_all(&tmp_path).await?; + } + } let status = installed.status; let main = match status.main { MainStatus::BackingUp { started, .. } => { diff --git a/backend/src/disk/mount/backup.rs b/backend/src/disk/mount/backup.rs index f725ea183..ce9e09784 100644 --- a/backend/src/disk/mount/backup.rs +++ b/backend/src/disk/mount/backup.rs @@ -1,6 +1,7 @@ use std::path::{Path, PathBuf}; use color_eyre::eyre::eyre; +use helpers::AtomicFile; use tokio::io::AsyncWriteExt; use tracing::instrument; @@ -14,9 +15,9 @@ use crate::disk::util::EmbassyOsRecoveryInfo; use crate::middleware::encrypt::{decrypt_slice, encrypt_slice}; use crate::s9pk::manifest::PackageId; use crate::util::serde::IoFormat; -use crate::util::{AtomicFile, FileLock}; +use crate::util::FileLock; use crate::volume::BACKUP_DIR; -use crate::{Error, ResultExt}; +use crate::{Error, ErrorKind, ResultExt}; pub struct BackupMountGuard { backup_disk_mount_guard: Option, @@ -162,16 +163,20 @@ impl BackupMountGuard { pub async fn save(&self) -> Result<(), Error> { let metadata_path = self.as_ref().join("metadata.cbor"); let backup_disk_path = self.backup_disk_path(); - let mut file = AtomicFile::new(&metadata_path).await?; + let mut file = AtomicFile::new(&metadata_path, None::) + .await + .with_kind(ErrorKind::Filesystem)?; file.write_all(&IoFormat::Cbor.to_vec(&self.metadata)?) .await?; - file.save().await?; + file.save().await.with_kind(ErrorKind::Filesystem)?; let unencrypted_metadata_path = backup_disk_path.join("EmbassyBackups/unencrypted-metadata.cbor"); - let mut file = AtomicFile::new(&unencrypted_metadata_path).await?; + let mut file = AtomicFile::new(&unencrypted_metadata_path, None::) + .await + .with_kind(ErrorKind::Filesystem)?; file.write_all(&IoFormat::Cbor.to_vec(&self.unencrypted_metadata)?) .await?; - file.save().await?; + file.save().await.with_kind(ErrorKind::Filesystem)?; Ok(()) } diff --git a/backend/src/disk/quirks.rs b/backend/src/disk/quirks.rs index d5a4b9ca8..8fb2f8fea 100644 --- a/backend/src/disk/quirks.rs +++ b/backend/src/disk/quirks.rs @@ -1,14 +1,14 @@ use std::collections::BTreeSet; use std::num::ParseIntError; -use std::path::Path; +use std::path::{Path, PathBuf}; use color_eyre::eyre::eyre; +use helpers::AtomicFile; use tokio::io::AsyncWriteExt; use tracing::instrument; use super::BOOT_RW_PATH; -use crate::util::AtomicFile; -use crate::Error; +use crate::{Error, ErrorKind, ResultExt}; pub const QUIRK_PATH: &'static str = "/sys/module/usb_storage/parameters/quirks"; @@ -160,11 +160,13 @@ pub async fn save_quirks(quirks: &Quirks) -> Result<(), Error> { tokio::fs::copy(&target_path, &orig_path).await?; } let cmdline = tokio::fs::read_to_string(&orig_path).await?; - let mut target = AtomicFile::new(&target_path).await?; + let mut target = AtomicFile::new(&target_path, None::) + .await + .with_kind(ErrorKind::Filesystem)?; target .write_all(format!("usb-storage.quirks={} {}", quirks, cmdline).as_bytes()) .await?; - target.save().await?; + target.save().await.with_kind(ErrorKind::Filesystem)?; Ok(()) } diff --git a/backend/src/util/mod.rs b/backend/src/util/mod.rs index 084523142..48fbe38b8 100644 --- a/backend/src/util/mod.rs +++ b/backend/src/util/mod.rs @@ -11,8 +11,7 @@ use async_trait::async_trait; use clap::ArgMatches; use color_eyre::eyre::{self, eyre}; use fd_lock_rs::FdLock; -use futures::future::BoxFuture; -use futures::FutureExt; +use helpers::canonicalize; pub use helpers::NonDetachingJoinHandle; use lazy_static::lazy_static; pub use models::Version; @@ -23,7 +22,7 @@ use tokio::sync::{Mutex, OwnedMutexGuard, RwLock}; use tracing::instrument; use crate::shutdown::Shutdown; -use crate::{Error, ResultExt as _}; +use crate::{Error, ErrorKind, ResultExt as _}; pub mod config; pub mod io; pub mod logger; @@ -273,40 +272,6 @@ impl T, T> Drop for GeneralGuard { } } -pub async fn canonicalize( - path: impl AsRef + Send + Sync, - create_parent: bool, -) -> Result { - fn create_canonical_folder<'a>( - path: impl AsRef + Send + Sync + 'a, - ) -> BoxFuture<'a, Result> { - async move { - let path = canonicalize(path, true).await?; - tokio::fs::create_dir(&path) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, path.display().to_string()))?; - Ok(path) - } - .boxed() - } - let path = path.as_ref(); - if tokio::fs::metadata(path).await.is_err() { - if let (Some(parent), Some(file_name)) = (path.parent(), path.file_name()) { - if create_parent && tokio::fs::metadata(parent).await.is_err() { - return Ok(create_canonical_folder(parent).await?.join(file_name)); - } else { - return Ok(tokio::fs::canonicalize(parent) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, parent.display().to_string()))? - .join(file_name)); - } - } - } - tokio::fs::canonicalize(&path) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, path.display().to_string())) -} - pub struct FileLock(OwnedMutexGuard<()>, Option>); impl Drop for FileLock { fn drop(&mut self) { @@ -322,7 +287,9 @@ impl FileLock { static ref INTERNAL_LOCKS: Mutex>>> = Mutex::new(BTreeMap::new()); } - let path = canonicalize(path.as_ref(), true).await?; + let path = canonicalize(path.as_ref(), true) + .await + .with_kind(ErrorKind::Filesystem)?; let mut internal_locks = INTERNAL_LOCKS.lock().await; if !internal_locks.contains_key(&path) { internal_locks.insert(path.clone(), Arc::new(Mutex::new(()))); @@ -362,58 +329,3 @@ impl FileLock { Ok(()) } } - -pub struct AtomicFile { - tmp_path: PathBuf, - path: PathBuf, - file: File, -} -impl AtomicFile { - pub async fn new(path: impl AsRef + Send + Sync) -> Result { - let path = canonicalize(&path, true).await?; - let tmp_path = if let (Some(parent), Some(file_name)) = - (path.parent(), path.file_name().and_then(|f| f.to_str())) - { - parent.join(format!(".{}.tmp", file_name)) - } else { - return Err(Error::new( - eyre!("invalid path: {}", path.display()), - crate::ErrorKind::Filesystem, - )); - }; - let file = File::create(&tmp_path) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, tmp_path.display().to_string()))?; - Ok(Self { - tmp_path, - path, - file, - }) - } - - pub async fn save(mut self) -> Result<(), Error> { - use tokio::io::AsyncWriteExt; - self.file.flush().await?; - self.file.shutdown().await?; - self.file.sync_all().await?; - tokio::fs::rename(&self.tmp_path, &self.path) - .await - .with_ctx(|_| { - ( - crate::ErrorKind::Filesystem, - format!("mv {} -> {}", self.tmp_path.display(), self.path.display()), - ) - }) - } -} -impl std::ops::Deref for AtomicFile { - type Target = File; - fn deref(&self) -> &Self::Target { - &self.file - } -} -impl std::ops::DerefMut for AtomicFile { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.file - } -} diff --git a/libs/Cargo.lock b/libs/Cargo.lock index c7b25e9b4..53af3efa4 100644 --- a/libs/Cargo.lock +++ b/libs/Cargo.lock @@ -12,6 +12,21 @@ dependencies = [ "regex", ] +[[package]] +name = "addr2line" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.7.6" @@ -75,6 +90,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.11.0" @@ -156,6 +186,33 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "color-eyre" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204" +dependencies = [ + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error 0.2.0", +] + +[[package]] +name = "color-spantrace" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ba75b3d9449ecdccb27ecbc479fdc0b87fa2dd43d2f8298f9bf0e59aacc8dce" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error 0.2.0", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -390,6 +447,16 @@ dependencies = [ "syn", ] +[[package]] +name = "eyre" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb" +dependencies = [ + "indenter", + "once_cell", +] + [[package]] name = "fastrand" version = "1.7.0" @@ -586,6 +653,12 @@ dependencies = [ "wasi 0.10.2+wasi-snapshot-preview1", ] +[[package]] +name = "gimli" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" + [[package]] name = "h2" version = "0.3.13" @@ -636,6 +709,8 @@ dependencies = [ name = "helpers" version = "0.1.0" dependencies = [ + "color-eyre", + "futures", "models", "pin-project", "tokio", @@ -758,6 +833,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + [[package]] name = "indexmap" version = "1.8.2" @@ -833,6 +914,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "swc_atoms", "swc_common", "swc_config", @@ -1022,6 +1104,15 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "miniz_oxide" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.4" @@ -1148,6 +1239,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.12.0" @@ -1199,6 +1299,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "owo-colors" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b" + [[package]] name = "parking_lot" version = "0.12.1" @@ -1241,7 +1347,7 @@ dependencies = [ "thiserror", "tokio", "tracing", - "tracing-error", + "tracing-error 0.1.2", ] [[package]] @@ -1525,6 +1631,12 @@ dependencies = [ "winreg", ] +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -1696,6 +1808,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -2369,7 +2492,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4d7c0b83d4a500748fa5879461652b361edf5c9d51ede2a2ac03875ca185e24" dependencies = [ "tracing", - "tracing-subscriber", + "tracing-subscriber 0.2.25", +] + +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber 0.3.11", ] [[package]] @@ -2383,6 +2516,17 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-subscriber" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", +] + [[package]] name = "treediff" version = "4.0.2" diff --git a/libs/helpers/Cargo.toml b/libs/helpers/Cargo.toml index e7fe8841b..1fb991afd 100644 --- a/libs/helpers/Cargo.toml +++ b/libs/helpers/Cargo.toml @@ -6,6 +6,8 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +color-eyre = "0.6.2" +futures = "0.3.21" pin-project = "1.0.11" tokio = { version = "1.19.2", features = ["full"] } models = { path = "../models" } diff --git a/libs/helpers/src/lib.rs b/libs/helpers/src/lib.rs index f46aa1f4d..51d3d5cb4 100644 --- a/libs/helpers/src/lib.rs +++ b/libs/helpers/src/lib.rs @@ -1,10 +1,60 @@ use std::future::Future; +use std::path::{Path, PathBuf}; +use color_eyre::eyre::{eyre, Context, Error}; +use futures::future::BoxFuture; +use futures::FutureExt; +use tokio::fs::File; use tokio::task::{JoinError, JoinHandle}; mod script_dir; pub use script_dir::*; +pub fn to_tmp_path(path: impl AsRef) -> Result { + let path = path.as_ref(); + if let (Some(parent), Some(file_name)) = + (path.parent(), path.file_name().and_then(|f| f.to_str())) + { + Ok(parent.join(format!(".{}.tmp", file_name))) + } else { + Err(eyre!("invalid path: {}", path.display())) + } +} + +pub async fn canonicalize( + path: impl AsRef + Send + Sync, + create_parent: bool, +) -> Result { + fn create_canonical_folder<'a>( + path: impl AsRef + Send + Sync + 'a, + ) -> BoxFuture<'a, Result> { + async move { + let path = canonicalize(path, true).await?; + tokio::fs::create_dir(&path) + .await + .with_context(|| path.display().to_string())?; + Ok(path) + } + .boxed() + } + let path = path.as_ref(); + if tokio::fs::metadata(path).await.is_err() { + if let (Some(parent), Some(file_name)) = (path.parent(), path.file_name()) { + if create_parent && tokio::fs::metadata(parent).await.is_err() { + return Ok(create_canonical_folder(parent).await?.join(file_name)); + } else { + return Ok(tokio::fs::canonicalize(parent) + .await + .with_context(|| parent.display().to_string())? + .join(file_name)); + } + } + } + tokio::fs::canonicalize(&path) + .await + .with_context(|| path.display().to_string()) +} + #[pin_project::pin_project(PinnedDrop)] pub struct NonDetachingJoinHandle(#[pin] JoinHandle); impl From> for NonDetachingJoinHandle { @@ -29,3 +79,74 @@ impl Future for NonDetachingJoinHandle { this.0.poll(cx) } } + +pub struct AtomicFile { + tmp_path: PathBuf, + path: PathBuf, + file: Option, +} +impl AtomicFile { + pub async fn new( + path: impl AsRef + Send + Sync, + tmp_path: Option + Send + Sync>, + ) -> Result { + let path = canonicalize(&path, true).await?; + let tmp_path = if let Some(tmp_path) = tmp_path { + canonicalize(&tmp_path, true).await? + } else { + to_tmp_path(&path)? + }; + let file = File::create(&tmp_path) + .await + .with_context(|| tmp_path.display().to_string())?; + Ok(Self { + tmp_path, + path, + file: Some(file), + }) + } + + pub async fn rollback(mut self) -> Result<(), Error> { + drop(self.file.take()); + tokio::fs::remove_file(&self.tmp_path) + .await + .with_context(|| format!("rm {}", self.tmp_path.display()))?; + Ok(()) + } + + pub async fn save(mut self) -> Result<(), Error> { + use tokio::io::AsyncWriteExt; + if let Some(file) = self.file.as_mut() { + file.flush().await?; + file.shutdown().await?; + file.sync_all().await?; + } + drop(self.file.take()); + tokio::fs::rename(&self.tmp_path, &self.path) + .await + .with_context(|| { + format!("mv {} -> {}", self.tmp_path.display(), self.path.display()) + })?; + Ok(()) + } +} +impl std::ops::Deref for AtomicFile { + type Target = File; + fn deref(&self) -> &Self::Target { + self.file.as_ref().unwrap() + } +} +impl std::ops::DerefMut for AtomicFile { + fn deref_mut(&mut self) -> &mut Self::Target { + self.file.as_mut().unwrap() + } +} +impl Drop for AtomicFile { + fn drop(&mut self) { + if let Some(file) = self.file.take() { + drop(file); + let path = std::mem::take(&mut self.tmp_path); + tokio::spawn(async move { tokio::fs::remove_file(path).await.unwrap() }); + } + } +} diff --git a/libs/js_engine/Cargo.toml b/libs/js_engine/Cargo.toml index 86fb1aac8..b62a79761 100644 --- a/libs/js_engine/Cargo.toml +++ b/libs/js_engine/Cargo.toml @@ -33,6 +33,7 @@ swc_eq_ignore_macros = "=0.1.0" swc_macros_common = "=0.3.5" swc_visit = "=0.3.0" swc_visit_macros = "=0.3.1" +sha2 = "0.10.2" models = { path = "../models" } helpers = { path = "../helpers" } serde = { version = "1.0", features = ["derive", "rc"] } diff --git a/libs/js_engine/src/lib.rs b/libs/js_engine/src/lib.rs index 4057f5fb9..36c9c50d5 100644 --- a/libs/js_engine/src/lib.rs +++ b/libs/js_engine/src/lib.rs @@ -347,8 +347,10 @@ mod fns { use deno_core::anyhow::{anyhow, bail}; use deno_core::error::AnyError; use deno_core::*; + use helpers::{to_tmp_path, AtomicFile}; use models::VolumeId; use serde_json::Value; + use tokio::io::AsyncWriteExt; use super::{AnswerState, JsContext}; use crate::{system_time_as_unix_ms, MetadataJs}; @@ -514,7 +516,7 @@ mod fns { bail!("Volume {} is readonly", volume_id); } - let new_file = volume_path.join(path_in); + let new_file = volume_path.join(&path_in); let parent_new_file = new_file .parent() .ok_or_else(|| anyhow!("Expecting that file is not root"))?; @@ -526,7 +528,22 @@ mod fns { volume_path.to_string_lossy(), ); } - tokio::fs::write(new_file, write).await?; + let new_volume_tmp = to_tmp_path(&volume_path).map_err(|e| anyhow!("{}", e))?; + let hashed_name = { + use sha2::{Digest, Sha256}; + use std::os::unix::ffi::OsStrExt; + let mut hasher = Sha256::new(); + + hasher.update(path_in.as_os_str().as_bytes()); + let result = hasher.finalize(); + format!("{:X}", result) + }; + let temp_file = new_volume_tmp.join(&hashed_name); + let mut file = AtomicFile::new(&new_file, Some(&temp_file)) + .await + .map_err(|e| anyhow!("{}", e))?; + file.write_all(write.as_bytes()).await?; + file.save().await.map_err(|e| anyhow!("{}", e))?; Ok(()) } #[op]