Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 04:01:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: atomic writing (#1673)

Browse Source 
* feat: atomic writing

* Apply suggestions from code review

* clean up temp files on error

Co-authored-by: Aiden McClelland <me@drbonez.dev>
This commit is contained in:
J M
2022-07-22 14:08:49 -06:00
committed by GitHub
parent 15af827cbc
commit c22c80d3b0
12 changed files with 357 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend/Cargo.lock generated
View File

@@ -558,9 +558,9 @@ dependencies = [
[[package]]
name = "color-eyre"
version = "0.6.1"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ebf286c900a6d5867aeff75cfee3192857bb7f24b547d4f0df2ed6baa812c90"
checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204"
dependencies = [
"backtrace",
"color-spantrace",
@@ -1619,6 +1619,8 @@ dependencies = [
name = "helpers"
version = "0.1.0"
dependencies = [
"color-eyre",
"futures",
"models",
"pin-project",
"tokio",
@@ -1940,6 +1942,7 @@ dependencies = [
"reqwest",
"serde",
"serde_json",
"sha2 0.10.2",
"swc_atoms",
"swc_common",
"swc_config",

18
backend/src/backup/backup_bulk.rs
View File

@@ -1,9 +1,11 @@
use std::collections::{BTreeMap, BTreeSet};
use std::path::PathBuf;
use std::sync::Arc;
use chrono::Utc;
use clap::ArgMatches;
use color_eyre::eyre::eyre;
use helpers::AtomicFile;
use openssl::pkey::{PKey, Private};
use openssl::x509::X509;
use patch_db::{DbHandle, LockType, PatchDbHandle, Revision};
@@ -27,10 +29,10 @@ use crate::disk::mount::guard::TmpMountGuard;
use crate::notifications::NotificationLevel;
use crate::s9pk::manifest::PackageId;
use crate::status::MainStatus;
use crate::util::display_none;
use crate::util::serde::IoFormat;
use crate::util::{display_none, AtomicFile};
use crate::version::VersionT;
use crate::Error;
use crate::{Error, ErrorKind, ResultExt};
#[derive(Debug)]
pub struct OsBackup {
@@ -424,7 +426,12 @@ async fn perform_backup<Db: DbHandle>(
.await?;
let (root_ca_key, root_ca_cert) = ctx.net_controller.ssl.export_root_ca().await?;
let mut os_backup_file = AtomicFile::new(backup_guard.as_ref().join("os-backup.cbor")).await?;
let mut os_backup_file = AtomicFile::new(
backup_guard.as_ref().join("os-backup.cbor"),
None::<PathBuf>,
)
.await
.with_kind(ErrorKind::Filesystem)?;
os_backup_file
.write_all(
&IoFormat::Cbor.to_vec(&OsBackup {
@@ -439,7 +446,10 @@ async fn perform_backup<Db: DbHandle>(
})?,
)
.await?;
os_backup_file.save().await?;
os_backup_file
.save()
.await
.with_kind(ErrorKind::Filesystem)?;
let timestamp = Some(Utc::now());

19
backend/src/backup/mod.rs
View File

@@ -1,8 +1,9 @@
use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use std::path::{Path, PathBuf};
use chrono::{DateTime, Utc};
use color_eyre::eyre::eyre;
use helpers::AtomicFile;
use patch_db::{DbHandle, HasModel, LockType};
use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
@@ -20,10 +21,10 @@ use crate::net::interface::{InterfaceId, Interfaces};
use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
use crate::util::serde::IoFormat;
use crate::util::{AtomicFile, Version};
use crate::util::Version;
use crate::version::{Current, VersionT};
use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR};
use crate::{Error, ResultExt};
use crate::{Error, ErrorKind, ResultExt};
pub mod backup_bulk;
pub mod restore;
@@ -129,7 +130,9 @@ impl BackupActions {
.join(pkg_version.as_str())
.join(format!("{}.s9pk", pkg_id));
let mut infile = File::open(&s9pk_path).await?;
let mut outfile = AtomicFile::new(&tmp_path).await?;
let mut outfile = AtomicFile::new(&tmp_path, None::<PathBuf>)
.await
.with_kind(ErrorKind::Filesystem)?;
tokio::io::copy(&mut infile, &mut *outfile)
.await
.with_ctx(|_| {
@@ -138,17 +141,19 @@ impl BackupActions {
format!("cp {} -> {}", s9pk_path.display(), tmp_path.display()),
)
})?;
outfile.save().await?;
outfile.save().await.with_kind(ErrorKind::Filesystem)?;
let timestamp = Utc::now();
let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
let mut outfile = AtomicFile::new(&metadata_path).await?;
let mut outfile = AtomicFile::new(&metadata_path, None::<PathBuf>)
.await
.with_kind(ErrorKind::Filesystem)?;
outfile
.write_all(&IoFormat::Cbor.to_vec(&BackupMetadata {
timestamp,
tor_keys,
})?)
.await?;
outfile.save().await?;
outfile.save().await.with_kind(ErrorKind::Filesystem)?;
Ok(PackageBackupInfo {
os_version: Current::new().semver().into(),
title: pkg_title.to_owned(),

15
backend/src/context/rpc.rs
View File

@@ -7,6 +7,7 @@ use std::sync::Arc;
use std::time::Duration;
use bollard::Docker;
use helpers::to_tmp_path;
use patch_db::json_ptr::JsonPointer;
use patch_db::{DbHandle, LockReceipt, LockType, PatchDb, Revision};
use reqwest::Url;
@@ -35,7 +36,7 @@ use crate::shutdown::Shutdown;
use crate::status::{MainStatus, Status};
use crate::util::io::from_yaml_async_reader;
use crate::util::{AsyncFileExt, Invoke};
use crate::{Error, ResultExt};
use crate::{volume, Error, ErrorKind, ResultExt};
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "kebab-case")]
@@ -336,6 +337,18 @@ impl RpcContext {
static_files,
manifest,
} => {
for (volume_id, volume_info) in &*manifest.volumes {
let tmp_path = to_tmp_path(volume_info.path_for(
&self.datadir,
&package_id,
&manifest.version,
&volume_id,
))
.with_kind(ErrorKind::Filesystem)?;
if tokio::fs::metadata(&tmp_path).await.is_ok() {
tokio::fs::remove_dir_all(&tmp_path).await?;
}
}
let status = installed.status;
let main = match status.main {
MainStatus::BackingUp { started, .. } => {

17
backend/src/disk/mount/backup.rs
View File

@@ -1,6 +1,7 @@
use std::path::{Path, PathBuf};
use color_eyre::eyre::eyre;
use helpers::AtomicFile;
use tokio::io::AsyncWriteExt;
use tracing::instrument;
@@ -14,9 +15,9 @@ use crate::disk::util::EmbassyOsRecoveryInfo;
use crate::middleware::encrypt::{decrypt_slice, encrypt_slice};
use crate::s9pk::manifest::PackageId;
use crate::util::serde::IoFormat;
use crate::util::{AtomicFile, FileLock};
use crate::util::FileLock;
use crate::volume::BACKUP_DIR;
use crate::{Error, ResultExt};
use crate::{Error, ErrorKind, ResultExt};
pub struct BackupMountGuard<G: GenericMountGuard> {
backup_disk_mount_guard: Option<G>,
@@ -162,16 +163,20 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
pub async fn save(&self) -> Result<(), Error> {
let metadata_path = self.as_ref().join("metadata.cbor");
let backup_disk_path = self.backup_disk_path();
let mut file = AtomicFile::new(&metadata_path).await?;
let mut file = AtomicFile::new(&metadata_path, None::<PathBuf>)
.await
.with_kind(ErrorKind::Filesystem)?;
file.write_all(&IoFormat::Cbor.to_vec(&self.metadata)?)
.await?;
file.save().await?;
file.save().await.with_kind(ErrorKind::Filesystem)?;
let unencrypted_metadata_path =
backup_disk_path.join("EmbassyBackups/unencrypted-metadata.cbor");
let mut file = AtomicFile::new(&unencrypted_metadata_path).await?;
let mut file = AtomicFile::new(&unencrypted_metadata_path, None::<PathBuf>)
.await
.with_kind(ErrorKind::Filesystem)?;
file.write_all(&IoFormat::Cbor.to_vec(&self.unencrypted_metadata)?)
.await?;
file.save().await?;
file.save().await.with_kind(ErrorKind::Filesystem)?;
Ok(())
}

12
backend/src/disk/quirks.rs
View File

@@ -1,14 +1,14 @@
use std::collections::BTreeSet;
use std::num::ParseIntError;
use std::path::Path;
use std::path::{Path, PathBuf};
use color_eyre::eyre::eyre;
use helpers::AtomicFile;
use tokio::io::AsyncWriteExt;
use tracing::instrument;
use super::BOOT_RW_PATH;
use crate::util::AtomicFile;
use crate::Error;
use crate::{Error, ErrorKind, ResultExt};
pub const QUIRK_PATH: &'static str = "/sys/module/usb_storage/parameters/quirks";
@@ -160,11 +160,13 @@ pub async fn save_quirks(quirks: &Quirks) -> Result<(), Error> {
tokio::fs::copy(&target_path, &orig_path).await?;
}
let cmdline = tokio::fs::read_to_string(&orig_path).await?;
let mut target = AtomicFile::new(&target_path).await?;
let mut target = AtomicFile::new(&target_path, None::<PathBuf>)
.await
.with_kind(ErrorKind::Filesystem)?;
target
.write_all(format!("usb-storage.quirks={} {}", quirks, cmdline).as_bytes())
.await?;
target.save().await?;
target.save().await.with_kind(ErrorKind::Filesystem)?;
Ok(())
}

98
backend/src/util/mod.rs
View File

@@ -11,8 +11,7 @@ use async_trait::async_trait;
use clap::ArgMatches;
use color_eyre::eyre::{self, eyre};
use fd_lock_rs::FdLock;
use futures::future::BoxFuture;
use futures::FutureExt;
use helpers::canonicalize;
pub use helpers::NonDetachingJoinHandle;
use lazy_static::lazy_static;
pub use models::Version;
@@ -23,7 +22,7 @@ use tokio::sync::{Mutex, OwnedMutexGuard, RwLock};
use tracing::instrument;
use crate::shutdown::Shutdown;
use crate::{Error, ResultExt as _};
use crate::{Error, ErrorKind, ResultExt as _};
pub mod config;
pub mod io;
pub mod logger;
@@ -273,40 +272,6 @@ impl<F: FnOnce() -> T, T> Drop for GeneralGuard<F, T> {
}
}
pub async fn canonicalize(
path: impl AsRef<Path> + Send + Sync,
create_parent: bool,
) -> Result<PathBuf, Error> {
fn create_canonical_folder<'a>(
path: impl AsRef<Path> + Send + Sync + 'a,
) -> BoxFuture<'a, Result<PathBuf, Error>> {
async move {
let path = canonicalize(path, true).await?;
tokio::fs::create_dir(&path)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, path.display().to_string()))?;
Ok(path)
}
.boxed()
}
let path = path.as_ref();
if tokio::fs::metadata(path).await.is_err() {
if let (Some(parent), Some(file_name)) = (path.parent(), path.file_name()) {
if create_parent && tokio::fs::metadata(parent).await.is_err() {
return Ok(create_canonical_folder(parent).await?.join(file_name));
} else {
return Ok(tokio::fs::canonicalize(parent)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, parent.display().to_string()))?
.join(file_name));
}
}
}
tokio::fs::canonicalize(&path)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, path.display().to_string()))
}
pub struct FileLock(OwnedMutexGuard<()>, Option<FdLock<File>>);
impl Drop for FileLock {
fn drop(&mut self) {
@@ -322,7 +287,9 @@ impl FileLock {
static ref INTERNAL_LOCKS: Mutex<BTreeMap<PathBuf, Arc<Mutex<()>>>> =
Mutex::new(BTreeMap::new());
}
let path = canonicalize(path.as_ref(), true).await?;
let path = canonicalize(path.as_ref(), true)
.await
.with_kind(ErrorKind::Filesystem)?;
let mut internal_locks = INTERNAL_LOCKS.lock().await;
if !internal_locks.contains_key(&path) {
internal_locks.insert(path.clone(), Arc::new(Mutex::new(())));
@@ -362,58 +329,3 @@ impl FileLock {
Ok(())
}
}
pub struct AtomicFile {
tmp_path: PathBuf,
path: PathBuf,
file: File,
}
impl AtomicFile {
pub async fn new(path: impl AsRef<Path> + Send + Sync) -> Result<Self, Error> {
let path = canonicalize(&path, true).await?;
let tmp_path = if let (Some(parent), Some(file_name)) =
(path.parent(), path.file_name().and_then(|f| f.to_str()))
{
parent.join(format!(".{}.tmp", file_name))
} else {
return Err(Error::new(
eyre!("invalid path: {}", path.display()),
crate::ErrorKind::Filesystem,
));
};
let file = File::create(&tmp_path)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, tmp_path.display().to_string()))?;
Ok(Self {
tmp_path,
path,
file,
})
}
pub async fn save(mut self) -> Result<(), Error> {
use tokio::io::AsyncWriteExt;
self.file.flush().await?;
self.file.shutdown().await?;
self.file.sync_all().await?;
tokio::fs::rename(&self.tmp_path, &self.path)
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("mv {} -> {}", self.tmp_path.display(), self.path.display()),
)
})
}
}
impl std::ops::Deref for AtomicFile {
type Target = File;
fn deref(&self) -> &Self::Target {
&self.file
}
}
impl std::ops::DerefMut for AtomicFile {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.file
}
}