From 1e7492e9151878cad758651970f5d0b510f78588 Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Wed, 27 Oct 2021 15:50:53 -0600 Subject: [PATCH] adds recovery (#731) * should work * works for real * fix build --- appmgr/Cargo.lock | 7 - appmgr/Cargo.toml | 1 - appmgr/src/backup/backup_bulk.rs | 63 ++---- appmgr/src/backup/mod.rs | 95 ++++---- appmgr/src/backup/restore.rs | 186 ++++++++++++++++ appmgr/src/bin/embassy-init.rs | 1 - appmgr/src/bin/embassyd.rs | 1 - appmgr/src/context/cli.rs | 39 ---- appmgr/src/db/model.rs | 14 +- appmgr/src/disk/mod.rs | 8 +- appmgr/src/disk/util.rs | 335 ++++++++++++++++------------- appmgr/src/inspect.rs | 43 +++- appmgr/src/install/cleanup.rs | 7 +- appmgr/src/install/mod.rs | 212 ++++++++++-------- appmgr/src/lib.rs | 3 +- appmgr/src/manager/health.rs | 1 - appmgr/src/s9pk/mod.rs | 2 +- appmgr/src/s9pk/reader.rs | 67 +++--- appmgr/src/setup.rs | 21 +- appmgr/src/sound.rs | 2 +- appmgr/src/status/mod.rs | 3 +- appmgr/src/util/logger.rs | 128 ----------- appmgr/src/util/mod.rs | 17 +- appmgr/src/version/v0_3_0.rs | 4 +- appmgr/src/volume.rs | 9 +- system-images/compat/Cargo.lock | 7 - system-images/compat/src/backup.rs | 16 +- 27 files changed, 694 insertions(+), 598 deletions(-) create mode 100644 appmgr/src/backup/restore.rs diff --git a/appmgr/Cargo.lock b/appmgr/Cargo.lock index a1828f8e5..060e4136e 100644 --- a/appmgr/Cargo.lock +++ b/appmgr/Cargo.lock @@ -872,7 +872,6 @@ dependencies = [ "rpc-toolkit", "rust-argon2", "scopeguard", - "sequence_trie", "serde", "serde_json", "serde_with", @@ -2763,12 +2762,6 @@ dependencies = [ "pest", ] -[[package]] -name = "sequence_trie" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee22067b7ccd072eeb64454b9c6e1b33b61cd0d49e895fd48676a184580e0c3" - [[package]] name = "serde" version = "1.0.130" diff --git a/appmgr/Cargo.toml b/appmgr/Cargo.toml index 3a8fb9341..ecfce890d 100644 --- a/appmgr/Cargo.toml +++ b/appmgr/Cargo.toml @@ -94,7 +94,6 @@ rpassword = "5.0.1" rpc-toolkit = { version = "*", path = "../rpc-toolkit/rpc-toolkit" } rust-argon2 = "0.8.3" scopeguard = "1.1" # because avahi-sys fucks your shit up -sequence_trie = "0.3.6" serde = { version = "1.0.130", features = ["derive", "rc"] } serde_cbor = { package = "ciborium", version = "0.1.0" } serde_json = "1.0.68" diff --git a/appmgr/src/backup/backup_bulk.rs b/appmgr/src/backup/backup_bulk.rs index a819321ee..9a94cd2fc 100644 --- a/appmgr/src/backup/backup_bulk.rs +++ b/appmgr/src/backup/backup_bulk.rs @@ -1,17 +1,15 @@ use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::Arc; use chrono::Utc; use color_eyre::eyre::eyre; -use futures::task::Spawn; use openssl::pkey::{PKey, Private}; use openssl::x509::X509; use patch_db::{DbHandle, LockType, PatchDbHandle, Revision}; use rpc_toolkit::command; use serde::{Deserialize, Serialize}; use serde_json::Value; -use tokio::fs::File; use tokio::io::AsyncWriteExt; use torut::onion::TorSecretKeyV3; use tracing::instrument; @@ -28,7 +26,7 @@ use crate::s9pk::manifest::PackageId; use crate::status::MainStatus; use crate::util::{display_none, AtomicFile, IoFormat}; use crate::version::VersionT; -use crate::{Error, ErrorKind, ResultExt}; +use crate::Error; #[derive(Debug)] pub struct OsBackup { @@ -93,6 +91,7 @@ impl Serialize for OsBackup { } #[command(rename = "create", display(display_none))] +#[instrument(skip(ctx, old_password, password))] pub async fn backup_all( #[context] ctx: RpcContext, #[arg] logicalname: PathBuf, @@ -101,17 +100,17 @@ pub async fn backup_all( ) -> Result, Error> { let mut db = ctx.db.handle(); check_password_against_db(&mut ctx.secret_store.acquire().await?, &password).await?; + let mut backup_guard = BackupMountGuard::mount( + TmpMountGuard::mount(&logicalname, None).await?, + old_password.as_ref().unwrap_or(&password), + ) + .await?; + if old_password.is_some() { + backup_guard.change_password(&password)?; + } let revision = assure_backing_up(&mut db).await?; tokio::task::spawn(async move { - match perform_backup( - &ctx, - &mut db, - logicalname, - old_password.as_deref(), - &password, - ) - .await - { + match perform_backup(&ctx, &mut db, backup_guard).await { Ok(report) => ctx .notification_manager .notify( @@ -200,46 +199,12 @@ async fn assure_backing_up(db: &mut PatchDbHandle) -> Result( - value: &T, - tmp_path: impl AsRef, - path: impl AsRef, -) -> Result<(), Error> { - let tmp_path = tmp_path.as_ref(); - let path = path.as_ref(); - let mut file = File::create(tmp_path) - .await - .with_ctx(|_| (ErrorKind::Filesystem, tmp_path.display().to_string()))?; - file.write_all(&IoFormat::Cbor.to_vec(value)?).await?; - file.flush().await?; - file.shutdown().await?; - file.sync_all().await?; - drop(file); - tokio::fs::rename(tmp_path, path).await.with_ctx(|_| { - ( - ErrorKind::Filesystem, - format!("mv {} -> {}", tmp_path.display(), path.display()), - ) - }) -} - -#[instrument(skip(ctx, db, password))] +#[instrument(skip(ctx, db, backup_guard))] async fn perform_backup( ctx: &RpcContext, mut db: Db, - logicalname: PathBuf, - old_password: Option<&str>, - password: &str, + mut backup_guard: BackupMountGuard, ) -> Result, Error> { - let mut backup_guard = BackupMountGuard::mount( - TmpMountGuard::mount(&logicalname).await?, - old_password.unwrap_or(password), - ) - .await?; - if old_password.is_some() { - backup_guard.change_password(password)?; - } - let mut backup_report = BTreeMap::new(); for package_id in crate::db::DatabaseModel::new() diff --git a/appmgr/src/backup/mod.rs b/appmgr/src/backup/mod.rs index bbfe7172b..78e295218 100644 --- a/appmgr/src/backup/mod.rs +++ b/appmgr/src/backup/mod.rs @@ -3,12 +3,12 @@ use std::path::Path; use chrono::{DateTime, Utc}; use color_eyre::eyre::eyre; -use patch_db::HasModel; +use patch_db::{DbHandle, HasModel}; use rpc_toolkit::command; use serde::{Deserialize, Serialize}; +use sqlx::{Executor, Sqlite}; use tokio::fs::File; use tokio::io::AsyncWriteExt; -use torut::onion::TorSecretKeyV3; use tracing::instrument; use crate::action::{ActionImplementation, NoOutput}; @@ -17,12 +17,13 @@ use crate::disk::PackageBackupInfo; use crate::install::PKG_ARCHIVE_DIR; use crate::net::interface::{InterfaceId, Interfaces}; use crate::s9pk::manifest::PackageId; -use crate::util::{IoFormat, Version}; +use crate::util::{AtomicFile, IoFormat, Version}; use crate::version::{Current, VersionT}; use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR}; use crate::{Error, ResultExt}; mod backup_bulk; +mod restore; #[derive(Debug, Deserialize, Serialize)] pub struct BackupReport { @@ -46,10 +47,15 @@ pub fn backup() -> Result<(), Error> { Ok(()) } +#[command(rename = "backup", subcommands(restore::restore_packages))] +pub fn package_backup() -> Result<(), Error> { + Ok(()) +} + #[derive(Deserialize, Serialize)] struct BackupMetadata { pub timestamp: DateTime, - pub tor_keys: BTreeMap, + pub tor_keys: BTreeMap, } #[derive(Clone, Debug, Deserialize, Serialize, HasModel)] @@ -89,13 +95,18 @@ impl BackupActions { .with_kind(crate::ErrorKind::Backup)?; let tor_keys = interfaces .tor_keys(&mut ctx.secret_store.acquire().await?, pkg_id) - .await?; + .await? + .into_iter() + .map(|(id, key)| { + ( + id, + base32::encode(base32::Alphabet::RFC4648 { padding: true }, &key.as_bytes()), + ) + }) + .collect(); let tmp_path = Path::new(BACKUP_DIR) .join(pkg_id) .join(format!("{}.s9pk", pkg_id)); - let real_path = Path::new(BACKUP_DIR) - .join(pkg_id) - .join(format!(".{}.s9pk.tmp", pkg_id)); let s9pk_path = ctx .datadir .join(PKG_ARCHIVE_DIR) @@ -103,8 +114,8 @@ impl BackupActions { .join(pkg_version.as_str()) .join(format!("{}.s9pk", pkg_id)); let mut infile = File::open(&s9pk_path).await?; - let mut outfile = File::create(&tmp_path).await?; - tokio::io::copy(&mut infile, &mut outfile) + let mut outfile = AtomicFile::new(&tmp_path).await?; + tokio::io::copy(&mut infile, &mut *outfile) .await .with_ctx(|_| { ( @@ -112,40 +123,17 @@ impl BackupActions { format!("cp {} -> {}", s9pk_path.display(), tmp_path.display()), ) })?; - outfile.flush().await?; - outfile.shutdown().await?; - outfile.sync_all().await?; - tokio::fs::rename(&tmp_path, &real_path) - .await - .with_ctx(|_| { - ( - crate::ErrorKind::Filesystem, - format!("mv {} -> {}", tmp_path.display(), real_path.display()), - ) - })?; + outfile.save().await?; let timestamp = Utc::now(); - let tmp_path = Path::new(BACKUP_DIR) - .join(pkg_id) - .join(".metadata.cbor.tmp"); - let real_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor"); - let mut outfile = File::create(&tmp_path).await?; + let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor"); + let mut outfile = AtomicFile::new(&metadata_path).await?; outfile .write_all(&IoFormat::Cbor.to_vec(&BackupMetadata { timestamp, tor_keys, })?) .await?; - outfile.flush().await?; - outfile.shutdown().await?; - outfile.sync_all().await?; - tokio::fs::rename(&tmp_path, &real_path) - .await - .with_ctx(|_| { - ( - crate::ErrorKind::Filesystem, - format!("mv {} -> {}", tmp_path.display(), real_path.display()), - ) - })?; + outfile.save().await?; Ok(PackageBackupInfo { os_version: Current::new().semver().into(), title: pkg_title.to_owned(), @@ -154,13 +142,20 @@ impl BackupActions { }) } - pub async fn restore( + #[instrument(skip(ctx, db, secrets))] + pub async fn restore( &self, ctx: &RpcContext, + db: &mut Db, + secrets: &mut Ex, pkg_id: &PackageId, pkg_version: &Version, + interfaces: &Interfaces, volumes: &Volumes, - ) -> Result<(), Error> { + ) -> Result<(), Error> + where + for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>, + { let mut volumes = volumes.clone(); volumes.insert(VolumeId::Backup, Volume::Backup { readonly: true }); self.restore @@ -185,18 +180,34 @@ impl BackupActions { ) })?, )?; - let mut sql_handle = ctx.secret_store.acquire().await?; for (iface, key) in metadata.tor_keys { - let key_vec = key.as_bytes().to_vec(); + let key_vec = base32::decode(base32::Alphabet::RFC4648 { padding: true }, &key) + .ok_or_else(|| { + Error::new( + eyre!("invalid base32 string"), + crate::ErrorKind::Deserialization, + ) + })?; sqlx::query!( "REPLACE INTO tor (package, interface, key) VALUES (?, ?, ?)", **pkg_id, *iface, key_vec, ) - .execute(&mut sql_handle) + .execute(&mut *secrets) .await?; } + crate::db::DatabaseModel::new() + .package_data() + .idx_model(pkg_id) + .expect(db) + .await? + .installed() + .expect(db) + .await? + .interface_addresses() + .put(db, &interfaces.install(&mut *secrets, pkg_id).await?) + .await?; Ok(()) } } diff --git a/appmgr/src/backup/restore.rs b/appmgr/src/backup/restore.rs new file mode 100644 index 000000000..ec9024eff --- /dev/null +++ b/appmgr/src/backup/restore.rs @@ -0,0 +1,186 @@ +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use clap::ArgMatches; +use color_eyre::eyre::eyre; +use patch_db::{DbHandle, PatchDbHandle, Revision}; +use rpc_toolkit::command; +use tokio::fs::File; +use tracing::instrument; + +use crate::auth::check_password_against_db; +use crate::context::RpcContext; +use crate::db::model::{PackageDataEntry, StaticFiles}; +use crate::db::util::WithRevision; +use crate::disk::util::{BackupMountGuard, PackageBackupMountGuard, TmpMountGuard}; +use crate::install::progress::{InstallProgress, InstallProgressTracker}; +use crate::install::{install_s9pk_or_cleanup, PKG_PUBLIC_DIR}; +use crate::s9pk::manifest::PackageId; +use crate::s9pk::reader::S9pkReader; +use crate::util::{display_none, Version}; +use crate::volume::BACKUP_DIR; +use crate::Error; + +fn parse_comma_separated(arg: &str, _: &ArgMatches<'_>) -> Result, Error> { + arg.split(",") + .map(|s| s.trim().parse().map_err(Error::from)) + .collect() +} + +#[command(rename = "restore", display(display_none))] +#[instrument(skip(ctx, old_password, password))] +pub async fn restore_packages( + #[context] ctx: RpcContext, + #[arg(parse(parse_comma_separated))] ids: Vec, + #[arg] logicalname: PathBuf, + #[arg(rename = "old-password", long = "old-password")] old_password: Option, + #[arg] password: String, +) -> Result, Error> { + let mut db = ctx.db.handle(); + check_password_against_db(&mut ctx.secret_store.acquire().await?, &password).await?; + let mut backup_guard = BackupMountGuard::mount( + TmpMountGuard::mount(&logicalname, None).await?, + old_password.as_ref().unwrap_or(&password), + ) + .await?; + if old_password.is_some() { + backup_guard.change_password(&password)?; + } + let (revision, guards) = assure_restoring(&ctx, &mut db, ids, &backup_guard).await?; + + let mut tasks = Vec::with_capacity(guards.len()); + for (id, version, progress, guard) in guards { + let ctx = ctx.clone(); + tasks.push(tokio::spawn(async move { + if let Err(e) = restore_package(&ctx, &id, &version, progress, guard).await { + tracing::error!("Error restoring package {}: {}", id, e); + tracing::debug!("{:?}", e); + } + })); + } + + tokio::spawn(async { + futures::future::join_all(tasks).await; + if let Err(e) = backup_guard.unmount().await { + tracing::error!("Error unmounting backup drive: {}", e); + tracing::debug!("{:?}", e); + } + }); + + Ok(WithRevision { + response: (), + revision, + }) +} + +#[instrument(skip(ctx, db, backup_guard))] +async fn assure_restoring( + ctx: &RpcContext, + db: &mut PatchDbHandle, + ids: Vec, + backup_guard: &BackupMountGuard, +) -> Result< + ( + Option>, + Vec<( + PackageId, + Version, + Arc, + PackageBackupMountGuard, + )>, + ), + Error, +> { + let mut tx = db.begin().await?; + + let mut guards = Vec::with_capacity(ids.len()); + + for id in ids { + let mut model = crate::db::DatabaseModel::new() + .package_data() + .idx_model(&id) + .get_mut(&mut tx) + .await?; + + if !model.is_none() { + return Err(Error::new( + eyre!("Can't restore over existing package: {}", id), + crate::ErrorKind::InvalidRequest, + )); + } + + let guard = backup_guard.mount_package_backup(&id).await?; + let s9pk_path = Path::new(BACKUP_DIR).join(&id).join(format!("{}.s9pk", id)); + let mut rdr = S9pkReader::open(&s9pk_path, false).await?; + + let manifest = rdr.manifest().await?; + let version = manifest.version.clone(); + let progress = InstallProgress::new(Some(tokio::fs::metadata(&s9pk_path).await?.len())); + progress + .download_complete + .store(true, std::sync::atomic::Ordering::SeqCst); + + let public_dir_path = ctx + .datadir + .join(PKG_PUBLIC_DIR) + .join(&id) + .join(version.as_str()); + tokio::fs::create_dir_all(&public_dir_path).await?; + + let license_path = public_dir_path.join("LICENSE.md"); + let mut dst = File::create(&license_path).await?; + tokio::io::copy(&mut rdr.license().await?, &mut dst).await?; + dst.sync_all().await?; + + let instructions_path = public_dir_path.join("INSTRUCTIONS.md"); + let mut dst = File::create(&instructions_path).await?; + tokio::io::copy(&mut rdr.instructions().await?, &mut dst).await?; + dst.sync_all().await?; + + let icon_path = Path::new("icon").with_extension(&manifest.assets.icon_type()); + let icon_path = public_dir_path.join(&icon_path); + let mut dst = File::create(&icon_path).await?; + tokio::io::copy(&mut rdr.icon().await?, &mut dst).await?; + dst.sync_all().await?; + + *model = Some(PackageDataEntry::Restoring { + install_progress: progress.clone(), + static_files: StaticFiles::local(&id, &version, manifest.assets.icon_type()), + manifest, + }); + model.save(&mut tx).await?; + + guards.push((id, version, progress, guard)); + } + + Ok((tx.commit(None).await?, guards)) +} + +#[instrument(skip(ctx, guard))] +async fn restore_package( + ctx: &RpcContext, + id: &PackageId, + version: &Version, + progress: Arc, + guard: PackageBackupMountGuard, +) -> Result<(), Error> { + let s9pk_path = Path::new(BACKUP_DIR).join(id).join(format!("{}.s9pk", id)); + let progress_reader = + InstallProgressTracker::new(File::open(&s9pk_path).await?, progress.clone()); + let mut s9pk_reader = progress + .track_read_during( + crate::db::DatabaseModel::new() + .package_data() + .idx_model(id) + .and_then(|pde| pde.install_progress()), + &ctx.db, + || S9pkReader::from_reader(progress_reader, true), + ) + .await?; + + install_s9pk_or_cleanup(ctx, id, version, &mut s9pk_reader, progress).await?; + + guard.unmount().await?; + + Ok(()) +} diff --git a/appmgr/src/bin/embassy-init.rs b/appmgr/src/bin/embassy-init.rs index 581cd59f7..15202c577 100644 --- a/appmgr/src/bin/embassy-init.rs +++ b/appmgr/src/bin/embassy-init.rs @@ -16,7 +16,6 @@ use embassy::util::logger::EmbassyLogger; use embassy::util::Invoke; use embassy::{Error, ResultExt}; use http::StatusCode; -use nix::sys::socket::shutdown; use rpc_toolkit::rpc_server; use tokio::process::Command; use tracing::instrument; diff --git a/appmgr/src/bin/embassyd.rs b/appmgr/src/bin/embassyd.rs index 22540a87a..43a8acf2d 100644 --- a/appmgr/src/bin/embassyd.rs +++ b/appmgr/src/bin/embassyd.rs @@ -3,7 +3,6 @@ use std::time::Duration; use color_eyre::eyre::eyre; use embassy::context::{DiagnosticContext, RpcContext}; use embassy::db::subscribe; -use embassy::hostname::get_hostname; use embassy::middleware::auth::auth; use embassy::middleware::cors::cors; use embassy::middleware::diagnostic::diagnostic; diff --git a/appmgr/src/context/cli.rs b/appmgr/src/context/cli.rs index 80dd05697..3ea300cc1 100644 --- a/appmgr/src/context/cli.rs +++ b/appmgr/src/context/cli.rs @@ -140,42 +140,3 @@ impl Context for CliContext { &self.0.client } } - -fn deserialize_host<'de, D: serde::de::Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { - struct Visitor; - impl<'de> serde::de::Visitor<'de> for Visitor { - type Value = Option; - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(formatter, "a parsable string") - } - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - Host::parse(v) - .map(Some) - .map_err(|e| serde::de::Error::custom(e)) - } - fn visit_some(self, deserializer: D) -> Result - where - D: serde::de::Deserializer<'de>, - { - deserializer.deserialize_str(Visitor) - } - fn visit_none(self) -> Result - where - E: serde::de::Error, - { - Ok(None) - } - fn visit_unit(self) -> Result - where - E: serde::de::Error, - { - Ok(None) - } - } - deserializer.deserialize_any(Visitor) -} diff --git a/appmgr/src/db/model.rs b/appmgr/src/db/model.rs index 8cdddd9e1..61667fba9 100644 --- a/appmgr/src/db/model.rs +++ b/appmgr/src/db/model.rs @@ -184,7 +184,7 @@ pub enum PackageDataEntry { static_files: StaticFiles, manifest: Manifest, install_progress: Arc, - }, // { state: "installing", 'install-progress': InstallProgress } + }, #[serde(rename_all = "kebab-case")] Updating { static_files: StaticFiles, @@ -193,6 +193,12 @@ pub enum PackageDataEntry { install_progress: Arc, }, #[serde(rename_all = "kebab-case")] + Restoring { + static_files: StaticFiles, + manifest: Manifest, + install_progress: Arc, + }, + #[serde(rename_all = "kebab-case")] Removing { static_files: StaticFiles, manifest: Manifest, @@ -207,19 +213,19 @@ pub enum PackageDataEntry { impl PackageDataEntry { pub fn installed(&self) -> Option<&InstalledPackageDataEntry> { match self { - Self::Installing { .. } | Self::Removing { .. } => None, + Self::Installing { .. } | Self::Restoring { .. } | Self::Removing { .. } => None, Self::Updating { installed, .. } | Self::Installed { installed, .. } => Some(installed), } } pub fn installed_mut(&mut self) -> Option<&mut InstalledPackageDataEntry> { match self { - Self::Installing { .. } | Self::Removing { .. } => None, + Self::Installing { .. } | Self::Restoring { .. } | Self::Removing { .. } => None, Self::Updating { installed, .. } | Self::Installed { installed, .. } => Some(installed), } } pub fn into_installed(self) -> Option { match self { - Self::Installing { .. } | Self::Removing { .. } => None, + Self::Installing { .. } | Self::Restoring { .. } | Self::Removing { .. } => None, Self::Updating { installed, .. } | Self::Installed { installed, .. } => Some(installed), } } diff --git a/appmgr/src/disk/mod.rs b/appmgr/src/disk/mod.rs index dec4c6fa6..00e32b5c1 100644 --- a/appmgr/src/disk/mod.rs +++ b/appmgr/src/disk/mod.rs @@ -8,7 +8,6 @@ use serde::{Deserialize, Serialize}; use tracing::instrument; use self::util::DiskInfo; -use crate::context::RpcContext; use crate::disk::util::{BackupMountGuard, TmpMountGuard}; use crate::s9pk::manifest::PackageId; use crate::util::{display_serializable, IoFormat, Version}; @@ -17,7 +16,7 @@ use crate::Error; pub mod main; pub mod util; -#[command(subcommands(list))] +#[command(subcommands(list, backup_info))] pub fn disk() -> Result<(), Error> { Ok(()) } @@ -139,14 +138,13 @@ fn display_backup_info(info: BackupInfo, matches: &ArgMatches<'_>) { } #[command(rename = "backup-info", display(display_backup_info))] -#[instrument(skip(ctx, password))] +#[instrument(skip(password))] pub async fn backup_info( - #[context] ctx: RpcContext, #[arg] logicalname: PathBuf, #[arg] password: String, ) -> Result { let guard = - BackupMountGuard::mount(TmpMountGuard::mount(logicalname).await?, &password).await?; + BackupMountGuard::mount(TmpMountGuard::mount(logicalname, None).await?, &password).await?; let res = guard.metadata.clone(); diff --git a/appmgr/src/disk/util.rs b/appmgr/src/disk/util.rs index b54767eb7..d1a192693 100644 --- a/appmgr/src/disk/util.rs +++ b/appmgr/src/disk/util.rs @@ -1,16 +1,19 @@ use std::collections::{BTreeMap, BTreeSet}; use std::os::unix::prelude::OsStrExt; use std::path::{Path, PathBuf}; +use std::sync::{Arc, Weak}; use color_eyre::eyre::{self, eyre}; use digest::Digest; use futures::TryStreamExt; use indexmap::IndexSet; +use lazy_static::lazy_static; use regex::Regex; use serde::{Deserialize, Serialize}; use tokio::fs::File; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; +use tokio::sync::Mutex; use tracing::instrument; use super::BackupInfo; @@ -18,7 +21,7 @@ use crate::auth::check_password; use crate::middleware::encrypt::{decrypt_slice, encrypt_slice}; use crate::s9pk::manifest::PackageId; use crate::util::io::from_yaml_async_reader; -use crate::util::{AtomicFile, FileLock, GeneralGuard, Invoke, IoFormat, Version}; +use crate::util::{AtomicFile, FileLock, Invoke, IoFormat, Version}; use crate::volume::BACKUP_DIR; use crate::{Error, ResultExt as _}; @@ -158,7 +161,10 @@ pub async fn list() -> Result, Error> { .filter_map(|c| c.get(1)) .map(|d| Path::new("/dev").join(d.as_str())) .collect(), - Err(e) => BTreeSet::new(), + Err(e) => { + tracing::warn!("`zpool status` returned error: {}", e); + BTreeSet::new() + } }; let disks = tokio_stream::wrappers::ReadDirStream::new( tokio::fs::read_dir(DISK_PATH) @@ -241,71 +247,67 @@ pub async fn list() -> Result, Error> { .unwrap_or_default(); let mut used = None; - let tmp_mountpoint = - Path::new(TMP_MOUNTPOINT).join(&part.strip_prefix("/").unwrap_or(&part)); - if let Err(e) = mount(&part, &tmp_mountpoint).await { - tracing::warn!("Could not collect usage information: {}", e.source) - } else { - let mount_guard = GeneralGuard::new(|| { - let path = tmp_mountpoint.clone(); - tokio::spawn(unmount(path)) - }); - used = get_used(&tmp_mountpoint) - .await - .map_err(|e| { - tracing::warn!( - "Could not get usage of {}: {}", - part.display(), - e.source - ) - }) - .ok(); - let backup_unencrypted_metadata_path = - tmp_mountpoint.join("EmbassyBackups/unencrypted-metadata.cbor"); - if tokio::fs::metadata(&backup_unencrypted_metadata_path) - .await - .is_ok() - { - embassy_os = match (|| async { - IoFormat::Cbor.from_slice( - &tokio::fs::read(&backup_unencrypted_metadata_path) - .await - .with_ctx(|_| { - ( - crate::ErrorKind::Filesystem, - backup_unencrypted_metadata_path.display().to_string(), - ) - })?, - ) - })() - .await + match TmpMountGuard::mount(&part, None).await { + Err(e) => tracing::warn!("Could not collect usage information: {}", e.source), + Ok(mount_guard) => { + used = get_used(&mount_guard) + .await + .map_err(|e| { + tracing::warn!( + "Could not get usage of {}: {}", + part.display(), + e.source + ) + }) + .ok(); + let backup_unencrypted_metadata_path = mount_guard + .as_ref() + .join("EmbassyBackups/unencrypted-metadata.cbor"); + if tokio::fs::metadata(&backup_unencrypted_metadata_path) + .await + .is_ok() { - Ok(a) => Some(a), - Err(e) => { - tracing::error!( - "Error fetching unencrypted backup metadata: {}", - e - ); - None - } - }; - } else if label.as_deref() == Some("rootfs") { - let version_path = - tmp_mountpoint.join("root").join("appmgr").join("version"); - if tokio::fs::metadata(&version_path).await.is_ok() { - embassy_os = Some(EmbassyOsRecoveryInfo { - version: from_yaml_async_reader(File::open(&version_path).await?) + embassy_os = match (|| async { + IoFormat::Cbor.from_slice( + &tokio::fs::read(&backup_unencrypted_metadata_path) + .await + .with_ctx(|_| { + ( + crate::ErrorKind::Filesystem, + backup_unencrypted_metadata_path + .display() + .to_string(), + ) + })?, + ) + })() + .await + { + Ok(a) => Some(a), + Err(e) => { + tracing::error!( + "Error fetching unencrypted backup metadata: {}", + e + ); + None + } + }; + } else if label.as_deref() == Some("rootfs") { + let version_path = mount_guard.as_ref().join("root/appmgr/version"); + if tokio::fs::metadata(&version_path).await.is_ok() { + embassy_os = Some(EmbassyOsRecoveryInfo { + version: from_yaml_async_reader( + File::open(&version_path).await?, + ) .await?, - full: true, - password_hash: None, - wrapped_key: None, - }); + full: true, + password_hash: None, + wrapped_key: None, + }); + } } + mount_guard.unmount().await?; } - mount_guard - .drop() - .await - .with_kind(crate::ErrorKind::Unknown)??; } partitions.push(PartitionInfo { @@ -361,11 +363,11 @@ pub async fn mount( Ok(()) } -#[instrument(skip(src, dst, password))] +#[instrument(skip(src, dst, key))] pub async fn mount_ecryptfs, P1: AsRef>( src: P0, dst: P1, - password: &str, + key: &str, ) -> Result<(), Error> { let is_mountpoint = tokio::process::Command::new("mountpoint") .arg(dst.as_ref()) @@ -383,7 +385,7 @@ pub async fn mount_ecryptfs, P1: AsRef>( .arg(src.as_ref()) .arg(dst.as_ref()) .arg("-o") - .arg(format!("key=passphrase,passwd={},ecryptfs_cipher=aes,ecryptfs_key_bytes=32,ecryptfs_passthrough=n,ecryptfs_enable_filename_crypto=y", password)) + .arg(format!("key=passphrase,passwd={},ecryptfs_cipher=aes,ecryptfs_key_bytes=32,ecryptfs_passthrough=n,ecryptfs_enable_filename_crypto=y", key)) .stdin(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn()?; @@ -422,6 +424,7 @@ pub async fn bind, P1: AsRef>( if is_mountpoint.success() { unmount(dst.as_ref()).await?; } + tokio::fs::create_dir_all(&src).await?; tokio::fs::create_dir_all(&dst).await?; let mut mount_cmd = tokio::process::Command::new("mount"); mount_cmd.arg("--bind"); @@ -432,17 +435,7 @@ pub async fn bind, P1: AsRef>( .arg(src.as_ref()) .arg(dst.as_ref()) .invoke(crate::ErrorKind::Filesystem) - .await - .map_err(|e| { - Error::new( - e.source.wrap_err(format!( - "Binding {} to {}", - src.as_ref().display(), - dst.as_ref().display(), - )), - e.kind, - ) - })?; + .await?; Ok(()) } @@ -450,6 +443,7 @@ pub async fn bind, P1: AsRef>( pub async fn unmount>(mountpoint: P) -> Result<(), Error> { tracing::debug!("Unmounting {}.", mountpoint.as_ref().display()); let umount_output = tokio::process::Command::new("umount") + .arg("-l") .arg(mountpoint.as_ref()) .output() .await?; @@ -472,10 +466,11 @@ pub async fn unmount>(mountpoint: P) -> Result<(), Error> { } #[async_trait::async_trait] -pub trait GenericMountGuard: AsRef { +pub trait GenericMountGuard: AsRef + std::fmt::Debug + Send + Sync + 'static { async fn unmount(mut self) -> Result<(), Error>; } +#[derive(Debug)] pub struct MountGuard { mountpoint: PathBuf, mounted: bool, @@ -484,9 +479,14 @@ impl MountGuard { pub async fn mount( logicalname: impl AsRef, mountpoint: impl AsRef, + encryption_key: Option<&str>, ) -> Result { let mountpoint = mountpoint.as_ref().to_owned(); - mount(logicalname, &mountpoint).await?; + if let Some(key) = encryption_key { + mount_ecryptfs(logicalname, &mountpoint, key).await?; + } else { + mount(logicalname, &mountpoint).await?; + } Ok(MountGuard { mountpoint, mounted: true, @@ -538,27 +538,45 @@ async fn tmp_mountpoint(source: impl AsRef) -> Result { ))) } +lazy_static! { + static ref TMP_MOUNTS: Mutex>> = Mutex::new(BTreeMap::new()); +} + +#[derive(Debug)] pub struct TmpMountGuard { - guard: MountGuard, - lock: FileLock, + guard: Arc, } impl TmpMountGuard { - pub async fn mount(logicalname: impl AsRef) -> Result { + #[instrument(skip(logicalname, encryption_key))] + pub async fn mount( + logicalname: impl AsRef, + encryption_key: Option<&str>, + ) -> Result { let mountpoint = tmp_mountpoint(&logicalname).await?; - let lock = FileLock::new(mountpoint.with_extension("lock")).await?; - let guard = MountGuard::mount(logicalname, &mountpoint).await?; - Ok(TmpMountGuard { guard, lock }) + let mut tmp_mounts = TMP_MOUNTS.lock().await; + if !tmp_mounts.contains_key(&mountpoint) { + tmp_mounts.insert(mountpoint.clone(), Weak::new()); + } + let weak_slot = tmp_mounts.get_mut(&mountpoint).unwrap(); + if let Some(guard) = weak_slot.upgrade() { + Ok(TmpMountGuard { guard }) + } else { + let guard = + Arc::new(MountGuard::mount(logicalname, &mountpoint, encryption_key).await?); + *weak_slot = Arc::downgrade(&guard); + Ok(TmpMountGuard { guard }) + } } pub async fn unmount(self) -> Result<(), Error> { - let TmpMountGuard { guard, lock } = self; - guard.unmount().await?; - lock.unlock().await?; + if let Ok(guard) = Arc::try_unwrap(self.guard) { + guard.unmount().await?; + } Ok(()) } } impl AsRef for TmpMountGuard { fn as_ref(&self) -> &Path { - self.guard.as_ref() + (&*self.guard).as_ref() } } #[async_trait::async_trait] @@ -570,11 +588,10 @@ impl GenericMountGuard for TmpMountGuard { pub struct BackupMountGuard { backup_disk_mount_guard: Option, + encrypted_guard: Option, enc_key: String, pub unencrypted_metadata: EmbassyOsRecoveryInfo, pub metadata: BackupInfo, - mountpoint: PathBuf, - mounted: bool, } impl BackupMountGuard { fn backup_disk_path(&self) -> &Path { @@ -585,12 +602,12 @@ impl BackupMountGuard { } } + #[instrument(skip(password))] pub async fn mount(backup_disk_mount_guard: G, password: &str) -> Result { - let mountpoint = tmp_mountpoint(&backup_disk_mount_guard).await?; let backup_disk_path = backup_disk_mount_guard.as_ref(); let unencrypted_metadata_path = backup_disk_path.join("EmbassyBackups/unencrypted-metadata.cbor"); - let unencrypted_metadata: EmbassyOsRecoveryInfo = + let mut unencrypted_metadata: EmbassyOsRecoveryInfo = if tokio::fs::metadata(&unencrypted_metadata_path) .await .is_ok() @@ -613,7 +630,7 @@ impl BackupMountGuard { unencrypted_metadata.wrapped_key.as_ref(), ) { let wrapped_key = - base32::decode(base32::Alphabet::RFC4648 { padding: false }, wrapped_key) + base32::decode(base32::Alphabet::RFC4648 { padding: true }, wrapped_key) .ok_or_else(|| { Error::new( eyre!("failed to decode wrapped key"), @@ -629,6 +646,23 @@ impl BackupMountGuard { ) }; + if unencrypted_metadata.password_hash.is_none() { + unencrypted_metadata.password_hash = Some( + argon2::hash_encoded( + password.as_bytes(), + &rand::random::<[u8; 16]>()[..], + &argon2::Config::default(), + ) + .with_kind(crate::ErrorKind::PasswordHashGeneration)?, + ); + } + if unencrypted_metadata.wrapped_key.is_none() { + unencrypted_metadata.wrapped_key = Some(base32::encode( + base32::Alphabet::RFC4648 { padding: true }, + &encrypt_slice(&enc_key, password), + )); + } + let crypt_path = backup_disk_path.join("EmbassyBackups/crypt"); if tokio::fs::metadata(&crypt_path).await.is_err() { tokio::fs::create_dir_all(&crypt_path).await.with_ctx(|_| { @@ -638,38 +672,26 @@ impl BackupMountGuard { ) })?; } - mount_ecryptfs(&crypt_path, &mountpoint, &enc_key).await?; - let metadata = match async { - let metadata_path = mountpoint.join("metadata.cbor"); - let metadata: BackupInfo = if tokio::fs::metadata(&metadata_path).await.is_ok() { - IoFormat::Cbor.from_slice(&tokio::fs::read(&metadata_path).await.with_ctx( - |_| { - ( - crate::ErrorKind::Filesystem, - metadata_path.display().to_string(), - ) - }, - )?)? - } else { - Default::default() - }; - Ok(metadata) - } - .await - { - Ok(a) => a, - Err(e) => { - unmount(&mountpoint).await?; - return Err(e); - } + let encrypted_guard = TmpMountGuard::mount(&crypt_path, Some(&enc_key)).await?; + + let metadata_path = encrypted_guard.as_ref().join("metadata.cbor"); + let metadata: BackupInfo = if tokio::fs::metadata(&metadata_path).await.is_ok() { + IoFormat::Cbor.from_slice(&tokio::fs::read(&metadata_path).await.with_ctx(|_| { + ( + crate::ErrorKind::Filesystem, + metadata_path.display().to_string(), + ) + })?)? + } else { + Default::default() }; + Ok(Self { backup_disk_mount_guard: Some(backup_disk_mount_guard), + encrypted_guard: Some(encrypted_guard), enc_key, unencrypted_metadata, metadata, - mountpoint, - mounted: true, }) } @@ -689,22 +711,23 @@ impl BackupMountGuard { Ok(()) } + #[instrument(skip(self))] pub async fn mount_package_backup( &self, id: &PackageId, ) -> Result { - let lock = FileLock::new(Path::new(BACKUP_DIR).join(format!("{}.lock", id))).await?; + let lock = FileLock::new(Path::new(BACKUP_DIR).join(format!("{}.lock", id)), false).await?; let mountpoint = Path::new(BACKUP_DIR).join(id); - bind(self.mountpoint.join(id), &mountpoint, false).await?; + bind(self.as_ref().join(id), &mountpoint, false).await?; Ok(PackageBackupMountGuard { - mountpoint, - lock, - mounted: true, + mountpoint: Some(mountpoint), + lock: Some(lock), }) } + #[instrument(skip(self))] pub async fn save(&self) -> Result<(), Error> { - let metadata_path = self.mountpoint.join("metadata.cbor"); + let metadata_path = self.as_ref().join("metadata.cbor"); let backup_disk_path = self.backup_disk_path(); let mut file = AtomicFile::new(&metadata_path).await?; file.write_all(&IoFormat::Cbor.to_vec(&self.metadata)?) @@ -719,10 +742,10 @@ impl BackupMountGuard { Ok(()) } + #[instrument(skip(self))] pub async fn unmount(mut self) -> Result<(), Error> { - if self.mounted { - unmount(&self.mountpoint).await?; - self.mounted = false; + if let Some(guard) = self.encrypted_guard.take() { + guard.unmount().await?; } if let Some(guard) = self.backup_disk_mount_guard.take() { guard.unmount().await?; @@ -730,6 +753,7 @@ impl BackupMountGuard { Ok(()) } + #[instrument(skip(self))] pub async fn save_and_unmount(self) -> Result<(), Error> { self.save().await?; self.unmount().await?; @@ -738,42 +762,63 @@ impl BackupMountGuard { } impl AsRef for BackupMountGuard { fn as_ref(&self) -> &Path { - &self.mountpoint + if let Some(guard) = &self.encrypted_guard { + guard.as_ref() + } else { + unreachable!() + } } } impl Drop for BackupMountGuard { fn drop(&mut self) { - if self.mounted { - let mountpoint = std::mem::take(&mut self.mountpoint); - tokio::spawn(async move { unmount(mountpoint).await.unwrap() }); - } + let first = self.encrypted_guard.take(); + let second = self.backup_disk_mount_guard.take(); + tokio::spawn(async move { + if let Some(guard) = first { + guard.unmount().await.unwrap(); + } + if let Some(guard) = second { + guard.unmount().await.unwrap(); + } + }); } } pub struct PackageBackupMountGuard { - mountpoint: PathBuf, - lock: FileLock, - mounted: bool, + mountpoint: Option, + lock: Option, } impl PackageBackupMountGuard { pub async fn unmount(mut self) -> Result<(), Error> { - if self.mounted { - unmount(&self.mountpoint).await?; - self.mounted = false; + if let Some(mountpoint) = self.mountpoint.take() { + unmount(&mountpoint).await?; + } + if let Some(lock) = self.lock.take() { + lock.unlock().await?; } Ok(()) } } impl AsRef for PackageBackupMountGuard { fn as_ref(&self) -> &Path { - &self.mountpoint + if let Some(mountpoint) = &self.mountpoint { + mountpoint + } else { + unreachable!() + } } } impl Drop for PackageBackupMountGuard { fn drop(&mut self) { - if self.mounted { - let mountpoint = std::mem::take(&mut self.mountpoint); - tokio::spawn(async move { unmount(mountpoint).await.unwrap() }); - } + let mountpoint = self.mountpoint.take(); + let lock = self.lock.take(); + tokio::spawn(async move { + if let Some(mountpoint) = mountpoint { + unmount(&mountpoint).await.unwrap(); + } + if let Some(lock) = lock { + lock.unlock().await.unwrap(); + } + }); } } diff --git a/appmgr/src/inspect.rs b/appmgr/src/inspect.rs index e2b3d1f21..6366230b0 100644 --- a/appmgr/src/inspect.rs +++ b/appmgr/src/inspect.rs @@ -14,23 +14,31 @@ pub fn inspect() -> Result<(), Error> { #[command(cli_only)] pub async fn hash(#[arg] path: PathBuf) -> Result { - Ok(S9pkReader::open(path).await?.hash_str().to_owned()) + Ok(S9pkReader::open(path, true) + .await? + .hash_str() + .unwrap() + .to_owned()) } #[command(cli_only, display(display_serializable))] pub async fn manifest( #[arg] path: PathBuf, + #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool, #[allow(unused_variables)] #[arg(long = "format")] format: Option, ) -> Result { - S9pkReader::open(path).await?.manifest().await + S9pkReader::open(path, !no_verify).await?.manifest().await } #[command(cli_only, display(display_none))] -pub async fn license(#[arg] path: PathBuf) -> Result<(), Error> { +pub async fn license( + #[arg] path: PathBuf, + #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool, +) -> Result<(), Error> { tokio::io::copy( - &mut S9pkReader::open(path).await?.license().await?, + &mut S9pkReader::open(path, !no_verify).await?.license().await?, &mut tokio::io::stdout(), ) .await?; @@ -38,9 +46,12 @@ pub async fn license(#[arg] path: PathBuf) -> Result<(), Error> { } #[command(cli_only, display(display_none))] -pub async fn icon(#[arg] path: PathBuf) -> Result<(), Error> { +pub async fn icon( + #[arg] path: PathBuf, + #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool, +) -> Result<(), Error> { tokio::io::copy( - &mut S9pkReader::open(path).await?.icon().await?, + &mut S9pkReader::open(path, !no_verify).await?.icon().await?, &mut tokio::io::stdout(), ) .await?; @@ -48,9 +59,15 @@ pub async fn icon(#[arg] path: PathBuf) -> Result<(), Error> { } #[command(cli_only, display(display_none))] -pub async fn instructions(#[arg] path: PathBuf) -> Result<(), Error> { +pub async fn instructions( + #[arg] path: PathBuf, + #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool, +) -> Result<(), Error> { tokio::io::copy( - &mut S9pkReader::open(path).await?.instructions().await?, + &mut S9pkReader::open(path, !no_verify) + .await? + .instructions() + .await?, &mut tokio::io::stdout(), ) .await?; @@ -58,9 +75,15 @@ pub async fn instructions(#[arg] path: PathBuf) -> Result<(), Error> { } #[command(cli_only, display(display_none), rename = "docker-images")] -pub async fn docker_images(#[arg] path: PathBuf) -> Result<(), Error> { +pub async fn docker_images( + #[arg] path: PathBuf, + #[arg(rename = "no-verify", long = "no-verify")] no_verify: bool, +) -> Result<(), Error> { tokio::io::copy( - &mut S9pkReader::open(path).await?.docker_images().await?, + &mut S9pkReader::open(path, !no_verify) + .await? + .docker_images() + .await?, &mut tokio::io::stdout(), ) .await?; diff --git a/appmgr/src/install/cleanup.rs b/appmgr/src/install/cleanup.rs index 0a8d021e1..073dad840 100644 --- a/appmgr/src/install/cleanup.rs +++ b/appmgr/src/install/cleanup.rs @@ -4,8 +4,7 @@ use bollard::image::ListImagesOptions; use patch_db::{DbHandle, PatchDbHandle}; use tracing::instrument; -use super::PKG_ARCHIVE_DIR; -use super::PKG_DOCKER_DIR; +use super::{PKG_ARCHIVE_DIR, PKG_DOCKER_DIR}; use crate::context::RpcContext; use crate::db::model::{CurrentDependencyInfo, InstalledPackageDataEntry, PackageDataEntry}; use crate::s9pk::manifest::PackageId; @@ -131,7 +130,7 @@ pub async fn cleanup_failed( .await? .into_owned(); if match &pde { - PackageDataEntry::Installing { .. } => true, + PackageDataEntry::Installing { .. } | PackageDataEntry::Restoring { .. } => true, PackageDataEntry::Updating { manifest, .. } => { if &manifest.version != version { true @@ -148,7 +147,7 @@ pub async fn cleanup_failed( } match pde { - PackageDataEntry::Installing { .. } => { + PackageDataEntry::Installing { .. } | PackageDataEntry::Restoring { .. } => { crate::db::DatabaseModel::new() .package_data() .remove(db, id) diff --git a/appmgr/src/install/mod.rs b/appmgr/src/install/mod.rs index 91836d4c5..facc32bfe 100644 --- a/appmgr/src/install/mod.rs +++ b/appmgr/src/install/mod.rs @@ -22,8 +22,8 @@ use tracing::instrument; use self::cleanup::cleanup_failed; use crate::context::RpcContext; use crate::db::model::{ - CurrentDependencyInfo, InstalledPackageDataEntry, PackageDataEntry, StaticDependencyInfo, - StaticFiles, + CurrentDependencyInfo, InstalledPackageDataEntry, PackageDataEntry, RecoveredPackageInfo, + StaticDependencyInfo, StaticFiles, }; use crate::db::util::WithRevision; use crate::dependencies::{ @@ -214,7 +214,7 @@ pub async fn uninstall_impl(ctx: RpcContext, id: PackageId) -> Result( + ctx: &RpcContext, + pkg_id: &PackageId, + version: &Version, + rdr: &mut S9pkReader>, + progress: Arc, +) -> Result<(), Error> { + if let Err(e) = install_s9pk(ctx, pkg_id, version, rdr, progress).await { let mut handle = ctx.db.handle(); let mut tx = handle.begin().await?; @@ -325,6 +331,7 @@ pub async fn install_s9pk( ) -> Result<(), Error> { rdr.validate().await?; rdr.validated(); + rdr.reset().await?; let model = crate::db::DatabaseModel::new() .package_data() .idx_model(pkg_id); @@ -672,16 +679,6 @@ pub async fn install_s9pk( .. } = prev { - update_dependents( - ctx, - &mut tx, - pkg_id, - current_dependents - .keys() - .chain(prev.current_dependents.keys()) - .collect::>(), - ) - .await?; let mut configured = prev.status.configured; if let Some(res) = prev_manifest .migrations @@ -739,17 +736,48 @@ pub async fn install_s9pk( *main_status = prev.status.main; main_status.save(&mut tx).await?; } + update_dependents( + ctx, + &mut tx, + pkg_id, + current_dependents + .keys() + .chain(prev.current_dependents.keys()) + .collect::>(), + ) + .await?; + } else if let PackageDataEntry::Restoring { .. } = prev { + manifest + .backup + .restore( + ctx, + &mut tx, + &mut sql_tx, + pkg_id, + version, + &manifest.interfaces, + &manifest.volumes, + ) + .await?; + update_dependents(ctx, &mut tx, pkg_id, current_dependents.keys()).await?; + } else if let Some(recovered) = crate::db::DatabaseModel::new() + .recovered_packages() + .idx_model(pkg_id) + .get(&mut tx, true) + .await? + .into_owned() + { + handle_recovered_package(recovered, manifest, ctx, pkg_id, version, &mut tx).await?; + update_dependents(ctx, &mut tx, pkg_id, current_dependents.keys()).await?; } else { update_dependents(ctx, &mut tx, pkg_id, current_dependents.keys()).await?; - let recovered = crate::db::DatabaseModel::new() - .recovered_packages() - .idx_model(pkg_id) - .get(&mut tx, true) - .await? - .into_owned(); - handle_recovered_package(recovered, manifest, ctx, pkg_id, version, &mut tx).await?; } + crate::db::DatabaseModel::new() + .recovered_packages() + .remove(&mut tx, pkg_id) + .await?; + sql_tx.commit().await?; tx.commit(None).await?; @@ -760,41 +788,37 @@ pub async fn install_s9pk( #[instrument(skip(ctx, tx))] async fn handle_recovered_package( - recovered: Option, + recovered: RecoveredPackageInfo, manifest: Manifest, ctx: &RpcContext, pkg_id: &PackageId, version: &Version, tx: &mut patch_db::Transaction<&mut patch_db::PatchDbHandle>, ) -> Result<(), Error> { - Ok(if let Some(recovered) = recovered { - let configured = if let Some(res) = manifest - .migrations - .from(ctx, &recovered.version, pkg_id, version, &manifest.volumes) - .await? - { - res.configured - } else { - false - }; - if configured { - crate::config::configure( - ctx, - tx, - pkg_id, - None, - &None, - false, - &mut BTreeMap::new(), - &mut BTreeMap::new(), - ) - .await?; - } - crate::db::DatabaseModel::new() - .recovered_packages() - .remove(tx, pkg_id) - .await? - }) + let configured = if let Some(res) = manifest + .migrations + .from(ctx, &recovered.version, pkg_id, version, &manifest.volumes) + .await? + { + res.configured + } else { + false + }; + if configured { + crate::config::configure( + ctx, + tx, + pkg_id, + None, + &None, + false, + &mut BTreeMap::new(), + &mut BTreeMap::new(), + ) + .await?; + } + + Ok(()) } #[instrument(skip(datadir))] diff --git a/appmgr/src/lib.rs b/appmgr/src/lib.rs index 8cbdf43b0..9f35f782a 100644 --- a/appmgr/src/lib.rs +++ b/appmgr/src/lib.rs @@ -90,7 +90,8 @@ pub fn server() -> Result<(), RpcError> { control::stop, logs::logs, properties::properties, - dependencies::dependency + dependencies::dependency, + backup::package_backup, ))] pub fn package() -> Result<(), RpcError> { Ok(()) diff --git a/appmgr/src/manager/health.rs b/appmgr/src/manager/health.rs index 465ea82d5..2bb373293 100644 --- a/appmgr/src/manager/health.rs +++ b/appmgr/src/manager/health.rs @@ -1,6 +1,5 @@ use std::collections::BTreeMap; -use itertools::Itertools; use patch_db::DbHandle; use tracing::instrument; diff --git a/appmgr/src/s9pk/mod.rs b/appmgr/src/s9pk/mod.rs index 737e62bcf..3ba3a9cc0 100644 --- a/appmgr/src/s9pk/mod.rs +++ b/appmgr/src/s9pk/mod.rs @@ -109,7 +109,7 @@ pub fn pack(#[context] ctx: SdkContext, #[arg] path: Option) -> Result< #[command(cli_only, display(display_none))] pub async fn verify(#[arg] path: PathBuf) -> Result<(), Error> { - let mut s9pk = S9pkReader::open(path).await?; + let mut s9pk = S9pkReader::open(path, true).await?; s9pk.validate().await?; Ok(()) diff --git a/appmgr/src/s9pk/reader.rs b/appmgr/src/s9pk/reader.rs index f5edcc9ad..a7adbe64f 100644 --- a/appmgr/src/s9pk/reader.rs +++ b/appmgr/src/s9pk/reader.rs @@ -45,20 +45,20 @@ impl<'a, R: AsyncRead + AsyncSeek + Unpin> AsyncRead for ReadHandle<'a, R> { } pub struct S9pkReader { - hash: Output, - hash_string: String, + hash: Option>, + hash_string: Option, toc: TableOfContents, pos: u64, rdr: R, } impl S9pkReader { - pub async fn open>(path: P) -> Result { + pub async fn open>(path: P, check_sig: bool) -> Result { let p = path.as_ref(); let rdr = File::open(p) .await .with_ctx(|_| (crate::error::ErrorKind::Filesystem, p.display().to_string()))?; - Self::from_reader(rdr).await + Self::from_reader(rdr, check_sig).await } } impl S9pkReader> { @@ -69,33 +69,41 @@ impl S9pkReader> { impl S9pkReader { #[instrument(skip(self))] pub async fn validate(&mut self) -> Result<(), Error> { - self.rdr.seek(SeekFrom::Start(0)).await?; Ok(()) } #[instrument(skip(rdr))] - pub async fn from_reader(mut rdr: R) -> Result { + pub async fn from_reader(mut rdr: R, check_sig: bool) -> Result { let header = Header::deserialize(&mut rdr).await?; - let mut hasher = Sha512::new(); - let mut buf = [0; 1024]; - let mut read; - while { - read = rdr.read(&mut buf).await?; - read != 0 - } { - hasher.update(&buf[0..read]); - } - let hash = hasher.clone().finalize(); - header - .pubkey - .verify_prehashed(hasher, Some(SIG_CONTEXT), &header.signature)?; + let (hash, hash_string) = if check_sig { + let mut hasher = Sha512::new(); + let mut buf = [0; 1024]; + let mut read; + while { + read = rdr.read(&mut buf).await?; + read != 0 + } { + hasher.update(&buf[0..read]); + } + let hash = hasher.clone().finalize(); + header + .pubkey + .verify_prehashed(hasher, Some(SIG_CONTEXT), &header.signature)?; + ( + Some(hash), + Some(base32::encode( + base32::Alphabet::RFC4648 { padding: false }, + hash.as_slice(), + )), + ) + } else { + (None, None) + }; + let pos = rdr.stream_position().await?; Ok(S9pkReader { - hash_string: base32::encode( - base32::Alphabet::RFC4648 { padding: false }, - hash.as_slice(), - ), + hash_string, hash, toc: header.table_of_contents, pos, @@ -103,12 +111,17 @@ impl S9pkReader { }) } - pub fn hash(&self) -> &Output { - &self.hash + pub fn hash(&self) -> Option<&Output> { + self.hash.as_ref() } - pub fn hash_str(&self) -> &str { - self.hash_string.as_str() + pub fn hash_str(&self) -> Option<&str> { + self.hash_string.as_ref().map(|s| s.as_str()) + } + + pub async fn reset(&mut self) -> Result<(), Error> { + self.rdr.seek(SeekFrom::Start(0)).await?; + Ok(()) } async fn read_handle<'a>( diff --git a/appmgr/src/setup.rs b/appmgr/src/setup.rs index e70993701..580a4ef15 100644 --- a/appmgr/src/setup.rs +++ b/appmgr/src/setup.rs @@ -305,14 +305,15 @@ fn dir_copy<'a, P0: AsRef + 'a + Send + Sync, P1: AsRef + 'a + Send format!("cp -P {} -> {}", src_path.display(), dst_path.display()), ) })?; - tokio::fs::set_permissions(&dst_path, m.permissions()) - .await - .with_ctx(|_| { - ( - crate::ErrorKind::Filesystem, - format!("chmod {}", dst_path.display()), - ) - })?; + // Removed (see https://unix.stackexchange.com/questions/87200/change-permissions-for-a-symbolic-link): + // tokio::fs::set_permissions(&dst_path, m.permissions()) + // .await + // .with_ctx(|_| { + // ( + // crate::ErrorKind::Filesystem, + // format!("chmod {}", dst_path.display()), + // ) + // })?; } Ok(()) }) @@ -324,7 +325,7 @@ fn dir_copy<'a, P0: AsRef + 'a + Send + Sync, P1: AsRef + 'a + Send #[instrument(skip(ctx))] async fn recover_v2(ctx: &SetupContext, recovery_partition: PartitionInfo) -> Result<(), Error> { - let recovery = TmpMountGuard::mount(&recovery_partition.logicalname).await?; + let recovery = TmpMountGuard::mount(&recovery_partition.logicalname, None).await?; let secret_store = ctx.secret_store().await?; let db = ctx.db(&secret_store).await?; @@ -371,7 +372,7 @@ async fn recover_v2(ctx: &SetupContext, recovery_partition: PartitionInfo) -> Re let volume_id = VolumeId::Custom(Id::try_from("main".to_owned())?); for (pkg_id, info) in packages { let volume_src_path = volume_path.join(&pkg_id); - let volume_dst_path = data_dir(&ctx.datadir, &pkg_id, &info.version, &volume_id); + let volume_dst_path = data_dir(&ctx.datadir, &pkg_id, &volume_id); tokio::fs::create_dir_all(&volume_dst_path) .await .with_ctx(|_| { diff --git a/appmgr/src/sound.rs b/appmgr/src/sound.rs index 853b6eaaa..4a560ad3c 100644 --- a/appmgr/src/sound.rs +++ b/appmgr/src/sound.rs @@ -26,7 +26,7 @@ struct SoundInterface(Option); impl SoundInterface { #[instrument] pub async fn lease() -> Result { - let guard = FileLock::new(SOUND_LOCK_FILE).await?; + let guard = FileLock::new(SOUND_LOCK_FILE, true).await?; tokio::fs::write(&*EXPORT_FILE, "0") .await .or_else(|e| { diff --git a/appmgr/src/status/mod.rs b/appmgr/src/status/mod.rs index 5c34864fd..4bacd6a31 100644 --- a/appmgr/src/status/mod.rs +++ b/appmgr/src/status/mod.rs @@ -2,8 +2,7 @@ use std::collections::BTreeMap; use chrono::{DateTime, Utc}; use color_eyre::eyre::eyre; -use futures::{FutureExt, StreamExt}; -use patch_db::{DbHandle, HasModel, Map}; +use patch_db::{DbHandle, HasModel}; use serde::{Deserialize, Serialize}; use tracing::instrument; diff --git a/appmgr/src/util/logger.rs b/appmgr/src/util/logger.rs index bdc8a48e5..6931c42ae 100644 --- a/appmgr/src/util/logger.rs +++ b/appmgr/src/util/logger.rs @@ -1,91 +1,13 @@ -use std::collections::BTreeMap; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use reqwest::{Client, Url}; -use sequence_trie::SequenceTrie; use serde::Serialize; use tracing::Subscriber; -use tracing_subscriber::filter::LevelFilter; use tracing_subscriber::Layer; use crate::version::COMMIT_HASH; -#[derive(Clone, Debug)] -pub struct ModuleMap { - trie: SequenceTrie, -} -impl ModuleMap { - fn new(vals: BTreeMap) -> Self { - let mut module_trie = SequenceTrie::new(); - for (k, v) in vals { - let mut include_submodules = false; - let module_key = k - .split("::") - .take_while(|&s| { - if s == "*" { - include_submodules = true; - false - } else { - true - } - }) - .collect::>(); - match module_trie.get_node(module_key.clone()) { - None => match module_trie.get_ancestor_node(module_key.clone()) { - None => { - module_trie.insert(module_key, (v, include_submodules)); - } - Some(ancestor) => match ancestor.value() { - None => { - module_trie.insert(module_key, (v, include_submodules)); - } - Some((_, sub)) => { - if !sub { - module_trie.insert(module_key, (v, include_submodules)); - } - } - }, - }, - Some(n) => match n.value() { - None => { - module_trie.insert(module_key.clone(), (v, include_submodules)); - if include_submodules { - let new_node = module_trie.get_node_mut(module_key).unwrap(); // we just inserted it - let child_keys = new_node - .children_with_keys() - .into_iter() - .map(|x| x.0.clone()) - .collect::>(); - for c in child_keys { - new_node.remove(std::iter::once(&c)); - } - } - } - Some(_) => unreachable!("Trie build failed on 'impossible' duplicate"), - }, - } - } - ModuleMap { trie: module_trie } - } - fn level_for<'a>(&'a self, k: &str) -> &'a LevelFilter { - let module_key = k.split("::"); - match self.trie.get(module_key.clone()) { - None => match self.trie.get_ancestor(module_key) { - None => &LevelFilter::OFF, - Some((level_filter, include_submodules)) => { - if *include_submodules { - level_filter - } else { - &LevelFilter::OFF - } - } - }, - Some((level_filter, _)) => level_filter, - } - } -} - pub struct SharingLayer { log_epoch: Arc, sharing: Arc, @@ -187,53 +109,3 @@ pub async fn order_level() { pub fn module() { println!("{}", module_path!()) } - -proptest::proptest! { - #[test] - fn submodules_handled_by_parent(s0 in "[a-z][a-z0-9_]+", s1 in "[a-z][a-z0-9_]+", level in filter_strategy()) { - proptest::prop_assume!(level > LevelFilter::OFF); - let mut hm = BTreeMap::new(); - hm.insert(format!("{}::*", s0.clone()), level); - let mod_map = ModuleMap::new(hm); - proptest::prop_assert_eq!(mod_map.level_for(&format!("{}::{}", s0, s1)), &level) - } - #[test] - fn submodules_ignored_by_parent(s0 in "[a-z][a-z0-9_]+", s1 in "[a-z][a-z0-9_]+", level in filter_strategy()) { - proptest::prop_assume!(level > LevelFilter::OFF); - let mut hm = BTreeMap::new(); - hm.insert(s0.clone(), level); - let mod_map = ModuleMap::new(hm); - proptest::prop_assert_eq!(mod_map.level_for(&format!("{}::{}", s0, s1)), &LevelFilter::OFF) - } - #[test] - fn duplicate_insertion_ignored(s0 in "[a-z][a-z0-9_]+", s1 in "[a-z][a-z0-9_]+", level in filter_strategy()) { - proptest::prop_assume!(level > LevelFilter::OFF); - let mut hm = BTreeMap::new(); - hm.insert(format!("{}::*", s0.clone()), level); - let sub = format!("{}::{}", s0, s1); - hm.insert(sub.clone(), level); - let mod_map = ModuleMap::new(hm); - proptest::prop_assert_eq!(mod_map.trie.get(sub.split("::")), None) - } - #[test] - fn parent_child_simul(s0 in "[a-z][a-z0-9_]+", s1 in "[a-z][a-z0-9_]+", level0 in filter_strategy(), level1 in filter_strategy()) { - let mut hm = BTreeMap::new(); - hm.insert(s0.clone(), level0); - let sub = format!("{}::{}", s0, s1); - hm.insert(sub.clone(), level1); - let mod_map = ModuleMap::new(hm); - proptest::prop_assert_eq!(mod_map.level_for(&s0), &level0); - proptest::prop_assert_eq!(mod_map.level_for(&sub), &level1); - } -} -fn filter_strategy() -> impl proptest::strategy::Strategy { - use proptest::strategy::Just; - proptest::prop_oneof![ - Just(LevelFilter::OFF), - Just(LevelFilter::ERROR), - Just(LevelFilter::WARN), - Just(LevelFilter::INFO), - Just(LevelFilter::DEBUG), - Just(LevelFilter::TRACE), - ] -} diff --git a/appmgr/src/util/mod.rs b/appmgr/src/util/mod.rs index 10c13bf80..b58d3ad09 100644 --- a/appmgr/src/util/mod.rs +++ b/appmgr/src/util/mod.rs @@ -23,6 +23,7 @@ use serde_json::Value; use tokio::fs::File; use tokio::sync::{Mutex, OwnedMutexGuard, RwLock}; use tokio::task::{JoinError, JoinHandle}; +use tracing::instrument; use crate::shutdown::Shutdown; use crate::{Error, ResultExt as _}; @@ -733,8 +734,6 @@ pub fn deserialize_number_permissive< >( deserializer: D, ) -> std::result::Result { - use num::cast::FromPrimitive; - struct Visitor + num::cast::FromPrimitive, E>(std::marker::PhantomData); impl<'de, T: FromStr + num::cast::FromPrimitive, Err: std::fmt::Display> serde::de::Visitor<'de> for Visitor @@ -998,7 +997,8 @@ impl Drop for FileLock { } } impl FileLock { - pub async fn new(path: impl AsRef + Send + Sync) -> Result { + #[instrument(skip(path))] + pub async fn new(path: impl AsRef + Send + Sync, blocking: bool) -> Result { lazy_static! { static ref INTERNAL_LOCKS: Mutex>>> = Mutex::new(BTreeMap::new()); @@ -1010,7 +1010,12 @@ impl FileLock { } let tex = internal_locks.get(&path).unwrap().clone(); drop(internal_locks); - let tex_guard = tex.lock_owned().await; + let tex_guard = if blocking { + tex.lock_owned().await + } else { + tex.try_lock_owned() + .with_kind(crate::ErrorKind::Filesystem)? + }; let parent = path.parent().unwrap_or(Path::new("/")); if tokio::fs::metadata(parent).await.is_err() { tokio::fs::create_dir_all(parent) @@ -1020,8 +1025,8 @@ impl FileLock { let f = File::create(&path) .await .with_ctx(|_| (crate::ErrorKind::Filesystem, path.display().to_string()))?; - let file_guard = tokio::task::spawn_blocking(|| { - fd_lock_rs::FdLock::lock(f, fd_lock_rs::LockType::Exclusive, true) + let file_guard = tokio::task::spawn_blocking(move || { + fd_lock_rs::FdLock::lock(f, fd_lock_rs::LockType::Exclusive, blocking) }) .await .with_kind(crate::ErrorKind::Unknown)? diff --git a/appmgr/src/version/v0_3_0.rs b/appmgr/src/version/v0_3_0.rs index d2fa260c7..d5b96e694 100644 --- a/appmgr/src/version/v0_3_0.rs +++ b/appmgr/src/version/v0_3_0.rs @@ -12,10 +12,10 @@ impl VersionT for Version { fn semver(&self) -> emver::Version { V0_3_0 } - async fn up(&self, db: &mut Db) -> Result<(), Error> { + async fn up(&self, _db: &mut Db) -> Result<(), Error> { Ok(()) } - async fn down(&self, db: &mut Db) -> Result<(), Error> { + async fn down(&self, _db: &mut Db) -> Result<(), Error> { Ok(()) } } diff --git a/appmgr/src/volume.rs b/appmgr/src/volume.rs index b2ef8c742..1a47ee1b4 100644 --- a/appmgr/src/volume.rs +++ b/appmgr/src/volume.rs @@ -133,12 +133,7 @@ impl HasModel for Volumes { type Model = MapModel; } -pub fn data_dir>( - datadir: P, - pkg_id: &PackageId, - version: &Version, - volume_id: &VolumeId, -) -> PathBuf { +pub fn data_dir>(datadir: P, pkg_id: &PackageId, volume_id: &VolumeId) -> PathBuf { datadir .as_ref() .join(PKG_VOLUME_DIR) @@ -208,7 +203,7 @@ impl Volume { volume_id: &VolumeId, ) -> PathBuf { match self { - Volume::Data { .. } => data_dir(&ctx.datadir, pkg_id, version, volume_id), + Volume::Data { .. } => data_dir(&ctx.datadir, pkg_id, volume_id), Volume::Assets {} => asset_dir(&ctx.datadir, pkg_id, version).join(volume_id), Volume::Pointer { package_id, diff --git a/system-images/compat/Cargo.lock b/system-images/compat/Cargo.lock index ddaff48c3..ad620478a 100644 --- a/system-images/compat/Cargo.lock +++ b/system-images/compat/Cargo.lock @@ -887,7 +887,6 @@ dependencies = [ "rpc-toolkit", "rust-argon2", "scopeguard", - "sequence_trie", "serde", "serde_json", "serde_with", @@ -2814,12 +2813,6 @@ dependencies = [ "pest", ] -[[package]] -name = "sequence_trie" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee22067b7ccd072eeb64454b9c6e1b33b61cd0d49e895fd48676a184580e0c3" - [[package]] name = "serde" version = "1.0.130" diff --git a/system-images/compat/src/backup.rs b/system-images/compat/src/backup.rs index a1c8bcf22..146455c83 100644 --- a/system-images/compat/src/backup.rs +++ b/system-images/compat/src/backup.rs @@ -1,5 +1,7 @@ use std::{path::Path, process::Stdio}; +use embassy::disk::main::DEFAULT_PASSWORD; + pub fn create_backup( mountpoint: impl AsRef, data_path: impl AsRef, @@ -30,11 +32,18 @@ pub fn create_backup( )); } } - let data_res = data_cmd + let data_output = data_cmd + .env("PASSPHRASE", DEFAULT_PASSWORD) .arg(data_path) .arg(format!("file://{}", mountpoint.display().to_string())) - .output(); - data_res?; + .stderr(Stdio::piped()) + .output()?; + if !data_output.status.success() { + return Err(anyhow::anyhow!( + "duplicity error: {}", + String::from_utf8(data_output.stderr).unwrap() + )); + } Ok(()) } @@ -47,6 +56,7 @@ pub fn restore_backup( let data_path = std::fs::canonicalize(data_path)?; let data_output = std::process::Command::new("duplicity") + .env("PASSPHRASE", DEFAULT_PASSWORD) .arg("--force") .arg(format!("file://{}", mountpoint.display().to_string())) .arg(&data_path)