adds recovery (#731)

* should work

* works for real

* fix build
This commit is contained in:
Aiden McClelland
2021-10-27 15:50:53 -06:00
parent d0e8211b24
commit 1e7492e915
27 changed files with 694 additions and 598 deletions

7
appmgr/Cargo.lock generated
View File

@@ -872,7 +872,6 @@ dependencies = [
"rpc-toolkit",
"rust-argon2",
"scopeguard",
"sequence_trie",
"serde",
"serde_json",
"serde_with",
@@ -2763,12 +2762,6 @@ dependencies = [
"pest",
]
[[package]]
name = "sequence_trie"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ee22067b7ccd072eeb64454b9c6e1b33b61cd0d49e895fd48676a184580e0c3"
[[package]]
name = "serde"
version = "1.0.130"

View File

@@ -94,7 +94,6 @@ rpassword = "5.0.1"
rpc-toolkit = { version = "*", path = "../rpc-toolkit/rpc-toolkit" }
rust-argon2 = "0.8.3"
scopeguard = "1.1" # because avahi-sys fucks your shit up
sequence_trie = "0.3.6"
serde = { version = "1.0.130", features = ["derive", "rc"] }
serde_cbor = { package = "ciborium", version = "0.1.0" }
serde_json = "1.0.68"

View File

@@ -1,17 +1,15 @@
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::sync::Arc;
use chrono::Utc;
use color_eyre::eyre::eyre;
use futures::task::Spawn;
use openssl::pkey::{PKey, Private};
use openssl::x509::X509;
use patch_db::{DbHandle, LockType, PatchDbHandle, Revision};
use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
use torut::onion::TorSecretKeyV3;
use tracing::instrument;
@@ -28,7 +26,7 @@ use crate::s9pk::manifest::PackageId;
use crate::status::MainStatus;
use crate::util::{display_none, AtomicFile, IoFormat};
use crate::version::VersionT;
use crate::{Error, ErrorKind, ResultExt};
use crate::Error;
#[derive(Debug)]
pub struct OsBackup {
@@ -93,6 +91,7 @@ impl Serialize for OsBackup {
}
#[command(rename = "create", display(display_none))]
#[instrument(skip(ctx, old_password, password))]
pub async fn backup_all(
#[context] ctx: RpcContext,
#[arg] logicalname: PathBuf,
@@ -101,17 +100,17 @@ pub async fn backup_all(
) -> Result<WithRevision<()>, Error> {
let mut db = ctx.db.handle();
check_password_against_db(&mut ctx.secret_store.acquire().await?, &password).await?;
let mut backup_guard = BackupMountGuard::mount(
TmpMountGuard::mount(&logicalname, None).await?,
old_password.as_ref().unwrap_or(&password),
)
.await?;
if old_password.is_some() {
backup_guard.change_password(&password)?;
}
let revision = assure_backing_up(&mut db).await?;
tokio::task::spawn(async move {
match perform_backup(
&ctx,
&mut db,
logicalname,
old_password.as_deref(),
&password,
)
.await
{
match perform_backup(&ctx, &mut db, backup_guard).await {
Ok(report) => ctx
.notification_manager
.notify(
@@ -200,46 +199,12 @@ async fn assure_backing_up(db: &mut PatchDbHandle) -> Result<Option<Arc<Revision
Ok(tx.commit(None).await?)
}
async fn write_cbor_file<T: Serialize>(
value: &T,
tmp_path: impl AsRef<Path>,
path: impl AsRef<Path>,
) -> Result<(), Error> {
let tmp_path = tmp_path.as_ref();
let path = path.as_ref();
let mut file = File::create(tmp_path)
.await
.with_ctx(|_| (ErrorKind::Filesystem, tmp_path.display().to_string()))?;
file.write_all(&IoFormat::Cbor.to_vec(value)?).await?;
file.flush().await?;
file.shutdown().await?;
file.sync_all().await?;
drop(file);
tokio::fs::rename(tmp_path, path).await.with_ctx(|_| {
(
ErrorKind::Filesystem,
format!("mv {} -> {}", tmp_path.display(), path.display()),
)
})
}
#[instrument(skip(ctx, db, password))]
#[instrument(skip(ctx, db, backup_guard))]
async fn perform_backup<Db: DbHandle>(
ctx: &RpcContext,
mut db: Db,
logicalname: PathBuf,
old_password: Option<&str>,
password: &str,
mut backup_guard: BackupMountGuard<TmpMountGuard>,
) -> Result<BTreeMap<PackageId, PackageBackupReport>, Error> {
let mut backup_guard = BackupMountGuard::mount(
TmpMountGuard::mount(&logicalname).await?,
old_password.unwrap_or(password),
)
.await?;
if old_password.is_some() {
backup_guard.change_password(password)?;
}
let mut backup_report = BTreeMap::new();
for package_id in crate::db::DatabaseModel::new()

View File

@@ -3,12 +3,12 @@ use std::path::Path;
use chrono::{DateTime, Utc};
use color_eyre::eyre::eyre;
use patch_db::HasModel;
use patch_db::{DbHandle, HasModel};
use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
use sqlx::{Executor, Sqlite};
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
use torut::onion::TorSecretKeyV3;
use tracing::instrument;
use crate::action::{ActionImplementation, NoOutput};
@@ -17,12 +17,13 @@ use crate::disk::PackageBackupInfo;
use crate::install::PKG_ARCHIVE_DIR;
use crate::net::interface::{InterfaceId, Interfaces};
use crate::s9pk::manifest::PackageId;
use crate::util::{IoFormat, Version};
use crate::util::{AtomicFile, IoFormat, Version};
use crate::version::{Current, VersionT};
use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR};
use crate::{Error, ResultExt};
mod backup_bulk;
mod restore;
#[derive(Debug, Deserialize, Serialize)]
pub struct BackupReport {
@@ -46,10 +47,15 @@ pub fn backup() -> Result<(), Error> {
Ok(())
}
#[command(rename = "backup", subcommands(restore::restore_packages))]
pub fn package_backup() -> Result<(), Error> {
Ok(())
}
#[derive(Deserialize, Serialize)]
struct BackupMetadata {
pub timestamp: DateTime<Utc>,
pub tor_keys: BTreeMap<InterfaceId, TorSecretKeyV3>,
pub tor_keys: BTreeMap<InterfaceId, String>,
}
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
@@ -89,13 +95,18 @@ impl BackupActions {
.with_kind(crate::ErrorKind::Backup)?;
let tor_keys = interfaces
.tor_keys(&mut ctx.secret_store.acquire().await?, pkg_id)
.await?;
.await?
.into_iter()
.map(|(id, key)| {
(
id,
base32::encode(base32::Alphabet::RFC4648 { padding: true }, &key.as_bytes()),
)
})
.collect();
let tmp_path = Path::new(BACKUP_DIR)
.join(pkg_id)
.join(format!("{}.s9pk", pkg_id));
let real_path = Path::new(BACKUP_DIR)
.join(pkg_id)
.join(format!(".{}.s9pk.tmp", pkg_id));
let s9pk_path = ctx
.datadir
.join(PKG_ARCHIVE_DIR)
@@ -103,8 +114,8 @@ impl BackupActions {
.join(pkg_version.as_str())
.join(format!("{}.s9pk", pkg_id));
let mut infile = File::open(&s9pk_path).await?;
let mut outfile = File::create(&tmp_path).await?;
tokio::io::copy(&mut infile, &mut outfile)
let mut outfile = AtomicFile::new(&tmp_path).await?;
tokio::io::copy(&mut infile, &mut *outfile)
.await
.with_ctx(|_| {
(
@@ -112,40 +123,17 @@ impl BackupActions {
format!("cp {} -> {}", s9pk_path.display(), tmp_path.display()),
)
})?;
outfile.flush().await?;
outfile.shutdown().await?;
outfile.sync_all().await?;
tokio::fs::rename(&tmp_path, &real_path)
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("mv {} -> {}", tmp_path.display(), real_path.display()),
)
})?;
outfile.save().await?;
let timestamp = Utc::now();
let tmp_path = Path::new(BACKUP_DIR)
.join(pkg_id)
.join(".metadata.cbor.tmp");
let real_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
let mut outfile = File::create(&tmp_path).await?;
let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
let mut outfile = AtomicFile::new(&metadata_path).await?;
outfile
.write_all(&IoFormat::Cbor.to_vec(&BackupMetadata {
timestamp,
tor_keys,
})?)
.await?;
outfile.flush().await?;
outfile.shutdown().await?;
outfile.sync_all().await?;
tokio::fs::rename(&tmp_path, &real_path)
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("mv {} -> {}", tmp_path.display(), real_path.display()),
)
})?;
outfile.save().await?;
Ok(PackageBackupInfo {
os_version: Current::new().semver().into(),
title: pkg_title.to_owned(),
@@ -154,13 +142,20 @@ impl BackupActions {
})
}
pub async fn restore(
#[instrument(skip(ctx, db, secrets))]
pub async fn restore<Ex, Db: DbHandle>(
&self,
ctx: &RpcContext,
db: &mut Db,
secrets: &mut Ex,
pkg_id: &PackageId,
pkg_version: &Version,
interfaces: &Interfaces,
volumes: &Volumes,
) -> Result<(), Error> {
) -> Result<(), Error>
where
for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>,
{
let mut volumes = volumes.clone();
volumes.insert(VolumeId::Backup, Volume::Backup { readonly: true });
self.restore
@@ -185,18 +180,34 @@ impl BackupActions {
)
})?,
)?;
let mut sql_handle = ctx.secret_store.acquire().await?;
for (iface, key) in metadata.tor_keys {
let key_vec = key.as_bytes().to_vec();
let key_vec = base32::decode(base32::Alphabet::RFC4648 { padding: true }, &key)
.ok_or_else(|| {
Error::new(
eyre!("invalid base32 string"),
crate::ErrorKind::Deserialization,
)
})?;
sqlx::query!(
"REPLACE INTO tor (package, interface, key) VALUES (?, ?, ?)",
**pkg_id,
*iface,
key_vec,
)
.execute(&mut sql_handle)
.execute(&mut *secrets)
.await?;
}
crate::db::DatabaseModel::new()
.package_data()
.idx_model(pkg_id)
.expect(db)
.await?
.installed()
.expect(db)
.await?
.interface_addresses()
.put(db, &interfaces.install(&mut *secrets, pkg_id).await?)
.await?;
Ok(())
}
}

View File

@@ -0,0 +1,186 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
use clap::ArgMatches;
use color_eyre::eyre::eyre;
use patch_db::{DbHandle, PatchDbHandle, Revision};
use rpc_toolkit::command;
use tokio::fs::File;
use tracing::instrument;
use crate::auth::check_password_against_db;
use crate::context::RpcContext;
use crate::db::model::{PackageDataEntry, StaticFiles};
use crate::db::util::WithRevision;
use crate::disk::util::{BackupMountGuard, PackageBackupMountGuard, TmpMountGuard};
use crate::install::progress::{InstallProgress, InstallProgressTracker};
use crate::install::{install_s9pk_or_cleanup, PKG_PUBLIC_DIR};
use crate::s9pk::manifest::PackageId;
use crate::s9pk::reader::S9pkReader;
use crate::util::{display_none, Version};
use crate::volume::BACKUP_DIR;
use crate::Error;
fn parse_comma_separated(arg: &str, _: &ArgMatches<'_>) -> Result<Vec<PackageId>, Error> {
arg.split(",")
.map(|s| s.trim().parse().map_err(Error::from))
.collect()
}
#[command(rename = "restore", display(display_none))]
#[instrument(skip(ctx, old_password, password))]
pub async fn restore_packages(
#[context] ctx: RpcContext,
#[arg(parse(parse_comma_separated))] ids: Vec<PackageId>,
#[arg] logicalname: PathBuf,
#[arg(rename = "old-password", long = "old-password")] old_password: Option<String>,
#[arg] password: String,
) -> Result<WithRevision<()>, Error> {
let mut db = ctx.db.handle();
check_password_against_db(&mut ctx.secret_store.acquire().await?, &password).await?;
let mut backup_guard = BackupMountGuard::mount(
TmpMountGuard::mount(&logicalname, None).await?,
old_password.as_ref().unwrap_or(&password),
)
.await?;
if old_password.is_some() {
backup_guard.change_password(&password)?;
}
let (revision, guards) = assure_restoring(&ctx, &mut db, ids, &backup_guard).await?;
let mut tasks = Vec::with_capacity(guards.len());
for (id, version, progress, guard) in guards {
let ctx = ctx.clone();
tasks.push(tokio::spawn(async move {
if let Err(e) = restore_package(&ctx, &id, &version, progress, guard).await {
tracing::error!("Error restoring package {}: {}", id, e);
tracing::debug!("{:?}", e);
}
}));
}
tokio::spawn(async {
futures::future::join_all(tasks).await;
if let Err(e) = backup_guard.unmount().await {
tracing::error!("Error unmounting backup drive: {}", e);
tracing::debug!("{:?}", e);
}
});
Ok(WithRevision {
response: (),
revision,
})
}
#[instrument(skip(ctx, db, backup_guard))]
async fn assure_restoring(
ctx: &RpcContext,
db: &mut PatchDbHandle,
ids: Vec<PackageId>,
backup_guard: &BackupMountGuard<TmpMountGuard>,
) -> Result<
(
Option<Arc<Revision>>,
Vec<(
PackageId,
Version,
Arc<InstallProgress>,
PackageBackupMountGuard,
)>,
),
Error,
> {
let mut tx = db.begin().await?;
let mut guards = Vec::with_capacity(ids.len());
for id in ids {
let mut model = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&id)
.get_mut(&mut tx)
.await?;
if !model.is_none() {
return Err(Error::new(
eyre!("Can't restore over existing package: {}", id),
crate::ErrorKind::InvalidRequest,
));
}
let guard = backup_guard.mount_package_backup(&id).await?;
let s9pk_path = Path::new(BACKUP_DIR).join(&id).join(format!("{}.s9pk", id));
let mut rdr = S9pkReader::open(&s9pk_path, false).await?;
let manifest = rdr.manifest().await?;
let version = manifest.version.clone();
let progress = InstallProgress::new(Some(tokio::fs::metadata(&s9pk_path).await?.len()));
progress
.download_complete
.store(true, std::sync::atomic::Ordering::SeqCst);
let public_dir_path = ctx
.datadir
.join(PKG_PUBLIC_DIR)
.join(&id)
.join(version.as_str());
tokio::fs::create_dir_all(&public_dir_path).await?;
let license_path = public_dir_path.join("LICENSE.md");
let mut dst = File::create(&license_path).await?;
tokio::io::copy(&mut rdr.license().await?, &mut dst).await?;
dst.sync_all().await?;
let instructions_path = public_dir_path.join("INSTRUCTIONS.md");
let mut dst = File::create(&instructions_path).await?;
tokio::io::copy(&mut rdr.instructions().await?, &mut dst).await?;
dst.sync_all().await?;
let icon_path = Path::new("icon").with_extension(&manifest.assets.icon_type());
let icon_path = public_dir_path.join(&icon_path);
let mut dst = File::create(&icon_path).await?;
tokio::io::copy(&mut rdr.icon().await?, &mut dst).await?;
dst.sync_all().await?;
*model = Some(PackageDataEntry::Restoring {
install_progress: progress.clone(),
static_files: StaticFiles::local(&id, &version, manifest.assets.icon_type()),
manifest,
});
model.save(&mut tx).await?;
guards.push((id, version, progress, guard));
}
Ok((tx.commit(None).await?, guards))
}
#[instrument(skip(ctx, guard))]
async fn restore_package(
ctx: &RpcContext,
id: &PackageId,
version: &Version,
progress: Arc<InstallProgress>,
guard: PackageBackupMountGuard,
) -> Result<(), Error> {
let s9pk_path = Path::new(BACKUP_DIR).join(id).join(format!("{}.s9pk", id));
let progress_reader =
InstallProgressTracker::new(File::open(&s9pk_path).await?, progress.clone());
let mut s9pk_reader = progress
.track_read_during(
crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|pde| pde.install_progress()),
&ctx.db,
|| S9pkReader::from_reader(progress_reader, true),
)
.await?;
install_s9pk_or_cleanup(ctx, id, version, &mut s9pk_reader, progress).await?;
guard.unmount().await?;
Ok(())
}

View File

@@ -16,7 +16,6 @@ use embassy::util::logger::EmbassyLogger;
use embassy::util::Invoke;
use embassy::{Error, ResultExt};
use http::StatusCode;
use nix::sys::socket::shutdown;
use rpc_toolkit::rpc_server;
use tokio::process::Command;
use tracing::instrument;

View File

@@ -3,7 +3,6 @@ use std::time::Duration;
use color_eyre::eyre::eyre;
use embassy::context::{DiagnosticContext, RpcContext};
use embassy::db::subscribe;
use embassy::hostname::get_hostname;
use embassy::middleware::auth::auth;
use embassy::middleware::cors::cors;
use embassy::middleware::diagnostic::diagnostic;

View File

@@ -140,42 +140,3 @@ impl Context for CliContext {
&self.0.client
}
}
fn deserialize_host<'de, D: serde::de::Deserializer<'de>>(
deserializer: D,
) -> Result<Option<Host>, D::Error> {
struct Visitor;
impl<'de> serde::de::Visitor<'de> for Visitor {
type Value = Option<Host>;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(formatter, "a parsable string")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Host::parse(v)
.map(Some)
.map_err(|e| serde::de::Error::custom(e))
}
fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
where
D: serde::de::Deserializer<'de>,
{
deserializer.deserialize_str(Visitor)
}
fn visit_none<E>(self) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(None)
}
fn visit_unit<E>(self) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
Ok(None)
}
}
deserializer.deserialize_any(Visitor)
}

View File

@@ -184,7 +184,7 @@ pub enum PackageDataEntry {
static_files: StaticFiles,
manifest: Manifest,
install_progress: Arc<InstallProgress>,
}, // { state: "installing", 'install-progress': InstallProgress }
},
#[serde(rename_all = "kebab-case")]
Updating {
static_files: StaticFiles,
@@ -193,6 +193,12 @@ pub enum PackageDataEntry {
install_progress: Arc<InstallProgress>,
},
#[serde(rename_all = "kebab-case")]
Restoring {
static_files: StaticFiles,
manifest: Manifest,
install_progress: Arc<InstallProgress>,
},
#[serde(rename_all = "kebab-case")]
Removing {
static_files: StaticFiles,
manifest: Manifest,
@@ -207,19 +213,19 @@ pub enum PackageDataEntry {
impl PackageDataEntry {
pub fn installed(&self) -> Option<&InstalledPackageDataEntry> {
match self {
Self::Installing { .. } | Self::Removing { .. } => None,
Self::Installing { .. } | Self::Restoring { .. } | Self::Removing { .. } => None,
Self::Updating { installed, .. } | Self::Installed { installed, .. } => Some(installed),
}
}
pub fn installed_mut(&mut self) -> Option<&mut InstalledPackageDataEntry> {
match self {
Self::Installing { .. } | Self::Removing { .. } => None,
Self::Installing { .. } | Self::Restoring { .. } | Self::Removing { .. } => None,
Self::Updating { installed, .. } | Self::Installed { installed, .. } => Some(installed),
}
}
pub fn into_installed(self) -> Option<InstalledPackageDataEntry> {
match self {
Self::Installing { .. } | Self::Removing { .. } => None,
Self::Installing { .. } | Self::Restoring { .. } | Self::Removing { .. } => None,
Self::Updating { installed, .. } | Self::Installed { installed, .. } => Some(installed),
}
}

View File

@@ -8,7 +8,6 @@ use serde::{Deserialize, Serialize};
use tracing::instrument;
use self::util::DiskInfo;
use crate::context::RpcContext;
use crate::disk::util::{BackupMountGuard, TmpMountGuard};
use crate::s9pk::manifest::PackageId;
use crate::util::{display_serializable, IoFormat, Version};
@@ -17,7 +16,7 @@ use crate::Error;
pub mod main;
pub mod util;
#[command(subcommands(list))]
#[command(subcommands(list, backup_info))]
pub fn disk() -> Result<(), Error> {
Ok(())
}
@@ -139,14 +138,13 @@ fn display_backup_info(info: BackupInfo, matches: &ArgMatches<'_>) {
}
#[command(rename = "backup-info", display(display_backup_info))]
#[instrument(skip(ctx, password))]
#[instrument(skip(password))]
pub async fn backup_info(
#[context] ctx: RpcContext,
#[arg] logicalname: PathBuf,
#[arg] password: String,
) -> Result<BackupInfo, Error> {
let guard =
BackupMountGuard::mount(TmpMountGuard::mount(logicalname).await?, &password).await?;
BackupMountGuard::mount(TmpMountGuard::mount(logicalname, None).await?, &password).await?;
let res = guard.metadata.clone();

View File

@@ -1,16 +1,19 @@
use std::collections::{BTreeMap, BTreeSet};
use std::os::unix::prelude::OsStrExt;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Weak};
use color_eyre::eyre::{self, eyre};
use digest::Digest;
use futures::TryStreamExt;
use indexmap::IndexSet;
use lazy_static::lazy_static;
use regex::Regex;
use serde::{Deserialize, Serialize};
use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::process::Command;
use tokio::sync::Mutex;
use tracing::instrument;
use super::BackupInfo;
@@ -18,7 +21,7 @@ use crate::auth::check_password;
use crate::middleware::encrypt::{decrypt_slice, encrypt_slice};
use crate::s9pk::manifest::PackageId;
use crate::util::io::from_yaml_async_reader;
use crate::util::{AtomicFile, FileLock, GeneralGuard, Invoke, IoFormat, Version};
use crate::util::{AtomicFile, FileLock, Invoke, IoFormat, Version};
use crate::volume::BACKUP_DIR;
use crate::{Error, ResultExt as _};
@@ -158,7 +161,10 @@ pub async fn list() -> Result<Vec<DiskInfo>, Error> {
.filter_map(|c| c.get(1))
.map(|d| Path::new("/dev").join(d.as_str()))
.collect(),
Err(e) => BTreeSet::new(),
Err(e) => {
tracing::warn!("`zpool status` returned error: {}", e);
BTreeSet::new()
}
};
let disks = tokio_stream::wrappers::ReadDirStream::new(
tokio::fs::read_dir(DISK_PATH)
@@ -241,16 +247,10 @@ pub async fn list() -> Result<Vec<DiskInfo>, Error> {
.unwrap_or_default();
let mut used = None;
let tmp_mountpoint =
Path::new(TMP_MOUNTPOINT).join(&part.strip_prefix("/").unwrap_or(&part));
if let Err(e) = mount(&part, &tmp_mountpoint).await {
tracing::warn!("Could not collect usage information: {}", e.source)
} else {
let mount_guard = GeneralGuard::new(|| {
let path = tmp_mountpoint.clone();
tokio::spawn(unmount(path))
});
used = get_used(&tmp_mountpoint)
match TmpMountGuard::mount(&part, None).await {
Err(e) => tracing::warn!("Could not collect usage information: {}", e.source),
Ok(mount_guard) => {
used = get_used(&mount_guard)
.await
.map_err(|e| {
tracing::warn!(
@@ -260,8 +260,9 @@ pub async fn list() -> Result<Vec<DiskInfo>, Error> {
)
})
.ok();
let backup_unencrypted_metadata_path =
tmp_mountpoint.join("EmbassyBackups/unencrypted-metadata.cbor");
let backup_unencrypted_metadata_path = mount_guard
.as_ref()
.join("EmbassyBackups/unencrypted-metadata.cbor");
if tokio::fs::metadata(&backup_unencrypted_metadata_path)
.await
.is_ok()
@@ -273,7 +274,9 @@ pub async fn list() -> Result<Vec<DiskInfo>, Error> {
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
backup_unencrypted_metadata_path.display().to_string(),
backup_unencrypted_metadata_path
.display()
.to_string(),
)
})?,
)
@@ -290,11 +293,12 @@ pub async fn list() -> Result<Vec<DiskInfo>, Error> {
}
};
} else if label.as_deref() == Some("rootfs") {
let version_path =
tmp_mountpoint.join("root").join("appmgr").join("version");
let version_path = mount_guard.as_ref().join("root/appmgr/version");
if tokio::fs::metadata(&version_path).await.is_ok() {
embassy_os = Some(EmbassyOsRecoveryInfo {
version: from_yaml_async_reader(File::open(&version_path).await?)
version: from_yaml_async_reader(
File::open(&version_path).await?,
)
.await?,
full: true,
password_hash: None,
@@ -302,10 +306,8 @@ pub async fn list() -> Result<Vec<DiskInfo>, Error> {
});
}
}
mount_guard
.drop()
.await
.with_kind(crate::ErrorKind::Unknown)??;
mount_guard.unmount().await?;
}
}
partitions.push(PartitionInfo {
@@ -361,11 +363,11 @@ pub async fn mount(
Ok(())
}
#[instrument(skip(src, dst, password))]
#[instrument(skip(src, dst, key))]
pub async fn mount_ecryptfs<P0: AsRef<Path>, P1: AsRef<Path>>(
src: P0,
dst: P1,
password: &str,
key: &str,
) -> Result<(), Error> {
let is_mountpoint = tokio::process::Command::new("mountpoint")
.arg(dst.as_ref())
@@ -383,7 +385,7 @@ pub async fn mount_ecryptfs<P0: AsRef<Path>, P1: AsRef<Path>>(
.arg(src.as_ref())
.arg(dst.as_ref())
.arg("-o")
.arg(format!("key=passphrase,passwd={},ecryptfs_cipher=aes,ecryptfs_key_bytes=32,ecryptfs_passthrough=n,ecryptfs_enable_filename_crypto=y", password))
.arg(format!("key=passphrase,passwd={},ecryptfs_cipher=aes,ecryptfs_key_bytes=32,ecryptfs_passthrough=n,ecryptfs_enable_filename_crypto=y", key))
.stdin(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()?;
@@ -422,6 +424,7 @@ pub async fn bind<P0: AsRef<Path>, P1: AsRef<Path>>(
if is_mountpoint.success() {
unmount(dst.as_ref()).await?;
}
tokio::fs::create_dir_all(&src).await?;
tokio::fs::create_dir_all(&dst).await?;
let mut mount_cmd = tokio::process::Command::new("mount");
mount_cmd.arg("--bind");
@@ -432,17 +435,7 @@ pub async fn bind<P0: AsRef<Path>, P1: AsRef<Path>>(
.arg(src.as_ref())
.arg(dst.as_ref())
.invoke(crate::ErrorKind::Filesystem)
.await
.map_err(|e| {
Error::new(
e.source.wrap_err(format!(
"Binding {} to {}",
src.as_ref().display(),
dst.as_ref().display(),
)),
e.kind,
)
})?;
.await?;
Ok(())
}
@@ -450,6 +443,7 @@ pub async fn bind<P0: AsRef<Path>, P1: AsRef<Path>>(
pub async fn unmount<P: AsRef<Path>>(mountpoint: P) -> Result<(), Error> {
tracing::debug!("Unmounting {}.", mountpoint.as_ref().display());
let umount_output = tokio::process::Command::new("umount")
.arg("-l")
.arg(mountpoint.as_ref())
.output()
.await?;
@@ -472,10 +466,11 @@ pub async fn unmount<P: AsRef<Path>>(mountpoint: P) -> Result<(), Error> {
}
#[async_trait::async_trait]
pub trait GenericMountGuard: AsRef<Path> {
pub trait GenericMountGuard: AsRef<Path> + std::fmt::Debug + Send + Sync + 'static {
async fn unmount(mut self) -> Result<(), Error>;
}
#[derive(Debug)]
pub struct MountGuard {
mountpoint: PathBuf,
mounted: bool,
@@ -484,9 +479,14 @@ impl MountGuard {
pub async fn mount(
logicalname: impl AsRef<Path>,
mountpoint: impl AsRef<Path>,
encryption_key: Option<&str>,
) -> Result<Self, Error> {
let mountpoint = mountpoint.as_ref().to_owned();
if let Some(key) = encryption_key {
mount_ecryptfs(logicalname, &mountpoint, key).await?;
} else {
mount(logicalname, &mountpoint).await?;
}
Ok(MountGuard {
mountpoint,
mounted: true,
@@ -538,27 +538,45 @@ async fn tmp_mountpoint(source: impl AsRef<Path>) -> Result<PathBuf, Error> {
)))
}
lazy_static! {
static ref TMP_MOUNTS: Mutex<BTreeMap<PathBuf, Weak<MountGuard>>> = Mutex::new(BTreeMap::new());
}
#[derive(Debug)]
pub struct TmpMountGuard {
guard: MountGuard,
lock: FileLock,
guard: Arc<MountGuard>,
}
impl TmpMountGuard {
pub async fn mount(logicalname: impl AsRef<Path>) -> Result<Self, Error> {
#[instrument(skip(logicalname, encryption_key))]
pub async fn mount(
logicalname: impl AsRef<Path>,
encryption_key: Option<&str>,
) -> Result<Self, Error> {
let mountpoint = tmp_mountpoint(&logicalname).await?;
let lock = FileLock::new(mountpoint.with_extension("lock")).await?;
let guard = MountGuard::mount(logicalname, &mountpoint).await?;
Ok(TmpMountGuard { guard, lock })
let mut tmp_mounts = TMP_MOUNTS.lock().await;
if !tmp_mounts.contains_key(&mountpoint) {
tmp_mounts.insert(mountpoint.clone(), Weak::new());
}
let weak_slot = tmp_mounts.get_mut(&mountpoint).unwrap();
if let Some(guard) = weak_slot.upgrade() {
Ok(TmpMountGuard { guard })
} else {
let guard =
Arc::new(MountGuard::mount(logicalname, &mountpoint, encryption_key).await?);
*weak_slot = Arc::downgrade(&guard);
Ok(TmpMountGuard { guard })
}
}
pub async fn unmount(self) -> Result<(), Error> {
let TmpMountGuard { guard, lock } = self;
if let Ok(guard) = Arc::try_unwrap(self.guard) {
guard.unmount().await?;
lock.unlock().await?;
}
Ok(())
}
}
impl AsRef<Path> for TmpMountGuard {
fn as_ref(&self) -> &Path {
self.guard.as_ref()
(&*self.guard).as_ref()
}
}
#[async_trait::async_trait]
@@ -570,11 +588,10 @@ impl GenericMountGuard for TmpMountGuard {
pub struct BackupMountGuard<G: GenericMountGuard> {
backup_disk_mount_guard: Option<G>,
encrypted_guard: Option<TmpMountGuard>,
enc_key: String,
pub unencrypted_metadata: EmbassyOsRecoveryInfo,
pub metadata: BackupInfo,
mountpoint: PathBuf,
mounted: bool,
}
impl<G: GenericMountGuard> BackupMountGuard<G> {
fn backup_disk_path(&self) -> &Path {
@@ -585,12 +602,12 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
}
}
#[instrument(skip(password))]
pub async fn mount(backup_disk_mount_guard: G, password: &str) -> Result<Self, Error> {
let mountpoint = tmp_mountpoint(&backup_disk_mount_guard).await?;
let backup_disk_path = backup_disk_mount_guard.as_ref();
let unencrypted_metadata_path =
backup_disk_path.join("EmbassyBackups/unencrypted-metadata.cbor");
let unencrypted_metadata: EmbassyOsRecoveryInfo =
let mut unencrypted_metadata: EmbassyOsRecoveryInfo =
if tokio::fs::metadata(&unencrypted_metadata_path)
.await
.is_ok()
@@ -613,7 +630,7 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
unencrypted_metadata.wrapped_key.as_ref(),
) {
let wrapped_key =
base32::decode(base32::Alphabet::RFC4648 { padding: false }, wrapped_key)
base32::decode(base32::Alphabet::RFC4648 { padding: true }, wrapped_key)
.ok_or_else(|| {
Error::new(
eyre!("failed to decode wrapped key"),
@@ -629,6 +646,23 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
)
};
if unencrypted_metadata.password_hash.is_none() {
unencrypted_metadata.password_hash = Some(
argon2::hash_encoded(
password.as_bytes(),
&rand::random::<[u8; 16]>()[..],
&argon2::Config::default(),
)
.with_kind(crate::ErrorKind::PasswordHashGeneration)?,
);
}
if unencrypted_metadata.wrapped_key.is_none() {
unencrypted_metadata.wrapped_key = Some(base32::encode(
base32::Alphabet::RFC4648 { padding: true },
&encrypt_slice(&enc_key, password),
));
}
let crypt_path = backup_disk_path.join("EmbassyBackups/crypt");
if tokio::fs::metadata(&crypt_path).await.is_err() {
tokio::fs::create_dir_all(&crypt_path).await.with_ctx(|_| {
@@ -638,38 +672,26 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
)
})?;
}
mount_ecryptfs(&crypt_path, &mountpoint, &enc_key).await?;
let metadata = match async {
let metadata_path = mountpoint.join("metadata.cbor");
let encrypted_guard = TmpMountGuard::mount(&crypt_path, Some(&enc_key)).await?;
let metadata_path = encrypted_guard.as_ref().join("metadata.cbor");
let metadata: BackupInfo = if tokio::fs::metadata(&metadata_path).await.is_ok() {
IoFormat::Cbor.from_slice(&tokio::fs::read(&metadata_path).await.with_ctx(
|_| {
IoFormat::Cbor.from_slice(&tokio::fs::read(&metadata_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
metadata_path.display().to_string(),
)
},
)?)?
})?)?
} else {
Default::default()
};
Ok(metadata)
}
.await
{
Ok(a) => a,
Err(e) => {
unmount(&mountpoint).await?;
return Err(e);
}
};
Ok(Self {
backup_disk_mount_guard: Some(backup_disk_mount_guard),
encrypted_guard: Some(encrypted_guard),
enc_key,
unencrypted_metadata,
metadata,
mountpoint,
mounted: true,
})
}
@@ -689,22 +711,23 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
Ok(())
}
#[instrument(skip(self))]
pub async fn mount_package_backup(
&self,
id: &PackageId,
) -> Result<PackageBackupMountGuard, Error> {
let lock = FileLock::new(Path::new(BACKUP_DIR).join(format!("{}.lock", id))).await?;
let lock = FileLock::new(Path::new(BACKUP_DIR).join(format!("{}.lock", id)), false).await?;
let mountpoint = Path::new(BACKUP_DIR).join(id);
bind(self.mountpoint.join(id), &mountpoint, false).await?;
bind(self.as_ref().join(id), &mountpoint, false).await?;
Ok(PackageBackupMountGuard {
mountpoint,
lock,
mounted: true,
mountpoint: Some(mountpoint),
lock: Some(lock),
})
}
#[instrument(skip(self))]
pub async fn save(&self) -> Result<(), Error> {
let metadata_path = self.mountpoint.join("metadata.cbor");
let metadata_path = self.as_ref().join("metadata.cbor");
let backup_disk_path = self.backup_disk_path();
let mut file = AtomicFile::new(&metadata_path).await?;
file.write_all(&IoFormat::Cbor.to_vec(&self.metadata)?)
@@ -719,10 +742,10 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
Ok(())
}
#[instrument(skip(self))]
pub async fn unmount(mut self) -> Result<(), Error> {
if self.mounted {
unmount(&self.mountpoint).await?;
self.mounted = false;
if let Some(guard) = self.encrypted_guard.take() {
guard.unmount().await?;
}
if let Some(guard) = self.backup_disk_mount_guard.take() {
guard.unmount().await?;
@@ -730,6 +753,7 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
Ok(())
}
#[instrument(skip(self))]
pub async fn save_and_unmount(self) -> Result<(), Error> {
self.save().await?;
self.unmount().await?;
@@ -738,42 +762,63 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
}
impl<G: GenericMountGuard> AsRef<Path> for BackupMountGuard<G> {
fn as_ref(&self) -> &Path {
&self.mountpoint
if let Some(guard) = &self.encrypted_guard {
guard.as_ref()
} else {
unreachable!()
}
}
}
impl<G: GenericMountGuard> Drop for BackupMountGuard<G> {
fn drop(&mut self) {
if self.mounted {
let mountpoint = std::mem::take(&mut self.mountpoint);
tokio::spawn(async move { unmount(mountpoint).await.unwrap() });
let first = self.encrypted_guard.take();
let second = self.backup_disk_mount_guard.take();
tokio::spawn(async move {
if let Some(guard) = first {
guard.unmount().await.unwrap();
}
if let Some(guard) = second {
guard.unmount().await.unwrap();
}
});
}
}
pub struct PackageBackupMountGuard {
mountpoint: PathBuf,
lock: FileLock,
mounted: bool,
mountpoint: Option<PathBuf>,
lock: Option<FileLock>,
}
impl PackageBackupMountGuard {
pub async fn unmount(mut self) -> Result<(), Error> {
if self.mounted {
unmount(&self.mountpoint).await?;
self.mounted = false;
if let Some(mountpoint) = self.mountpoint.take() {
unmount(&mountpoint).await?;
}
if let Some(lock) = self.lock.take() {
lock.unlock().await?;
}
Ok(())
}
}
impl AsRef<Path> for PackageBackupMountGuard {
fn as_ref(&self) -> &Path {
&self.mountpoint
if let Some(mountpoint) = &self.mountpoint {
mountpoint
} else {
unreachable!()
}
}
}
impl Drop for PackageBackupMountGuard {
fn drop(&mut self) {
if self.mounted {
let mountpoint = std::mem::take(&mut self.mountpoint);
tokio::spawn(async move { unmount(mountpoint).await.unwrap() });
}
let mountpoint = self.mountpoint.take();
let lock = self.lock.take();
tokio::spawn(async move {
if let Some(mountpoint) = mountpoint {
unmount(&mountpoint).await.unwrap();
}
if let Some(lock) = lock {
lock.unlock().await.unwrap();
}
});
}
}

View File

@@ -14,23 +14,31 @@ pub fn inspect() -> Result<(), Error> {
#[command(cli_only)]
pub async fn hash(#[arg] path: PathBuf) -> Result<String, Error> {
Ok(S9pkReader::open(path).await?.hash_str().to_owned())
Ok(S9pkReader::open(path, true)
.await?
.hash_str()
.unwrap()
.to_owned())
}
#[command(cli_only, display(display_serializable))]
pub async fn manifest(
#[arg] path: PathBuf,
#[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
#[allow(unused_variables)]
#[arg(long = "format")]
format: Option<IoFormat>,
) -> Result<Manifest, Error> {
S9pkReader::open(path).await?.manifest().await
S9pkReader::open(path, !no_verify).await?.manifest().await
}
#[command(cli_only, display(display_none))]
pub async fn license(#[arg] path: PathBuf) -> Result<(), Error> {
pub async fn license(
#[arg] path: PathBuf,
#[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
) -> Result<(), Error> {
tokio::io::copy(
&mut S9pkReader::open(path).await?.license().await?,
&mut S9pkReader::open(path, !no_verify).await?.license().await?,
&mut tokio::io::stdout(),
)
.await?;
@@ -38,9 +46,12 @@ pub async fn license(#[arg] path: PathBuf) -> Result<(), Error> {
}
#[command(cli_only, display(display_none))]
pub async fn icon(#[arg] path: PathBuf) -> Result<(), Error> {
pub async fn icon(
#[arg] path: PathBuf,
#[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
) -> Result<(), Error> {
tokio::io::copy(
&mut S9pkReader::open(path).await?.icon().await?,
&mut S9pkReader::open(path, !no_verify).await?.icon().await?,
&mut tokio::io::stdout(),
)
.await?;
@@ -48,9 +59,15 @@ pub async fn icon(#[arg] path: PathBuf) -> Result<(), Error> {
}
#[command(cli_only, display(display_none))]
pub async fn instructions(#[arg] path: PathBuf) -> Result<(), Error> {
pub async fn instructions(
#[arg] path: PathBuf,
#[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
) -> Result<(), Error> {
tokio::io::copy(
&mut S9pkReader::open(path).await?.instructions().await?,
&mut S9pkReader::open(path, !no_verify)
.await?
.instructions()
.await?,
&mut tokio::io::stdout(),
)
.await?;
@@ -58,9 +75,15 @@ pub async fn instructions(#[arg] path: PathBuf) -> Result<(), Error> {
}
#[command(cli_only, display(display_none), rename = "docker-images")]
pub async fn docker_images(#[arg] path: PathBuf) -> Result<(), Error> {
pub async fn docker_images(
#[arg] path: PathBuf,
#[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
) -> Result<(), Error> {
tokio::io::copy(
&mut S9pkReader::open(path).await?.docker_images().await?,
&mut S9pkReader::open(path, !no_verify)
.await?
.docker_images()
.await?,
&mut tokio::io::stdout(),
)
.await?;

View File

@@ -4,8 +4,7 @@ use bollard::image::ListImagesOptions;
use patch_db::{DbHandle, PatchDbHandle};
use tracing::instrument;
use super::PKG_ARCHIVE_DIR;
use super::PKG_DOCKER_DIR;
use super::{PKG_ARCHIVE_DIR, PKG_DOCKER_DIR};
use crate::context::RpcContext;
use crate::db::model::{CurrentDependencyInfo, InstalledPackageDataEntry, PackageDataEntry};
use crate::s9pk::manifest::PackageId;
@@ -131,7 +130,7 @@ pub async fn cleanup_failed<Db: DbHandle>(
.await?
.into_owned();
if match &pde {
PackageDataEntry::Installing { .. } => true,
PackageDataEntry::Installing { .. } | PackageDataEntry::Restoring { .. } => true,
PackageDataEntry::Updating { manifest, .. } => {
if &manifest.version != version {
true
@@ -148,7 +147,7 @@ pub async fn cleanup_failed<Db: DbHandle>(
}
match pde {
PackageDataEntry::Installing { .. } => {
PackageDataEntry::Installing { .. } | PackageDataEntry::Restoring { .. } => {
crate::db::DatabaseModel::new()
.package_data()
.remove(db, id)

View File

@@ -22,8 +22,8 @@ use tracing::instrument;
use self::cleanup::cleanup_failed;
use crate::context::RpcContext;
use crate::db::model::{
CurrentDependencyInfo, InstalledPackageDataEntry, PackageDataEntry, StaticDependencyInfo,
StaticFiles,
CurrentDependencyInfo, InstalledPackageDataEntry, PackageDataEntry, RecoveredPackageInfo,
StaticDependencyInfo, StaticFiles,
};
use crate::db::util::WithRevision;
use crate::dependencies::{
@@ -214,7 +214,7 @@ pub async fn uninstall_impl(ctx: RpcContext, id: PackageId) -> Result<WithRevisi
})
}
#[instrument(skip(ctx))]
#[instrument(skip(ctx, temp_manifest))]
pub async fn download_install_s9pk(
ctx: &RpcContext,
temp_manifest: &Manifest,
@@ -235,7 +235,6 @@ pub async fn download_install_s9pk(
.package_data()
.idx_model(pkg_id);
let res = (|| async {
let progress = InstallProgress::new(s9pk.content_length());
let progress_model = pkg_data_entry.and_then(|pde| pde.install_progress());
@@ -276,17 +275,24 @@ pub async fn download_install_s9pk(
let progress_reader = InstallProgressTracker::new(dst, progress.clone());
let mut s9pk_reader = progress
.track_read_during(progress_model.clone(), &ctx.db, || {
S9pkReader::from_reader(progress_reader)
S9pkReader::from_reader(progress_reader, true)
})
.await?;
install_s9pk(&ctx, pkg_id, version, &mut s9pk_reader, progress).await?;
install_s9pk_or_cleanup(&ctx, pkg_id, version, &mut s9pk_reader, progress).await?;
Ok(())
})()
.await;
}
if let Err(e) = res {
#[instrument(skip(ctx, rdr))]
pub async fn install_s9pk_or_cleanup<R: AsyncRead + AsyncSeek + Unpin>(
ctx: &RpcContext,
pkg_id: &PackageId,
version: &Version,
rdr: &mut S9pkReader<InstallProgressTracker<R>>,
progress: Arc<InstallProgress>,
) -> Result<(), Error> {
if let Err(e) = install_s9pk(ctx, pkg_id, version, rdr, progress).await {
let mut handle = ctx.db.handle();
let mut tx = handle.begin().await?;
@@ -325,6 +331,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
) -> Result<(), Error> {
rdr.validate().await?;
rdr.validated();
rdr.reset().await?;
let model = crate::db::DatabaseModel::new()
.package_data()
.idx_model(pkg_id);
@@ -672,16 +679,6 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
..
} = prev
{
update_dependents(
ctx,
&mut tx,
pkg_id,
current_dependents
.keys()
.chain(prev.current_dependents.keys())
.collect::<BTreeSet<_>>(),
)
.await?;
let mut configured = prev.status.configured;
if let Some(res) = prev_manifest
.migrations
@@ -739,17 +736,48 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
*main_status = prev.status.main;
main_status.save(&mut tx).await?;
}
} else {
update_dependents(
ctx,
&mut tx,
pkg_id,
current_dependents
.keys()
.chain(prev.current_dependents.keys())
.collect::<BTreeSet<_>>(),
)
.await?;
} else if let PackageDataEntry::Restoring { .. } = prev {
manifest
.backup
.restore(
ctx,
&mut tx,
&mut sql_tx,
pkg_id,
version,
&manifest.interfaces,
&manifest.volumes,
)
.await?;
update_dependents(ctx, &mut tx, pkg_id, current_dependents.keys()).await?;
let recovered = crate::db::DatabaseModel::new()
} else if let Some(recovered) = crate::db::DatabaseModel::new()
.recovered_packages()
.idx_model(pkg_id)
.get(&mut tx, true)
.await?
.into_owned();
.into_owned()
{
handle_recovered_package(recovered, manifest, ctx, pkg_id, version, &mut tx).await?;
update_dependents(ctx, &mut tx, pkg_id, current_dependents.keys()).await?;
} else {
update_dependents(ctx, &mut tx, pkg_id, current_dependents.keys()).await?;
}
crate::db::DatabaseModel::new()
.recovered_packages()
.remove(&mut tx, pkg_id)
.await?;
sql_tx.commit().await?;
tx.commit(None).await?;
@@ -760,14 +788,13 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
#[instrument(skip(ctx, tx))]
async fn handle_recovered_package(
recovered: Option<crate::db::model::RecoveredPackageInfo>,
recovered: RecoveredPackageInfo,
manifest: Manifest,
ctx: &RpcContext,
pkg_id: &PackageId,
version: &Version,
tx: &mut patch_db::Transaction<&mut patch_db::PatchDbHandle>,
) -> Result<(), Error> {
Ok(if let Some(recovered) = recovered {
let configured = if let Some(res) = manifest
.migrations
.from(ctx, &recovered.version, pkg_id, version, &manifest.volumes)
@@ -790,11 +817,8 @@ async fn handle_recovered_package(
)
.await?;
}
crate::db::DatabaseModel::new()
.recovered_packages()
.remove(tx, pkg_id)
.await?
})
Ok(())
}
#[instrument(skip(datadir))]

View File

@@ -90,7 +90,8 @@ pub fn server() -> Result<(), RpcError> {
control::stop,
logs::logs,
properties::properties,
dependencies::dependency
dependencies::dependency,
backup::package_backup,
))]
pub fn package() -> Result<(), RpcError> {
Ok(())

View File

@@ -1,6 +1,5 @@
use std::collections::BTreeMap;
use itertools::Itertools;
use patch_db::DbHandle;
use tracing::instrument;

View File

@@ -109,7 +109,7 @@ pub fn pack(#[context] ctx: SdkContext, #[arg] path: Option<PathBuf>) -> Result<
#[command(cli_only, display(display_none))]
pub async fn verify(#[arg] path: PathBuf) -> Result<(), Error> {
let mut s9pk = S9pkReader::open(path).await?;
let mut s9pk = S9pkReader::open(path, true).await?;
s9pk.validate().await?;
Ok(())

View File

@@ -45,20 +45,20 @@ impl<'a, R: AsyncRead + AsyncSeek + Unpin> AsyncRead for ReadHandle<'a, R> {
}
pub struct S9pkReader<R: AsyncRead + AsyncSeek + Unpin = File> {
hash: Output<Sha512>,
hash_string: String,
hash: Option<Output<Sha512>>,
hash_string: Option<String>,
toc: TableOfContents,
pos: u64,
rdr: R,
}
impl S9pkReader {
pub async fn open<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
pub async fn open<P: AsRef<Path>>(path: P, check_sig: bool) -> Result<Self, Error> {
let p = path.as_ref();
let rdr = File::open(p)
.await
.with_ctx(|_| (crate::error::ErrorKind::Filesystem, p.display().to_string()))?;
Self::from_reader(rdr).await
Self::from_reader(rdr, check_sig).await
}
}
impl<R: AsyncRead + AsyncSeek + Unpin> S9pkReader<InstallProgressTracker<R>> {
@@ -69,13 +69,13 @@ impl<R: AsyncRead + AsyncSeek + Unpin> S9pkReader<InstallProgressTracker<R>> {
impl<R: AsyncRead + AsyncSeek + Unpin> S9pkReader<R> {
#[instrument(skip(self))]
pub async fn validate(&mut self) -> Result<(), Error> {
self.rdr.seek(SeekFrom::Start(0)).await?;
Ok(())
}
#[instrument(skip(rdr))]
pub async fn from_reader(mut rdr: R) -> Result<Self, Error> {
pub async fn from_reader(mut rdr: R, check_sig: bool) -> Result<Self, Error> {
let header = Header::deserialize(&mut rdr).await?;
let (hash, hash_string) = if check_sig {
let mut hasher = Sha512::new();
let mut buf = [0; 1024];
let mut read;
@@ -89,13 +89,21 @@ impl<R: AsyncRead + AsyncSeek + Unpin> S9pkReader<R> {
header
.pubkey
.verify_prehashed(hasher, Some(SIG_CONTEXT), &header.signature)?;
(
Some(hash),
Some(base32::encode(
base32::Alphabet::RFC4648 { padding: false },
hash.as_slice(),
)),
)
} else {
(None, None)
};
let pos = rdr.stream_position().await?;
Ok(S9pkReader {
hash_string: base32::encode(
base32::Alphabet::RFC4648 { padding: false },
hash.as_slice(),
),
hash_string,
hash,
toc: header.table_of_contents,
pos,
@@ -103,12 +111,17 @@ impl<R: AsyncRead + AsyncSeek + Unpin> S9pkReader<R> {
})
}
pub fn hash(&self) -> &Output<Sha512> {
&self.hash
pub fn hash(&self) -> Option<&Output<Sha512>> {
self.hash.as_ref()
}
pub fn hash_str(&self) -> &str {
self.hash_string.as_str()
pub fn hash_str(&self) -> Option<&str> {
self.hash_string.as_ref().map(|s| s.as_str())
}
pub async fn reset(&mut self) -> Result<(), Error> {
self.rdr.seek(SeekFrom::Start(0)).await?;
Ok(())
}
async fn read_handle<'a>(

View File

@@ -305,14 +305,15 @@ fn dir_copy<'a, P0: AsRef<Path> + 'a + Send + Sync, P1: AsRef<Path> + 'a + Send
format!("cp -P {} -> {}", src_path.display(), dst_path.display()),
)
})?;
tokio::fs::set_permissions(&dst_path, m.permissions())
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("chmod {}", dst_path.display()),
)
})?;
// Removed (see https://unix.stackexchange.com/questions/87200/change-permissions-for-a-symbolic-link):
// tokio::fs::set_permissions(&dst_path, m.permissions())
// .await
// .with_ctx(|_| {
// (
// crate::ErrorKind::Filesystem,
// format!("chmod {}", dst_path.display()),
// )
// })?;
}
Ok(())
})
@@ -324,7 +325,7 @@ fn dir_copy<'a, P0: AsRef<Path> + 'a + Send + Sync, P1: AsRef<Path> + 'a + Send
#[instrument(skip(ctx))]
async fn recover_v2(ctx: &SetupContext, recovery_partition: PartitionInfo) -> Result<(), Error> {
let recovery = TmpMountGuard::mount(&recovery_partition.logicalname).await?;
let recovery = TmpMountGuard::mount(&recovery_partition.logicalname, None).await?;
let secret_store = ctx.secret_store().await?;
let db = ctx.db(&secret_store).await?;
@@ -371,7 +372,7 @@ async fn recover_v2(ctx: &SetupContext, recovery_partition: PartitionInfo) -> Re
let volume_id = VolumeId::Custom(Id::try_from("main".to_owned())?);
for (pkg_id, info) in packages {
let volume_src_path = volume_path.join(&pkg_id);
let volume_dst_path = data_dir(&ctx.datadir, &pkg_id, &info.version, &volume_id);
let volume_dst_path = data_dir(&ctx.datadir, &pkg_id, &volume_id);
tokio::fs::create_dir_all(&volume_dst_path)
.await
.with_ctx(|_| {

View File

@@ -26,7 +26,7 @@ struct SoundInterface(Option<FileLock>);
impl SoundInterface {
#[instrument]
pub async fn lease() -> Result<Self, Error> {
let guard = FileLock::new(SOUND_LOCK_FILE).await?;
let guard = FileLock::new(SOUND_LOCK_FILE, true).await?;
tokio::fs::write(&*EXPORT_FILE, "0")
.await
.or_else(|e| {

View File

@@ -2,8 +2,7 @@ use std::collections::BTreeMap;
use chrono::{DateTime, Utc};
use color_eyre::eyre::eyre;
use futures::{FutureExt, StreamExt};
use patch_db::{DbHandle, HasModel, Map};
use patch_db::{DbHandle, HasModel};
use serde::{Deserialize, Serialize};
use tracing::instrument;

View File

@@ -1,91 +1,13 @@
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use reqwest::{Client, Url};
use sequence_trie::SequenceTrie;
use serde::Serialize;
use tracing::Subscriber;
use tracing_subscriber::filter::LevelFilter;
use tracing_subscriber::Layer;
use crate::version::COMMIT_HASH;
#[derive(Clone, Debug)]
pub struct ModuleMap {
trie: SequenceTrie<String, (LevelFilter, bool)>,
}
impl ModuleMap {
fn new(vals: BTreeMap<String, LevelFilter>) -> Self {
let mut module_trie = SequenceTrie::new();
for (k, v) in vals {
let mut include_submodules = false;
let module_key = k
.split("::")
.take_while(|&s| {
if s == "*" {
include_submodules = true;
false
} else {
true
}
})
.collect::<Vec<&str>>();
match module_trie.get_node(module_key.clone()) {
None => match module_trie.get_ancestor_node(module_key.clone()) {
None => {
module_trie.insert(module_key, (v, include_submodules));
}
Some(ancestor) => match ancestor.value() {
None => {
module_trie.insert(module_key, (v, include_submodules));
}
Some((_, sub)) => {
if !sub {
module_trie.insert(module_key, (v, include_submodules));
}
}
},
},
Some(n) => match n.value() {
None => {
module_trie.insert(module_key.clone(), (v, include_submodules));
if include_submodules {
let new_node = module_trie.get_node_mut(module_key).unwrap(); // we just inserted it
let child_keys = new_node
.children_with_keys()
.into_iter()
.map(|x| x.0.clone())
.collect::<Vec<String>>();
for c in child_keys {
new_node.remove(std::iter::once(&c));
}
}
}
Some(_) => unreachable!("Trie build failed on 'impossible' duplicate"),
},
}
}
ModuleMap { trie: module_trie }
}
fn level_for<'a>(&'a self, k: &str) -> &'a LevelFilter {
let module_key = k.split("::");
match self.trie.get(module_key.clone()) {
None => match self.trie.get_ancestor(module_key) {
None => &LevelFilter::OFF,
Some((level_filter, include_submodules)) => {
if *include_submodules {
level_filter
} else {
&LevelFilter::OFF
}
}
},
Some((level_filter, _)) => level_filter,
}
}
}
pub struct SharingLayer {
log_epoch: Arc<AtomicU64>,
sharing: Arc<AtomicBool>,
@@ -187,53 +109,3 @@ pub async fn order_level() {
pub fn module() {
println!("{}", module_path!())
}
proptest::proptest! {
#[test]
fn submodules_handled_by_parent(s0 in "[a-z][a-z0-9_]+", s1 in "[a-z][a-z0-9_]+", level in filter_strategy()) {
proptest::prop_assume!(level > LevelFilter::OFF);
let mut hm = BTreeMap::new();
hm.insert(format!("{}::*", s0.clone()), level);
let mod_map = ModuleMap::new(hm);
proptest::prop_assert_eq!(mod_map.level_for(&format!("{}::{}", s0, s1)), &level)
}
#[test]
fn submodules_ignored_by_parent(s0 in "[a-z][a-z0-9_]+", s1 in "[a-z][a-z0-9_]+", level in filter_strategy()) {
proptest::prop_assume!(level > LevelFilter::OFF);
let mut hm = BTreeMap::new();
hm.insert(s0.clone(), level);
let mod_map = ModuleMap::new(hm);
proptest::prop_assert_eq!(mod_map.level_for(&format!("{}::{}", s0, s1)), &LevelFilter::OFF)
}
#[test]
fn duplicate_insertion_ignored(s0 in "[a-z][a-z0-9_]+", s1 in "[a-z][a-z0-9_]+", level in filter_strategy()) {
proptest::prop_assume!(level > LevelFilter::OFF);
let mut hm = BTreeMap::new();
hm.insert(format!("{}::*", s0.clone()), level);
let sub = format!("{}::{}", s0, s1);
hm.insert(sub.clone(), level);
let mod_map = ModuleMap::new(hm);
proptest::prop_assert_eq!(mod_map.trie.get(sub.split("::")), None)
}
#[test]
fn parent_child_simul(s0 in "[a-z][a-z0-9_]+", s1 in "[a-z][a-z0-9_]+", level0 in filter_strategy(), level1 in filter_strategy()) {
let mut hm = BTreeMap::new();
hm.insert(s0.clone(), level0);
let sub = format!("{}::{}", s0, s1);
hm.insert(sub.clone(), level1);
let mod_map = ModuleMap::new(hm);
proptest::prop_assert_eq!(mod_map.level_for(&s0), &level0);
proptest::prop_assert_eq!(mod_map.level_for(&sub), &level1);
}
}
fn filter_strategy() -> impl proptest::strategy::Strategy<Value = LevelFilter> {
use proptest::strategy::Just;
proptest::prop_oneof![
Just(LevelFilter::OFF),
Just(LevelFilter::ERROR),
Just(LevelFilter::WARN),
Just(LevelFilter::INFO),
Just(LevelFilter::DEBUG),
Just(LevelFilter::TRACE),
]
}

View File

@@ -23,6 +23,7 @@ use serde_json::Value;
use tokio::fs::File;
use tokio::sync::{Mutex, OwnedMutexGuard, RwLock};
use tokio::task::{JoinError, JoinHandle};
use tracing::instrument;
use crate::shutdown::Shutdown;
use crate::{Error, ResultExt as _};
@@ -733,8 +734,6 @@ pub fn deserialize_number_permissive<
>(
deserializer: D,
) -> std::result::Result<T, D::Error> {
use num::cast::FromPrimitive;
struct Visitor<T: FromStr<Err = E> + num::cast::FromPrimitive, E>(std::marker::PhantomData<T>);
impl<'de, T: FromStr<Err = Err> + num::cast::FromPrimitive, Err: std::fmt::Display>
serde::de::Visitor<'de> for Visitor<T, Err>
@@ -998,7 +997,8 @@ impl Drop for FileLock {
}
}
impl FileLock {
pub async fn new(path: impl AsRef<Path> + Send + Sync) -> Result<Self, Error> {
#[instrument(skip(path))]
pub async fn new(path: impl AsRef<Path> + Send + Sync, blocking: bool) -> Result<Self, Error> {
lazy_static! {
static ref INTERNAL_LOCKS: Mutex<BTreeMap<PathBuf, Arc<Mutex<()>>>> =
Mutex::new(BTreeMap::new());
@@ -1010,7 +1010,12 @@ impl FileLock {
}
let tex = internal_locks.get(&path).unwrap().clone();
drop(internal_locks);
let tex_guard = tex.lock_owned().await;
let tex_guard = if blocking {
tex.lock_owned().await
} else {
tex.try_lock_owned()
.with_kind(crate::ErrorKind::Filesystem)?
};
let parent = path.parent().unwrap_or(Path::new("/"));
if tokio::fs::metadata(parent).await.is_err() {
tokio::fs::create_dir_all(parent)
@@ -1020,8 +1025,8 @@ impl FileLock {
let f = File::create(&path)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, path.display().to_string()))?;
let file_guard = tokio::task::spawn_blocking(|| {
fd_lock_rs::FdLock::lock(f, fd_lock_rs::LockType::Exclusive, true)
let file_guard = tokio::task::spawn_blocking(move || {
fd_lock_rs::FdLock::lock(f, fd_lock_rs::LockType::Exclusive, blocking)
})
.await
.with_kind(crate::ErrorKind::Unknown)?

View File

@@ -12,10 +12,10 @@ impl VersionT for Version {
fn semver(&self) -> emver::Version {
V0_3_0
}
async fn up<Db: DbHandle>(&self, db: &mut Db) -> Result<(), Error> {
async fn up<Db: DbHandle>(&self, _db: &mut Db) -> Result<(), Error> {
Ok(())
}
async fn down<Db: DbHandle>(&self, db: &mut Db) -> Result<(), Error> {
async fn down<Db: DbHandle>(&self, _db: &mut Db) -> Result<(), Error> {
Ok(())
}
}

View File

@@ -133,12 +133,7 @@ impl HasModel for Volumes {
type Model = MapModel<Self>;
}
pub fn data_dir<P: AsRef<Path>>(
datadir: P,
pkg_id: &PackageId,
version: &Version,
volume_id: &VolumeId,
) -> PathBuf {
pub fn data_dir<P: AsRef<Path>>(datadir: P, pkg_id: &PackageId, volume_id: &VolumeId) -> PathBuf {
datadir
.as_ref()
.join(PKG_VOLUME_DIR)
@@ -208,7 +203,7 @@ impl Volume {
volume_id: &VolumeId,
) -> PathBuf {
match self {
Volume::Data { .. } => data_dir(&ctx.datadir, pkg_id, version, volume_id),
Volume::Data { .. } => data_dir(&ctx.datadir, pkg_id, volume_id),
Volume::Assets {} => asset_dir(&ctx.datadir, pkg_id, version).join(volume_id),
Volume::Pointer {
package_id,

View File

@@ -887,7 +887,6 @@ dependencies = [
"rpc-toolkit",
"rust-argon2",
"scopeguard",
"sequence_trie",
"serde",
"serde_json",
"serde_with",
@@ -2814,12 +2813,6 @@ dependencies = [
"pest",
]
[[package]]
name = "sequence_trie"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ee22067b7ccd072eeb64454b9c6e1b33b61cd0d49e895fd48676a184580e0c3"
[[package]]
name = "serde"
version = "1.0.130"

View File

@@ -1,5 +1,7 @@
use std::{path::Path, process::Stdio};
use embassy::disk::main::DEFAULT_PASSWORD;
pub fn create_backup(
mountpoint: impl AsRef<Path>,
data_path: impl AsRef<Path>,
@@ -30,11 +32,18 @@ pub fn create_backup(
));
}
}
let data_res = data_cmd
let data_output = data_cmd
.env("PASSPHRASE", DEFAULT_PASSWORD)
.arg(data_path)
.arg(format!("file://{}", mountpoint.display().to_string()))
.output();
data_res?;
.stderr(Stdio::piped())
.output()?;
if !data_output.status.success() {
return Err(anyhow::anyhow!(
"duplicity error: {}",
String::from_utf8(data_output.stderr).unwrap()
));
}
Ok(())
}
@@ -47,6 +56,7 @@ pub fn restore_backup(
let data_path = std::fs::canonicalize(data_path)?;
let data_output = std::process::Command::new("duplicity")
.env("PASSPHRASE", DEFAULT_PASSWORD)
.arg("--force")
.arg(format!("file://{}", mountpoint.display().to_string()))
.arg(&data_path)