rename appmgr

This commit is contained in:
Aiden McClelland
2022-01-21 19:02:23 -07:00
committed by Aiden McClelland
parent 9cf379f9ee
commit edde478382
124 changed files with 25 additions and 45 deletions

View File

@@ -0,0 +1,399 @@
use std::collections::BTreeMap;
use std::sync::Arc;
use chrono::Utc;
use color_eyre::eyre::eyre;
use openssl::pkey::{PKey, Private};
use openssl::x509::X509;
use patch_db::{DbHandle, LockType, PatchDbHandle, Revision};
use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio::io::AsyncWriteExt;
use torut::onion::TorSecretKeyV3;
use tracing::instrument;
use super::target::BackupTargetId;
use super::PackageBackupReport;
use crate::auth::check_password_against_db;
use crate::backup::{BackupReport, ServerBackupReport};
use crate::context::RpcContext;
use crate::db::model::ServerStatus;
use crate::db::util::WithRevision;
use crate::disk::mount::backup::BackupMountGuard;
use crate::disk::mount::guard::TmpMountGuard;
use crate::notifications::NotificationLevel;
use crate::s9pk::manifest::PackageId;
use crate::status::MainStatus;
use crate::util::serde::IoFormat;
use crate::util::{display_none, AtomicFile};
use crate::version::VersionT;
use crate::Error;
#[derive(Debug)]
pub struct OsBackup {
pub tor_key: TorSecretKeyV3,
pub root_ca_key: PKey<Private>,
pub root_ca_cert: X509,
pub ui: Value,
}
impl<'de> Deserialize<'de> for OsBackup {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(rename = "kebab-case")]
struct OsBackupDe {
tor_key: String,
root_ca_key: String,
root_ca_cert: String,
ui: Value,
}
let int = OsBackupDe::deserialize(deserializer)?;
let key_vec = base32::decode(base32::Alphabet::RFC4648 { padding: true }, &int.tor_key)
.ok_or(serde::de::Error::invalid_value(
serde::de::Unexpected::Str(&int.tor_key),
&"an RFC4648 encoded string",
))?;
if key_vec.len() != 64 {
return Err(serde::de::Error::invalid_value(
serde::de::Unexpected::Str(&int.tor_key),
&"a 64 byte value encoded as an RFC4648 string",
));
}
let mut key_slice = [0; 64];
key_slice.clone_from_slice(&key_vec);
Ok(OsBackup {
tor_key: TorSecretKeyV3::from(key_slice),
root_ca_key: PKey::<Private>::private_key_from_pem(int.root_ca_key.as_bytes())
.map_err(serde::de::Error::custom)?,
root_ca_cert: X509::from_pem(int.root_ca_cert.as_bytes())
.map_err(serde::de::Error::custom)?,
ui: int.ui,
})
}
}
impl Serialize for OsBackup {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
#[derive(Serialize)]
#[serde(rename = "kebab-case")]
struct OsBackupSer<'a> {
tor_key: String,
root_ca_key: String,
root_ca_cert: String,
ui: &'a Value,
}
OsBackupSer {
tor_key: base32::encode(
base32::Alphabet::RFC4648 { padding: true },
&self.tor_key.as_bytes(),
),
root_ca_key: String::from_utf8(
self.root_ca_key
.private_key_to_pem_pkcs8()
.map_err(serde::ser::Error::custom)?,
)
.map_err(serde::ser::Error::custom)?,
root_ca_cert: String::from_utf8(
self.root_ca_cert
.to_pem()
.map_err(serde::ser::Error::custom)?,
)
.map_err(serde::ser::Error::custom)?,
ui: &self.ui,
}
.serialize(serializer)
}
}
#[command(rename = "create", display(display_none))]
#[instrument(skip(ctx, old_password, password))]
pub async fn backup_all(
#[context] ctx: RpcContext,
#[arg(rename = "target-id")] target_id: BackupTargetId,
#[arg(rename = "old-password", long = "old-password")] old_password: Option<String>,
#[arg] password: String,
) -> Result<WithRevision<()>, Error> {
let mut db = ctx.db.handle();
check_password_against_db(&mut ctx.secret_store.acquire().await?, &password).await?;
let fs = target_id
.load(&mut ctx.secret_store.acquire().await?)
.await?;
let mut backup_guard = BackupMountGuard::mount(
TmpMountGuard::mount(&fs).await?,
old_password.as_ref().unwrap_or(&password),
)
.await?;
if old_password.is_some() {
backup_guard.change_password(&password)?;
}
let revision = assure_backing_up(&mut db).await?;
tokio::task::spawn(async move {
let backup_res = perform_backup(&ctx, &mut db, backup_guard).await;
let status_model = crate::db::DatabaseModel::new().server_info().status();
status_model
.clone()
.lock(&mut db, LockType::Write)
.await
.expect("failed to lock server status");
match backup_res {
Ok(report) if report.iter().all(|(_, rep)| rep.error.is_none()) => ctx
.notification_manager
.notify(
&mut db,
None,
NotificationLevel::Success,
"Backup Complete".to_owned(),
"Your backup has completed".to_owned(),
BackupReport {
server: ServerBackupReport {
attempted: true,
error: None,
},
packages: report,
},
None,
)
.await
.expect("failed to send notification"),
Ok(report) => ctx
.notification_manager
.notify(
&mut db,
None,
NotificationLevel::Warning,
"Backup Complete".to_owned(),
"Your backup has completed, but some package(s) failed to backup".to_owned(),
BackupReport {
server: ServerBackupReport {
attempted: true,
error: None,
},
packages: report,
},
None,
)
.await
.expect("failed to send notification"),
Err(e) => {
tracing::error!("Backup Failed: {}", e);
tracing::debug!("{:?}", e);
ctx.notification_manager
.notify(
&mut db,
None,
NotificationLevel::Error,
"Backup Failed".to_owned(),
"Your backup failed to complete.".to_owned(),
BackupReport {
server: ServerBackupReport {
attempted: true,
error: Some(e.to_string()),
},
packages: BTreeMap::new(),
},
None,
)
.await
.expect("failed to send notification");
}
}
status_model
.put(&mut db, &ServerStatus::Running)
.await
.expect("failed to change server status");
});
Ok(WithRevision {
response: (),
revision,
})
}
#[instrument(skip(db))]
async fn assure_backing_up(db: &mut PatchDbHandle) -> Result<Option<Arc<Revision>>, Error> {
let mut tx = db.begin().await?;
let mut info = crate::db::DatabaseModel::new()
.server_info()
.get_mut(&mut tx)
.await?;
match &info.status {
ServerStatus::Updating => {
return Err(Error::new(
eyre!("Server is updating!"),
crate::ErrorKind::InvalidRequest,
))
}
ServerStatus::Updated => {
return Err(Error::new(
eyre!("Server is updated and needs to be reset"),
crate::ErrorKind::InvalidRequest,
))
}
ServerStatus::BackingUp => {
return Err(Error::new(
eyre!("Server is already backing up!"),
crate::ErrorKind::InvalidRequest,
))
}
ServerStatus::Running => (),
}
info.status = ServerStatus::BackingUp;
info.save(&mut tx).await?;
Ok(tx.commit(None).await?)
}
#[instrument(skip(ctx, db, backup_guard))]
async fn perform_backup<Db: DbHandle>(
ctx: &RpcContext,
mut db: Db,
mut backup_guard: BackupMountGuard<TmpMountGuard>,
) -> Result<BTreeMap<PackageId, PackageBackupReport>, Error> {
let mut backup_report = BTreeMap::new();
for package_id in crate::db::DatabaseModel::new()
.package_data()
.keys(&mut db, true)
.await?
{
let installed_model = if let Some(installed_model) = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&package_id)
.and_then(|m| m.installed())
.check(&mut db)
.await?
{
installed_model
} else {
continue;
};
let main_status_model = installed_model.clone().status().main();
let mut tx = db.begin().await?; // for lock scope
main_status_model.lock(&mut tx, LockType::Write).await?;
let (started, health) = match main_status_model.get(&mut tx, true).await?.into_owned() {
MainStatus::Starting => (Some(Utc::now()), Default::default()),
MainStatus::Running { started, health } => (Some(started), health.clone()),
MainStatus::Stopped | MainStatus::Stopping => (None, Default::default()),
MainStatus::BackingUp { .. } => {
backup_report.insert(
package_id,
PackageBackupReport {
error: Some(
"Can't do backup because service is in a backing up state".to_owned(),
),
},
);
continue;
}
};
main_status_model
.put(
&mut tx,
&MainStatus::BackingUp {
started: started.clone(),
health: health.clone(),
},
)
.await?;
tx.save().await?; // drop locks
let manifest = installed_model
.clone()
.manifest()
.get(&mut db, true)
.await?;
ctx.managers
.get(&(manifest.id.clone(), manifest.version.clone()))
.await
.ok_or_else(|| {
Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest)
})?
.synchronize()
.await;
installed_model.lock(&mut db, LockType::Write).await?;
let guard = backup_guard.mount_package_backup(&package_id).await?;
let res = manifest
.backup
.create(
&ctx,
&package_id,
&manifest.title,
&manifest.version,
&manifest.interfaces,
&manifest.volumes,
)
.await;
drop(guard);
backup_report.insert(
package_id.clone(),
PackageBackupReport {
error: res.as_ref().err().map(|e| e.to_string()),
},
);
let mut tx = db.begin().await?;
if let Ok(pkg_meta) = res {
installed_model
.last_backup()
.put(&mut tx, &Some(pkg_meta.timestamp))
.await?;
backup_guard
.metadata
.package_backups
.insert(package_id, pkg_meta);
}
main_status_model
.put(
&mut tx,
&match started {
Some(started) => MainStatus::Running { started, health },
None => MainStatus::Stopped,
},
)
.await?;
tx.save().await?;
}
let (root_ca_key, root_ca_cert) = ctx.net_controller.ssl.export_root_ca().await?;
let mut os_backup_file = AtomicFile::new(backup_guard.as_ref().join("os-backup.cbor")).await?;
os_backup_file
.write_all(
&IoFormat::Cbor.to_vec(&OsBackup {
tor_key: ctx.net_controller.tor.embassyd_tor_key().await,
root_ca_key,
root_ca_cert,
ui: crate::db::DatabaseModel::new()
.ui()
.get(&mut db, true)
.await?
.into_owned(),
})?,
)
.await?;
os_backup_file.save().await?;
let timestamp = Some(Utc::now());
backup_guard.unencrypted_metadata.version = crate::version::Current::new().semver().into();
backup_guard.unencrypted_metadata.full = true;
backup_guard.metadata.version = crate::version::Current::new().semver().into();
backup_guard.metadata.timestamp = timestamp;
backup_guard.save_and_unmount().await?;
crate::db::DatabaseModel::new()
.server_info()
.last_backup()
.put(&mut db, &timestamp)
.await?;
Ok(backup_report)
}

236
backend/src/backup/mod.rs Normal file
View File

@@ -0,0 +1,236 @@
use std::collections::BTreeMap;
use std::path::Path;
use chrono::{DateTime, Utc};
use color_eyre::eyre::eyre;
use patch_db::{DbHandle, HasModel, LockType};
use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
use sqlx::{Executor, Sqlite};
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
use tracing::instrument;
use self::target::PackageBackupInfo;
use crate::action::{ActionImplementation, NoOutput};
use crate::context::RpcContext;
use crate::dependencies::reconfigure_dependents_with_live_pointers;
use crate::install::PKG_ARCHIVE_DIR;
use crate::net::interface::{InterfaceId, Interfaces};
use crate::s9pk::manifest::PackageId;
use crate::util::serde::IoFormat;
use crate::util::{AtomicFile, Version};
use crate::version::{Current, VersionT};
use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR};
use crate::{Error, ResultExt};
pub mod backup_bulk;
pub mod restore;
pub mod target;
#[derive(Debug, Deserialize, Serialize)]
pub struct BackupReport {
server: ServerBackupReport,
packages: BTreeMap<PackageId, PackageBackupReport>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct ServerBackupReport {
attempted: bool,
error: Option<String>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct PackageBackupReport {
error: Option<String>,
}
#[command(subcommands(backup_bulk::backup_all, target::target))]
pub fn backup() -> Result<(), Error> {
Ok(())
}
#[command(rename = "backup", subcommands(restore::restore_packages_rpc))]
pub fn package_backup() -> Result<(), Error> {
Ok(())
}
#[derive(Deserialize, Serialize)]
struct BackupMetadata {
pub timestamp: DateTime<Utc>,
pub tor_keys: BTreeMap<InterfaceId, String>,
}
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
pub struct BackupActions {
pub create: ActionImplementation,
pub restore: ActionImplementation,
}
impl BackupActions {
#[instrument(skip(ctx))]
pub async fn create(
&self,
ctx: &RpcContext,
pkg_id: &PackageId,
pkg_title: &str,
pkg_version: &Version,
interfaces: &Interfaces,
volumes: &Volumes,
) -> Result<PackageBackupInfo, Error> {
let mut volumes = volumes.to_readonly();
volumes.insert(VolumeId::Backup, Volume::Backup { readonly: false });
let backup_dir = backup_dir(pkg_id);
if tokio::fs::metadata(&backup_dir).await.is_err() {
tokio::fs::create_dir_all(&backup_dir).await?
}
self.create
.execute::<(), NoOutput>(
ctx,
pkg_id,
pkg_version,
Some("CreateBackup"),
&volumes,
None,
false,
None,
)
.await?
.map_err(|e| eyre!("{}", e.1))
.with_kind(crate::ErrorKind::Backup)?;
let tor_keys = interfaces
.tor_keys(&mut ctx.secret_store.acquire().await?, pkg_id)
.await?
.into_iter()
.map(|(id, key)| {
(
id,
base32::encode(base32::Alphabet::RFC4648 { padding: true }, &key.as_bytes()),
)
})
.collect();
let tmp_path = Path::new(BACKUP_DIR)
.join(pkg_id)
.join(format!("{}.s9pk", pkg_id));
let s9pk_path = ctx
.datadir
.join(PKG_ARCHIVE_DIR)
.join(pkg_id)
.join(pkg_version.as_str())
.join(format!("{}.s9pk", pkg_id));
let mut infile = File::open(&s9pk_path).await?;
let mut outfile = AtomicFile::new(&tmp_path).await?;
tokio::io::copy(&mut infile, &mut *outfile)
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("cp {} -> {}", s9pk_path.display(), tmp_path.display()),
)
})?;
outfile.save().await?;
let timestamp = Utc::now();
let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
let mut outfile = AtomicFile::new(&metadata_path).await?;
outfile
.write_all(&IoFormat::Cbor.to_vec(&BackupMetadata {
timestamp,
tor_keys,
})?)
.await?;
outfile.save().await?;
Ok(PackageBackupInfo {
os_version: Current::new().semver().into(),
title: pkg_title.to_owned(),
version: pkg_version.clone(),
timestamp,
})
}
#[instrument(skip(ctx, db, secrets))]
pub async fn restore<Ex, Db: DbHandle>(
&self,
ctx: &RpcContext,
db: &mut Db,
secrets: &mut Ex,
pkg_id: &PackageId,
pkg_version: &Version,
interfaces: &Interfaces,
volumes: &Volumes,
) -> Result<(), Error>
where
for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>,
{
let mut volumes = volumes.clone();
volumes.insert(VolumeId::Backup, Volume::Backup { readonly: true });
self.restore
.execute::<(), NoOutput>(
ctx,
pkg_id,
pkg_version,
Some("RestoreBackup"),
&volumes,
None,
false,
None,
)
.await?
.map_err(|e| eyre!("{}", e.1))
.with_kind(crate::ErrorKind::Restore)?;
let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
let metadata: BackupMetadata = IoFormat::Cbor.from_slice(
&tokio::fs::read(&metadata_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
metadata_path.display().to_string(),
)
})?,
)?;
for (iface, key) in metadata.tor_keys {
let key_vec = base32::decode(base32::Alphabet::RFC4648 { padding: true }, &key)
.ok_or_else(|| {
Error::new(
eyre!("invalid base32 string"),
crate::ErrorKind::Deserialization,
)
})?;
sqlx::query!(
"REPLACE INTO tor (package, interface, key) VALUES (?, ?, ?)",
**pkg_id,
*iface,
key_vec,
)
.execute(&mut *secrets)
.await?;
}
crate::db::DatabaseModel::new()
.package_data()
.lock(db, LockType::Write)
.await?;
crate::db::DatabaseModel::new()
.package_data()
.idx_model(pkg_id)
.expect(db)
.await?
.installed()
.expect(db)
.await?
.interface_addresses()
.put(db, &interfaces.install(&mut *secrets, pkg_id).await?)
.await?;
let entry = crate::db::DatabaseModel::new()
.package_data()
.idx_model(pkg_id)
.expect(db)
.await?
.installed()
.expect(db)
.await?
.get(db, true)
.await?;
reconfigure_dependents_with_live_pointers(ctx, db, &entry).await?;
Ok(())
}
}

View File

@@ -0,0 +1,388 @@
use std::collections::BTreeMap;
use std::path::Path;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;
use clap::ArgMatches;
use color_eyre::eyre::eyre;
use futures::future::BoxFuture;
use futures::FutureExt;
use openssl::x509::X509;
use patch_db::{DbHandle, PatchDbHandle, Revision};
use rpc_toolkit::command;
use tokio::fs::File;
use tokio::task::JoinHandle;
use torut::onion::OnionAddressV3;
use tracing::instrument;
use super::target::BackupTargetId;
use crate::auth::check_password_against_db;
use crate::backup::backup_bulk::OsBackup;
use crate::context::{RpcContext, SetupContext};
use crate::db::model::{PackageDataEntry, StaticFiles};
use crate::db::util::WithRevision;
use crate::disk::mount::backup::{BackupMountGuard, PackageBackupMountGuard};
use crate::disk::mount::guard::TmpMountGuard;
use crate::install::progress::InstallProgress;
use crate::install::{download_install_s9pk, PKG_PUBLIC_DIR};
use crate::net::ssl::SslManager;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::s9pk::reader::S9pkReader;
use crate::setup::RecoveryStatus;
use crate::util::display_none;
use crate::util::io::dir_size;
use crate::util::serde::IoFormat;
use crate::volume::{backup_dir, BACKUP_DIR, PKG_VOLUME_DIR};
use crate::{Error, ResultExt};
fn parse_comma_separated(arg: &str, _: &ArgMatches<'_>) -> Result<Vec<PackageId>, Error> {
arg.split(",")
.map(|s| s.trim().parse().map_err(Error::from))
.collect()
}
#[command(rename = "restore", display(display_none))]
#[instrument(skip(ctx, old_password, password))]
pub async fn restore_packages_rpc(
#[context] ctx: RpcContext,
#[arg(parse(parse_comma_separated))] ids: Vec<PackageId>,
#[arg(rename = "target-id")] target_id: BackupTargetId,
#[arg(rename = "old-password", long = "old-password")] old_password: Option<String>,
#[arg] password: String,
) -> Result<WithRevision<()>, Error> {
let mut db = ctx.db.handle();
check_password_against_db(&mut ctx.secret_store.acquire().await?, &password).await?;
let fs = target_id
.load(&mut ctx.secret_store.acquire().await?)
.await?;
let mut backup_guard = BackupMountGuard::mount(
TmpMountGuard::mount(&fs).await?,
old_password.as_ref().unwrap_or(&password),
)
.await?;
if old_password.is_some() {
backup_guard.change_password(&password)?;
}
let (revision, backup_guard, tasks, _) =
restore_packages(&ctx, &mut db, backup_guard, ids).await?;
tokio::spawn(async {
futures::future::join_all(tasks).await;
if let Err(e) = backup_guard.unmount().await {
tracing::error!("Error unmounting backup drive: {}", e);
tracing::debug!("{:?}", e);
}
});
Ok(WithRevision {
response: (),
revision,
})
}
async fn approximate_progress(
rpc_ctx: &RpcContext,
progress: &mut ProgressInfo,
) -> Result<(), Error> {
for (id, size) in &mut progress.target_volume_size {
let dir = rpc_ctx.datadir.join(PKG_VOLUME_DIR).join(id).join("data");
if tokio::fs::metadata(&dir).await.is_err() {
*size = 0;
} else {
*size = dir_size(&dir).await?;
}
}
Ok(())
}
async fn approximate_progress_loop(
ctx: &SetupContext,
rpc_ctx: &RpcContext,
mut starting_info: ProgressInfo,
) {
loop {
if let Err(e) = approximate_progress(rpc_ctx, &mut starting_info).await {
tracing::error!("Failed to approximate restore progress: {}", e);
tracing::debug!("{:?}", e);
} else {
*ctx.recovery_status.write().await = Some(Ok(starting_info.flatten()));
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
}
#[derive(Debug, Default)]
struct ProgressInfo {
package_installs: BTreeMap<PackageId, Arc<InstallProgress>>,
src_volume_size: BTreeMap<PackageId, u64>,
target_volume_size: BTreeMap<PackageId, u64>,
}
impl ProgressInfo {
fn flatten(&self) -> RecoveryStatus {
let mut total_bytes = 0;
let mut bytes_transferred = 0;
for (_, progress) in &self.package_installs {
total_bytes += ((progress.size.unwrap_or(0) as f64) * 2.2) as u64;
bytes_transferred += progress.downloaded.load(Ordering::SeqCst);
bytes_transferred += ((progress.validated.load(Ordering::SeqCst) as f64) * 0.2) as u64;
bytes_transferred += progress.unpacked.load(Ordering::SeqCst);
}
for (_, size) in &self.src_volume_size {
total_bytes += *size;
}
for (_, size) in &self.target_volume_size {
bytes_transferred += *size;
}
if bytes_transferred > total_bytes {
bytes_transferred = total_bytes;
}
RecoveryStatus {
total_bytes,
bytes_transferred,
complete: false,
}
}
}
#[instrument(skip(ctx))]
pub async fn recover_full_embassy(
ctx: SetupContext,
disk_guid: Arc<String>,
embassy_password: String,
recovery_source: TmpMountGuard,
recovery_password: Option<String>,
) -> Result<(OnionAddressV3, X509, BoxFuture<'static, Result<(), Error>>), Error> {
let backup_guard = BackupMountGuard::mount(
recovery_source,
recovery_password.as_deref().unwrap_or_default(),
)
.await?;
let os_backup_path = backup_guard.as_ref().join("os-backup.cbor");
let os_backup: OsBackup =
IoFormat::Cbor.from_slice(&tokio::fs::read(&os_backup_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
os_backup_path.display().to_string(),
)
})?)?;
let password = argon2::hash_encoded(
embassy_password.as_bytes(),
&rand::random::<[u8; 16]>()[..],
&argon2::Config::default(),
)
.with_kind(crate::ErrorKind::PasswordHashGeneration)?;
let key_vec = os_backup.tor_key.as_bytes().to_vec();
let secret_store = ctx.secret_store().await?;
sqlx::query!(
"REPLACE INTO account (id, password, tor_key) VALUES (?, ?, ?)",
0,
password,
key_vec,
)
.execute(&mut secret_store.acquire().await?)
.await?;
SslManager::import_root_ca(
secret_store.clone(),
os_backup.root_ca_key,
os_backup.root_ca_cert.clone(),
)
.await?;
secret_store.close().await;
Ok((
os_backup.tor_key.public().get_onion_address(),
os_backup.root_ca_cert,
async move {
let rpc_ctx = RpcContext::init(ctx.config_path.as_ref(), disk_guid).await?;
let mut db = rpc_ctx.db.handle();
let ids = backup_guard
.metadata
.package_backups
.keys()
.cloned()
.collect();
let (_, backup_guard, tasks, progress_info) = restore_packages(
&rpc_ctx,
&mut db,
backup_guard,
ids,
)
.await?;
tokio::select! {
res = futures::future::join_all(tasks) => res.into_iter().map(|res| res.with_kind(crate::ErrorKind::Unknown).and_then(|a|a)).collect::<Result<(), Error>>()?,
_ = approximate_progress_loop(&ctx, &rpc_ctx, progress_info) => unreachable!(concat!(module_path!(), "::approximate_progress_loop should not terminate")),
}
backup_guard.unmount().await?;
rpc_ctx.shutdown().await
}.boxed()
))
}
async fn restore_packages(
ctx: &RpcContext,
db: &mut PatchDbHandle,
backup_guard: BackupMountGuard<TmpMountGuard>,
ids: Vec<PackageId>,
) -> Result<
(
Option<Arc<Revision>>,
BackupMountGuard<TmpMountGuard>,
Vec<JoinHandle<Result<(), Error>>>,
ProgressInfo,
),
Error,
> {
let (revision, guards) = assure_restoring(&ctx, db, ids, &backup_guard).await?;
let mut progress_info = ProgressInfo::default();
let mut tasks = Vec::with_capacity(guards.len());
for (manifest, guard) in guards {
let id = manifest.id.clone();
let (progress, task) = restore_package(ctx.clone(), manifest, guard).await?;
progress_info.package_installs.insert(id.clone(), progress);
progress_info
.src_volume_size
.insert(id.clone(), dir_size(backup_dir(&id)).await?);
progress_info.target_volume_size.insert(id.clone(), 0);
tasks.push(tokio::spawn(async move {
if let Err(e) = task.await {
tracing::error!("Error restoring package {}: {}", id, e);
tracing::debug!("{:?}", e);
Err(e)
} else {
Ok(())
}
}));
}
Ok((revision, backup_guard, tasks, progress_info))
}
#[instrument(skip(ctx, db, backup_guard))]
async fn assure_restoring(
ctx: &RpcContext,
db: &mut PatchDbHandle,
ids: Vec<PackageId>,
backup_guard: &BackupMountGuard<TmpMountGuard>,
) -> Result<
(
Option<Arc<Revision>>,
Vec<(Manifest, PackageBackupMountGuard)>,
),
Error,
> {
let mut tx = db.begin().await?;
let mut guards = Vec::with_capacity(ids.len());
for id in ids {
let mut model = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&id)
.get_mut(&mut tx)
.await?;
if !model.is_none() {
return Err(Error::new(
eyre!("Can't restore over existing package: {}", id),
crate::ErrorKind::InvalidRequest,
));
}
let guard = backup_guard.mount_package_backup(&id).await?;
let s9pk_path = Path::new(BACKUP_DIR).join(&id).join(format!("{}.s9pk", id));
let mut rdr = S9pkReader::open(&s9pk_path, false).await?;
let manifest = rdr.manifest().await?;
let version = manifest.version.clone();
let progress = InstallProgress::new(Some(tokio::fs::metadata(&s9pk_path).await?.len()));
let public_dir_path = ctx
.datadir
.join(PKG_PUBLIC_DIR)
.join(&id)
.join(version.as_str());
tokio::fs::create_dir_all(&public_dir_path).await?;
let license_path = public_dir_path.join("LICENSE.md");
let mut dst = File::create(&license_path).await?;
tokio::io::copy(&mut rdr.license().await?, &mut dst).await?;
dst.sync_all().await?;
let instructions_path = public_dir_path.join("INSTRUCTIONS.md");
let mut dst = File::create(&instructions_path).await?;
tokio::io::copy(&mut rdr.instructions().await?, &mut dst).await?;
dst.sync_all().await?;
let icon_path = Path::new("icon").with_extension(&manifest.assets.icon_type());
let icon_path = public_dir_path.join(&icon_path);
let mut dst = File::create(&icon_path).await?;
tokio::io::copy(&mut rdr.icon().await?, &mut dst).await?;
dst.sync_all().await?;
*model = Some(PackageDataEntry::Restoring {
install_progress: progress.clone(),
static_files: StaticFiles::local(&id, &version, manifest.assets.icon_type()),
manifest: manifest.clone(),
});
model.save(&mut tx).await?;
guards.push((manifest, guard));
}
Ok((tx.commit(None).await?, guards))
}
#[instrument(skip(ctx, guard))]
async fn restore_package<'a>(
ctx: RpcContext,
manifest: Manifest,
guard: PackageBackupMountGuard,
) -> Result<(Arc<InstallProgress>, BoxFuture<'static, Result<(), Error>>), Error> {
let s9pk_path = Path::new(BACKUP_DIR)
.join(&manifest.id)
.join(format!("{}.s9pk", manifest.id));
let len = tokio::fs::metadata(&s9pk_path)
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
s9pk_path.display().to_string(),
)
})?
.len();
let file = File::open(&s9pk_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
s9pk_path.display().to_string(),
)
})?;
let progress = InstallProgress::new(Some(len));
Ok((
progress.clone(),
async move {
download_install_s9pk(&ctx, &manifest, progress, file).await?;
guard.unmount().await?;
Ok(())
}
.boxed(),
))
}

View File

@@ -0,0 +1,211 @@
use std::path::{Path, PathBuf};
use color_eyre::eyre::eyre;
use futures::TryStreamExt;
use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
use sqlx::{Executor, Sqlite};
use super::{BackupTarget, BackupTargetId};
use crate::context::RpcContext;
use crate::disk::mount::filesystem::cifs::Cifs;
use crate::disk::mount::guard::TmpMountGuard;
use crate::disk::util::{recovery_info, EmbassyOsRecoveryInfo};
use crate::util::display_none;
use crate::util::serde::KeyVal;
use crate::Error;
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct CifsBackupTarget {
hostname: String,
path: PathBuf,
username: String,
mountable: bool,
embassy_os: Option<EmbassyOsRecoveryInfo>,
}
#[command(subcommands(add, update, remove))]
pub fn cifs() -> Result<(), Error> {
Ok(())
}
#[command(display(display_none))]
pub async fn add(
#[context] ctx: RpcContext,
#[arg] hostname: String,
#[arg] path: PathBuf,
#[arg] username: String,
#[arg] password: Option<String>,
) -> Result<KeyVal<BackupTargetId, BackupTarget>, Error> {
let cifs = Cifs {
hostname,
path,
username,
password,
};
let guard = TmpMountGuard::mount(&cifs).await?;
let embassy_os = recovery_info(&guard).await?;
guard.unmount().await?;
let path_string = Path::new("/").join(&cifs.path).display().to_string();
let id: u32 = sqlx::query!(
"INSERT INTO cifs_shares (hostname, path, username, password) VALUES (?, ?, ?, ?) RETURNING id AS \"id: u32\"",
cifs.hostname,
path_string,
cifs.username,
cifs.password,
)
.fetch_one(&ctx.secret_store)
.await?.id;
Ok(KeyVal {
key: BackupTargetId::Cifs { id },
value: BackupTarget::Cifs(CifsBackupTarget {
hostname: cifs.hostname,
path: cifs.path,
username: cifs.username,
mountable: true,
embassy_os,
}),
})
}
#[command(display(display_none))]
pub async fn update(
#[context] ctx: RpcContext,
#[arg] id: BackupTargetId,
#[arg] hostname: String,
#[arg] path: PathBuf,
#[arg] username: String,
#[arg] password: Option<String>,
) -> Result<KeyVal<BackupTargetId, BackupTarget>, Error> {
let id = if let BackupTargetId::Cifs { id } = id {
id
} else {
return Err(Error::new(
eyre!("Backup Target ID {} Not Found", id),
crate::ErrorKind::NotFound,
));
};
let cifs = Cifs {
hostname,
path,
username,
password,
};
let guard = TmpMountGuard::mount(&cifs).await?;
let embassy_os = recovery_info(&guard).await?;
guard.unmount().await?;
let path_string = Path::new("/").join(&cifs.path).display().to_string();
if sqlx::query!(
"UPDATE cifs_shares SET hostname = ?, path = ?, username = ?, password = ? WHERE id = ?",
cifs.hostname,
path_string,
cifs.username,
cifs.password,
id,
)
.execute(&ctx.secret_store)
.await?
.rows_affected()
== 0
{
return Err(Error::new(
eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
crate::ErrorKind::NotFound,
));
};
Ok(KeyVal {
key: BackupTargetId::Cifs { id },
value: BackupTarget::Cifs(CifsBackupTarget {
hostname: cifs.hostname,
path: cifs.path,
username: cifs.username,
mountable: true,
embassy_os,
}),
})
}
#[command(display(display_none))]
pub async fn remove(#[context] ctx: RpcContext, #[arg] id: BackupTargetId) -> Result<(), Error> {
let id = if let BackupTargetId::Cifs { id } = id {
id
} else {
return Err(Error::new(
eyre!("Backup Target ID {} Not Found", id),
crate::ErrorKind::NotFound,
));
};
if sqlx::query!("DELETE FROM cifs_shares WHERE id = ?", id)
.execute(&ctx.secret_store)
.await?
.rows_affected()
== 0
{
return Err(Error::new(
eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
crate::ErrorKind::NotFound,
));
};
Ok(())
}
pub async fn load<Ex>(secrets: &mut Ex, id: u32) -> Result<Cifs, Error>
where
for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>,
{
let record = sqlx::query!(
"SELECT hostname, path, username, password FROM cifs_shares WHERE id = ?",
id
)
.fetch_one(secrets)
.await?;
Ok(Cifs {
hostname: record.hostname,
path: PathBuf::from(record.path),
username: record.username,
password: record.password,
})
}
pub async fn list<Ex>(secrets: &mut Ex) -> Result<Vec<(u32, CifsBackupTarget)>, Error>
where
for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>,
{
let mut records = sqlx::query!(
"SELECT id AS \"id: u32\", hostname, path, username, password FROM cifs_shares"
)
.fetch_many(secrets);
let mut cifs = Vec::new();
while let Some(query_result) = records.try_next().await? {
if let Some(record) = query_result.right() {
let mount_info = Cifs {
hostname: record.hostname,
path: PathBuf::from(record.path),
username: record.username,
password: record.password,
};
let embassy_os = async {
let guard = TmpMountGuard::mount(&mount_info).await?;
let embassy_os = recovery_info(&guard).await?;
guard.unmount().await?;
Ok::<_, Error>(embassy_os)
}
.await;
cifs.push((
record.id,
CifsBackupTarget {
hostname: mount_info.hostname,
path: mount_info.path,
username: mount_info.username,
mountable: embassy_os.is_ok(),
embassy_os: embassy_os.ok().and_then(|a| a),
},
));
}
}
Ok(cifs)
}

View File

@@ -0,0 +1,242 @@
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use clap::ArgMatches;
use color_eyre::eyre::eyre;
use digest::generic_array::GenericArray;
use digest::Digest;
use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
use sha2::Sha256;
use sqlx::{Executor, Sqlite};
use tracing::instrument;
use self::cifs::CifsBackupTarget;
use crate::context::RpcContext;
use crate::disk::mount::backup::BackupMountGuard;
use crate::disk::mount::filesystem::block_dev::BlockDev;
use crate::disk::mount::filesystem::cifs::Cifs;
use crate::disk::mount::filesystem::FileSystem;
use crate::disk::mount::guard::TmpMountGuard;
use crate::disk::util::PartitionInfo;
use crate::s9pk::manifest::PackageId;
use crate::util::serde::{deserialize_from_str, display_serializable, serialize_display};
use crate::util::Version;
use crate::Error;
pub mod cifs;
#[derive(Debug, Deserialize, Serialize)]
#[serde(tag = "type")]
#[serde(rename_all = "kebab-case")]
pub enum BackupTarget {
#[serde(rename_all = "kebab-case")]
Disk {
vendor: Option<String>,
model: Option<String>,
#[serde(flatten)]
partition_info: PartitionInfo,
},
Cifs(CifsBackupTarget),
}
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum BackupTargetId {
Disk { logicalname: PathBuf },
Cifs { id: u32 },
}
impl BackupTargetId {
pub async fn load<Ex>(self, secrets: &mut Ex) -> Result<BackupTargetFS, Error>
where
for<'a> &'a mut Ex: Executor<'a, Database = Sqlite>,
{
Ok(match self {
BackupTargetId::Disk { logicalname } => {
BackupTargetFS::Disk(BlockDev::new(logicalname))
}
BackupTargetId::Cifs { id } => BackupTargetFS::Cifs(cifs::load(secrets, id).await?),
})
}
}
impl std::fmt::Display for BackupTargetId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
BackupTargetId::Disk { logicalname } => write!(f, "disk-{}", logicalname.display()),
BackupTargetId::Cifs { id } => write!(f, "cifs-{}", id),
}
}
}
impl std::str::FromStr for BackupTargetId {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.split_once("-") {
Some(("disk", logicalname)) => Ok(BackupTargetId::Disk {
logicalname: Path::new(logicalname).to_owned(),
}),
Some(("cifs", id)) => Ok(BackupTargetId::Cifs { id: id.parse()? }),
_ => Err(Error::new(
eyre!("Invalid Backup Target ID"),
crate::ErrorKind::InvalidBackupTargetId,
)),
}
}
}
impl<'de> Deserialize<'de> for BackupTargetId {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
deserialize_from_str(deserializer)
}
}
impl Serialize for BackupTargetId {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serialize_display(self, serializer)
}
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(tag = "type")]
#[serde(rename_all = "kebab-case")]
pub enum BackupTargetFS {
Disk(BlockDev<PathBuf>),
Cifs(Cifs),
}
#[async_trait]
impl FileSystem for BackupTargetFS {
async fn mount<P: AsRef<Path> + Send + Sync>(&self, mountpoint: P) -> Result<(), Error> {
match self {
BackupTargetFS::Disk(a) => a.mount(mountpoint).await,
BackupTargetFS::Cifs(a) => a.mount(mountpoint).await,
}
}
async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
match self {
BackupTargetFS::Disk(a) => a.source_hash().await,
BackupTargetFS::Cifs(a) => a.source_hash().await,
}
}
}
#[command(subcommands(cifs::cifs, list, info))]
pub fn target() -> Result<(), Error> {
Ok(())
}
// TODO: incorporate reconnect into this response as well
#[command(display(display_serializable))]
pub async fn list(
#[context] ctx: RpcContext,
) -> Result<BTreeMap<BackupTargetId, BackupTarget>, Error> {
let mut sql_handle = ctx.secret_store.acquire().await?;
let (disks_res, cifs) =
tokio::try_join!(crate::disk::util::list(), cifs::list(&mut sql_handle),)?;
Ok(disks_res
.disks
.into_iter()
.flat_map(|mut disk| {
std::mem::take(&mut disk.partitions)
.into_iter()
.map(|part| {
(
BackupTargetId::Disk {
logicalname: part.logicalname.clone(),
},
BackupTarget::Disk {
vendor: disk.vendor.clone(),
model: disk.model.clone(),
partition_info: part,
},
)
})
.collect::<Vec<_>>()
})
.chain(
cifs.into_iter()
.map(|(id, cifs)| (BackupTargetId::Cifs { id }, BackupTarget::Cifs(cifs))),
)
.collect())
}
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct BackupInfo {
pub version: Version,
pub timestamp: Option<DateTime<Utc>>,
pub package_backups: BTreeMap<PackageId, PackageBackupInfo>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct PackageBackupInfo {
pub title: String,
pub version: Version,
pub os_version: Version,
pub timestamp: DateTime<Utc>,
}
fn display_backup_info(info: BackupInfo, matches: &ArgMatches<'_>) {
use prettytable::*;
if matches.is_present("format") {
return display_serializable(info, matches);
}
let mut table = Table::new();
table.add_row(row![bc =>
"ID",
"VERSION",
"OS VERSION",
"TIMESTAMP",
]);
table.add_row(row![
"EMBASSY OS",
info.version.as_str(),
info.version.as_str(),
&if let Some(ts) = &info.timestamp {
ts.to_string()
} else {
"N/A".to_owned()
},
]);
for (id, info) in info.package_backups {
let row = row![
id.as_str(),
info.version.as_str(),
info.os_version.as_str(),
&info.timestamp.to_string(),
];
table.add_row(row);
}
table.print_tty(false);
}
#[command(display(display_backup_info))]
#[instrument(skip(ctx, password))]
pub async fn info(
#[context] ctx: RpcContext,
#[arg(rename = "target-id")] target_id: BackupTargetId,
#[arg] password: String,
) -> Result<BackupInfo, Error> {
let guard = BackupMountGuard::mount(
TmpMountGuard::mount(
&target_id
.load(&mut ctx.secret_store.acquire().await?)
.await?,
)
.await?,
&password,
)
.await?;
let res = guard.metadata.clone();
guard.unmount().await?;
Ok(res)
}