Feature/setup migrate (#1841)

* add migrate component

* finish out migrate page and adjust recover options

* fix typo

* rename migrate -> transfer, adjust copy and typos, update transfer component logic

* add alert for old drive data when transferring

* comments for clarity

* auto adjust swiper slide height

* cleanup uneeded imports from transfer module

* pr feedback suggestions

* remove 02x from setup wiz

* clean up copy/styling for recover flow

* add support for migrating from old drive

* add RecoverySource lifted type

Co-authored-by: Matt Hill <matthewonthemoon@gmail.com>
Co-authored-by: Aiden McClelland <me@drbonez.dev>
This commit is contained in:
Lucy C
2022-11-01 09:00:25 -06:00
committed by Aiden McClelland
parent 1d151d8fa6
commit 74af03408f
25 changed files with 502 additions and 695 deletions

View File

@@ -1,23 +1,15 @@
use std::collections::BTreeMap;
use std::os::unix::prelude::MetadataExt;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use color_eyre::eyre::eyre;
use digest::generic_array::GenericArray;
use digest::OutputSizeUser;
use futures::future::BoxFuture;
use futures::{FutureExt, TryFutureExt, TryStreamExt};
use futures::{StreamExt, TryFutureExt};
use josekit::jwk::Jwk;
use nix::unistd::{Gid, Uid};
use openssl::x509::X509;
use patch_db::{DbHandle, LockType};
use patch_db::DbHandle;
use rpc_toolkit::command;
use rpc_toolkit::yajrc::RpcError;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use sqlx::{Connection, Executor, Postgres};
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
@@ -29,27 +21,20 @@ use crate::backup::target::BackupTargetFS;
use crate::context::rpc::RpcContextConfig;
use crate::context::setup::SetupResult;
use crate::context::SetupContext;
use crate::db::model::RecoveredPackageInfo;
use crate::disk::fsck::RepairStrategy;
use crate::disk::main::DEFAULT_PASSWORD;
use crate::disk::mount::filesystem::block_dev::BlockDev;
use crate::disk::mount::filesystem::cifs::Cifs;
use crate::disk::mount::filesystem::ReadOnly;
use crate::disk::mount::guard::TmpMountGuard;
use crate::disk::util::{pvscan, recovery_info, DiskInfo, EmbassyOsRecoveryInfo};
use crate::disk::REPAIR_DISK_PATH;
use crate::hostname::{get_hostname, HostNameReceipt, Hostname};
use crate::id::Id;
use crate::init::init;
use crate::install::PKG_PUBLIC_DIR;
use crate::middleware::encrypt::EncryptedWire;
use crate::net::ssl::SslManager;
use crate::s9pk::manifest::PackageId;
use crate::sound::BEETHOVEN;
use crate::util::io::{dir_size, from_yaml_async_reader};
use crate::util::Version;
use crate::volume::{data_dir, VolumeId};
use crate::{ensure_code, Error, ErrorKind, ResultExt};
use crate::util::rsync::{Rsync, RsyncOptions};
use crate::{Error, ErrorKind, ResultExt};
#[instrument(skip(secrets))]
pub async fn password_hash<Ex>(secrets: &mut Ex) -> Result<String, Error>
@@ -92,6 +77,44 @@ pub async fn list_disks(#[context] ctx: SetupContext) -> Result<Vec<DiskInfo>, E
crate::disk::util::list(&ctx.os_partitions).await
}
async fn setup_init(
ctx: &SetupContext,
password: Option<String>,
) -> Result<(Hostname, OnionAddressV3, X509), Error> {
init(&RpcContextConfig::load(ctx.config_path.clone()).await?).await?;
let secrets = ctx.secret_store().await?;
let db = ctx.db(&secrets).await?;
let mut secrets_handle = secrets.acquire().await?;
let mut db_handle = db.handle();
let mut secrets_tx = secrets_handle.begin().await?;
let mut db_tx = db_handle.begin().await?;
if let Some(password) = password {
let set_password_receipt = crate::auth::SetPasswordReceipt::new(&mut db_tx).await?;
crate::auth::set_password(
&mut db_tx,
&set_password_receipt,
&mut secrets_tx,
&password,
)
.await?;
}
let tor_key = crate::net::tor::os_key(&mut secrets_tx).await?;
db_tx.commit().await?;
secrets_tx.commit().await?;
let hostname_receipts = HostNameReceipt::new(&mut db_handle).await?;
let hostname = get_hostname(&mut db_handle, &hostname_receipts).await?;
let (_, root_ca) = SslManager::init(secrets, &mut db_handle)
.await?
.export_root_ca()
.await?;
Ok((hostname, tor_key.public().get_onion_address(), root_ca))
}
#[command(rpc_only)]
pub async fn attach(
#[context] ctx: SetupContext,
@@ -135,39 +158,9 @@ pub async fn attach(
ErrorKind::DiskManagement,
));
}
init(&RpcContextConfig::load(ctx.config_path.clone()).await?).await?;
let secrets = ctx.secret_store().await?;
let db = ctx.db(&secrets).await?;
let mut secrets_handle = secrets.acquire().await?;
let mut db_handle = db.handle();
let mut secrets_tx = secrets_handle.begin().await?;
let mut db_tx = db_handle.begin().await?;
if let Some(password) = password {
let set_password_receipt = crate::auth::SetPasswordReceipt::new(&mut db_tx).await?;
crate::auth::set_password(
&mut db_tx,
&set_password_receipt,
&mut secrets_tx,
&password,
)
.await?;
}
let tor_key = crate::net::tor::os_key(&mut secrets_tx).await?;
db_tx.commit().await?;
secrets_tx.commit().await?;
let hostname_receipts = HostNameReceipt::new(&mut db_handle).await?;
let hostname = get_hostname(&mut db_handle, &hostname_receipts).await?;
let (_, root_ca) = SslManager::init(secrets, &mut db_handle)
.await?
.export_root_ca()
.await?;
let (hostname, tor_addr, root_ca) = setup_init(&ctx, password).await?;
let setup_result = SetupResult {
tor_address: format!("http://{}", tor_key.public().get_onion_address()),
tor_address: format!("http://{}", tor_addr),
lan_address: hostname.lan_address(),
root_ca: String::from_utf8(root_ca.to_pem()?)?,
};
@@ -175,29 +168,11 @@ pub async fn attach(
Ok(setup_result)
}
#[command(subcommands(v2, recovery_status))]
#[command(subcommands(recovery_status))]
pub fn recovery() -> Result<(), Error> {
Ok(())
}
#[command(subcommands(set))]
pub fn v2() -> Result<(), Error> {
Ok(())
}
#[command(rpc_only, metadata(authenticated = false))]
pub async fn set(#[context] ctx: SetupContext, #[arg] logicalname: PathBuf) -> Result<(), Error> {
let guard = TmpMountGuard::mount(&BlockDev::new(&logicalname), ReadOnly).await?;
let product_key = tokio::fs::read_to_string(guard.as_ref().join("root/agent/product_key"))
.await?
.trim()
.to_owned();
guard.unmount().await?;
*ctx.cached_product_key.write().await = Some(Arc::new(product_key));
*ctx.selected_v2_drive.write().await = Some(logicalname);
Ok(())
}
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct RecoveryStatus {
@@ -253,19 +228,27 @@ pub async fn verify_cifs(
embassy_os.ok_or_else(|| Error::new(eyre!("No Backup Found"), crate::ErrorKind::NotFound))
}
#[derive(Debug, Deserialize)]
#[serde(tag = "type")]
#[serde(rename_all = "kebab-case")]
pub enum RecoverySource {
Migrate { guid: String },
Backup { target: BackupTargetFS },
}
#[command(rpc_only)]
pub async fn execute(
#[context] ctx: SetupContext,
#[arg(rename = "embassy-logicalname")] embassy_logicalname: PathBuf,
#[arg(rename = "embassy-password")] embassy_password: EncryptedWire,
#[arg(rename = "recovery-source")] mut recovery_source: Option<BackupTargetFS>,
#[arg(rename = "recovery-source")] recovery_source: Option<RecoverySource>,
#[arg(rename = "recovery-password")] recovery_password: Option<EncryptedWire>,
) -> Result<SetupResult, Error> {
let embassy_password = match embassy_password.decrypt(&*ctx) {
Some(a) => a,
None => {
return Err(Error::new(
color_eyre::eyre::eyre!("Couldn't decode embassy_password"),
color_eyre::eyre::eyre!("Couldn't decode embassy-password"),
crate::ErrorKind::Unknown,
))
}
@@ -275,16 +258,13 @@ pub async fn execute(
Some(a) => Some(a),
None => {
return Err(Error::new(
color_eyre::eyre::eyre!("Couldn't decode recovery_password"),
color_eyre::eyre::eyre!("Couldn't decode recovery-password"),
crate::ErrorKind::Unknown,
))
}
},
None => None,
};
if let Some(v2_drive) = &*ctx.selected_v2_drive.read().await {
recovery_source = Some(BackupTargetFS::Disk(BlockDev::new(v2_drive.clone())))
}
match execute_inner(
ctx.clone(),
embassy_logicalname,
@@ -343,7 +323,7 @@ pub async fn execute_inner(
ctx: SetupContext,
embassy_logicalname: PathBuf,
embassy_password: String,
recovery_source: Option<BackupTargetFS>,
recovery_source: Option<RecoverySource>,
recovery_password: Option<String>,
) -> Result<(Hostname, OnionAddressV3, X509), Error> {
if ctx.recovery_status.read().await.is_some() {
@@ -369,12 +349,12 @@ pub async fn execute_inner(
)
.await?;
let res = if let Some(recovery_source) = recovery_source {
let res = if let Some(RecoverySource::Backup { target }) = recovery_source {
let (tor_addr, root_ca, recover_fut) = recover(
ctx.clone(),
guid.clone(),
embassy_password,
recovery_source,
target,
recovery_password,
)
.await?;
@@ -414,6 +394,87 @@ pub async fn execute_inner(
}
});
res
} else if let Some(RecoverySource::Migrate { guid: old_guid }) = recovery_source {
let _ = crate::disk::main::mount_fs(
&old_guid,
"/media/embassy/migrate",
"main",
RepairStrategy::Preen,
DEFAULT_PASSWORD,
)
.await?;
Rsync::new(
"/media/embassy/migrate/main",
"/embassy-data/main",
RsyncOptions {
delete: true,
force: true,
ignore_existing: false,
},
)?
.wait()
.await?;
let _ = crate::disk::main::mount_fs(
&old_guid,
"/media/embassy/migrate",
"package-data",
RepairStrategy::Preen,
DEFAULT_PASSWORD,
)
.await?;
let mut package_data_transfer = Rsync::new(
"/media/embassy/migrate/package-data",
"/embassy-data/package-data",
RsyncOptions {
delete: true,
force: true,
ignore_existing: false,
},
)?;
*ctx.recovery_status.write().await = Some(Ok(RecoveryStatus {
bytes_transferred: 0,
total_bytes: 100,
complete: false,
}));
let (hostname, tor_addr, root_ca) = setup_init(&ctx, Some(embassy_password)).await?;
let res = (hostname.clone(), tor_addr.clone(), root_ca.clone());
tokio::spawn(async move {
if let Err(e) = async {
while let Some(progress) = package_data_transfer.progress.next().await {
*ctx.recovery_status.write().await = Some(Ok(RecoveryStatus {
bytes_transferred: (progress * 100.0) as u64,
total_bytes: 100,
complete: false,
}));
}
package_data_transfer.wait().await?;
Ok::<_, Error>(())
}
.and_then(|_| async {
*ctx.setup_result.write().await = Some((
guid,
SetupResult {
tor_address: format!("http://{}", tor_addr),
lan_address: hostname.lan_address(),
root_ca: String::from_utf8(root_ca.to_pem()?)?,
},
));
if let Some(Ok(recovery_status)) = &mut *ctx.recovery_status.write().await {
recovery_status.complete = true;
}
Ok(())
})
.await
{
(&BEETHOVEN).play().await.unwrap_or_default(); // ignore error in playing the song
tracing::error!("Error recovering drive!: {}", e);
tracing::debug!("{:?}", e);
*ctx.recovery_status.write().await = Some(Err(e.into()));
} else {
tracing::info!("Recovery Complete!");
}
});
res
} else {
let (tor_addr, root_ca) = fresh_setup(&ctx, &embassy_password).await?;
let db = init(&RpcContextConfig::load(ctx.config_path.clone()).await?)
@@ -475,385 +536,12 @@ async fn recover(
recovery_password: Option<String>,
) -> Result<(OnionAddressV3, X509, BoxFuture<'static, Result<(), Error>>), Error> {
let recovery_source = TmpMountGuard::mount(&recovery_source, ReadOnly).await?;
let recovery_version = recovery_info(&recovery_source)
.await?
.as_ref()
.map(|i| i.version.clone())
.unwrap_or_else(|| emver::Version::new(0, 2, 0, 0).into());
let res = if recovery_version.major() == 0 && recovery_version.minor() == 2 {
recover_v2(ctx.clone(), &embassy_password, recovery_source).await?
} else if recovery_version.major() == 0 && recovery_version.minor() == 3 {
recover_full_embassy(
ctx.clone(),
guid.clone(),
embassy_password,
recovery_source,
recovery_password,
)
.await?
} else {
return Err(Error::new(
eyre!("Unsupported version of embassyOS: {}", recovery_version),
crate::ErrorKind::VersionIncompatible,
));
};
Ok(res)
}
async fn shasum(
path: impl AsRef<Path>,
) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
use tokio::io::AsyncReadExt;
let mut rdr = tokio::fs::File::open(path).await?;
let mut hasher = Sha256::new();
let mut buf = [0; 1024];
let mut read;
while {
read = rdr.read(&mut buf).await?;
read != 0
} {
hasher.update(&buf[0..read]);
}
Ok(hasher.finalize())
}
async fn validated_copy(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> Result<(), Error> {
let src_path = src.as_ref();
let dst_path = dst.as_ref();
tokio::fs::copy(src_path, dst_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("cp {} -> {}", src_path.display(), dst_path.display()),
)
})?;
let (src_hash, dst_hash) = tokio::try_join!(shasum(src_path), shasum(dst_path))?;
if src_hash != dst_hash {
Err(Error::new(
eyre!(
"source hash does not match destination hash for {}",
dst_path.display()
),
crate::ErrorKind::Filesystem,
))
} else {
Ok(())
}
}
fn dir_copy<'a, P0: AsRef<Path> + 'a + Send + Sync, P1: AsRef<Path> + 'a + Send + Sync>(
src: P0,
dst: P1,
ctr: &'a AtomicU64,
) -> BoxFuture<'a, Result<(), Error>> {
async move {
let m = tokio::fs::metadata(&src).await?;
let dst_path = dst.as_ref();
tokio::fs::create_dir_all(&dst_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("mkdir {}", dst_path.display()),
)
})?;
tokio::fs::set_permissions(&dst_path, m.permissions())
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("chmod {}", dst_path.display()),
)
})?;
let tmp_dst_path = dst_path.to_owned();
tokio::task::spawn_blocking(move || {
nix::unistd::chown(
&tmp_dst_path,
Some(Uid::from_raw(m.uid())),
Some(Gid::from_raw(m.gid())),
)
})
.await
.with_kind(crate::ErrorKind::Unknown)?
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("chown {}", dst_path.display()),
)
})?;
tokio_stream::wrappers::ReadDirStream::new(tokio::fs::read_dir(src.as_ref()).await?)
.map_err(|e| Error::new(e, crate::ErrorKind::Filesystem))
.try_for_each(|e| async move {
let m = e.metadata().await?;
let src_path = e.path();
let dst_path = dst_path.join(e.file_name());
if m.is_file() {
let len = m.len();
let mut cp_res = Ok(());
for _ in 0..10 {
cp_res = validated_copy(&src_path, &dst_path).await;
if cp_res.is_ok() {
break;
}
}
cp_res?;
let tmp_dst_path = dst_path.clone();
tokio::task::spawn_blocking(move || {
nix::unistd::chown(
&tmp_dst_path,
Some(Uid::from_raw(m.uid())),
Some(Gid::from_raw(m.gid())),
)
})
.await
.with_kind(crate::ErrorKind::Unknown)?
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("chown {}", dst_path.display()),
)
})?;
ctr.fetch_add(len, Ordering::Relaxed);
} else if m.is_dir() {
dir_copy(src_path, dst_path, ctr).await?;
} else if m.file_type().is_symlink() {
tokio::fs::symlink(
tokio::fs::read_link(&src_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("readlink {}", src_path.display()),
)
})?,
&dst_path,
)
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("cp -P {} -> {}", src_path.display(), dst_path.display()),
)
})?;
// Do not set permissions (see https://unix.stackexchange.com/questions/87200/change-permissions-for-a-symbolic-link)
}
Ok(())
})
.await?;
Ok(())
}
.boxed()
}
#[instrument(skip(ctx))]
async fn recover_v2(
ctx: SetupContext,
embassy_password: &str,
recovery_source: TmpMountGuard,
) -> Result<(OnionAddressV3, X509, BoxFuture<'static, Result<(), Error>>), Error> {
let secret_store = ctx.secret_store().await?;
// migrate the root CA
let root_ca_key_path = recovery_source
.as_ref()
.join("root")
.join("agent")
.join("ca")
.join("private")
.join("embassy-root-ca.key.pem");
let root_ca_cert_path = recovery_source
.as_ref()
.join("root")
.join("agent")
.join("ca")
.join("certs")
.join("embassy-root-ca.cert.pem");
let (root_ca_key_bytes, root_ca_cert_bytes) = tokio::try_join!(
tokio::fs::read(root_ca_key_path),
tokio::fs::read(root_ca_cert_path)
)?;
let root_ca_key = openssl::pkey::PKey::private_key_from_pem(&root_ca_key_bytes)?;
let root_ca_cert = openssl::x509::X509::from_pem(&root_ca_cert_bytes)?;
crate::net::ssl::SslManager::import_root_ca(
secret_store.clone(),
root_ca_key,
root_ca_cert.clone(),
recover_full_embassy(
ctx.clone(),
guid.clone(),
embassy_password,
recovery_source,
recovery_password,
)
.await?;
// migrate the tor address
let tor_key_path = recovery_source
.as_ref()
.join("var")
.join("lib")
.join("tor")
.join("agent")
.join("hs_ed25519_secret_key");
let tor_key_bytes = tokio::fs::read(tor_key_path).await?;
let mut tor_key_array_tmp = [0u8; 64];
tor_key_array_tmp.clone_from_slice(&tor_key_bytes[32..]);
let tor_key: TorSecretKeyV3 = tor_key_array_tmp.into();
let key_vec = tor_key.as_bytes().to_vec();
let password = argon2::hash_encoded(
embassy_password.as_bytes(),
&rand::random::<[u8; 16]>()[..],
&argon2::Config::default(),
)
.with_kind(crate::ErrorKind::PasswordHashGeneration)?;
let sqlite_pool = ctx.secret_store().await?;
sqlx::query!(
"INSERT INTO account (id, password, tor_key) VALUES ($1, $2, $3) ON CONFLICT (id) DO UPDATE SET password = $2, tor_key = $3",
0,
password,
key_vec
)
.execute(&mut sqlite_pool.acquire().await?)
.await?;
// rest of migration as future
let fut = async move {
let db = ctx.db(&secret_store).await?;
let mut handle = db.handle();
// lock everything to avoid issues with renamed packages (bitwarden)
crate::db::DatabaseModel::new()
.lock(&mut handle, LockType::Write)
.await?;
let apps_yaml_path = recovery_source
.as_ref()
.join("root")
.join("appmgr")
.join("apps.yaml");
#[derive(Deserialize)]
struct LegacyAppInfo {
title: String,
version: Version,
}
let packages: BTreeMap<PackageId, LegacyAppInfo> =
from_yaml_async_reader(File::open(&apps_yaml_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
apps_yaml_path.display().to_string(),
)
})?)
.await?;
let volume_path = recovery_source.as_ref().join("root/volumes");
let mut total_bytes = 0;
for (pkg_id, _) in &packages {
let volume_src_path = volume_path.join(&pkg_id);
total_bytes += dir_size(&volume_src_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
volume_src_path.display().to_string(),
)
})?;
}
*ctx.recovery_status.write().await = Some(Ok(RecoveryStatus {
bytes_transferred: 0,
total_bytes,
complete: false,
}));
let bytes_transferred = AtomicU64::new(0);
let volume_id = VolumeId::Custom(Id::try_from("main".to_owned())?);
for (pkg_id, info) in packages {
let (src_id, dst_id) = rename_pkg_id(pkg_id);
let volume_src_path = volume_path.join(&src_id);
let volume_dst_path = data_dir(&ctx.datadir, &dst_id, &volume_id);
tokio::select!(
res = dir_copy(
&volume_src_path,
&volume_dst_path,
&bytes_transferred
) => res?,
_ = async {
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
*ctx.recovery_status.write().await = Some(Ok(RecoveryStatus {
bytes_transferred: bytes_transferred.load(Ordering::Relaxed),
total_bytes,
complete: false
}));
}
} => (),
);
let tor_src_path = recovery_source
.as_ref()
.join("var/lib/tor")
.join(format!("app-{}", src_id))
.join("hs_ed25519_secret_key");
let key_vec = tokio::fs::read(&tor_src_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
tor_src_path.display().to_string(),
)
})?;
ensure_code!(
key_vec.len() == 96,
crate::ErrorKind::Tor,
"{} not 96 bytes",
tor_src_path.display()
);
let key_vec = key_vec[32..].to_vec();
sqlx::query!(
"INSERT INTO tor (package, interface, key) VALUES ($1, 'main', $2) ON CONFLICT (package, interface) DO UPDATE SET key = $2",
*dst_id,
key_vec,
)
.execute(&mut secret_store.acquire().await?)
.await?;
let icon_leaf = AsRef::<Path>::as_ref(&dst_id)
.join(info.version.as_str())
.join("icon.png");
let icon_src_path = recovery_source
.as_ref()
.join("root/agent/icons")
.join(format!("{}.png", src_id));
let icon_dst_path = ctx.datadir.join(PKG_PUBLIC_DIR).join(&icon_leaf);
if let Some(parent) = icon_dst_path.parent() {
tokio::fs::create_dir_all(&parent)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, parent.display().to_string()))?;
}
tokio::fs::copy(&icon_src_path, &icon_dst_path)
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!(
"cp {} -> {}",
icon_src_path.display(),
icon_dst_path.display()
),
)
})?;
let icon_url = Path::new("/public/package-data").join(&icon_leaf);
crate::db::DatabaseModel::new()
.recovered_packages()
.idx_model(&dst_id)
.put(
&mut handle,
&RecoveredPackageInfo {
title: info.title,
icon: icon_url.display().to_string(),
version: info.version,
},
)
.await?;
}
secret_store.close().await;
recovery_source.unmount().await?;
Ok(())
};
Ok((
tor_key.public().get_onion_address(),
root_ca_cert,
fut.boxed(),
))
}
fn rename_pkg_id(src_pkg_id: PackageId) -> (PackageId, PackageId) {
if &*src_pkg_id == "bitwarden" {
(src_pkg_id, "vaultwarden".parse().unwrap())
} else {
(src_pkg_id.clone(), src_pkg_id)
}
.await
}