Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-31 20:43:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor/networking (#2189)

Browse Source 
* refactor networking and account

* add interfaces from manifest automatically

* use nistp256 to satisfy firefox

* use ed25519 if available

* fix ip signing

* fix SQL error

* update prettytable to fix segfault

* fix migration

* fix migration

* bump welcome-ack

* add redirect if connecting to https over http

* misc rebase fixes

* fix compression

* bump rustc version
This commit is contained in:
Aiden McClelland
2023-03-08 19:30:46 -07:00
committed by GitHub
parent da55d6f7cd
commit bbb9980941
79 changed files with 3577 additions and 3587 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
backend/src/backup/backup_bulk.rs
View File

@@ -5,21 +5,15 @@ use chrono::Utc;
use clap::ArgMatches;
use color_eyre::eyre::eyre;
use helpers::AtomicFile;
use openssl::pkey::{PKey, Private};
use openssl::x509::X509;
use patch_db::{DbHandle, LockType, PatchDbHandle};
use rand::random;
use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use ssh_key::private::Ed25519PrivateKey;
use tokio::io::AsyncWriteExt;
use torut::onion::TorSecretKeyV3;
use tracing::instrument;
use super::target::BackupTargetId;
use super::PackageBackupReport;
use crate::auth::check_password_against_db;
use crate::backup::os::OsBackup;
use crate::backup::{BackupReport, ServerBackupReport};
use crate::context::RpcContext;
use crate::db::model::BackupProgress;
@@ -34,109 +28,6 @@ use crate::util::serde::IoFormat;
use crate::version::VersionT;
use crate::{Error, ErrorKind, ResultExt};
#[derive(Debug)]
pub struct OsBackup {
pub tor_key: TorSecretKeyV3,
pub ssh_key: Ed25519PrivateKey,
pub root_ca_key: PKey<Private>,
pub root_ca_cert: X509,
pub ui: Value,
}
impl<'de> Deserialize<'de> for OsBackup {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(rename = "kebab-case")]
struct OsBackupDe {
tor_key: String,
ssh_key: Option<String>,
root_ca_key: String,
root_ca_cert: String,
ui: Value,
}
fn vec_from_base32<E: serde::de::Error>(base32: &str, len: usize) -> Result<Vec<u8>, E> {
let key_vec = base32::decode(base32::Alphabet::RFC4648 { padding: true }, base32)
.ok_or_else(|| {
serde::de::Error::invalid_value(
serde::de::Unexpected::Str(base32),
&"an RFC4648 encoded string",
)
})?;
if key_vec.len() != 64 {
return Err(serde::de::Error::invalid_value(
serde::de::Unexpected::Str(base32),
&"a 64 byte value encoded as an RFC4648 string",
));
}
Ok(key_vec)
}
let int = OsBackupDe::deserialize(deserializer)?;
let tor_key = {
let mut key_slice = [0; 64];
key_slice.clone_from_slice(&vec_from_base32(&int.tor_key, 64)?);
TorSecretKeyV3::from(key_slice)
};
let ssh_key = int
.ssh_key
.as_ref()
.map(|ssh_key| {
let mut key_slice = [0; 32];
key_slice.clone_from_slice(&vec_from_base32(ssh_key, 32)?);
Ok(Ed25519PrivateKey::from_bytes(&key_slice))
})
.transpose()?
.unwrap_or_else(|| Ed25519PrivateKey::from_bytes(&random()));
let root_ca_key = PKey::<Private>::private_key_from_pem(int.root_ca_key.as_bytes())
.map_err(serde::de::Error::custom)?;
let root_ca_cert =
X509::from_pem(int.root_ca_cert.as_bytes()).map_err(serde::de::Error::custom)?;
Ok(OsBackup {
tor_key,
ssh_key,
root_ca_key,
root_ca_cert,
ui: int.ui,
})
}
}
impl Serialize for OsBackup {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
#[derive(Serialize)]
#[serde(rename = "kebab-case")]
struct OsBackupSer<'a> {
tor_key: String,
root_ca_key: String,
root_ca_cert: String,
ui: &'a Value,
}
OsBackupSer {
tor_key: base32::encode(
base32::Alphabet::RFC4648 { padding: true },
&self.tor_key.as_bytes(),
),
root_ca_key: String::from_utf8(
self.root_ca_key
.private_key_to_pem_pkcs8()
.map_err(serde::ser::Error::custom)?,
)
.map_err(serde::ser::Error::custom)?,
root_ca_cert: String::from_utf8(
self.root_ca_cert
.to_pem()
.map_err(serde::ser::Error::custom)?,
)
.map_err(serde::ser::Error::custom)?,
ui: &self.ui,
}
.serialize(serializer)
}
}
fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<BTreeSet<PackageId>, Error> {
arg.split(',')
.map(|s| s.trim().parse().map_err(Error::from))
@@ -317,7 +208,6 @@ async fn perform_backup<Db: DbHandle>(
package_ids: &BTreeSet<PackageId>,
) -> Result<BTreeMap<PackageId, PackageBackupReport>, Error> {
let mut backup_report = BTreeMap::new();
for package_id in crate::db::DatabaseModel::new()
.package_data()
.keys(&mut db)
@@ -445,11 +335,12 @@ async fn perform_backup<Db: DbHandle>(
tx.save().await?;
}
crate::db::DatabaseModel::new()
.lock(&mut db, LockType::Write)
.await?;
let ui = crate::db::DatabaseModel::new()
.ui()
.get(&mut db)
.await?
.into_owned();
let (root_ca_key, root_ca_cert) = ctx.net_controller.ssl.export_root_ca().await?;
let mut os_backup_file = AtomicFile::new(
backup_guard.as_ref().join("os-backup.cbor"),
None::<PathBuf>,
@@ -457,19 +348,10 @@ async fn perform_backup<Db: DbHandle>(
.await
.with_kind(ErrorKind::Filesystem)?;
os_backup_file
.write_all(
&IoFormat::Cbor.to_vec(&OsBackup {
tor_key: ctx.net_controller.tor.embassyd_tor_key().await,
ssh_key: crate::ssh::os_key(&mut ctx.secret_store.acquire().await?).await?,
root_ca_key,
root_ca_cert,
ui: crate::db::DatabaseModel::new()
.ui()
.get(&mut db)
.await?
.into_owned(),
})?,
)
.write_all(&IoFormat::Cbor.to_vec(&OsBackup {
account: ctx.account.read().await.clone(),
ui,
})?)
.await?;
os_backup_file
.save()

28
backend/src/backup/mod.rs
View File

@@ -4,6 +4,7 @@ use std::path::{Path, PathBuf};
use chrono::{DateTime, Utc};
use color_eyre::eyre::eyre;
use helpers::AtomicFile;
use models::ImageId;
use patch_db::{DbHandle, HasModel};
use reqwest::Url;
use rpc_toolkit::command;
@@ -15,19 +16,20 @@ use tracing::instrument;
use self::target::PackageBackupInfo;
use crate::context::RpcContext;
use crate::dependencies::reconfigure_dependents_with_live_pointers;
use crate::id::ImageId;
use crate::install::PKG_ARCHIVE_DIR;
use crate::net::interface::{InterfaceId, Interfaces};
use crate::net::keys::Key;
use crate::procedure::docker::DockerContainers;
use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
use crate::util::serde::IoFormat;
use crate::util::serde::{Base32, Base64, IoFormat};
use crate::util::Version;
use crate::version::{Current, VersionT};
use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR};
use crate::{Error, ErrorKind, ResultExt};
pub mod backup_bulk;
pub mod os;
pub mod restore;
pub mod target;
@@ -61,7 +63,10 @@ pub fn package_backup() -> Result<(), Error> {
#[derive(Deserialize, Serialize)]
struct BackupMetadata {
pub timestamp: DateTime<Utc>,
pub tor_keys: BTreeMap<InterfaceId, String>,
#[serde(default)]
pub network_keys: BTreeMap<InterfaceId, Base64<[u8; 32]>>,
#[serde(default)]
pub tor_keys: BTreeMap<InterfaceId, Base32<[u8; 64]>>, // DEPRECATED
pub marketplace_url: Option<Url>,
}
@@ -117,17 +122,17 @@ impl BackupActions {
.await?
.map_err(|e| eyre!("{}", e.1))
.with_kind(crate::ErrorKind::Backup)?;
let tor_keys = interfaces
.tor_keys(&mut ctx.secret_store.acquire().await?, pkg_id)
let (network_keys, tor_keys) = Key::for_package(&ctx.secret_store, pkg_id)
.await?
.into_iter()
.map(|(id, key)| {
(
id,
base32::encode(base32::Alphabet::RFC4648 { padding: true }, &key.as_bytes()),
)
.filter_map(|k| {
let interface = k.interface().map(|(_, i)| i)?;
Some((
(interface.clone(), Base64(k.as_bytes())),
(interface, Base32(k.tor_key().as_bytes())),
))
})
.collect();
.unzip();
let marketplace_url = crate::db::DatabaseModel::new()
.package_data()
.idx_model(pkg_id)
@@ -170,6 +175,7 @@ impl BackupActions {
outfile
.write_all(&IoFormat::Cbor.to_vec(&BackupMetadata {
timestamp,
network_keys,
tor_keys,
marketplace_url,
})?)

121
backend/src/backup/os.rs Normal file
View File

@@ -0,0 +1,121 @@
use crate::account::AccountInfo;
use crate::hostname::{generate_hostname, generate_id, Hostname};
use crate::net::keys::Key;
use crate::util::serde::Base64;
use crate::Error;
use openssl::pkey::PKey;
use openssl::x509::X509;
use serde::{Deserialize, Serialize};
use serde_json::Value;
pub struct OsBackup {
pub account: AccountInfo,
pub ui: Value,
}
impl<'de> Deserialize<'de> for OsBackup {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let tagged = OsBackupSerDe::deserialize(deserializer)?;
match tagged.version {
0 => serde_json::from_value::<OsBackupV0>(tagged.rest)
.map_err(serde::de::Error::custom)?
.project()
.map_err(serde::de::Error::custom),
1 => serde_json::from_value::<OsBackupV1>(tagged.rest)
.map_err(serde::de::Error::custom)?
.project()
.map_err(serde::de::Error::custom),
v => Err(serde::de::Error::custom(&format!(
"Unknown backup version {v}"
))),
}
}
}
impl Serialize for OsBackup {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
OsBackupSerDe {
version: 1,
rest: serde_json::to_value(
&OsBackupV1::unproject(self).map_err(serde::ser::Error::custom)?,
)
.map_err(serde::ser::Error::custom)?,
}
.serialize(serializer)
}
}
#[derive(Deserialize, Serialize)]
struct OsBackupSerDe {
#[serde(default)]
version: usize,
#[serde(flatten)]
rest: Value,
}
/// V0
#[derive(Deserialize)]
#[serde(rename = "kebab-case")]
struct OsBackupV0 {
// tor_key: Base32<[u8; 64]>,
root_ca_key: String, // PEM Encoded OpenSSL Key
root_ca_cert: String, // PEM Encoded OpenSSL X509 Certificate
ui: Value, // JSON Value
}
impl OsBackupV0 {
fn project(self) -> Result<OsBackup, Error> {
Ok(OsBackup {
account: AccountInfo {
server_id: generate_id(),
hostname: generate_hostname(),
password: Default::default(),
key: Key::new(None),
root_ca_key: PKey::private_key_from_pem(self.root_ca_key.as_bytes())?,
root_ca_cert: X509::from_pem(self.root_ca_cert.as_bytes())?,
},
ui: self.ui,
})
}
}
/// V1
#[derive(Deserialize, Serialize)]
#[serde(rename = "kebab-case")]
struct OsBackupV1 {
server_id: String, // uuidv4
hostname: String, // embassy-<adjective>-<noun>
net_key: Base64<[u8; 32]>, // Ed25519 Secret Key
root_ca_key: String, // PEM Encoded OpenSSL Key
root_ca_cert: String, // PEM Encoded OpenSSL X509 Certificate
ui: Value, // JSON Value
// TODO add more
}
impl OsBackupV1 {
fn project(self) -> Result<OsBackup, Error> {
Ok(OsBackup {
account: AccountInfo {
server_id: self.server_id,
hostname: Hostname(self.hostname),
password: Default::default(),
key: Key::from_bytes(None, self.net_key.0),
root_ca_key: PKey::private_key_from_pem(self.root_ca_key.as_bytes())?,
root_ca_cert: X509::from_pem(self.root_ca_cert.as_bytes())?,
},
ui: self.ui,
})
}
fn unproject(backup: &OsBackup) -> Result<Self, Error> {
Ok(Self {
server_id: backup.account.server_id.clone(),
hostname: backup.account.hostname.0.clone(),
net_key: Base64(backup.account.key.as_bytes()),
root_ca_key: String::from_utf8(backup.account.root_ca_key.private_key_to_pem_pkcs8()?)?,
root_ca_cert: String::from_utf8(backup.account.root_ca_cert.to_pem()?)?,
ui: backup.ui.clone(),
})
}
}

82
backend/src/backup/restore.rs
View File

@@ -11,12 +11,13 @@ use futures::{FutureExt, StreamExt};
use openssl::x509::X509;
use patch_db::{DbHandle, PatchDbHandle};
use rpc_toolkit::command;
use sqlx::Connection;
use tokio::fs::File;
use torut::onion::OnionAddressV3;
use tracing::instrument;
use super::target::BackupTargetId;
use crate::backup::backup_bulk::OsBackup;
use crate::backup::os::OsBackup;
use crate::backup::BackupMetadata;
use crate::context::rpc::RpcContextConfig;
use crate::context::{RpcContext, SetupContext};
@@ -24,11 +25,10 @@ use crate::db::model::{PackageDataEntry, StaticFiles};
use crate::disk::mount::backup::{BackupMountGuard, PackageBackupMountGuard};
use crate::disk::mount::filesystem::ReadWrite;
use crate::disk::mount::guard::TmpMountGuard;
use crate::hostname::{get_hostname, Hostname};
use crate::hostname::Hostname;
use crate::init::init;
use crate::install::progress::InstallProgress;
use crate::install::{download_install_s9pk, PKG_PUBLIC_DIR};
use crate::net::ssl::SslManager;
use crate::notifications::NotificationLevel;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::s9pk::reader::S9pkReader;
@@ -184,7 +184,7 @@ pub async fn recover_full_embassy(
.await?;
let os_backup_path = backup_guard.as_ref().join("os-backup.cbor");
let os_backup: OsBackup =
let mut os_backup: OsBackup =
IoFormat::Cbor.from_slice(&tokio::fs::read(&os_backup_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
@@ -192,31 +192,17 @@ pub async fn recover_full_embassy(
)
})?)?;
let password = argon2::hash_encoded(
os_backup.account.password = argon2::hash_encoded(
embassy_password.as_bytes(),
&rand::random::<[u8; 16]>()[..],
&argon2::Config::default(),
)
.with_kind(crate::ErrorKind::PasswordHashGeneration)?;
let tor_key_bytes = os_backup.tor_key.as_bytes().to_vec();
let ssh_key_bytes = os_backup.ssh_key.to_bytes().to_vec();
let secret_store = ctx.secret_store().await?;
sqlx::query!(
"INSERT INTO account (id, password, tor_key, ssh_key) VALUES ($1, $2, $3, $4) ON CONFLICT (id) DO UPDATE SET password = $2, tor_key = $3, ssh_key = $4",
0,
password,
tor_key_bytes,
ssh_key_bytes,
)
.execute(&mut secret_store.acquire().await?)
.await?;
SslManager::import_root_ca(
secret_store.clone(),
os_backup.root_ca_key,
os_backup.root_ca_cert.clone(),
)
.await?;
let secret_store = ctx.secret_store().await?;
os_backup.account.save(&secret_store).await?;
secret_store.close().await;
let cfg = RpcContextConfig::load(ctx.config_path.clone()).await?;
@@ -224,12 +210,7 @@ pub async fn recover_full_embassy(
init(&cfg).await?;
let rpc_ctx = RpcContext::init(ctx.config_path.clone(), disk_guid.clone()).await?;
let mut db = rpc_ctx.db.handle();
let receipts = crate::hostname::HostNameReceipt::new(&mut db).await?;
let hostname = get_hostname(&mut db, &receipts).await?;
drop(db);
let mut db = rpc_ctx.db.handle();
let ids = backup_guard
@@ -274,9 +255,9 @@ pub async fn recover_full_embassy(
Ok((
disk_guid,
hostname,
os_backup.tor_key.public().get_onion_address(),
os_backup.root_ca_cert,
os_backup.account.hostname,
os_backup.account.key.tor_address(),
os_backup.account.root_ca_cert,
))
}
@@ -414,23 +395,32 @@ async fn restore_package<'a>(
metadata_path.display().to_string(),
)
})?)?;
for (iface, key) in metadata.tor_keys {
let key_vec = base32::decode(base32::Alphabet::RFC4648 { padding: true }, &key)
.ok_or_else(|| {
Error::new(
eyre!("invalid base32 string"),
crate::ErrorKind::Deserialization,
)
})?;
let mut secrets = ctx.secret_store.acquire().await?;
let mut secrets_tx = secrets.begin().await?;
for (iface, key) in metadata.network_keys {
let k = key.0.as_slice();
sqlx::query!(
"INSERT INTO tor (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO UPDATE SET key = $3",
*id,
*iface,
key_vec,
)
.execute(&ctx.secret_store)
.await?;
"INSERT INTO network_keys (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
*id,
*iface,
k,
)
.execute(&mut secrets_tx).await?;
}
// DEPRECATED
for (iface, key) in metadata.tor_keys {
let k = key.0.as_slice();
sqlx::query!(
"INSERT INTO tor (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
*id,
*iface,
k,
)
.execute(&mut secrets_tx).await?;
}
secrets_tx.commit().await?;
drop(secrets);
let len = tokio::fs::metadata(&s9pk_path)
.await