Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-04-01 21:13:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'next' of github.com:Start9Labs/start-os into rebase/integration/refactors

Browse Source
This commit is contained in:
Aiden McClelland
2023-09-28 13:27:41 -06:00
parent 72ffedead7 f5da5f4ef0
commit d43767b945
167 changed files with 20082 additions and 7978 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
backend/src/action.rs
View File

@@ -11,6 +11,7 @@ use tracing::instrument;
use crate::config::{Config, ConfigSpec};
use crate::context::RpcContext;
use crate::prelude::*;
use crate::procedure::docker::DockerContainers;
use crate::procedure::{PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
@@ -59,13 +60,13 @@ impl Action {
#[instrument(skip_all)]
pub fn validate(
&self,
container: &Option<DockerContainers>,
_container: &Option<DockerContainers>,
eos_version: &Version,
volumes: &Volumes,
image_ids: &BTreeSet<ImageId>,
) -> Result<(), Error> {
self.implementation
.validate(container, eos_version, volumes, image_ids, true)
.validate(eos_version, volumes, image_ids, true)
.with_ctx(|_| {
(
crate::ErrorKind::ValidateS9pk,
@@ -130,18 +131,17 @@ pub async fn action(
#[arg(long = "format")]
format: Option<IoFormat>,
) -> Result<ActionResult, Error> {
let mut db = ctx.db.handle();
let manifest = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&pkg_id)
.and_then(|p| p.installed())
.expect(&mut db)
.await
.with_kind(crate::ErrorKind::NotFound)?
.manifest()
.get(&mut db)
let manifest = ctx
.db
.peek()
.await?
.to_owned();
.as_package_data()
.as_idx(&pkg_id)
.or_not_found(&pkg_id)?
.as_installed()
.or_not_found(&pkg_id)?
.as_manifest()
.de()?;
if let Some(action) = manifest.actions.0.get(&action_id) {
action

38
backend/src/auth.rs
View File

@@ -5,7 +5,6 @@ use chrono::{DateTime, Utc};
use clap::ArgMatches;
use color_eyre::eyre::eyre;
use josekit::jwk::Jwk;
use patch_db::{DbHandle, LockReceipt};
use rpc_toolkit::command;
use rpc_toolkit::command_helpers::prelude::{RequestParts, ResponseParts};
use rpc_toolkit::yajrc::RpcError;
@@ -17,6 +16,7 @@ use tracing::instrument;
use crate::context::{CliContext, RpcContext};
use crate::middleware::auth::{AsLogoutSessionId, HasLoggedOutSessions, HashSessionToken};
use crate::middleware::encrypt::EncryptedWire;
use crate::prelude::*;
use crate::util::display_none;
use crate::util::serde::{display_serializable, IoFormat};
use crate::{ensure_code, Error, ResultExt};
@@ -343,27 +343,6 @@ async fn cli_reset_password(
Ok(())
}
pub struct SetPasswordReceipt(LockReceipt<String, ()>);
impl SetPasswordReceipt {
pub async fn new<Db: DbHandle>(db: &mut Db) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(
locks: &mut Vec<patch_db::LockTargetId>,
) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
let password_hash = crate::db::DatabaseModel::new()
.server_info()
.password_hash()
.make_locker(patch_db::LockType::Write)
.add_to_keys(locks);
move |skeleton_key| Ok(Self(password_hash.verify(skeleton_key)?))
}
}
#[command(
rename = "reset-password",
custom_cli(cli_reset_password(async, context(CliContext))),
@@ -389,13 +368,14 @@ pub async fn reset_password(
}
account.set_password(&new_password)?;
account.save(&ctx.secret_store).await?;
crate::db::DatabaseModel::new()
.server_info()
.password_hash()
.put(&mut ctx.db.handle(), &account.password)
.await?;
Ok(())
let account_password = &account.password;
ctx.db
.mutate(|d| {
d.as_server_info_mut()
.as_password_hash_mut()
.ser(account_password)
})
.await
}
#[command(

243
backend/src/backup/backup_bulk.rs
View File

@@ -1,4 +1,5 @@
use std::collections::{BTreeMap, BTreeSet};
use std::collections::BTreeMap;
use std::panic::UnwindSafe;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@@ -6,9 +7,11 @@ use chrono::Utc;
use clap::ArgMatches;
use color_eyre::eyre::eyre;
use helpers::AtomicFile;
use patch_db::{DbHandle, LockType, PatchDbHandle};
use imbl::OrdSet;
use models::Version;
use rpc_toolkit::command;
use tokio::{io::AsyncWriteExt, sync::Mutex};
use tokio::io::AsyncWriteExt;
use tokio::sync::Mutex;
use tracing::instrument;
use super::target::BackupTargetId;
@@ -18,26 +21,27 @@ use crate::backup::os::OsBackup;
use crate::backup::{BackupReport, ServerBackupReport};
use crate::context::RpcContext;
use crate::db::model::BackupProgress;
use crate::db::package::get_packages;
use crate::disk::mount::backup::BackupMountGuard;
use crate::disk::mount::filesystem::ReadWrite;
use crate::disk::mount::guard::TmpMountGuard;
use crate::manager::BackupReturn;
use crate::notifications::NotificationLevel;
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::util::display_none;
use crate::util::io::dir_copy;
use crate::util::serde::IoFormat;
use crate::util::{display_none, Invoke};
use crate::version::VersionT;
use crate::{Error, ErrorKind, ResultExt};
fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<BTreeSet<PackageId>, Error> {
fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<OrdSet<PackageId>, Error> {
arg.split(',')
.map(|s| s.trim().parse().map_err(Error::from))
.map(|s| s.trim().parse::<PackageId>().map_err(Error::from))
.collect()
}
#[command(rename = "create", display(display_none))]
#[instrument(skip_all)]
#[instrument(skip(ctx, old_password, password))]
pub async fn backup_all(
#[context] ctx: RpcContext,
#[arg(rename = "target-id")] target_id: BackupTargetId,
@@ -49,10 +53,10 @@ pub async fn backup_all(
long = "package-ids",
parse(parse_comma_separated)
)]
package_ids: Option<BTreeSet<PackageId>>,
package_ids: Option<OrdSet<PackageId>>,
#[arg] password: crate::auth::PasswordType,
) -> Result<(), Error> {
let mut db = ctx.db.handle();
let db = ctx.db.peek().await?;
let old_password_decrypted = old_password
.as_ref()
.unwrap_or(&password)
@@ -68,36 +72,33 @@ pub async fn backup_all(
&old_password_decrypted,
)
.await?;
let all_packages = crate::db::DatabaseModel::new()
.package_data()
.get(&mut db)
.await?
.0
.keys()
.into_iter()
.cloned()
.collect();
let package_ids = package_ids.unwrap_or(all_packages);
let package_ids = if let Some(ids) = package_ids {
ids.into_iter()
.flat_map(|package_id| {
let version = db
.as_package_data()
.as_idx(&package_id)?
.as_manifest()
.as_version()
.de()
.ok()?;
Some((package_id, version))
})
.collect()
} else {
get_packages(db.clone())?.into_iter().collect()
};
if old_password.is_some() {
backup_guard.change_password(&password)?;
}
assure_backing_up(&mut db, &package_ids).await?;
assure_backing_up(&ctx.db, &package_ids).await?;
tokio::task::spawn(async move {
let backup_res = perform_backup(&ctx, &mut db, backup_guard, &package_ids).await;
let backup_progress = crate::db::DatabaseModel::new()
.server_info()
.status_info()
.backup_progress();
backup_progress
.clone()
.lock(&mut db, LockType::Write)
.await
.expect("failed to lock server status");
let backup_res = perform_backup(&ctx, backup_guard, &package_ids).await;
match backup_res {
Ok(report) if report.iter().all(|(_, rep)| rep.error.is_none()) => ctx
.notification_manager
.notify(
&mut db,
ctx.db.clone(),
None,
NotificationLevel::Success,
"Backup Complete".to_owned(),
@@ -107,7 +108,10 @@ pub async fn backup_all(
attempted: true,
error: None,
},
packages: report,
packages: report
.into_iter()
.map(|((package_id, _), value)| (package_id, value))
.collect(),
},
None,
)
@@ -116,7 +120,7 @@ pub async fn backup_all(
Ok(report) => ctx
.notification_manager
.notify(
&mut db,
ctx.db.clone(),
None,
NotificationLevel::Warning,
"Backup Complete".to_owned(),
@@ -126,7 +130,10 @@ pub async fn backup_all(
attempted: true,
error: None,
},
packages: report,
packages: report
.into_iter()
.map(|((package_id, _), value)| (package_id, value))
.collect(),
},
None,
)
@@ -137,7 +144,7 @@ pub async fn backup_all(
tracing::debug!("{:?}", e);
ctx.notification_manager
.notify(
&mut db,
ctx.db.clone(),
None,
NotificationLevel::Error,
"Backup Failed".to_owned(),
@@ -155,106 +162,85 @@ pub async fn backup_all(
.expect("failed to send notification");
}
}
backup_progress
.delete(&mut db)
.await
.expect("failed to change server status");
ctx.db
.mutate(|v| {
v.as_server_info_mut()
.as_status_info_mut()
.as_backup_progress_mut()
.ser(&None)
})
.await?;
Ok::<(), Error>(())
});
Ok(())
}
#[instrument(skip_all)]
#[instrument(skip(db, packages))]
async fn assure_backing_up(
db: &mut PatchDbHandle,
packages: impl IntoIterator<Item = &PackageId>,
db: &PatchDb,
packages: impl IntoIterator<Item = &(PackageId, Version)> + UnwindSafe + Send,
) -> Result<(), Error> {
let mut tx = db.begin().await?;
let mut backing_up = crate::db::DatabaseModel::new()
.server_info()
.status_info()
.backup_progress()
.get_mut(&mut tx)
.await?;
if backing_up
.iter()
.flat_map(|x| x.values())
.fold(false, |acc, x| {
if !x.complete {
return true;
}
acc
})
{
return Err(Error::new(
eyre!("Server is already backing up!"),
crate::ErrorKind::InvalidRequest,
));
}
*backing_up = Some(
packages
.into_iter()
.map(|x| (x.clone(), BackupProgress { complete: false }))
.collect(),
);
backing_up.save(&mut tx).await?;
tx.commit().await?;
Ok(())
db.mutate(|v| {
let backing_up = v
.as_server_info_mut()
.as_status_info_mut()
.as_backup_progress_mut();
if backing_up
.clone()
.de()?
.iter()
.flat_map(|x| x.values())
.fold(false, |acc, x| {
if !x.complete {
return true;
}
acc
})
{
return Err(Error::new(
eyre!("Server is already backing up!"),
ErrorKind::InvalidRequest,
));
}
backing_up.ser(&Some(
packages
.into_iter()
.map(|(x, _)| (x.clone(), BackupProgress { complete: false }))
.collect(),
))?;
Ok(())
})
.await
}
#[instrument(skip_all)]
async fn perform_backup<Db: DbHandle>(
#[instrument(skip(ctx, backup_guard))]
async fn perform_backup(
ctx: &RpcContext,
mut db: Db,
backup_guard: BackupMountGuard<TmpMountGuard>,
package_ids: &BTreeSet<PackageId>,
) -> Result<BTreeMap<PackageId, PackageBackupReport>, Error> {
package_ids: &OrdSet<(PackageId, Version)>,
) -> Result<BTreeMap<(PackageId, Version), PackageBackupReport>, Error> {
let mut backup_report = BTreeMap::new();
let backup_guard = Arc::new(Mutex::new(backup_guard));
for package_id in crate::db::DatabaseModel::new()
.package_data()
.keys(&mut db)
.await?
.into_iter()
.filter(|id| package_ids.contains(id))
{
let mut tx = db.begin().await?; // for lock scope
let installed_model = if let Some(installed_model) = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&package_id)
.and_then(|m| m.installed())
.check(&mut tx)
.await?
{
installed_model
} else {
continue;
};
let main_status_model = installed_model.clone().status().main();
let manifest = installed_model.clone().manifest().get(&mut tx).await?;
let (response, report) = match ctx
for package_id in package_ids {
let (response, _report) = match ctx
.managers
.get(&(manifest.id.clone(), manifest.version.clone()))
.get(package_id)
.await
.ok_or_else(|| {
Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest)
})?
.ok_or_else(|| Error::new(eyre!("Manager not found"), ErrorKind::InvalidRequest))?
.backup(backup_guard.clone())
.await
{
BackupReturn::Ran { report, res } => (res, report),
BackupReturn::AlreadyRunning(report) => {
backup_report.insert(package_id, report);
backup_report.insert(package_id.clone(), report);
continue;
}
BackupReturn::Error(error) => {
tracing::warn!("Backup thread error");
tracing::debug!("{error:?}");
backup_report.insert(
package_id,
package_id.clone(),
PackageBackupReport {
error: Some("Backup thread error".to_owned()),
},
@@ -270,42 +256,16 @@ async fn perform_backup<Db: DbHandle>(
);
if let Ok(pkg_meta) = response {
installed_model
.last_backup()
.put(&mut tx, &Some(pkg_meta.timestamp))
.await?;
backup_guard
.lock()
.await
.metadata
.package_backups
.insert(package_id.clone(), pkg_meta);
.insert(package_id.0.clone(), pkg_meta);
}
let mut backup_progress = crate::db::DatabaseModel::new()
.server_info()
.status_info()
.backup_progress()
.get_mut(&mut tx)
.await?;
if backup_progress.is_none() {
*backup_progress = Some(Default::default());
}
if let Some(mut backup_progress) = backup_progress
.as_mut()
.and_then(|bp| bp.get_mut(&package_id))
{
(*backup_progress).complete = true;
}
backup_progress.save(&mut tx).await?;
tx.save().await?;
}
let ui = crate::db::DatabaseModel::new()
.ui()
.get(&mut db)
.await?
.into_owned();
let ui = ctx.db.peek().await?.into_ui().de()?;
let mut os_backup_file = AtomicFile::new(
backup_guard.lock().await.as_ref().join("os-backup.cbor"),
@@ -354,10 +314,9 @@ async fn perform_backup<Db: DbHandle>(
backup_guard.save_and_unmount().await?;
crate::db::DatabaseModel::new()
.server_info()
.last_backup()
.put(&mut db, &timestamp)
ctx.db
.mutate(|v| v.as_server_info_mut().as_last_backup_mut().ser(&timestamp))
.await?;
Ok(backup_report)
}

116
backend/src/backup/mod.rs
View File

@@ -1,11 +1,11 @@
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use chrono::{DateTime, Utc};
use color_eyre::eyre::eyre;
use helpers::AtomicFile;
use models::ImageId;
use patch_db::{DbHandle, HasModel};
use models::{ImageId, OptionExt};
use reqwest::Url;
use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
@@ -15,10 +15,11 @@ use tracing::instrument;
use self::target::PackageBackupInfo;
use crate::context::RpcContext;
use crate::dependencies::reconfigure_dependents_with_live_pointers;
use crate::install::PKG_ARCHIVE_DIR;
use crate::net::interface::{InterfaceId, Interfaces};
use crate::manager::manager_seed::ManagerSeed;
use crate::net::interface::InterfaceId;
use crate::net::keys::Key;
use crate::prelude::*;
use crate::procedure::docker::DockerContainers;
use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
@@ -71,6 +72,7 @@ struct BackupMetadata {
}
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
#[model = "Model<Self>"]
pub struct BackupActions {
pub create: PackageProcedure,
pub restore: PackageProcedure,
@@ -78,34 +80,29 @@ pub struct BackupActions {
impl BackupActions {
pub fn validate(
&self,
container: &Option<DockerContainers>,
_container: &Option<DockerContainers>,
eos_version: &Version,
volumes: &Volumes,
image_ids: &BTreeSet<ImageId>,
) -> Result<(), Error> {
self.create
.validate(container, eos_version, volumes, image_ids, false)
.validate(eos_version, volumes, image_ids, false)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Create"))?;
self.restore
.validate(container, eos_version, volumes, image_ids, false)
.validate(eos_version, volumes, image_ids, false)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Restore"))?;
Ok(())
}
#[instrument(skip_all)]
pub async fn create<Db: DbHandle>(
&self,
ctx: &RpcContext,
db: &mut Db,
pkg_id: &PackageId,
pkg_title: &str,
pkg_version: &Version,
interfaces: &Interfaces,
volumes: &Volumes,
) -> Result<PackageBackupInfo, Error> {
let mut volumes = volumes.to_readonly();
pub async fn create(&self, seed: Arc<ManagerSeed>) -> Result<PackageBackupInfo, Error> {
let manifest = &seed.manifest;
let mut volumes = seed.manifest.volumes.to_readonly();
let ctx = &seed.ctx;
let pkg_id = &manifest.id;
let pkg_version = &manifest.version;
volumes.insert(VolumeId::Backup, Volume::Backup { readonly: false });
let backup_dir = backup_dir(pkg_id);
let backup_dir = backup_dir(&manifest.id);
if tokio::fs::metadata(&backup_dir).await.is_err() {
tokio::fs::create_dir_all(&backup_dir).await?
}
@@ -122,29 +119,29 @@ impl BackupActions {
.await?
.map_err(|e| eyre!("{}", e.1))
.with_kind(crate::ErrorKind::Backup)?;
let (network_keys, tor_keys) = Key::for_package(&ctx.secret_store, pkg_id)
let (network_keys, tor_keys): (Vec<_>, Vec<_>) =
Key::for_package(&ctx.secret_store, pkg_id)
.await?
.into_iter()
.filter_map(|k| {
let interface = k.interface().map(|(_, i)| i)?;
Some((
(interface.clone(), Base64(k.as_bytes())),
(interface, Base32(k.tor_key().as_bytes())),
))
})
.unzip();
let marketplace_url = ctx
.db
.peek()
.await?
.into_iter()
.filter_map(|k| {
let interface = k.interface().map(|(_, i)| i)?;
Some((
(interface.clone(), Base64(k.as_bytes())),
(interface, Base32(k.tor_key().as_bytes())),
))
})
.unzip();
let marketplace_url = crate::db::DatabaseModel::new()
.package_data()
.idx_model(pkg_id)
.expect(db)
.await?
.installed()
.expect(db)
.await?
.marketplace_url()
.get(db)
.await?
.into_owned();
.as_package_data()
.as_idx(&pkg_id)
.or_not_found(pkg_id)?
.expect_as_installed()?
.as_installed()
.as_marketplace_url()
.de()?;
let tmp_path = Path::new(BACKUP_DIR)
.join(pkg_id)
.join(format!("{}.s9pk", pkg_id));
@@ -172,6 +169,8 @@ impl BackupActions {
let mut outfile = AtomicFile::new(&metadata_path, None::<PathBuf>)
.await
.with_kind(ErrorKind::Filesystem)?;
let network_keys = network_keys.into_iter().collect();
let tor_keys = tor_keys.into_iter().collect();
outfile
.write_all(&IoFormat::Cbor.to_vec(&BackupMetadata {
timestamp,
@@ -183,22 +182,20 @@ impl BackupActions {
outfile.save().await.with_kind(ErrorKind::Filesystem)?;
Ok(PackageBackupInfo {
os_version: Current::new().semver().into(),
title: pkg_title.to_owned(),
title: manifest.title.clone(),
version: pkg_version.clone(),
timestamp,
})
}
#[instrument(skip_all)]
pub async fn restore<Db: DbHandle>(
pub async fn restore(
&self,
ctx: &RpcContext,
db: &mut Db,
pkg_id: &PackageId,
pkg_version: &Version,
interfaces: &Interfaces,
volumes: &Volumes,
) -> Result<(), Error> {
) -> Result<Option<Url>, Error> {
let mut volumes = volumes.clone();
volumes.insert(VolumeId::Backup, Volume::Backup { readonly: true });
self.restore
@@ -223,32 +220,7 @@ impl BackupActions {
)
})?,
)?;
let pde = crate::db::DatabaseModel::new()
.package_data()
.idx_model(pkg_id)
.expect(db)
.await?
.installed()
.expect(db)
.await?;
pde.marketplace_url()
.put(db, &metadata.marketplace_url)
.await?;
let entry = crate::db::DatabaseModel::new()
.package_data()
.idx_model(pkg_id)
.expect(db)
.await?
.installed()
.expect(db)
.await?
.get(db)
.await?;
let receipts = crate::config::ConfigReceipts::new(db).await?;
reconfigure_dependents_with_live_pointers(ctx, db, &receipts, &entry).await?;
Ok(())
Ok(metadata.marketplace_url)
}
}

10
backend/src/backup/os.rs
View File

@@ -1,13 +1,13 @@
use openssl::pkey::PKey;
use openssl::x509::X509;
use patch_db::Value;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::account::AccountInfo;
use crate::hostname::{generate_hostname, generate_id, Hostname};
use crate::net::keys::Key;
use crate::prelude::*;
use crate::util::serde::Base64;
use crate::Error;
pub struct OsBackup {
pub account: AccountInfo,
@@ -20,11 +20,11 @@ impl<'de> Deserialize<'de> for OsBackup {
{
let tagged = OsBackupSerDe::deserialize(deserializer)?;
match tagged.version {
0 => serde_json::from_value::<OsBackupV0>(tagged.rest)
0 => patch_db::value::from_value::<OsBackupV0>(tagged.rest)
.map_err(serde::de::Error::custom)?
.project()
.map_err(serde::de::Error::custom),
1 => serde_json::from_value::<OsBackupV1>(tagged.rest)
1 => patch_db::value::from_value::<OsBackupV1>(tagged.rest)
.map_err(serde::de::Error::custom)?
.project()
.map_err(serde::de::Error::custom),
@@ -41,7 +41,7 @@ impl Serialize for OsBackup {
{
OsBackupSerDe {
version: 1,
rest: serde_json::to_value(
rest: patch_db::value::to_value(
&OsBackupV1::unproject(self).map_err(serde::ser::Error::custom)?,
)
.map_err(serde::ser::Error::custom)?,

152
backend/src/backup/restore.rs
View File

@@ -5,11 +5,9 @@ use std::sync::Arc;
use std::time::Duration;
use clap::ArgMatches;
use color_eyre::eyre::eyre;
use futures::future::BoxFuture;
use futures::{stream, FutureExt, StreamExt};
use openssl::x509::X509;
use patch_db::{DbHandle, PatchDbHandle};
use rpc_toolkit::command;
use sqlx::Connection;
use tokio::fs::File;
@@ -21,7 +19,7 @@ use crate::backup::os::OsBackup;
use crate::backup::BackupMetadata;
use crate::context::rpc::RpcContextConfig;
use crate::context::{RpcContext, SetupContext};
use crate::db::model::{PackageDataEntry, StaticFiles};
use crate::db::model::{PackageDataEntry, PackageDataEntryRestoring, StaticFiles};
use crate::disk::mount::backup::{BackupMountGuard, PackageBackupMountGuard};
use crate::disk::mount::filesystem::ReadWrite;
use crate::disk::mount::guard::TmpMountGuard;
@@ -30,6 +28,7 @@ use crate::init::init;
use crate::install::progress::InstallProgress;
use crate::install::{download_install_s9pk, PKG_PUBLIC_DIR};
use crate::notifications::NotificationLevel;
use crate::prelude::*;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::s9pk::reader::S9pkReader;
use crate::setup::SetupStatus;
@@ -37,7 +36,6 @@ use crate::util::display_none;
use crate::util::io::dir_size;
use crate::util::serde::IoFormat;
use crate::volume::{backup_dir, BACKUP_DIR, PKG_VOLUME_DIR};
use crate::{Error, ResultExt};
fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<Vec<PackageId>, Error> {
arg.split(',')
@@ -46,33 +44,31 @@ fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<Vec<PackageId>, Er
}
#[command(rename = "restore", display(display_none))]
#[instrument(skip_all)]
#[instrument(skip(ctx, password))]
pub async fn restore_packages_rpc(
#[context] ctx: RpcContext,
#[arg(parse(parse_comma_separated))] ids: Vec<PackageId>,
#[arg(rename = "target-id")] target_id: BackupTargetId,
#[arg] password: String,
) -> Result<(), Error> {
let mut db = ctx.db.handle();
let fs = target_id
.load(&mut ctx.secret_store.acquire().await?)
.await?;
let backup_guard =
BackupMountGuard::mount(TmpMountGuard::mount(&fs, ReadWrite).await?, &password).await?;
let (backup_guard, tasks, _) = restore_packages(&ctx, &mut db, backup_guard, ids).await?;
let (backup_guard, tasks, _) = restore_packages(&ctx, backup_guard, ids).await?;
tokio::spawn(async move {
stream::iter(tasks.into_iter().map(|x| (x, ctx.clone())))
.for_each_concurrent(5, |(res, ctx)| async move {
let mut db = ctx.db.handle();
match res.await {
(Ok(_), _) => (),
(Err(err), package_id) => {
if let Err(err) = ctx
.notification_manager
.notify(
&mut db,
ctx.db.clone(),
Some(package_id.clone()),
NotificationLevel::Error,
"Restoration Failure".to_string(),
@@ -169,7 +165,7 @@ impl ProgressInfo {
}
}
#[instrument(skip_all)]
#[instrument(skip(ctx))]
pub async fn recover_full_embassy(
ctx: SetupContext,
disk_guid: Arc<String>,
@@ -184,20 +180,18 @@ pub async fn recover_full_embassy(
.await?;
let os_backup_path = backup_guard.as_ref().join("os-backup.cbor");
let mut os_backup: OsBackup =
IoFormat::Cbor.from_slice(&tokio::fs::read(&os_backup_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
os_backup_path.display().to_string(),
)
})?)?;
let mut os_backup: OsBackup = IoFormat::Cbor.from_slice(
&tokio::fs::read(&os_backup_path)
.await
.with_ctx(|_| (ErrorKind::Filesystem, os_backup_path.display().to_string()))?,
)?;
os_backup.account.password = argon2::hash_encoded(
embassy_password.as_bytes(),
&rand::random::<[u8; 16]>()[..],
&argon2::Config::default(),
)
.with_kind(crate::ErrorKind::PasswordHashGeneration)?;
.with_kind(ErrorKind::PasswordHashGeneration)?;
let secret_store = ctx.secret_store().await?;
@@ -211,27 +205,24 @@ pub async fn recover_full_embassy(
let rpc_ctx = RpcContext::init(ctx.config_path.clone(), disk_guid.clone()).await?;
let mut db = rpc_ctx.db.handle();
let ids = backup_guard
let ids: Vec<_> = backup_guard
.metadata
.package_backups
.keys()
.cloned()
.collect();
let (backup_guard, tasks, progress_info) =
restore_packages(&rpc_ctx, &mut db, backup_guard, ids).await?;
restore_packages(&rpc_ctx, backup_guard, ids).await?;
let task_consumer_rpc_ctx = rpc_ctx.clone();
tokio::select! {
_ = async move {
stream::iter(tasks.into_iter().map(|x| (x, task_consumer_rpc_ctx.clone())))
.for_each_concurrent(5, |(res, ctx)| async move {
let mut db = ctx.db.handle();
match res.await {
(Ok(_), _) => (),
(Err(err), package_id) => {
if let Err(err) = ctx.notification_manager.notify(
&mut db,
ctx.db.clone(),
Some(package_id.clone()),
NotificationLevel::Error,
"Restoration Failure".to_string(), format!("Error restoring package {}: {}", package_id,err), (), None).await{
@@ -261,9 +252,9 @@ pub async fn recover_full_embassy(
))
}
#[instrument(skip(ctx, backup_guard))]
async fn restore_packages(
ctx: &RpcContext,
db: &mut PatchDbHandle,
backup_guard: BackupMountGuard<TmpMountGuard>,
ids: Vec<PackageId>,
) -> Result<
@@ -274,7 +265,7 @@ async fn restore_packages(
),
Error,
> {
let guards = assure_restoring(ctx, db, ids, &backup_guard).await?;
let guards = assure_restoring(ctx, ids, &backup_guard).await?;
let mut progress_info = ProgressInfo::default();
@@ -282,7 +273,9 @@ async fn restore_packages(
for (manifest, guard) in guards {
let id = manifest.id.clone();
let (progress, task) = restore_package(ctx.clone(), manifest, guard).await?;
progress_info.package_installs.insert(id.clone(), progress);
progress_info
.package_installs
.insert(id.clone(), progress.clone());
progress_info
.src_volume_size
.insert(id.clone(), dir_size(backup_dir(&id), None).await?);
@@ -306,23 +299,20 @@ async fn restore_packages(
Ok((backup_guard, tasks, progress_info))
}
#[instrument(skip_all)]
#[instrument(skip(ctx, backup_guard))]
async fn assure_restoring(
ctx: &RpcContext,
db: &mut PatchDbHandle,
ids: Vec<PackageId>,
backup_guard: &BackupMountGuard<TmpMountGuard>,
) -> Result<Vec<(Manifest, PackageBackupMountGuard)>, Error> {
let mut tx = db.begin().await?;
let mut guards = Vec::with_capacity(ids.len());
let mut insert_packages = BTreeMap::new();
for id in ids {
let mut model = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&id)
.get_mut(&mut tx)
.await?;
let peek = ctx.db.peek().await?;
let model = peek.as_package_data().as_idx(&id);
if !model.is_none() {
return Err(Error::new(
@@ -330,14 +320,15 @@ async fn assure_restoring(
crate::ErrorKind::InvalidRequest,
));
}
let guard = backup_guard.mount_package_backup(&id).await?;
let s9pk_path = Path::new(BACKUP_DIR).join(&id).join(format!("{}.s9pk", id));
let mut rdr = S9pkReader::open(&s9pk_path, false).await?;
let manifest = rdr.manifest().await?;
let version = manifest.version.clone();
let progress = InstallProgress::new(Some(tokio::fs::metadata(&s9pk_path).await?.len()));
let progress = Arc::new(InstallProgress::new(Some(
tokio::fs::metadata(&s9pk_path).await?.len(),
)));
let public_dir_path = ctx
.datadir
@@ -361,22 +352,29 @@ async fn assure_restoring(
let mut dst = File::create(&icon_path).await?;
tokio::io::copy(&mut rdr.icon().await?, &mut dst).await?;
dst.sync_all().await?;
*model = Some(PackageDataEntry::Restoring {
install_progress: progress.clone(),
static_files: StaticFiles::local(&id, &version, manifest.assets.icon_type()),
manifest: manifest.clone(),
});
model.save(&mut tx).await?;
insert_packages.insert(
id.clone(),
PackageDataEntry::Restoring(PackageDataEntryRestoring {
install_progress: progress.clone(),
static_files: StaticFiles::local(&id, &version, manifest.assets.icon_type()),
manifest: manifest.clone(),
}),
);
guards.push((manifest, guard));
}
tx.commit().await?;
ctx.db
.mutate(|db| {
for (id, package) in insert_packages {
db.as_package_data_mut().insert(&id, &package)?;
}
Ok(())
})
.await?;
Ok(guards)
}
#[instrument(skip_all)]
#[instrument(skip(ctx, guard))]
async fn restore_package<'a>(
ctx: RpcContext,
manifest: Manifest,
@@ -388,13 +386,11 @@ async fn restore_package<'a>(
.join(format!("{}.s9pk", id));
let metadata_path = Path::new(BACKUP_DIR).join(&id).join("metadata.cbor");
let metadata: BackupMetadata =
IoFormat::Cbor.from_slice(&tokio::fs::read(&metadata_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
metadata_path.display().to_string(),
)
})?)?;
let metadata: BackupMetadata = IoFormat::Cbor.from_slice(
&tokio::fs::read(&metadata_path)
.await
.with_ctx(|_| (ErrorKind::Filesystem, metadata_path.display().to_string()))?,
)?;
let mut secrets = ctx.secret_store.acquire().await?;
let mut secrets_tx = secrets.begin().await?;
@@ -402,8 +398,8 @@ async fn restore_package<'a>(
let k = key.0.as_slice();
sqlx::query!(
"INSERT INTO network_keys (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
*id,
*iface,
id.to_string(),
iface.to_string(),
k,
)
.execute(&mut secrets_tx).await?;
@@ -413,8 +409,8 @@ async fn restore_package<'a>(
let k = key.0.as_slice();
sqlx::query!(
"INSERT INTO tor (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
*id,
*iface,
id.to_string(),
iface.to_string(),
k,
)
.execute(&mut secrets_tx).await?;
@@ -424,26 +420,38 @@ async fn restore_package<'a>(
let len = tokio::fs::metadata(&s9pk_path)
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
s9pk_path.display().to_string(),
)
})?
.with_ctx(|_| (ErrorKind::Filesystem, s9pk_path.display().to_string()))?
.len();
let file = File::open(&s9pk_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
s9pk_path.display().to_string(),
)
})?;
let file = File::open(&s9pk_path)
.await
.with_ctx(|_| (ErrorKind::Filesystem, s9pk_path.display().to_string()))?;
let progress = InstallProgress::new(Some(len));
let marketplace_url = metadata.marketplace_url;
let progress = Arc::new(progress);
ctx.db
.mutate(|db| {
db.as_package_data_mut().insert(
&id,
&PackageDataEntry::Restoring(PackageDataEntryRestoring {
install_progress: progress.clone(),
static_files: StaticFiles::local(
&id,
&manifest.version,
manifest.assets.icon_type(),
),
manifest: manifest.clone(),
}),
)
})
.await?;
Ok((
progress.clone(),
async move {
download_install_s9pk(&ctx, &manifest, None, progress, file, None).await?;
download_install_s9pk(ctx, manifest, marketplace_url, progress, file, None).await?;
guard.unmount().await?;
Ok(())

10
backend/src/backup/target/cifs.rs
View File

@@ -12,9 +12,9 @@ use crate::disk::mount::filesystem::cifs::Cifs;
use crate::disk::mount::filesystem::ReadOnly;
use crate::disk::mount::guard::TmpMountGuard;
use crate::disk::util::{recovery_info, EmbassyOsRecoveryInfo};
use crate::prelude::*;
use crate::util::display_none;
use crate::util::serde::KeyVal;
use crate::Error;
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
@@ -84,7 +84,7 @@ pub async fn update(
} else {
return Err(Error::new(
eyre!("Backup Target ID {} Not Found", id),
crate::ErrorKind::NotFound,
ErrorKind::NotFound,
));
};
let cifs = Cifs {
@@ -112,7 +112,7 @@ pub async fn update(
{
return Err(Error::new(
eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
crate::ErrorKind::NotFound,
ErrorKind::NotFound,
));
};
Ok(KeyVal {
@@ -134,7 +134,7 @@ pub async fn remove(#[context] ctx: RpcContext, #[arg] id: BackupTargetId) -> Re
} else {
return Err(Error::new(
eyre!("Backup Target ID {} Not Found", id),
crate::ErrorKind::NotFound,
ErrorKind::NotFound,
));
};
if sqlx::query!("DELETE FROM cifs_shares WHERE id = $1", id)
@@ -145,7 +145,7 @@ pub async fn remove(#[context] ctx: RpcContext, #[arg] id: BackupTargetId) -> Re
{
return Err(Error::new(
eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
crate::ErrorKind::NotFound,
ErrorKind::NotFound,
));
};
Ok(())

18
backend/src/backup/target/mod.rs
View File

@@ -7,7 +7,6 @@ use clap::ArgMatches;
use color_eyre::eyre::eyre;
use digest::generic_array::GenericArray;
use digest::OutputSizeUser;
use lazy_static::lazy_static;
use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
use sha2::Sha256;
@@ -23,10 +22,10 @@ use crate::disk::mount::filesystem::cifs::Cifs;
use crate::disk::mount::filesystem::{FileSystem, MountType, ReadWrite};
use crate::disk::mount::guard::TmpMountGuard;
use crate::disk::util::PartitionInfo;
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::util::serde::{deserialize_from_str, display_serializable, serialize_display};
use crate::util::{display_none, Version};
use crate::Error;
pub mod cifs;
@@ -44,7 +43,7 @@ pub enum BackupTarget {
Cifs(CifsBackupTarget),
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
pub enum BackupTargetId {
Disk { logicalname: PathBuf },
Cifs { id: i32 },
@@ -73,14 +72,14 @@ impl std::fmt::Display for BackupTargetId {
impl std::str::FromStr for BackupTargetId {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.split_once("-") {
match s.split_once('-') {
Some(("disk", logicalname)) => Ok(BackupTargetId::Disk {
logicalname: Path::new(logicalname).to_owned(),
}),
Some(("cifs", id)) => Ok(BackupTargetId::Cifs { id: id.parse()? }),
_ => Err(Error::new(
eyre!("Invalid Backup Target ID"),
crate::ErrorKind::InvalidBackupTargetId,
ErrorKind::InvalidBackupTargetId,
)),
}
}
@@ -214,7 +213,7 @@ fn display_backup_info(info: BackupInfo, matches: &ArgMatches) {
]);
for (id, info) in info.package_backups {
let row = row![
id.as_str(),
&*id,
info.version.as_str(),
info.os_version.as_str(),
&info.timestamp.to_string(),
@@ -225,7 +224,7 @@ fn display_backup_info(info: BackupInfo, matches: &ArgMatches) {
}
#[command(display(display_backup_info))]
#[instrument(skip_all)]
#[instrument(skip(ctx, password))]
pub async fn info(
#[context] ctx: RpcContext,
#[arg(rename = "target-id")] target_id: BackupTargetId,
@@ -250,7 +249,7 @@ pub async fn info(
Ok(res)
}
lazy_static! {
lazy_static::lazy_static! {
static ref USER_MOUNTS: Mutex<BTreeMap<BackupTargetId, BackupMountGuard<TmpMountGuard>>> =
Mutex::new(BTreeMap::new());
}
@@ -287,11 +286,10 @@ pub async fn mount(
Ok(res)
}
#[command(display(display_none))]
#[instrument(skip_all)]
pub async fn umount(
#[context] ctx: RpcContext,
#[context] _ctx: RpcContext,
#[arg(rename = "target-id")] target_id: Option<BackupTargetId>,
) -> Result<(), Error> {
let mut mounts = USER_MOUNTS.lock().await;

1
backend/src/bins/start_init.rs
View File

@@ -15,7 +15,6 @@ use crate::init::STANDBY_MODE_PATH;
use crate::net::web_server::WebServer;
use crate::shutdown::Shutdown;
use crate::sound::CHIME;
use crate::util::logger::EmbassyLogger;
use crate::util::Invoke;
use crate::{Error, ErrorKind, ResultExt, OS_ARCH};

16
backend/src/config/action.rs
View File

@@ -2,7 +2,6 @@ use std::collections::{BTreeMap, BTreeSet};
use color_eyre::eyre::eyre;
use models::ImageId;
use nix::sys::signal::Signal;
use patch_db::HasModel;
use serde::{Deserialize, Serialize};
use tracing::instrument;
@@ -10,6 +9,7 @@ use tracing::instrument;
use super::{Config, ConfigSpec};
use crate::context::RpcContext;
use crate::dependencies::Dependencies;
use crate::prelude::*;
use crate::procedure::docker::DockerContainers;
use crate::procedure::{PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
@@ -18,7 +18,7 @@ use crate::util::Version;
use crate::volume::Volumes;
use crate::{Error, ResultExt};
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct ConfigRes {
pub config: Option<Config>,
@@ -26,6 +26,7 @@ pub struct ConfigRes {
}
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
#[model = "Model<Self>"]
pub struct ConfigActions {
pub get: PackageProcedure,
pub set: PackageProcedure,
@@ -34,16 +35,16 @@ impl ConfigActions {
#[instrument(skip_all)]
pub fn validate(
&self,
container: &Option<DockerContainers>,
_container: &Option<DockerContainers>,
eos_version: &Version,
volumes: &Volumes,
image_ids: &BTreeSet<ImageId>,
) -> Result<(), Error> {
self.get
.validate(container, eos_version, volumes, image_ids, true)
.validate(eos_version, volumes, image_ids, true)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Get"))?;
self.set
.validate(container, eos_version, volumes, image_ids, true)
.validate(eos_version, volumes, image_ids, true)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Set"))?;
Ok(())
}
@@ -99,7 +100,6 @@ impl ConfigActions {
})
})?;
Ok(SetResult {
signal: res.signal,
depends_on: res
.depends_on
.into_iter()
@@ -112,9 +112,5 @@ impl ConfigActions {
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct SetResult {
#[serde(default)]
#[serde(deserialize_with = "crate::util::serde::deserialize_from_str_opt")]
#[serde(serialize_with = "crate::util::serde::serialize_display_opt")]
pub signal: Option<Signal>,
pub depends_on: BTreeMap<PackageId, BTreeSet<HealthCheckId>>,
}

356
backend/src/config/mod.rs
View File

@@ -1,42 +1,37 @@
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use color_eyre::eyre::eyre;
use indexmap::IndexSet;
use itertools::Itertools;
use models::ErrorKind;
use patch_db::{DbHandle, LockReceipt, LockTarget, LockTargetId, LockType, Verifier};
use models::{ErrorKind, OptionExt};
use patch_db::value::InternedString;
use patch_db::Value;
use regex::Regex;
use rpc_toolkit::command;
use serde_json::Value;
use tracing::instrument;
use crate::context::RpcContext;
use crate::db::model::{CurrentDependencies, CurrentDependents};
use crate::dependencies::{
BreakTransitiveReceipts, BreakageRes, Dependencies, DependencyConfig, DependencyErrors,
DependencyReceipt, TaggedDependencyError, TryHealReceipts,
};
use crate::install::cleanup::UpdateDependencyReceipts;
use crate::procedure::docker::DockerContainers;
use crate::db::model::CurrentDependencies;
use crate::prelude::*;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::util::display_none;
use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
use crate::Error;
pub mod action;
pub mod hook;
pub mod spec;
pub mod util;
pub use spec::{ConfigSpec, Defaultable};
use util::NumRange;
use self::action::{ConfigActions, ConfigRes};
use self::spec::{ConfigPointerReceipts, ValueSpecPointer};
use self::action::ConfigRes;
use self::spec::ValueSpecPointer;
pub type Config = serde_json::Map<String, Value>;
pub type Config = patch_db::value::InOMap<InternedString, Value>;
pub trait TypeOf {
fn type_of(&self) -> &'static str;
}
@@ -80,7 +75,7 @@ pub struct TimeoutError;
#[derive(Clone, Debug, thiserror::Error)]
pub struct NoMatchWithPath {
pub path: Vec<String>,
pub path: Vec<InternedString>,
pub error: MatchError,
}
impl NoMatchWithPath {
@@ -90,7 +85,7 @@ impl NoMatchWithPath {
error,
}
}
pub fn prepend(mut self, seg: String) -> Self {
pub fn prepend(mut self, seg: InternedString) -> Self {
self.path.push(seg);
self
}
@@ -109,9 +104,9 @@ impl From<NoMatchWithPath> for Error {
#[derive(Clone, Debug, thiserror::Error)]
pub enum MatchError {
#[error("String {0:?} Does Not Match Pattern {1}")]
Pattern(String, Regex),
Pattern(Arc<String>, Regex),
#[error("String {0:?} Is Not In Enum {1:?}")]
Enum(String, IndexSet<String>),
Enum(Arc<String>, IndexSet<String>),
#[error("Field Is Not Nullable")]
NotNullable,
#[error("Length Mismatch: expected {0}, actual: {1}")]
@@ -123,11 +118,11 @@ pub enum MatchError {
#[error("Number Is Not Integral: {0}")]
NonIntegral(f64),
#[error("Variant {0:?} Is Not In Union {1:?}")]
Union(String, IndexSet<String>),
Union(Arc<String>, IndexSet<String>),
#[error("Variant Is Missing Tag {0:?}")]
MissingTag(String),
MissingTag(InternedString),
#[error("Property {0:?} Of Variant {1:?} Conflicts With Union Tag")]
PropertyMatchesUnionTag(String, String),
PropertyMatchesUnionTag(InternedString, String),
#[error("Name of Property {0:?} Conflicts With Map Tag Name")]
PropertyNameMatchesMapTag(String),
#[error("Pointer Is Invalid: {0}")]
@@ -163,55 +158,6 @@ pub fn config(#[arg] id: PackageId) -> Result<PackageId, Error> {
Ok(id)
}
pub struct ConfigGetReceipts {
manifest_volumes: LockReceipt<crate::volume::Volumes, ()>,
manifest_version: LockReceipt<crate::util::Version, ()>,
manifest_config: LockReceipt<Option<ConfigActions>, ()>,
}
impl ConfigGetReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks, id);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(
locks: &mut Vec<LockTargetId>,
id: &PackageId,
) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
let manifest_version = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.manifest().version())
.make_locker(LockType::Write)
.add_to_keys(locks);
let manifest_volumes = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.manifest().volumes())
.make_locker(LockType::Write)
.add_to_keys(locks);
let manifest_config = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.manifest().config())
.make_locker(LockType::Write)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
manifest_volumes: manifest_volumes.verify(skeleton_key)?,
manifest_version: manifest_version.verify(skeleton_key)?,
manifest_config: manifest_config.verify(skeleton_key)?,
})
}
}
}
#[command(display(display_serializable))]
#[instrument(skip_all)]
pub async fn get(
@@ -221,16 +167,21 @@ pub async fn get(
#[arg(long = "format")]
format: Option<IoFormat>,
) -> Result<ConfigRes, Error> {
let mut db = ctx.db.handle();
let receipts = ConfigGetReceipts::new(&mut db, &id).await?;
let action = receipts
.manifest_config
.get(&mut db)
.await?
let db = ctx.db.peek().await?;
let manifest = db
.as_package_data()
.as_idx(&id)
.or_not_found(&id)?
.as_installed()
.or_not_found(&id)?
.as_manifest();
let action = manifest
.as_config()
.de()?
.ok_or_else(|| Error::new(eyre!("{} has no config", id), crate::ErrorKind::NotFound))?;
let volumes = receipts.manifest_volumes.get(&mut db).await?;
let version = receipts.manifest_version.get(&mut db).await?;
let volumes = manifest.as_volumes().de()?;
let version = manifest.as_version().de()?;
action.get(&ctx, &id, &version, &volumes).await
}
@@ -251,172 +202,12 @@ pub fn set(
Ok((id, config, timeout.map(|d| *d)))
}
/// So, the new locking finds all the possible locks and lifts them up into a bundle of locks.
/// Then this bundle will be passed down into the functions that will need to touch the db, and
/// instead of doing the locks down in the system, we have already done the locks and can
/// do the operation on the db.
/// An UnlockedLock has two types, the type of setting and getting from the db, and the second type
/// is the keys that we need to insert on getting/setting because we have included wild cards into the paths.
pub struct ConfigReceipts {
pub dependency_receipt: DependencyReceipt,
pub config_receipts: ConfigPointerReceipts,
pub update_dependency_receipts: UpdateDependencyReceipts,
pub try_heal_receipts: TryHealReceipts,
pub break_transitive_receipts: BreakTransitiveReceipts,
pub configured: LockReceipt<bool, String>,
pub config_actions: LockReceipt<ConfigActions, String>,
pub dependencies: LockReceipt<Dependencies, String>,
pub volumes: LockReceipt<crate::volume::Volumes, String>,
pub version: LockReceipt<crate::util::Version, String>,
pub manifest: LockReceipt<Manifest, String>,
pub system_pointers: LockReceipt<Vec<spec::SystemPointerSpec>, String>,
pub current_dependents: LockReceipt<CurrentDependents, String>,
pub current_dependencies: LockReceipt<CurrentDependencies, String>,
pub dependency_errors: LockReceipt<DependencyErrors, String>,
pub manifest_dependencies_config: LockReceipt<DependencyConfig, (String, String)>,
pub docker_containers: LockReceipt<DockerContainers, String>,
}
impl ConfigReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
let dependency_receipt = DependencyReceipt::setup(locks);
let config_receipts = ConfigPointerReceipts::setup(locks);
let update_dependency_receipts = UpdateDependencyReceipts::setup(locks);
let break_transitive_receipts = BreakTransitiveReceipts::setup(locks);
let try_heal_receipts = TryHealReceipts::setup(locks);
let configured: LockTarget<bool, String> = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.status().configured())
.make_locker(LockType::Write)
.add_to_keys(locks);
let config_actions = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.and_then(|x| x.manifest().config())
.make_locker(LockType::Read)
.add_to_keys(locks);
let dependencies = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest().dependencies())
.make_locker(LockType::Read)
.add_to_keys(locks);
let volumes = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest().volumes())
.make_locker(LockType::Read)
.add_to_keys(locks);
let version = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest().version())
.make_locker(LockType::Read)
.add_to_keys(locks);
let manifest = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest())
.make_locker(LockType::Read)
.add_to_keys(locks);
let system_pointers = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.system_pointers())
.make_locker(LockType::Write)
.add_to_keys(locks);
let current_dependents = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.current_dependents())
.make_locker(LockType::Write)
.add_to_keys(locks);
let current_dependencies = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.current_dependencies())
.make_locker(LockType::Write)
.add_to_keys(locks);
let dependency_errors = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.status().dependency_errors())
.make_locker(LockType::Write)
.add_to_keys(locks);
let manifest_dependencies_config = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.and_then(|x| x.manifest().dependencies().star().config())
.make_locker(LockType::Write)
.add_to_keys(locks);
let docker_containers = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.and_then(|x| x.manifest().containers())
.make_locker(LockType::Write)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
dependency_receipt: dependency_receipt(skeleton_key)?,
config_receipts: config_receipts(skeleton_key)?,
try_heal_receipts: try_heal_receipts(skeleton_key)?,
break_transitive_receipts: break_transitive_receipts(skeleton_key)?,
update_dependency_receipts: update_dependency_receipts(skeleton_key)?,
configured: configured.verify(skeleton_key)?,
config_actions: config_actions.verify(skeleton_key)?,
dependencies: dependencies.verify(skeleton_key)?,
volumes: volumes.verify(skeleton_key)?,
version: version.verify(skeleton_key)?,
manifest: manifest.verify(skeleton_key)?,
system_pointers: system_pointers.verify(skeleton_key)?,
current_dependents: current_dependents.verify(skeleton_key)?,
current_dependencies: current_dependencies.verify(skeleton_key)?,
dependency_errors: dependency_errors.verify(skeleton_key)?,
manifest_dependencies_config: manifest_dependencies_config.verify(skeleton_key)?,
docker_containers: docker_containers.verify(skeleton_key)?,
})
}
}
}
#[command(rename = "dry", display(display_serializable))]
#[instrument(skip_all)]
pub async fn set_dry(
#[context] ctx: RpcContext,
#[parent_data] (id, config, timeout): (PackageId, Option<Config>, Option<Duration>),
) -> Result<BreakageRes, Error> {
) -> Result<BTreeMap<PackageId, String>, Error> {
let breakages = BTreeMap::new();
let overrides = Default::default();
@@ -429,11 +220,11 @@ pub async fn set_dry(
};
let breakages = configure(&ctx, &id, configure_context).await?;
Ok(BreakageRes(breakages))
Ok(breakages)
}
pub struct ConfigureContext {
pub breakages: BTreeMap<PackageId, TaggedDependencyError>,
pub breakages: BTreeMap<PackageId, String>,
pub timeout: Option<Duration>,
pub config: Option<Config>,
pub overrides: BTreeMap<PackageId, Config>,
@@ -464,20 +255,15 @@ pub async fn configure(
ctx: &RpcContext,
id: &PackageId,
configure_context: ConfigureContext,
) -> Result<BTreeMap<PackageId, TaggedDependencyError>, Error> {
let mut db = ctx.db.handle();
let version = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.expect(&mut db)
.await?
.installed()
.expect(&mut db)
.await?
.manifest()
.version()
.get(&mut ctx.db.handle())
.await?;
) -> Result<BTreeMap<PackageId, String>, Error> {
let db = ctx.db.peek().await?;
let package = db
.as_package_data()
.as_idx(id)
.or_not_found(&id)?
.as_installed()
.or_not_found(&id)?;
let version = package.as_manifest().as_version().de()?;
ctx.managers
.get(&(id.clone(), version.clone()))
.await
@@ -500,63 +286,3 @@ macro_rules! not_found {
};
}
pub(crate) use not_found;
/// We want to have a double check that the paths are what we expect them to be.
/// Found that earlier the paths where not what we expected them to be.
#[tokio::test]
async fn ensure_creation_of_config_paths_makes_sense() {
let mut fake = patch_db::test_utils::NoOpDb();
let config_locks = ConfigReceipts::new(&mut fake).await.unwrap();
assert_eq!(
&format!("{}", config_locks.configured.lock.glob),
"/package-data/*/installed/status/configured"
);
assert_eq!(
&format!("{}", config_locks.config_actions.lock.glob),
"/package-data/*/installed/manifest/config"
);
assert_eq!(
&format!("{}", config_locks.dependencies.lock.glob),
"/package-data/*/installed/manifest/dependencies"
);
assert_eq!(
&format!("{}", config_locks.volumes.lock.glob),
"/package-data/*/installed/manifest/volumes"
);
assert_eq!(
&format!("{}", config_locks.version.lock.glob),
"/package-data/*/installed/manifest/version"
);
assert_eq!(
&format!("{}", config_locks.volumes.lock.glob),
"/package-data/*/installed/manifest/volumes"
);
assert_eq!(
&format!("{}", config_locks.manifest.lock.glob),
"/package-data/*/installed/manifest"
);
assert_eq!(
&format!("{}", config_locks.manifest.lock.glob),
"/package-data/*/installed/manifest"
);
assert_eq!(
&format!("{}", config_locks.system_pointers.lock.glob),
"/package-data/*/installed/system-pointers"
);
assert_eq!(
&format!("{}", config_locks.current_dependents.lock.glob),
"/package-data/*/installed/current-dependents"
);
assert_eq!(
&format!("{}", config_locks.dependency_errors.lock.glob),
"/package-data/*/installed/status/dependency-errors"
);
assert_eq!(
&format!("{}", config_locks.manifest_dependencies_config.lock.glob),
"/package-data/*/installed/manifest/dependencies/*/config"
);
assert_eq!(
&format!("{}", config_locks.system_pointers.lock.glob),
"/package-data/*/installed/system-pointers"
);
}

446
backend/src/config/spec.rs
View File

@@ -9,15 +9,16 @@ use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use imbl::Vector;
use imbl_value::InternedString;
use indexmap::{IndexMap, IndexSet};
use itertools::Itertools;
use jsonpath_lib::Compiled as CompiledJsonPath;
use patch_db::{DbHandle, LockReceipt, LockType};
use patch_db::value::{Number, Value};
use rand::{CryptoRng, Rng};
use regex::Regex;
use serde::de::{MapAccess, Visitor};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::{Number, Value};
use sqlx::PgPool;
use super::util::{self, CharSet, NumRange, UniqueBy, STATIC_NULL};
@@ -26,8 +27,8 @@ use crate::config::ConfigurationError;
use crate::context::RpcContext;
use crate::net::interface::InterfaceId;
use crate::net::keys::Key;
use crate::prelude::*;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::Error;
// Config Value Specifications
#[async_trait]
@@ -39,14 +40,12 @@ pub trait ValueSpec {
// since not all inVariant can be checked by the type
fn validate(&self, manifest: &Manifest) -> Result<(), NoMatchWithPath>;
// update is to fill in values for environment pointers recursively
async fn update<Db: DbHandle>(
async fn update(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError>;
// returns all pointers that are live in the provided config
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath>;
@@ -156,17 +155,15 @@ where
fn validate(&self, manifest: &Manifest) -> Result<(), NoMatchWithPath> {
self.inner.validate(manifest)
}
async fn update<Db: DbHandle>(
async fn update(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
self.inner
.update(ctx, db, manifest, config_overrides, value, receipts)
.update(ctx, manifest, config_overrides, value)
.await
}
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -201,17 +198,15 @@ where
fn validate(&self, manifest: &Manifest) -> Result<(), NoMatchWithPath> {
self.inner.validate(manifest)
}
async fn update<Db: DbHandle>(
async fn update(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
self.inner
.update(ctx, db, manifest, config_overrides, value, receipts)
.update(ctx, manifest, config_overrides, value)
.await
}
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -279,17 +274,15 @@ where
fn validate(&self, manifest: &Manifest) -> Result<(), NoMatchWithPath> {
self.inner.validate(manifest)
}
async fn update<Db: DbHandle>(
async fn update(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
self.inner
.update(ctx, db, manifest, config_overrides, value, receipts)
.update(ctx, manifest, config_overrides, value)
.await
}
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -394,48 +387,22 @@ impl ValueSpec for ValueSpecAny {
ValueSpecAny::Pointer(a) => a.validate(manifest),
}
}
async fn update<Db: DbHandle>(
async fn update(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
match self {
ValueSpecAny::Boolean(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::Enum(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::List(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::Number(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::Object(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::String(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::Union(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::Pointer(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::Boolean(a) => a.update(ctx, manifest, config_overrides, value).await,
ValueSpecAny::Enum(a) => a.update(ctx, manifest, config_overrides, value).await,
ValueSpecAny::List(a) => a.update(ctx, manifest, config_overrides, value).await,
ValueSpecAny::Number(a) => a.update(ctx, manifest, config_overrides, value).await,
ValueSpecAny::Object(a) => a.update(ctx, manifest, config_overrides, value).await,
ValueSpecAny::String(a) => a.update(ctx, manifest, config_overrides, value).await,
ValueSpecAny::Union(a) => a.update(ctx, manifest, config_overrides, value).await,
ValueSpecAny::Pointer(a) => a.update(ctx, manifest, config_overrides, value).await,
}
}
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -513,14 +480,12 @@ impl ValueSpec for ValueSpecBoolean {
fn validate(&self, _manifest: &Manifest) -> Result<(), NoMatchWithPath> {
Ok(())
}
async fn update<Db: DbHandle>(
async fn update(
&self,
_ctx: &RpcContext,
_db: &mut Db,
_manifest: &Manifest,
_config_overrides: &BTreeMap<PackageId, Config>,
_value: &mut Value,
_receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
Ok(())
}
@@ -584,7 +549,7 @@ impl ValueSpec for ValueSpecEnum {
fn matches(&self, val: &Value) -> Result<(), NoMatchWithPath> {
match val {
Value::String(b) => {
if self.values.contains(b) {
if self.values.contains(&**b) {
Ok(())
} else {
Err(NoMatchWithPath::new(MatchError::Enum(
@@ -603,14 +568,12 @@ impl ValueSpec for ValueSpecEnum {
fn validate(&self, _manifest: &Manifest) -> Result<(), NoMatchWithPath> {
Ok(())
}
async fn update<Db: DbHandle>(
async fn update(
&self,
_ctx: &RpcContext,
_db: &mut Db,
_manifest: &Manifest,
_config_overrides: &BTreeMap<PackageId, Config>,
_value: &mut Value,
_receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
Ok(())
}
@@ -628,7 +591,7 @@ impl ValueSpec for ValueSpecEnum {
}
}
impl DefaultableWith for ValueSpecEnum {
type DefaultSpec = String;
type DefaultSpec = Arc<String>;
type Error = crate::util::Never;
fn gen_with<R: Rng + CryptoRng + Sync + Send + Send>(
@@ -666,13 +629,13 @@ where
.map(|(i, v)| {
self.spec
.matches(v)
.map_err(|e| e.prepend(format!("{}", i)))?;
.map_err(|e| e.prepend(InternedString::from_display(&i)))?;
if l.iter()
.enumerate()
.any(|(i2, v2)| i != i2 && self.spec.eq(v, v2))
{
Err(NoMatchWithPath::new(MatchError::ListUniquenessViolation)
.prepend(format!("{}", i)))
.prepend(InternedString::from_display(&i)))
} else {
Ok(())
}
@@ -690,25 +653,19 @@ where
fn validate(&self, manifest: &Manifest) -> Result<(), NoMatchWithPath> {
self.spec.validate(manifest)
}
async fn update<Db: DbHandle>(
async fn update(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
if let Value::Array(ref mut ls) = value {
for (i, val) in ls.into_iter().enumerate() {
match self
.spec
.update(ctx, db, manifest, config_overrides, val, receipts)
.await
{
Err(ConfigurationError::NoMatch(e)) => {
Err(ConfigurationError::NoMatch(e.prepend(format!("{}", i))))
}
for (i, val) in ls.iter_mut().enumerate() {
match self.spec.update(ctx, manifest, config_overrides, val).await {
Err(ConfigurationError::NoMatch(e)) => Err(ConfigurationError::NoMatch(
e.prepend(InternedString::from_display(&i)),
)),
a => a,
}?;
}
@@ -755,9 +712,9 @@ where
rng: &mut R,
timeout: &Option<Duration>,
) -> Result<Value, Self::Error> {
let mut res = Vec::new();
let mut res = Vector::new();
for spec_member in spec.iter() {
res.push(self.spec.gen_with(spec_member, rng, timeout)?);
res.push_back(self.spec.gen_with(spec_member, rng, timeout)?);
}
Ok(Value::Array(res))
}
@@ -798,36 +755,19 @@ impl ValueSpec for ValueSpecList {
ValueSpecList::Union(a) => a.validate(manifest),
}
}
async fn update<Db: DbHandle>(
async fn update(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
match self {
ValueSpecList::Enum(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecList::Number(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecList::Object(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecList::String(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecList::Union(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecList::Enum(a) => a.update(ctx, manifest, config_overrides, value).await,
ValueSpecList::Number(a) => a.update(ctx, manifest, config_overrides, value).await,
ValueSpecList::Object(a) => a.update(ctx, manifest, config_overrides, value).await,
ValueSpecList::String(a) => a.update(ctx, manifest, config_overrides, value).await,
ValueSpecList::Union(a) => a.update(ctx, manifest, config_overrides, value).await,
}
}
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -885,7 +825,7 @@ impl Defaultable for ValueSpecList {
)
.contains(&ret.len())
{
ret.push(
ret.push_back(
a.inner
.inner
.spec
@@ -941,14 +881,12 @@ impl ValueSpec for ValueSpecNumber {
fn validate(&self, _manifest: &Manifest) -> Result<(), NoMatchWithPath> {
Ok(())
}
async fn update<Db: DbHandle>(
async fn update(
&self,
_ctx: &RpcContext,
_db: &mut Db,
_manifest: &Manifest,
_config_overrides: &BTreeMap<PackageId, Config>,
_value: &mut Value,
_receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
Ok(())
}
@@ -1005,19 +943,15 @@ impl ValueSpec for ValueSpecObject {
fn validate(&self, manifest: &Manifest) -> Result<(), NoMatchWithPath> {
self.spec.validate(manifest)
}
async fn update<Db: DbHandle>(
async fn update(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
if let Value::Object(o) = value {
self.spec
.update(ctx, db, manifest, config_overrides, o, receipts)
.await
self.spec.update(ctx, manifest, config_overrides, o).await
} else {
Err(ConfigurationError::NoMatch(NoMatchWithPath::new(
MatchError::InvalidType("object", value.type_of()),
@@ -1074,11 +1008,11 @@ impl Defaultable for ValueSpecObject {
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ConfigSpec(pub IndexMap<String, ValueSpecAny>);
pub struct ConfigSpec(pub IndexMap<InternedString, ValueSpecAny>);
impl ConfigSpec {
pub fn matches(&self, value: &Config) -> Result<(), NoMatchWithPath> {
for (key, val) in self.0.iter() {
if let Some(v) = value.get(key) {
if let Some(v) = value.get(&**key) {
val.matches(v).map_err(|e| e.prepend(key.clone()))?;
} else {
val.matches(&Value::Null)
@@ -1108,27 +1042,21 @@ impl ConfigSpec {
Ok(())
}
pub async fn update<Db: DbHandle>(
pub async fn update(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
cfg: &mut Config,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
for (k, vs) in self.0.iter() {
match cfg.get_mut(k) {
None => {
let mut v = Value::Null;
vs.update(ctx, db, manifest, config_overrides, &mut v, receipts)
.await?;
vs.update(ctx, manifest, config_overrides, &mut v).await?;
cfg.insert(k.clone(), v);
}
Some(v) => match vs
.update(ctx, db, manifest, config_overrides, v, receipts)
.await
{
Some(v) => match vs.update(ctx, manifest, config_overrides, v).await {
Err(ConfigurationError::NoMatch(e)) => {
Err(ConfigurationError::NoMatch(e.prepend(k.clone())))
}
@@ -1247,7 +1175,7 @@ impl<'de> Deserialize<'de> for ValueSpecString {
})
}
}
const FIELDS: &'static [&'static str] = &[
const FIELDS: &[&str] = &[
"pattern",
"pattern-description",
"textarea",
@@ -1268,7 +1196,7 @@ impl ValueSpec for ValueSpecString {
Ok(())
} else {
Err(NoMatchWithPath::new(MatchError::Pattern(
s.to_owned(),
s.clone(),
pattern.pattern.clone(),
)))
}
@@ -1286,14 +1214,12 @@ impl ValueSpec for ValueSpecString {
fn validate(&self, _manifest: &Manifest) -> Result<(), NoMatchWithPath> {
Ok(())
}
async fn update<Db: DbHandle>(
async fn update(
&self,
_ctx: &RpcContext,
_db: &mut Db,
_manifest: &Manifest,
_config_overrides: &BTreeMap<PackageId, Config>,
_value: &mut Value,
_receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
Ok(())
}
@@ -1352,11 +1278,11 @@ pub enum DefaultString {
Entropy(Entropy),
}
impl DefaultString {
pub fn gen<R: Rng + CryptoRng + Sync + Send>(&self, rng: &mut R) -> String {
match self {
pub fn gen<R: Rng + CryptoRng + Sync + Send>(&self, rng: &mut R) -> Arc<String> {
Arc::new(match self {
DefaultString::Literal(s) => s.clone(),
DefaultString::Entropy(e) => e.gen(rng),
}
})
}
}
@@ -1380,7 +1306,7 @@ impl Entropy {
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct UnionTag {
pub id: String,
pub id: InternedString,
pub name: String,
pub description: Option<String>,
pub variant_names: BTreeMap<String, String>,
@@ -1401,7 +1327,7 @@ impl<'de> serde::de::Deserialize<'de> for ValueSpecUnion {
#[serde(rename_all = "kebab-case")]
#[serde(untagged)]
pub enum _UnionTag {
Old(String),
Old(InternedString),
New(UnionTag),
}
#[derive(Deserialize)]
@@ -1419,7 +1345,7 @@ impl<'de> serde::de::Deserialize<'de> for ValueSpecUnion {
tag: match u.tag {
_UnionTag::Old(id) => UnionTag {
id: id.clone(),
name: id,
name: id.to_string(),
description: None,
variant_names: u
.variants
@@ -1461,10 +1387,10 @@ impl ValueSpec for ValueSpecUnion {
fn matches(&self, value: &Value) -> Result<(), NoMatchWithPath> {
match value {
Value::Object(o) => {
if let Some(Value::String(ref tag)) = o.get(&self.tag.id) {
if let Some(obj_spec) = self.variants.get(tag) {
if let Some(Value::String(ref tag)) = o.get(&*self.tag.id) {
if let Some(obj_spec) = self.variants.get(&**tag) {
let mut without_tag = o.clone();
without_tag.remove(&self.tag.id);
without_tag.remove(&*self.tag.id);
obj_spec.matches(&without_tag)
} else {
Err(NoMatchWithPath::new(MatchError::Union(
@@ -1487,7 +1413,7 @@ impl ValueSpec for ValueSpecUnion {
}
fn validate(&self, manifest: &Manifest) -> Result<(), NoMatchWithPath> {
for (name, variant) in &self.variants {
if variant.0.get(&self.tag.id).is_some() {
if variant.0.get(&*self.tag.id).is_some() {
return Err(NoMatchWithPath::new(MatchError::PropertyMatchesUnionTag(
self.tag.id.clone(),
name.clone(),
@@ -1497,28 +1423,23 @@ impl ValueSpec for ValueSpecUnion {
}
Ok(())
}
async fn update<Db: DbHandle>(
async fn update(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
if let Value::Object(o) = value {
match o.get(&self.tag.id) {
match o.get(&*self.tag.id) {
None => Err(ConfigurationError::NoMatch(NoMatchWithPath::new(
MatchError::MissingTag(self.tag.id.clone()),
))),
Some(Value::String(tag)) => match self.variants.get(tag) {
Some(Value::String(tag)) => match self.variants.get(&**tag) {
None => Err(ConfigurationError::NoMatch(NoMatchWithPath::new(
MatchError::Union(tag.clone(), self.variants.keys().cloned().collect()),
))),
Some(spec) => {
spec.update(ctx, db, manifest, config_overrides, o, receipts)
.await
}
Some(spec) => spec.update(ctx, manifest, config_overrides, o).await,
},
Some(other) => Err(ConfigurationError::NoMatch(
NoMatchWithPath::new(MatchError::InvalidType("string", other.type_of()))
@@ -1533,11 +1454,11 @@ impl ValueSpec for ValueSpecUnion {
}
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
if let Value::Object(o) = value {
match o.get(&self.tag.id) {
match o.get(&*self.tag.id) {
None => Err(NoMatchWithPath::new(MatchError::MissingTag(
self.tag.id.clone(),
))),
Some(Value::String(tag)) => match self.variants.get(tag) {
Some(Value::String(tag)) => match self.variants.get(&**tag) {
None => Err(NoMatchWithPath::new(MatchError::Union(
tag.clone(),
self.variants.keys().cloned().collect(),
@@ -1559,8 +1480,8 @@ impl ValueSpec for ValueSpecUnion {
}
fn requires(&self, id: &PackageId, value: &Value) -> bool {
if let Value::Object(o) = value {
match o.get(&self.tag.id) {
Some(Value::String(tag)) => match self.variants.get(tag) {
match o.get(&*self.tag.id) {
Some(Value::String(tag)) => match self.variants.get(&**tag) {
None => false,
Some(spec) => spec.requires(id, o),
},
@@ -1578,7 +1499,7 @@ impl ValueSpec for ValueSpecUnion {
}
}
impl DefaultableWith for ValueSpecUnion {
type DefaultSpec = String;
type DefaultSpec = Arc<String>;
type Error = ConfigurationError;
fn gen_with<R: Rng + CryptoRng + Sync + Send>(
@@ -1587,7 +1508,7 @@ impl DefaultableWith for ValueSpecUnion {
rng: &mut R,
timeout: &Option<Duration>,
) -> Result<Value, Self::Error> {
let variant = if let Some(v) = self.variants.get(spec) {
let variant = if let Some(v) = self.variants.get(&**spec) {
v
} else {
return Err(ConfigurationError::NoMatch(NoMatchWithPath::new(
@@ -1643,24 +1564,16 @@ impl ValueSpec for ValueSpecPointer {
ValueSpecPointer::System(a) => a.validate(manifest),
}
}
async fn update<Db: DbHandle>(
async fn update(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
match self {
ValueSpecPointer::Package(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecPointer::System(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecPointer::Package(a) => a.update(ctx, manifest, config_overrides, value).await,
ValueSpecPointer::System(a) => a.update(ctx, manifest, config_overrides, value).await,
}
}
fn pointers(&self, _value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -1697,23 +1610,17 @@ impl PackagePointerSpec {
PackagePointerSpec::Config(ConfigPointer { package_id, .. }) => package_id,
}
}
async fn deref<Db: DbHandle>(
async fn deref(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
receipts: &ConfigPointerReceipts,
) -> Result<Value, ConfigurationError> {
match &self {
PackagePointerSpec::TorKey(key) => key.deref(&manifest.id, &ctx.secret_store).await,
PackagePointerSpec::TorAddress(tor) => {
tor.deref(db, &receipts.interface_addresses_receipt).await
}
PackagePointerSpec::LanAddress(lan) => {
lan.deref(db, &receipts.interface_addresses_receipt).await
}
PackagePointerSpec::Config(cfg) => cfg.deref(ctx, db, config_overrides, receipts).await,
PackagePointerSpec::TorAddress(tor) => tor.deref(ctx).await,
PackagePointerSpec::LanAddress(lan) => lan.deref(ctx).await,
PackagePointerSpec::Config(cfg) => cfg.deref(ctx, config_overrides).await,
}
}
}
@@ -1754,18 +1661,14 @@ impl ValueSpec for PackagePointerSpec {
_ => Ok(()),
}
}
async fn update<Db: DbHandle>(
async fn update(
&self,
ctx: &RpcContext,
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
*value = self
.deref(ctx, db, manifest, config_overrides, receipts)
.await?;
*value = self.deref(ctx, manifest, config_overrides).await?;
Ok(())
}
fn pointers(&self, _value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -1788,18 +1691,20 @@ pub struct TorAddressPointer {
interface: InterfaceId,
}
impl TorAddressPointer {
async fn deref<Db: DbHandle>(
&self,
db: &mut Db,
receipt: &InterfaceAddressesReceipt,
) -> Result<Value, ConfigurationError> {
let addr = receipt
.interface_addresses
.get(db, (&self.package_id, &self.interface))
async fn deref(&self, ctx: &RpcContext) -> Result<Value, ConfigurationError> {
let addr = ctx
.db
.peek()
.await
.map_err(|e| ConfigurationError::SystemError(Error::from(e)))?
.and_then(|addresses| addresses.tor_address);
Ok(addr.to_owned().map(Value::String).unwrap_or(Value::Null))
.map_err(|e| ConfigurationError::SystemError(e))?
.as_package_data()
.as_idx(&self.package_id)
.and_then(|pde| pde.as_installed())
.and_then(|i| i.as_interface_addresses().as_idx(&self.interface))
.and_then(|a| a.as_tor_address().de().transpose())
.transpose()
.map_err(|e| ConfigurationError::SystemError(e))?;
Ok(addr.map(Arc::new).map(Value::String).unwrap_or(Value::Null))
}
}
impl fmt::Display for TorAddressPointer {
@@ -1813,39 +1718,6 @@ impl fmt::Display for TorAddressPointer {
}
}
pub struct InterfaceAddressesReceipt {
interface_addresses: LockReceipt<crate::db::model::InterfaceAddresses, (String, String)>,
}
impl InterfaceAddressesReceipt {
pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(
locks: &mut Vec<patch_db::LockTargetId>,
) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
// let cleanup_receipts = CleanupFailedReceipts::setup(locks);
let interface_addresses = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.interface_addresses().star())
.make_locker(LockType::Read)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
// cleanup_receipts: cleanup_receipts(skeleton_key)?,
interface_addresses: interface_addresses.verify(skeleton_key)?,
})
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct LanAddressPointer {
@@ -1862,73 +1734,27 @@ impl fmt::Display for LanAddressPointer {
}
}
impl LanAddressPointer {
async fn deref<Db: DbHandle>(
&self,
db: &mut Db,
receipts: &InterfaceAddressesReceipt,
) -> Result<Value, ConfigurationError> {
let addr = receipts
.interface_addresses
.get(db, (&self.package_id, &self.interface))
async fn deref(&self, ctx: &RpcContext) -> Result<Value, ConfigurationError> {
let addr = ctx
.db
.peek()
.await
.ok()
.flatten()
.and_then(|x| x.lan_address);
Ok(addr.to_owned().map(Value::String).unwrap_or(Value::Null))
.map_err(|e| ConfigurationError::SystemError(e))?
.as_package_data()
.as_idx(&self.package_id)
.and_then(|pde| pde.as_installed())
.and_then(|i| i.as_interface_addresses().as_idx(&self.interface))
.and_then(|a| a.as_lan_address().de().transpose())
.transpose()
.map_err(|e| ConfigurationError::SystemError(e))?;
Ok(addr
.to_owned()
.map(Arc::new)
.map(Value::String)
.unwrap_or(Value::Null))
}
}
pub struct ConfigPointerReceipts {
interface_addresses_receipt: InterfaceAddressesReceipt,
manifest_volumes: LockReceipt<crate::volume::Volumes, String>,
manifest_version: LockReceipt<crate::util::Version, String>,
config_actions: LockReceipt<super::action::ConfigActions, String>,
}
impl ConfigPointerReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(
locks: &mut Vec<patch_db::LockTargetId>,
) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
let interface_addresses_receipt = InterfaceAddressesReceipt::setup(locks);
let manifest_volumes = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest().volumes())
.make_locker(LockType::Read)
.add_to_keys(locks);
let manifest_version = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest().version())
.make_locker(LockType::Read)
.add_to_keys(locks);
let config_actions = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.and_then(|x| x.manifest().config())
.make_locker(LockType::Read)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
interface_addresses_receipt: interface_addresses_receipt(skeleton_key)?,
manifest_volumes: manifest_volumes.verify(skeleton_key)?,
config_actions: config_actions.verify(skeleton_key)?,
manifest_version: manifest_version.verify(skeleton_key)?,
})
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct ConfigPointer {
@@ -1940,25 +1766,38 @@ impl ConfigPointer {
pub fn select(&self, val: &Value) -> Value {
self.selector.select(self.multi, val)
}
async fn deref<Db: DbHandle>(
async fn deref(
&self,
ctx: &RpcContext,
db: &mut Db,
config_overrides: &BTreeMap<PackageId, Config>,
receipts: &ConfigPointerReceipts,
) -> Result<Value, ConfigurationError> {
if let Some(cfg) = config_overrides.get(&self.package_id) {
Ok(self.select(&Value::Object(cfg.clone())))
} else {
let id = &self.package_id;
let version = receipts.manifest_version.get(db, id).await.ok().flatten();
let cfg_actions = receipts.config_actions.get(db, id).await.ok().flatten();
let volumes = receipts.manifest_volumes.get(db, id).await.ok().flatten();
if let (Some(version), Some(cfg_actions), Some(volumes)) =
(&version, &cfg_actions, &volumes)
{
let db = ctx
.db
.peek()
.await
.map_err(|e| ConfigurationError::SystemError(e))?;
let manifest = db.as_package_data().as_idx(id).map(|pde| pde.as_manifest());
let cfg_actions = manifest.and_then(|m| m.as_config().transpose_ref());
if let (Some(manifest), Some(cfg_actions)) = (manifest, cfg_actions) {
let cfg_res = cfg_actions
.get(ctx, &self.package_id, version, volumes)
.de()
.map_err(|e| ConfigurationError::SystemError(e))?
.get(
ctx,
&self.package_id,
&manifest
.as_version()
.de()
.map_err(|e| ConfigurationError::SystemError(e))?,
&manifest
.as_volumes()
.de()
.map_err(|e| ConfigurationError::SystemError(e))?,
)
.await
.map_err(|e| ConfigurationError::SystemError(e))?;
if let Some(cfg) = cfg_res.config {
@@ -1990,7 +1829,7 @@ pub struct ConfigSelector {
}
impl ConfigSelector {
fn select(&self, multi: bool, val: &Value) -> Value {
let selected = self.compiled.select(&val).ok().unwrap_or_else(Vec::new);
let selected = self.compiled.select(&val).ok().unwrap_or_else(Vector::new);
if multi {
Value::Array(selected.into_iter().cloned().collect())
} else {
@@ -2069,10 +1908,10 @@ impl TorKeyPointer {
)
.await
.map_err(ConfigurationError::SystemError)?;
Ok(Value::String(base32::encode(
Ok(Value::String(Arc::new(base32::encode(
base32::Alphabet::RFC4648 { padding: false },
&key.tor_key().as_bytes(),
)))
))))
}
}
impl fmt::Display for TorKeyPointer {
@@ -2092,7 +1931,7 @@ impl fmt::Display for SystemPointerSpec {
}
}
impl SystemPointerSpec {
async fn deref<Db: DbHandle>(&self, _db: &mut Db) -> Result<Value, ConfigurationError> {
async fn deref(&self, _ctx: &RpcContext) -> Result<Value, ConfigurationError> {
#[allow(unreachable_code)]
Ok(match *self {})
}
@@ -2115,17 +1954,14 @@ impl ValueSpec for SystemPointerSpec {
fn validate(&self, _manifest: &Manifest) -> Result<(), NoMatchWithPath> {
Ok(())
}
async fn update<Db: DbHandle>(
async fn update(
&self,
_ctx: &RpcContext,
db: &mut Db,
ctx: &RpcContext,
_manifest: &Manifest,
_config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
_receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
*value = self.deref(db).await?;
*value = self.deref(ctx).await?;
Ok(())
}
fn pointers(&self, _value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {

4
backend/src/config/util.rs
View File

@@ -1,9 +1,9 @@
use std::borrow::Cow;
use std::ops::{Bound, RangeBounds, RangeInclusive};
use patch_db::Value;
use rand::distributions::Distribution;
use rand::Rng;
use serde_json::Value;
use super::Config;
@@ -321,7 +321,7 @@ impl UniqueBy {
match self {
UniqueBy::Any(any) => any.iter().any(|u| u.eq(lhs, rhs)),
UniqueBy::All(all) => all.iter().all(|u| u.eq(lhs, rhs)),
UniqueBy::Exactly(key) => lhs.get(key) == rhs.get(key),
UniqueBy::Exactly(key) => lhs.get(&**key) == rhs.get(&**key),
UniqueBy::NotUnique => false,
}
}

268
backend/src/context/rpc.rs
View File

@@ -4,14 +4,11 @@ use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
use bollard::Docker;
use helpers::to_tmp_path;
use josekit::jwk::Jwk;
use models::PackageId;
use patch_db::json_ptr::JsonPointer;
use patch_db::{DbHandle, LockReceipt, LockType, PatchDb};
use patch_db::PatchDb;
use reqwest::{Client, Proxy, Url};
use rpc_toolkit::Context;
use serde::Deserialize;
@@ -22,12 +19,12 @@ use tracing::instrument;
use super::setup::CURRENT_SECRET;
use crate::account::AccountInfo;
use crate::config::hook::ConfigHook;
use crate::core::rpc_continuations::{RequestGuid, RestHandler, RpcContinuation};
use crate::db::model::{CurrentDependents, Database, InstalledPackageDataEntry, PackageDataEntry};
use crate::db::model::{CurrentDependents, Database, PackageDataEntryMatchModelRef};
use crate::db::prelude::PatchDbExt;
use crate::disk::OsPartitionInfo;
use crate::init::init_postgres;
use crate::install::cleanup::{cleanup_failed, uninstall, CleanupFailedReceipts};
use crate::install::cleanup::{cleanup_failed, uninstall};
use crate::manager::ManagerMap;
use crate::middleware::auth::HashSessionToken;
use crate::net::net_controller::NetController;
@@ -35,7 +32,7 @@ use crate::net::ssl::SslManager;
use crate::net::wifi::WpaCli;
use crate::notifications::NotificationManager;
use crate::shutdown::Shutdown;
use crate::status::{MainStatus, Status};
use crate::status::MainStatus;
use crate::system::get_mem_info;
use crate::util::config::load_config_from_paths;
use crate::util::lshw::{lshw, LshwDevice};
@@ -113,7 +110,6 @@ pub struct RpcContextSeed {
pub db: PatchDb,
pub secret_store: PgPool,
pub account: RwLock<AccountInfo>,
pub docker: Docker,
pub net_controller: Arc<NetController>,
pub managers: ManagerMap,
pub metrics_cache: RwLock<Option<crate::system::Metrics>>,
@@ -124,7 +120,6 @@ pub struct RpcContextSeed {
pub rpc_stream_continuations: Mutex<BTreeMap<RequestGuid, RpcContinuation>>,
pub wifi_manager: Option<Arc<RwLock<WpaCli>>>,
pub current_secret: Arc<Jwk>,
pub config_hooks: Mutex<BTreeMap<PackageId, Vec<ConfigHook>>>,
pub client: Client,
pub hardware: Hardware,
}
@@ -134,49 +129,11 @@ pub struct Hardware {
pub ram: u64,
}
pub struct RpcCleanReceipts {
cleanup_receipts: CleanupFailedReceipts,
packages: LockReceipt<crate::db::model::AllPackageData, ()>,
package: LockReceipt<crate::db::model::PackageDataEntry, String>,
}
impl RpcCleanReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(
locks: &mut Vec<patch_db::LockTargetId>,
) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
let cleanup_receipts = CleanupFailedReceipts::setup(locks);
let packages = crate::db::DatabaseModel::new()
.package_data()
.make_locker(LockType::Write)
.add_to_keys(locks);
let package = crate::db::DatabaseModel::new()
.package_data()
.star()
.make_locker(LockType::Write)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
cleanup_receipts: cleanup_receipts(skeleton_key)?,
packages: packages.verify(skeleton_key)?,
package: package.verify(skeleton_key)?,
})
}
}
}
#[derive(Clone)]
pub struct RpcContext(Arc<RpcContextSeed>);
impl RpcContext {
#[instrument(skip_all)]
pub async fn init<P: AsRef<Path> + Send + 'static>(
pub async fn init<P: AsRef<Path> + Send + Sync + 'static>(
cfg_path: Option<P>,
disk_guid: Arc<String>,
) -> Result<Self, Error> {
@@ -192,9 +149,6 @@ impl RpcContext {
let account = AccountInfo::load(&secret_store).await?;
let db = base.db(&account).await?;
tracing::info!("Opened PatchDB");
let mut docker = Docker::connect_with_unix_defaults()?;
docker.set_timeout(Duration::from_secs(600));
tracing::info!("Connected to Docker");
let net_controller = Arc::new(
NetController::init(
base.tor_control
@@ -212,7 +166,7 @@ impl RpcContext {
);
tracing::info!("Initialized Net Controller");
let managers = ManagerMap::default();
let metrics_cache = RwLock::new(None);
let metrics_cache = RwLock::<Option<crate::system::Metrics>>::new(None);
let notification_manager = NotificationManager::new(secret_store.clone());
tracing::info!("Initialized Notification Manager");
let tor_proxy_url = format!("socks5h://{tor_proxy}");
@@ -228,7 +182,6 @@ impl RpcContext {
db,
secret_store,
account: RwLock::new(account),
docker,
net_controller,
managers,
metrics_cache,
@@ -250,7 +203,6 @@ impl RpcContext {
)
})?,
),
config_hooks: Mutex::new(BTreeMap::new()),
client: Client::builder()
.proxy(Proxy::custom(move |url| {
if url.host_str().map_or(false, |h| h.ends_with(".onion")) {
@@ -264,16 +216,11 @@ impl RpcContext {
hardware: Hardware { devices, ram },
});
let res = Self(seed);
let res = Self(seed.clone());
res.cleanup().await?;
tracing::info!("Cleaned up transient states");
res.managers
.init(
&res,
&mut res.db.handle(),
&mut res.secret_store.acquire().await?,
)
.await?;
let peeked = res.db.peek().await?;
res.managers.init(res.clone(), peeked).await?;
tracing::info!("Initialized Package Managers");
Ok(res)
}
@@ -288,118 +235,103 @@ impl RpcContext {
Ok(())
}
#[instrument(skip_all)]
#[instrument(skip(self))]
pub async fn cleanup(&self) -> Result<(), Error> {
let mut db = self.db.handle();
let receipts = RpcCleanReceipts::new(&mut db).await?;
let packages = receipts.packages.get(&mut db).await?.0;
let mut current_dependents = packages
.keys()
.map(|k| (k.clone(), BTreeMap::new()))
.collect::<BTreeMap<_, _>>();
for (package_id, package) in packages {
for (k, v) in package
.into_installed()
.into_iter()
.flat_map(|i| i.current_dependencies.0)
{
let mut entry: BTreeMap<_, _> = current_dependents.remove(&k).unwrap_or_default();
entry.insert(package_id.clone(), v);
current_dependents.insert(k, entry);
}
}
for (package_id, current_dependents) in current_dependents {
if let Some(deps) = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&package_id)
.and_then(|pde| pde.installed())
.map::<_, CurrentDependents>(|i| i.current_dependents())
.check(&mut db)
.await?
{
deps.put(&mut db, &CurrentDependents(current_dependents))
.await?;
} else if let Some(deps) = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&package_id)
.and_then(|pde| pde.removing())
.map::<_, CurrentDependents>(|i| i.current_dependents())
.check(&mut db)
.await?
{
deps.put(&mut db, &CurrentDependents(current_dependents))
.await?;
}
}
for (package_id, package) in receipts.packages.get(&mut db).await?.0 {
if let Err(e) = async {
match package {
PackageDataEntry::Installing { .. }
| PackageDataEntry::Restoring { .. }
| PackageDataEntry::Updating { .. } => {
cleanup_failed(self, &mut db, &package_id, &receipts.cleanup_receipts)
.await?;
}
PackageDataEntry::Removing { .. } => {
uninstall(
self,
&mut db,
&mut self.secret_store.acquire().await?,
&package_id,
)
.await?;
}
PackageDataEntry::Installed {
installed,
static_files,
manifest,
} => {
for (volume_id, volume_info) in &*manifest.volumes {
let tmp_path = to_tmp_path(volume_info.path_for(
&self.datadir,
&package_id,
&manifest.version,
&volume_id,
))
.with_kind(ErrorKind::Filesystem)?;
if tokio::fs::metadata(&tmp_path).await.is_ok() {
tokio::fs::remove_dir_all(&tmp_path).await?;
}
}
let status = installed.status;
let main = match status.main {
MainStatus::BackingUp { started, .. } => {
if let Some(_) = started {
MainStatus::Starting
} else {
MainStatus::Stopped
}
}
MainStatus::Running { .. } => MainStatus::Starting,
a => a.clone(),
};
let new_package = PackageDataEntry::Installed {
installed: InstalledPackageDataEntry {
status: Status { main, ..status },
..installed
},
static_files,
manifest,
};
receipts
.package
.set(&mut db, new_package, &package_id)
.await?;
self.db
.mutate(|f| {
let mut current_dependents = f
.as_package_data()
.keys()?
.into_iter()
.map(|k| (k.clone(), BTreeMap::new()))
.collect::<BTreeMap<_, _>>();
for (package_id, package) in f.as_package_data_mut().as_entries_mut()? {
for (k, v) in package
.as_installed_mut()
.into_iter()
.flat_map(|i| i.clone().into_current_dependencies().into_entries())
.flatten()
{
let mut entry: BTreeMap<_, _> =
current_dependents.remove(&k).unwrap_or_default();
entry.insert(package_id.clone(), v.de()?);
current_dependents.insert(k, entry);
}
}
Ok::<_, Error>(())
}
.await
{
for (package_id, current_dependents) in current_dependents {
if let Some(deps) = f
.as_package_data_mut()
.as_idx_mut(&package_id)
.and_then(|pde| pde.expect_as_installed_mut().ok())
.map(|i| i.as_installed_mut().as_current_dependents_mut())
{
deps.ser(&CurrentDependents(current_dependents))?;
} else if let Some(deps) = f
.as_package_data_mut()
.as_idx_mut(&package_id)
.and_then(|pde| pde.expect_as_removing_mut().ok())
.map(|i| i.as_removing_mut().as_current_dependents_mut())
{
deps.ser(&CurrentDependents(current_dependents))?;
}
}
Ok(())
})
.await?;
let peek = self.db.peek().await?;
for (package_id, package) in peek.as_package_data().as_entries()?.into_iter() {
let package = package.clone();
let action = match package.as_match() {
PackageDataEntryMatchModelRef::Installing(_)
| PackageDataEntryMatchModelRef::Restoring(_)
| PackageDataEntryMatchModelRef::Updating(_) => {
cleanup_failed(self, &package_id).await
}
PackageDataEntryMatchModelRef::Removing(_) => {
uninstall(self, &mut self.secret_store.acquire().await?, &package_id).await
}
PackageDataEntryMatchModelRef::Installed(m) => {
let version = m.as_manifest().as_version().clone().de()?;
let volumes = m.as_manifest().as_volumes().de()?;
for (volume_id, volume_info) in &*volumes {
let tmp_path = to_tmp_path(volume_info.path_for(
&self.datadir,
&package_id,
&version,
&volume_id,
))
.with_kind(ErrorKind::Filesystem)?;
if tokio::fs::metadata(&tmp_path).await.is_ok() {
tokio::fs::remove_dir_all(&tmp_path).await?;
}
}
Ok(())
}
_ => continue,
};
if let Err(e) = action {
tracing::error!("Failed to clean up package {}: {}", package_id, e);
tracing::debug!("{:?}", e);
}
}
self.db
.mutate(|v| {
for (_, pde) in v.as_package_data_mut().as_entries_mut()? {
let status = pde
.expect_as_installed_mut()?
.as_installed_mut()
.as_status_mut()
.as_main_mut();
let running = status.clone().de()?.running();
status.ser(&if running {
MainStatus::Starting
} else {
MainStatus::Stopped
})?;
}
Ok(())
})
.await?;
Ok(())
}

228
backend/src/control.rs
View File

@@ -1,77 +1,27 @@
use std::collections::BTreeMap;
use color_eyre::eyre::eyre;
use patch_db::{DbHandle, LockReceipt, LockType};
use rpc_toolkit::command;
use tracing::instrument;
use crate::context::RpcContext;
use crate::dependencies::{
break_all_dependents_transitive, heal_all_dependents_transitive, BreakageRes, DependencyError,
DependencyReceipt, TaggedDependencyError,
};
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::status::MainStatus;
use crate::util::display_none;
use crate::util::serde::display_serializable;
use crate::Error;
#[derive(Clone)]
pub struct StartReceipts {
dependency_receipt: DependencyReceipt,
status: LockReceipt<MainStatus, ()>,
version: LockReceipt<crate::util::Version, ()>,
}
impl StartReceipts {
pub async fn new(db: &mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks, id);
setup(&db.lock_all(locks).await?)
}
pub fn setup(
locks: &mut Vec<patch_db::LockTargetId>,
id: &PackageId,
) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
let dependency_receipt = DependencyReceipt::setup(locks);
let status = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.status().main())
.make_locker(LockType::Write)
.add_to_keys(locks);
let version = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.manifest().version())
.make_locker(LockType::Read)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
dependency_receipt: dependency_receipt(skeleton_key)?,
status: status.verify(skeleton_key)?,
version: version.verify(skeleton_key)?,
})
}
}
}
#[command(display(display_none), metadata(sync_db = true))]
#[instrument(skip_all)]
pub async fn start(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<(), Error> {
let mut db = ctx.db.handle();
let mut tx = db.begin().await?;
let receipts = StartReceipts::new(&mut tx, &id).await?;
let version = receipts.version.get(&mut tx).await?;
receipts.status.set(&mut tx, MainStatus::Starting).await?;
heal_all_dependents_transitive(&ctx, &mut tx, &id, &receipts.dependency_receipt).await?;
tx.commit().await?;
drop(receipts);
let peek = ctx.db.peek().await?;
let version = peek
.as_package_data()
.as_idx(&id)
.or_not_found(&id)?
.as_installed()
.or_not_found(&id)?
.as_manifest()
.as_version()
.de()?;
ctx.managers
.get(&(id, version))
@@ -81,112 +31,33 @@ pub async fn start(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<(
Ok(())
}
#[derive(Clone)]
pub struct StopReceipts {
breaks: crate::dependencies::BreakTransitiveReceipts,
status: LockReceipt<MainStatus, ()>,
}
impl StopReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
let mut locks = Vec::new();
#[command(display(display_none), metadata(sync_db = true))]
pub async fn stop(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<MainStatus, Error> {
let peek = ctx.db.peek().await?;
let version = peek
.as_package_data()
.as_idx(&id)
.or_not_found(&id)?
.as_installed()
.or_not_found(&id)?
.as_manifest()
.as_version()
.de()?;
let setup = Self::setup(&mut locks, id);
setup(&db.lock_all(locks).await?)
}
let last_statuts = ctx
.db
.mutate(|v| {
v.as_package_data_mut()
.as_idx_mut(&id)
.and_then(|x| x.as_installed_mut())
.ok_or_else(|| Error::new(eyre!("{} is not installed", id), ErrorKind::NotFound))?
.as_status_mut()
.as_main_mut()
.replace(&MainStatus::Stopping)
})
.await?;
pub fn setup(
locks: &mut Vec<patch_db::LockTargetId>,
id: &PackageId,
) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
let breaks = crate::dependencies::BreakTransitiveReceipts::setup(locks);
let status = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.status().main())
.make_locker(LockType::Write)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
breaks: breaks(skeleton_key)?,
status: status.verify(skeleton_key)?,
})
}
}
}
#[instrument(skip_all)]
pub async fn stop_common<Db: DbHandle>(
db: &mut Db,
id: &PackageId,
breakages: &mut BTreeMap<PackageId, TaggedDependencyError>,
) -> Result<MainStatus, Error> {
let mut tx = db.begin().await?;
let receipts = StopReceipts::new(&mut tx, id).await?;
let last_status = receipts.status.get(&mut tx).await?;
receipts.status.set(&mut tx, MainStatus::Stopping).await?;
tx.save().await?;
break_all_dependents_transitive(
db,
id,
DependencyError::NotRunning,
breakages,
&receipts.breaks,
)
.await?;
Ok(last_status)
}
#[command(
subcommands(self(stop_impl(async)), stop_dry),
display(display_none),
metadata(sync_db = true)
)]
pub fn stop(#[arg] id: PackageId) -> Result<PackageId, Error> {
Ok(id)
}
#[command(rename = "dry", display(display_serializable))]
#[instrument(skip_all)]
pub async fn stop_dry(
#[context] ctx: RpcContext,
#[parent_data] id: PackageId,
) -> Result<BreakageRes, Error> {
let mut db = ctx.db.handle();
let mut tx = db.begin().await?;
let mut breakages = BTreeMap::new();
stop_common(&mut tx, &id, &mut breakages).await?;
tx.abort().await?;
Ok(BreakageRes(breakages))
}
#[instrument(skip_all)]
pub async fn stop_impl(ctx: RpcContext, id: PackageId) -> Result<MainStatus, Error> {
let mut db = ctx.db.handle();
let mut tx = db.begin().await?;
let version = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&id)
.expect(&mut tx)
.await?
.installed()
.expect(&mut tx)
.await?
.manifest()
.version()
.get(&mut tx)
.await?
.clone();
let last_statuts = stop_common(&mut tx, &id, &mut BTreeMap::new()).await?;
tx.commit().await?;
ctx.managers
.get(&(id, version))
.await
@@ -198,30 +69,21 @@ pub async fn stop_impl(ctx: RpcContext, id: PackageId) -> Result<MainStatus, Err
#[command(display(display_none), metadata(sync_db = true))]
pub async fn restart(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<(), Error> {
let mut db = ctx.db.handle();
let mut tx = db.begin().await?;
let version = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&id)
.expect(&mut tx)
.await?
.installed()
.expect(&mut tx)
.await?
.manifest()
.version()
.get(&mut tx)
.await?
.clone();
tx.commit().await?;
let peek = ctx.db.peek().await?;
let version = peek
.as_package_data()
.as_idx(&id)
.or_not_found(&id)?
.expect_as_installed()?
.as_manifest()
.as_version()
.de()?;
ctx.managers
.get(&(id, version))
.await
.ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
.restart()
.await;
.restart();
Ok(())
}

153
backend/src/db/mod.rs
View File

@@ -1,13 +1,14 @@
pub mod model;
pub mod package;
pub mod prelude;
use std::future::Future;
use std::path::PathBuf;
use std::sync::Arc;
use color_eyre::eyre::eyre;
use futures::{FutureExt, SinkExt, StreamExt};
use patch_db::json_ptr::JsonPointer;
use patch_db::{DbHandle, Dump, LockType, Revision};
use patch_db::{Dump, Revision};
use rpc_toolkit::command;
use rpc_toolkit::hyper::upgrade::Upgraded;
use rpc_toolkit::hyper::{Body, Error as HyperError, Request, Response};
@@ -22,12 +23,11 @@ use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::WebSocketStream;
use tracing::instrument;
pub use self::model::DatabaseModel;
use crate::context::RpcContext;
use crate::context::{CliContext, RpcContext};
use crate::middleware::auth::{HasValidSession, HashSessionToken};
use crate::prelude::*;
use crate::util::display_none;
use crate::util::serde::{display_serializable, IoFormat};
use crate::{Error, ResultExt};
#[instrument(skip_all)]
async fn ws_handler<
@@ -40,8 +40,8 @@ async fn ws_handler<
let (dump, sub) = ctx.db.dump_and_sub().await?;
let mut stream = ws_fut
.await
.with_kind(crate::ErrorKind::Network)?
.with_kind(crate::ErrorKind::Unknown)?;
.with_kind(ErrorKind::Network)?
.with_kind(ErrorKind::Unknown)?;
if let Some((session, token)) = session {
let kill = subscribe_to_session_kill(&ctx, token).await;
@@ -55,7 +55,7 @@ async fn ws_handler<
reason: "UNAUTHORIZED".into(),
}))
.await
.with_kind(crate::ErrorKind::Network)?;
.with_kind(ErrorKind::Network)?;
}
Ok(())
@@ -92,18 +92,18 @@ async fn deal_with_messages(
reason: "UNAUTHORIZED".into(),
}))
.await
.with_kind(crate::ErrorKind::Network)?;
.with_kind(ErrorKind::Network)?;
return Ok(())
}
new_rev = sub.recv().fuse() => {
let rev = new_rev.expect("UNREACHABLE: patch-db is dropped");
stream
.send(Message::Text(serde_json::to_string(&rev).with_kind(crate::ErrorKind::Serialization)?))
.send(Message::Text(serde_json::to_string(&rev).with_kind(ErrorKind::Serialization)?))
.await
.with_kind(crate::ErrorKind::Network)?;
.with_kind(ErrorKind::Network)?;
}
message = stream.next().fuse() => {
let message = message.transpose().with_kind(crate::ErrorKind::Network)?;
let message = message.transpose().with_kind(ErrorKind::Network)?;
match message {
None => {
tracing::info!("Closing WebSocket: Stream Finished");
@@ -123,10 +123,10 @@ async fn send_dump(
) -> Result<(), Error> {
stream
.send(Message::Text(
serde_json::to_string(&dump).with_kind(crate::ErrorKind::Serialization)?,
serde_json::to_string(&dump).with_kind(ErrorKind::Serialization)?,
))
.await
.with_kind(crate::ErrorKind::Network)?;
.with_kind(ErrorKind::Network)?;
Ok(())
}
@@ -141,7 +141,7 @@ pub async fn subscribe(ctx: RpcContext, req: Request<Body>) -> Result<Response<B
{
Ok(a) => Some(a),
Err(e) => {
if e.kind != crate::ErrorKind::Authorization {
if e.kind != ErrorKind::Authorization {
tracing::error!("Error Authenticating Websocket: {}", e);
tracing::debug!("{:?}", e);
}
@@ -149,7 +149,7 @@ pub async fn subscribe(ctx: RpcContext, req: Request<Body>) -> Result<Response<B
}
};
let req = Request::from_parts(parts, body);
let (res, ws_fut) = hyper_ws_listener::create_ws(req).with_kind(crate::ErrorKind::Network)?;
let (res, ws_fut) = hyper_ws_listener::create_ws(req).with_kind(ErrorKind::Network)?;
if let Some(ws_fut) = ws_fut {
tokio::task::spawn(async move {
match ws_handler(ctx, session, ws_fut).await {
@@ -191,12 +191,40 @@ pub async fn revisions(
})
}
#[command(display(display_serializable))]
#[instrument(skip_all)]
async fn cli_dump(
ctx: CliContext,
_format: Option<IoFormat>,
path: Option<PathBuf>,
) -> Result<Dump, RpcError> {
let dump = if let Some(path) = path {
PatchDb::open(path).await?.dump().await?
} else {
rpc_toolkit::command_helpers::call_remote(
ctx,
"db.dump",
serde_json::json!({}),
std::marker::PhantomData::<Dump>,
)
.await?
.result?
};
Ok(dump)
}
#[command(
custom_cli(cli_dump(async, context(CliContext))),
display(display_serializable)
)]
pub async fn dump(
#[context] ctx: RpcContext,
#[allow(unused_variables)]
#[arg(long = "format")]
format: Option<IoFormat>,
#[allow(unused_variables)]
#[arg]
path: Option<PathBuf>,
) -> Result<Dump, Error> {
Ok(ctx.db.dump().await?)
}
@@ -252,34 +280,77 @@ fn apply_expr(input: jaq_core::Val, expr: &str) -> Result<jaq_core::Val, Error>
Ok(res)
}
#[command(display(display_none))]
pub async fn apply(#[context] ctx: RpcContext, #[arg] expr: String) -> Result<(), Error> {
let mut db = ctx.db.handle();
#[instrument(skip_all)]
async fn cli_apply(ctx: CliContext, expr: String, path: Option<PathBuf>) -> Result<(), RpcError> {
if let Some(path) = path {
PatchDb::open(path)
.await?
.mutate(|db| {
let res = apply_expr(
serde_json::to_value(patch_db::Value::from(db.clone()))
.with_kind(ErrorKind::Deserialization)?
.into(),
&expr,
)?;
DatabaseModel::new().lock(&mut db, LockType::Write).await?;
let root_ptr = JsonPointer::<String>::default();
let input = db.get_value(&root_ptr, None).await?;
let res = (|| {
let res = apply_expr(input.into(), &expr)?;
serde_json::from_value::<model::Database>(res.clone().into()).with_ctx(|_| {
(
crate::ErrorKind::Deserialization,
"result does not match database model",
)
})?;
Ok::<serde_json::Value, Error>(res.into())
})()?;
db.put_value(&root_ptr, &res).await?;
db.ser(
&serde_json::from_value::<model::Database>(res.clone().into()).with_ctx(
|_| {
(
crate::ErrorKind::Deserialization,
"result does not match database model",
)
},
)?,
)
})
.await?;
} else {
rpc_toolkit::command_helpers::call_remote(
ctx,
"db.apply",
serde_json::json!({ "expr": expr }),
std::marker::PhantomData::<()>,
)
.await?
.result?;
}
Ok(())
}
#[command(
custom_cli(cli_apply(async, context(CliContext))),
display(display_none)
)]
pub async fn apply(
#[context] ctx: RpcContext,
#[arg] expr: String,
#[allow(unused_variables)]
#[arg]
path: Option<PathBuf>,
) -> Result<(), Error> {
ctx.db
.mutate(|db| {
let res = apply_expr(
serde_json::to_value(patch_db::Value::from(db.clone()))
.with_kind(ErrorKind::Deserialization)?
.into(),
&expr,
)?;
db.ser(
&serde_json::from_value::<model::Database>(res.clone().into()).with_ctx(|_| {
(
crate::ErrorKind::Deserialization,
"result does not match database model",
)
})?,
)
})
.await
}
#[command(subcommands(ui))]
pub fn put() -> Result<(), RpcError> {
Ok(())
@@ -297,7 +368,7 @@ pub async fn ui(
) -> Result<(), Error> {
let ptr = "/ui"
.parse::<JsonPointer>()
.with_kind(crate::ErrorKind::Database)?
.with_kind(ErrorKind::Database)?
+ &pointer;
ctx.db.put(&ptr, &value).await?;
Ok(())

355
backend/src/db/model.rs
View File

@@ -7,33 +7,30 @@ use emver::VersionRange;
use ipnet::{Ipv4Net, Ipv6Net};
use isocountry::CountryCode;
use itertools::Itertools;
use models::{DataUrl, HealthCheckId, InterfaceId};
use openssl::hash::MessageDigest;
use patch_db::json_ptr::JsonPointer;
use patch_db::{HasModel, Map, MapModel, OptionModel};
use patch_db::{HasModel, Value};
use reqwest::Url;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use ssh_key::public::Ed25519PublicKey;
use crate::account::AccountInfo;
use crate::config::spec::{PackagePointerSpec, SystemPointerSpec};
use crate::config::spec::PackagePointerSpec;
use crate::install::progress::InstallProgress;
use crate::net::forward::LanPortForwards;
use crate::net::interface::InterfaceId;
use crate::net::utils::{get_iface_ipv4_addr, get_iface_ipv6_addr};
use crate::s9pk::manifest::{Manifest, ManifestModel, PackageId};
use crate::status::health_check::HealthCheckId;
use crate::prelude::*;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::status::Status;
use crate::util::Version;
use crate::version::{Current, VersionT};
use crate::Error;
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
// #[macro_debug]
pub struct Database {
#[model]
pub server_info: ServerInfo,
#[model]
pub package_data: AllPackageData,
pub lan_port_forwards: LanPortForwards,
pub ui: Value,
@@ -41,7 +38,6 @@ pub struct Database {
impl Database {
pub fn init(account: &AccountInfo) -> Self {
let lan_address = account.hostname.lan_address().parse().unwrap();
// TODO
Database {
server_info: ServerInfo {
id: account.server_id.clone(),
@@ -51,7 +47,7 @@ impl Database {
last_wifi_region: None,
eos_version_compat: Current::new().compat().clone(),
lan_address,
tor_address: format!("https://{}", account.key.tor_address())
tor_address: format!("http://{}", account.key.tor_address())
.parse()
.unwrap(),
ip_info: BTreeMap::new(),
@@ -91,14 +87,12 @@ impl Database {
}
}
}
impl DatabaseModel {
pub fn new() -> Self {
Self::from(JsonPointer::default())
}
}
pub type DatabaseModel = Model<Database>;
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct ServerInfo {
pub id: String,
pub hostname: String,
@@ -109,10 +103,7 @@ pub struct ServerInfo {
pub eos_version_compat: VersionRange,
pub lan_address: Url,
pub tor_address: Url,
#[model]
#[serde(default)]
pub ip_info: BTreeMap<String, IpInfo>,
#[model]
#[serde(default)]
pub status_info: ServerStatus,
pub wifi: WifiInfo,
@@ -128,6 +119,7 @@ pub struct ServerInfo {
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct IpInfo {
pub ipv4_range: Option<Ipv4Net>,
pub ipv4: Option<Ipv4Addr>,
@@ -148,29 +140,31 @@ impl IpInfo {
}
#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
#[model = "Model<Self>"]
pub struct BackupProgress {
pub complete: bool,
}
#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct ServerStatus {
#[model]
pub backup_progress: Option<BTreeMap<PackageId, BackupProgress>>,
pub updated: bool,
#[model]
pub update_progress: Option<UpdateProgress>,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct UpdateProgress {
pub size: Option<u64>,
pub downloaded: u64,
}
#[derive(Debug, Deserialize, Serialize)]
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct WifiInfo {
pub ssids: Vec<String>,
pub selected: Option<String>,
@@ -197,16 +191,11 @@ pub struct AllPackageData(pub BTreeMap<PackageId, PackageDataEntry>);
impl Map for AllPackageData {
type Key = PackageId;
type Value = PackageDataEntry;
fn get(&self, key: &Self::Key) -> Option<&Self::Value> {
self.0.get(key)
}
}
impl HasModel for AllPackageData {
type Model = MapModel<Self>;
}
#[derive(Debug, Deserialize, Serialize)]
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct StaticFiles {
license: String,
instructions: String,
@@ -222,120 +211,231 @@ impl StaticFiles {
}
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct PackageDataEntryInstalling {
pub static_files: StaticFiles,
pub manifest: Manifest,
pub install_progress: Arc<InstallProgress>,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct PackageDataEntryUpdating {
pub static_files: StaticFiles,
pub manifest: Manifest,
pub installed: InstalledPackageInfo,
pub install_progress: Arc<InstallProgress>,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct PackageDataEntryRestoring {
pub static_files: StaticFiles,
pub manifest: Manifest,
pub install_progress: Arc<InstallProgress>,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct PackageDataEntryRemoving {
pub static_files: StaticFiles,
pub manifest: Manifest,
pub removing: InstalledPackageInfo,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct PackageDataEntryInstalled {
pub static_files: StaticFiles,
pub manifest: Manifest,
pub installed: InstalledPackageInfo,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(tag = "state")]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
// #[macro_debug]
pub enum PackageDataEntry {
#[serde(rename_all = "kebab-case")]
Installing {
static_files: StaticFiles,
manifest: Manifest,
install_progress: Arc<InstallProgress>,
},
#[serde(rename_all = "kebab-case")]
Updating {
static_files: StaticFiles,
manifest: Manifest,
installed: InstalledPackageDataEntry,
install_progress: Arc<InstallProgress>,
},
#[serde(rename_all = "kebab-case")]
Restoring {
static_files: StaticFiles,
manifest: Manifest,
install_progress: Arc<InstallProgress>,
},
#[serde(rename_all = "kebab-case")]
Removing {
static_files: StaticFiles,
manifest: Manifest,
removing: InstalledPackageDataEntry,
},
#[serde(rename_all = "kebab-case")]
Installed {
static_files: StaticFiles,
manifest: Manifest,
installed: InstalledPackageDataEntry,
},
Installing(PackageDataEntryInstalling),
Updating(PackageDataEntryUpdating),
Restoring(PackageDataEntryRestoring),
Removing(PackageDataEntryRemoving),
Installed(PackageDataEntryInstalled),
}
impl PackageDataEntry {
pub fn installed(&self) -> Option<&InstalledPackageDataEntry> {
match self {
Self::Installing { .. } | Self::Restoring { .. } | Self::Removing { .. } => None,
Self::Updating { installed, .. } | Self::Installed { installed, .. } => Some(installed),
impl Model<PackageDataEntry> {
pub fn expect_into_installed(self) -> Result<Model<PackageDataEntryInstalled>, Error> {
if let PackageDataEntryMatchModel::Installed(a) = self.into_match() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in installed state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn installed_mut(&mut self) -> Option<&mut InstalledPackageDataEntry> {
match self {
Self::Installing { .. } | Self::Restoring { .. } | Self::Removing { .. } => None,
Self::Updating { installed, .. } | Self::Installed { installed, .. } => Some(installed),
pub fn expect_as_installed(&self) -> Result<&Model<PackageDataEntryInstalled>, Error> {
if let PackageDataEntryMatchModelRef::Installed(a) = self.as_match() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in installed state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn into_installed(self) -> Option<InstalledPackageDataEntry> {
match self {
Self::Installing { .. } | Self::Restoring { .. } | Self::Removing { .. } => None,
Self::Updating { installed, .. } | Self::Installed { installed, .. } => Some(installed),
pub fn expect_as_installed_mut(
&mut self,
) -> Result<&mut Model<PackageDataEntryInstalled>, Error> {
if let PackageDataEntryMatchModelMut::Installed(a) = self.as_match_mut() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in installed state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn manifest(self) -> Manifest {
match self {
PackageDataEntry::Installing { manifest, .. } => manifest,
PackageDataEntry::Updating { manifest, .. } => manifest,
PackageDataEntry::Restoring { manifest, .. } => manifest,
PackageDataEntry::Removing { manifest, .. } => manifest,
PackageDataEntry::Installed { manifest, .. } => manifest,
pub fn expect_into_removing(self) -> Result<Model<PackageDataEntryRemoving>, Error> {
if let PackageDataEntryMatchModel::Removing(a) = self.into_match() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in removing state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn manifest_borrow(&self) -> &Manifest {
match self {
PackageDataEntry::Installing { manifest, .. } => manifest,
PackageDataEntry::Updating { manifest, .. } => manifest,
PackageDataEntry::Restoring { manifest, .. } => manifest,
PackageDataEntry::Removing { manifest, .. } => manifest,
PackageDataEntry::Installed { manifest, .. } => manifest,
pub fn expect_as_removing(&self) -> Result<&Model<PackageDataEntryRemoving>, Error> {
if let PackageDataEntryMatchModelRef::Removing(a) = self.as_match() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in removing state"),
ErrorKind::InvalidRequest,
))
}
}
}
impl PackageDataEntryModel {
pub fn installed(self) -> OptionModel<InstalledPackageDataEntry> {
self.0.child("installed").into()
pub fn expect_as_removing_mut(
&mut self,
) -> Result<&mut Model<PackageDataEntryRemoving>, Error> {
if let PackageDataEntryMatchModelMut::Removing(a) = self.as_match_mut() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in removing state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn removing(self) -> OptionModel<InstalledPackageDataEntry> {
self.0.child("removing").into()
pub fn expect_as_installing_mut(
&mut self,
) -> Result<&mut Model<PackageDataEntryInstalling>, Error> {
if let PackageDataEntryMatchModelMut::Installing(a) = self.as_match_mut() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in installing state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn install_progress(self) -> OptionModel<InstallProgress> {
self.0.child("install-progress").into()
pub fn into_manifest(self) -> Model<Manifest> {
match self.into_match() {
PackageDataEntryMatchModel::Installing(a) => a.into_manifest(),
PackageDataEntryMatchModel::Updating(a) => a.into_installed().into_manifest(),
PackageDataEntryMatchModel::Restoring(a) => a.into_manifest(),
PackageDataEntryMatchModel::Removing(a) => a.into_manifest(),
PackageDataEntryMatchModel::Installed(a) => a.into_manifest(),
PackageDataEntryMatchModel::Error(_) => Model::from(Value::Null),
}
}
pub fn manifest(self) -> ManifestModel {
self.0.child("manifest").into()
pub fn as_manifest(&self) -> &Model<Manifest> {
match self.as_match() {
PackageDataEntryMatchModelRef::Installing(a) => a.as_manifest(),
PackageDataEntryMatchModelRef::Updating(a) => a.as_installed().as_manifest(),
PackageDataEntryMatchModelRef::Restoring(a) => a.as_manifest(),
PackageDataEntryMatchModelRef::Removing(a) => a.as_manifest(),
PackageDataEntryMatchModelRef::Installed(a) => a.as_manifest(),
PackageDataEntryMatchModelRef::Error(_) => (&Value::Null).into(),
}
}
pub fn into_installed(self) -> Option<Model<InstalledPackageInfo>> {
match self.into_match() {
PackageDataEntryMatchModel::Installing(_) => None,
PackageDataEntryMatchModel::Updating(a) => Some(a.into_installed()),
PackageDataEntryMatchModel::Restoring(_) => None,
PackageDataEntryMatchModel::Removing(_) => None,
PackageDataEntryMatchModel::Installed(a) => Some(a.into_installed()),
PackageDataEntryMatchModel::Error(_) => None,
}
}
pub fn as_installed(&self) -> Option<&Model<InstalledPackageInfo>> {
match self.as_match() {
PackageDataEntryMatchModelRef::Installing(_) => None,
PackageDataEntryMatchModelRef::Updating(a) => Some(a.as_installed()),
PackageDataEntryMatchModelRef::Restoring(_) => None,
PackageDataEntryMatchModelRef::Removing(_) => None,
PackageDataEntryMatchModelRef::Installed(a) => Some(a.as_installed()),
PackageDataEntryMatchModelRef::Error(_) => None,
}
}
pub fn as_installed_mut(&mut self) -> Option<&mut Model<InstalledPackageInfo>> {
match self.as_match_mut() {
PackageDataEntryMatchModelMut::Installing(_) => None,
PackageDataEntryMatchModelMut::Updating(a) => Some(a.as_installed_mut()),
PackageDataEntryMatchModelMut::Restoring(_) => None,
PackageDataEntryMatchModelMut::Removing(_) => None,
PackageDataEntryMatchModelMut::Installed(a) => Some(a.as_installed_mut()),
PackageDataEntryMatchModelMut::Error(_) => None,
}
}
pub fn as_install_progress(&self) -> Option<&Model<Arc<InstallProgress>>> {
match self.as_match() {
PackageDataEntryMatchModelRef::Installing(a) => Some(a.as_install_progress()),
PackageDataEntryMatchModelRef::Updating(a) => Some(a.as_install_progress()),
PackageDataEntryMatchModelRef::Restoring(a) => Some(a.as_install_progress()),
PackageDataEntryMatchModelRef::Removing(_) => None,
PackageDataEntryMatchModelRef::Installed(_) => None,
PackageDataEntryMatchModelRef::Error(_) => None,
}
}
pub fn as_install_progress_mut(&mut self) -> Option<&mut Model<Arc<InstallProgress>>> {
match self.as_match_mut() {
PackageDataEntryMatchModelMut::Installing(a) => Some(a.as_install_progress_mut()),
PackageDataEntryMatchModelMut::Updating(a) => Some(a.as_install_progress_mut()),
PackageDataEntryMatchModelMut::Restoring(a) => Some(a.as_install_progress_mut()),
PackageDataEntryMatchModelMut::Removing(_) => None,
PackageDataEntryMatchModelMut::Installed(_) => None,
PackageDataEntryMatchModelMut::Error(_) => None,
}
}
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
pub struct InstalledPackageDataEntry {
#[model]
#[model = "Model<Self>"]
pub struct InstalledPackageInfo {
pub status: Status,
pub marketplace_url: Option<Url>,
#[serde(default)]
#[serde(with = "crate::util::serde::ed25519_pubkey")]
pub developer_key: ed25519_dalek::PublicKey,
#[model]
pub manifest: Manifest,
pub last_backup: Option<DateTime<Utc>>,
#[model]
pub system_pointers: Vec<SystemPointerSpec>,
#[model]
pub dependency_info: BTreeMap<PackageId, StaticDependencyInfo>,
#[model]
pub current_dependents: CurrentDependents,
#[model]
pub current_dependencies: CurrentDependencies,
#[model]
pub interface_addresses: InterfaceAddressMap,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
pub struct CurrentDependents(pub BTreeMap<PackageId, CurrentDependencyInfo>);
impl CurrentDependents {
pub fn map(
@@ -351,12 +451,6 @@ impl CurrentDependents {
impl Map for CurrentDependents {
type Key = PackageId;
type Value = CurrentDependencyInfo;
fn get(&self, key: &Self::Key) -> Option<&Self::Value> {
self.0.get(key)
}
}
impl HasModel for CurrentDependents {
type Model = MapModel<Self>;
}
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
@@ -375,25 +469,21 @@ impl CurrentDependencies {
impl Map for CurrentDependencies {
type Key = PackageId;
type Value = CurrentDependencyInfo;
fn get(&self, key: &Self::Key) -> Option<&Self::Value> {
self.0.get(key)
}
}
impl HasModel for CurrentDependencies {
type Model = MapModel<Self>;
}
#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)]
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct StaticDependencyInfo {
pub manifest: Option<Manifest>,
pub icon: String,
pub title: String,
pub icon: DataUrl<'static>,
}
#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct CurrentDependencyInfo {
pub pointers: Vec<PackagePointerSpec>,
pub pointers: BTreeSet<PackagePointerSpec>,
pub health_checks: BTreeSet<HealthCheckId>,
}
@@ -402,27 +492,12 @@ pub struct InterfaceAddressMap(pub BTreeMap<InterfaceId, InterfaceAddresses>);
impl Map for InterfaceAddressMap {
type Key = InterfaceId;
type Value = InterfaceAddresses;
fn get(&self, key: &Self::Key) -> Option<&Self::Value> {
self.0.get(key)
}
}
impl HasModel for InterfaceAddressMap {
type Model = MapModel<Self>;
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct InterfaceAddresses {
#[model]
pub tor_address: Option<String>,
#[model]
pub lan_address: Option<String>,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
pub struct RecoveredPackageInfo {
pub title: String,
pub icon: String,
pub version: Version,
}

91
backend/src/db/package.rs
View File

@@ -1,75 +1,22 @@
use patch_db::{DbHandle, LockReceipt, LockTargetId, LockType, Verifier};
use models::Version;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::Error;
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
pub struct PackageReceipts {
package_data: LockReceipt<super::model::AllPackageData, ()>,
}
impl PackageReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
let package_data = crate::db::DatabaseModel::new()
.package_data()
.make_locker(LockType::Read)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
package_data: package_data.verify(&skeleton_key)?,
})
}
}
}
pub async fn get_packages<Db: DbHandle>(
db: &mut Db,
receipts: &PackageReceipts,
) -> Result<Vec<PackageId>, Error> {
let packages = receipts.package_data.get(db).await?;
Ok(packages.0.keys().cloned().collect())
}
pub struct ManifestReceipts {
manifest: LockReceipt<Manifest, String>,
}
impl ManifestReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks, id);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(
locks: &mut Vec<LockTargetId>,
_id: &PackageId,
) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
let manifest = crate::db::DatabaseModel::new()
.package_data()
.star()
.manifest()
.make_locker(LockType::Read)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
manifest: manifest.verify(&skeleton_key)?,
})
}
}
}
pub async fn get_manifest<Db: DbHandle>(
db: &mut Db,
pkg: &PackageId,
receipts: &ManifestReceipts,
) -> Result<Option<Manifest>, Error> {
Ok(receipts.manifest.get(db, pkg).await?)
pub fn get_packages(db: Peeked) -> Result<Vec<(PackageId, Version)>, Error> {
Ok(db
.as_package_data()
.keys()?
.into_iter()
.flat_map(|package_id| {
let version = db
.as_package_data()
.as_idx(&package_id)?
.as_manifest()
.as_version()
.de()
.ok()?;
Some((package_id, version))
})
.collect())
}

382
backend/src/db/prelude.rs Normal file
View File

@@ -0,0 +1,382 @@
use std::collections::BTreeMap;
use std::marker::PhantomData;
use std::panic::UnwindSafe;
use patch_db::value::InternedString;
pub use patch_db::{HasModel, PatchDb, Value};
use serde::de::DeserializeOwned;
use serde::Serialize;
use crate::db::model::DatabaseModel;
use crate::prelude::*;
pub type Peeked = Model<super::model::Database>;
pub fn to_value<T>(value: &T) -> Result<Value, Error>
where
T: Serialize,
{
patch_db::value::to_value(value).with_kind(ErrorKind::Serialization)
}
pub fn from_value<T>(value: Value) -> Result<T, Error>
where
T: DeserializeOwned,
{
patch_db::value::from_value(value).with_kind(ErrorKind::Deserialization)
}
#[async_trait::async_trait]
pub trait PatchDbExt {
async fn peek(&self) -> Result<DatabaseModel, Error>;
async fn mutate<U: UnwindSafe + Send>(
&self,
f: impl FnOnce(&mut DatabaseModel) -> Result<U, Error> + UnwindSafe + Send,
) -> Result<U, Error>;
async fn map_mutate(
&self,
f: impl FnOnce(DatabaseModel) -> Result<DatabaseModel, Error> + UnwindSafe + Send,
) -> Result<DatabaseModel, Error>;
}
#[async_trait::async_trait]
impl PatchDbExt for PatchDb {
async fn peek(&self) -> Result<DatabaseModel, Error> {
Ok(DatabaseModel::from(self.dump().await?.value))
}
async fn mutate<U: UnwindSafe + Send>(
&self,
f: impl FnOnce(&mut DatabaseModel) -> Result<U, Error> + UnwindSafe + Send,
) -> Result<U, Error> {
Ok(self
.apply_function(|mut v| {
let model = <&mut DatabaseModel>::from(&mut v);
let res = f(model)?;
Ok::<_, Error>((v, res))
})
.await?
.1)
}
async fn map_mutate(
&self,
f: impl FnOnce(DatabaseModel) -> Result<DatabaseModel, Error> + UnwindSafe + Send,
) -> Result<DatabaseModel, Error> {
Ok(DatabaseModel::from(
self.apply_function(|v| f(DatabaseModel::from(v)).map(|a| (a.into(), ())))
.await?
.0,
))
}
}
/// &mut Model<T> <=> &mut Value
#[repr(transparent)]
#[derive(Debug)]
pub struct Model<T> {
value: Value,
phantom: PhantomData<T>,
}
impl<T: DeserializeOwned> Model<T> {
pub fn de(&self) -> Result<T, Error> {
from_value(self.value.clone())
}
}
impl<T: Serialize> Model<T> {
pub fn new(value: &T) -> Result<Self, Error> {
Ok(Self::from(to_value(value)?))
}
pub fn ser(&mut self, value: &T) -> Result<(), Error> {
self.value = to_value(value)?;
Ok(())
}
}
impl<T: Serialize + DeserializeOwned> Model<T> {
pub fn replace(&mut self, value: &T) -> Result<T, Error> {
let orig = self.de()?;
self.ser(value)?;
Ok(orig)
}
}
impl<T> Clone for Model<T> {
fn clone(&self) -> Self {
Self {
value: self.value.clone(),
phantom: PhantomData,
}
}
}
impl<T> From<Value> for Model<T> {
fn from(value: Value) -> Self {
Self {
value,
phantom: PhantomData,
}
}
}
impl<T> From<Model<T>> for Value {
fn from(value: Model<T>) -> Self {
value.value
}
}
impl<'a, T> From<&'a Value> for &'a Model<T> {
fn from(value: &'a Value) -> Self {
unsafe { std::mem::transmute(value) }
}
}
impl<'a, T> From<&'a Model<T>> for &'a Value {
fn from(value: &'a Model<T>) -> Self {
unsafe { std::mem::transmute(value) }
}
}
impl<'a, T> From<&'a mut Value> for &mut Model<T> {
fn from(value: &'a mut Value) -> Self {
unsafe { std::mem::transmute(value) }
}
}
impl<'a, T> From<&'a mut Model<T>> for &mut Value {
fn from(value: &'a mut Model<T>) -> Self {
unsafe { std::mem::transmute(value) }
}
}
impl<T> patch_db::Model<T> for Model<T> {
type Model<U> = Model<U>;
}
impl<T> Model<Option<T>> {
pub fn transpose(self) -> Option<Model<T>> {
use patch_db::ModelExt;
if self.value.is_null() {
None
} else {
Some(self.transmute(|a| a))
}
}
pub fn transpose_ref(&self) -> Option<&Model<T>> {
use patch_db::ModelExt;
if self.value.is_null() {
None
} else {
Some(self.transmute_ref(|a| a))
}
}
pub fn transpose_mut(&mut self) -> Option<&mut Model<T>> {
use patch_db::ModelExt;
if self.value.is_null() {
None
} else {
Some(self.transmute_mut(|a| a))
}
}
pub fn from_option(opt: Option<Model<T>>) -> Self {
use patch_db::ModelExt;
match opt {
Some(a) => a.transmute(|a| a),
None => Self::from_value(Value::Null),
}
}
}
pub trait Map: DeserializeOwned + Serialize {
type Key;
type Value;
}
impl<A, B> Map for BTreeMap<A, B>
where
A: serde::Serialize + serde::de::DeserializeOwned + Ord,
B: serde::Serialize + serde::de::DeserializeOwned,
{
type Key = A;
type Value = B;
}
impl<T: Map> Model<T>
where
T::Key: AsRef<str>,
T::Value: Serialize,
{
pub fn insert(&mut self, key: &T::Key, value: &T::Value) -> Result<(), Error> {
use serde::ser::Error;
let v = patch_db::value::to_value(value)?;
match &mut self.value {
Value::Object(o) => {
o.insert(InternedString::intern(key.as_ref()), v);
Ok(())
}
v => Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
kind: patch_db::value::ErrorKind::Serialization,
}
.into()),
}
}
pub fn insert_model(&mut self, key: &T::Key, value: Model<T::Value>) -> Result<(), Error> {
use patch_db::ModelExt;
use serde::ser::Error;
let v = value.into_value();
match &mut self.value {
Value::Object(o) => {
o.insert(InternedString::intern(key.as_ref()), v);
Ok(())
}
v => Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
kind: patch_db::value::ErrorKind::Serialization,
}
.into()),
}
}
}
impl<T: Map> Model<T>
where
T::Key: DeserializeOwned + Ord + Clone,
{
pub fn keys(&self) -> Result<Vec<T::Key>, Error> {
use serde::de::Error;
use serde::Deserialize;
match &self.value {
Value::Object(o) => o
.keys()
.cloned()
.map(|k| {
T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(k))
.map_err(|e| {
patch_db::value::Error {
kind: patch_db::value::ErrorKind::Deserialization,
source: e,
}
.into()
})
})
.collect(),
v => Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
kind: patch_db::value::ErrorKind::Deserialization,
}
.into()),
}
}
pub fn into_entries(self) -> Result<Vec<(T::Key, Model<T::Value>)>, Error> {
use patch_db::ModelExt;
use serde::de::Error;
use serde::Deserialize;
match self.value {
Value::Object(o) => o
.into_iter()
.map(|(k, v)| {
Ok((
T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(
k,
))
.with_kind(ErrorKind::Deserialization)?,
Model::from_value(v),
))
})
.collect(),
v => Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
kind: patch_db::value::ErrorKind::Deserialization,
}
.into()),
}
}
pub fn as_entries(&self) -> Result<Vec<(T::Key, &Model<T::Value>)>, Error> {
use patch_db::ModelExt;
use serde::de::Error;
use serde::Deserialize;
match &self.value {
Value::Object(o) => o
.iter()
.map(|(k, v)| {
Ok((
T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(
k.clone(),
))
.with_kind(ErrorKind::Deserialization)?,
Model::value_as(v),
))
})
.collect(),
v => Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
kind: patch_db::value::ErrorKind::Deserialization,
}
.into()),
}
}
pub fn as_entries_mut(&mut self) -> Result<Vec<(T::Key, &mut Model<T::Value>)>, Error> {
use patch_db::ModelExt;
use serde::de::Error;
use serde::Deserialize;
match &mut self.value {
Value::Object(o) => o
.iter_mut()
.map(|(k, v)| {
Ok((
T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(
k.clone(),
))
.with_kind(ErrorKind::Deserialization)?,
Model::value_as_mut(v),
))
})
.collect(),
v => Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
kind: patch_db::value::ErrorKind::Deserialization,
}
.into()),
}
}
}
impl<T: Map> Model<T>
where
T::Key: AsRef<str>,
{
pub fn into_idx(self, key: &T::Key) -> Option<Model<T::Value>> {
use patch_db::ModelExt;
match &self.value {
Value::Object(o) if o.contains_key(key.as_ref()) => Some(self.transmute(|v| {
use patch_db::value::index::Index;
key.as_ref().index_into_owned(v).unwrap()
})),
_ => None,
}
}
pub fn as_idx<'a>(&'a self, key: &T::Key) -> Option<&'a Model<T::Value>> {
use patch_db::ModelExt;
match &self.value {
Value::Object(o) if o.contains_key(key.as_ref()) => Some(self.transmute_ref(|v| {
use patch_db::value::index::Index;
key.as_ref().index_into(v).unwrap()
})),
_ => None,
}
}
pub fn as_idx_mut<'a>(&'a mut self, key: &T::Key) -> Option<&'a mut Model<T::Value>> {
use patch_db::ModelExt;
match &mut self.value {
Value::Object(o) if o.contains_key(key.as_ref()) => Some(self.transmute_mut(|v| {
use patch_db::value::index::Index;
key.as_ref().index_or_insert(v)
})),
_ => None,
}
}
pub fn remove(&mut self, key: &T::Key) -> Result<Option<Model<T::Value>>, Error> {
use serde::ser::Error;
match &mut self.value {
Value::Object(o) => {
let v = o.remove(key.as_ref());
Ok(v.map(patch_db::ModelExt::from_value))
}
v => Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
kind: patch_db::value::ErrorKind::Serialization,
}
.into()),
}
}
}

1040
backend/src/dependencies.rs
View File

File diff suppressed because it is too large Load Diff

9
backend/src/developer/mod.rs
View File

@@ -35,6 +35,15 @@ pub fn init(#[context] ctx: SdkContext) -> Result<(), Error> {
.as_bytes(),
)?;
dev_key_file.sync_all()?;
println!(
"New developer key generated at {}",
ctx.developer_key_path.display()
);
} else {
println!(
"Developer key already exists at {}",
ctx.developer_key_path.display()
);
}
Ok(())
}

1
backend/src/disk/mount/filesystem/cifs.rs
View File

@@ -19,7 +19,6 @@ async fn resolve_hostname(hostname: &str) -> Result<IpAddr, Error> {
if let Ok(addr) = hostname.parse() {
return Ok(addr);
}
#[cfg(feature = "avahi")]
if hostname.ends_with(".local") {
return Ok(IpAddr::V4(crate::net::mdns::resolve_mdns(hostname).await?));
}

2
backend/src/disk/mount/filesystem/efivarfs.rs
View File

@@ -7,7 +7,7 @@ use sha2::Sha256;
use super::{FileSystem, MountType, ReadOnly};
use crate::util::Invoke;
use crate::{Error, ResultExt};
use crate::Error;
pub struct EfiVarFs;
#[async_trait]

2
backend/src/disk/mount/util.rs
View File

@@ -3,7 +3,7 @@ use std::path::Path;
use tracing::instrument;
use crate::util::Invoke;
use crate::{Error, ResultExt};
use crate::Error;
#[instrument(skip_all)]
pub async fn bind<P0: AsRef<Path>, P1: AsRef<Path>>(

5
backend/src/disk/util.rs
View File

@@ -3,7 +3,6 @@ use std::path::{Path, PathBuf};
use color_eyre::eyre::{self, eyre};
use futures::TryStreamExt;
use indexmap::IndexSet;
use nom::bytes::complete::{tag, take_till1};
use nom::character::complete::multispace1;
use nom::character::is_space;
@@ -62,8 +61,8 @@ pub struct EmbassyOsRecoveryInfo {
pub wrapped_key: Option<String>,
}
const DISK_PATH: &'static str = "/dev/disk/by-path";
const SYS_BLOCK_PATH: &'static str = "/sys/block";
const DISK_PATH: &str = "/dev/disk/by-path";
const SYS_BLOCK_PATH: &str = "/sys/block";
lazy_static::lazy_static! {
static ref PARTITION_REGEX: Regex = Regex::new("-part[0-9]+$").unwrap();

4
backend/src/error.rs
View File

@@ -1,5 +1,5 @@
use color_eyre::eyre::eyre;
pub use models::{Error, ErrorKind, ResultExt};
pub use models::{Error, ErrorKind, OptionExt, ResultExt};
#[derive(Debug, Default)]
pub struct ErrorCollection(Vec<Error>);
@@ -54,7 +54,7 @@ impl std::fmt::Display for ErrorCollection {
macro_rules! ensure_code {
($x:expr, $c:expr, $fmt:expr $(, $arg:expr)*) => {
if !($x) {
return Err(crate::Error::new(color_eyre::eyre::eyre!($fmt, $($arg, )*), $c));
return Err(crate::error::Error::new(color_eyre::eyre::eyre!($fmt, $($arg, )*), $c));
}
};
}

10
backend/src/hostname.rs
View File

@@ -1,9 +1,7 @@
use patch_db::DbHandle;
use rand::{thread_rng, Rng};
use tokio::process::Command;
use tracing::instrument;
use crate::account::AccountInfo;
use crate::util::Invoke;
use crate::{Error, ErrorKind};
#[derive(Clone, serde::Deserialize, serde::Serialize, Debug)]
@@ -62,6 +60,14 @@ pub async fn set_hostname(hostname: &Hostname) -> Result<(), Error> {
.arg(hostname)
.invoke(ErrorKind::ParseSysInfo)
.await?;
Command::new("sed")
.arg("-i")
.arg(format!(
"s/\\(\\s\\)localhost\\( {hostname}\\)\\?/\\1localhost {hostname}/g"
))
.arg("/etc/hosts")
.invoke(ErrorKind::ParseSysInfo)
.await?;
Ok(())
}

159
backend/src/init.rs
View File

@@ -1,4 +1,3 @@
use std::collections::HashMap;
use std::fs::Permissions;
use std::os::unix::fs::PermissionsExt;
use std::path::Path;
@@ -7,19 +6,21 @@ use std::time::Duration;
use color_eyre::eyre::eyre;
use helpers::NonDetachingJoinHandle;
use models::ResultExt;
use patch_db::{DbHandle, LockReceipt, LockType};
use rand::random;
use sqlx::{Pool, Postgres};
use tokio::process::Command;
use tracing::instrument;
use crate::account::AccountInfo;
use crate::context::rpc::RpcContextConfig;
use crate::db::model::{ServerInfo, ServerStatus};
use crate::db::model::ServerStatus;
use crate::disk::mount::util::unmount;
use crate::install::PKG_ARCHIVE_DIR;
use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
use crate::prelude::*;
use crate::sound::BEP;
use crate::system::time;
use crate::util::docker::{create_bridge_network, CONTAINER_DATADIR, CONTAINER_TOOL};
use crate::util::Invoke;
use crate::{Error, ARCH};
@@ -39,40 +40,8 @@ pub async fn check_time_is_synchronized() -> Result<bool, Error> {
== "NTPSynchronized=yes")
}
pub struct InitReceipts {
pub server_info: LockReceipt<ServerInfo, ()>,
pub server_version: LockReceipt<crate::util::Version, ()>,
pub version_range: LockReceipt<emver::VersionRange, ()>,
}
impl InitReceipts {
pub async fn new(db: &mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let server_info = crate::db::DatabaseModel::new()
.server_info()
.make_locker(LockType::Write)
.add_to_keys(&mut locks);
let server_version = crate::db::DatabaseModel::new()
.server_info()
.version()
.make_locker(LockType::Write)
.add_to_keys(&mut locks);
let version_range = crate::db::DatabaseModel::new()
.server_info()
.eos_version_compat()
.make_locker(LockType::Write)
.add_to_keys(&mut locks);
let skeleton_key = db.lock_all(locks).await?;
Ok(Self {
server_info: server_info.verify(&skeleton_key)?,
server_version: server_version.verify(&skeleton_key)?,
version_range: version_range.verify(&skeleton_key)?,
})
}
}
// must be idempotent
#[tracing::instrument(skip_all)]
pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
let db_dir = datadir.as_ref().join("main/postgresql");
if tokio::process::Command::new("mountpoint")
@@ -133,7 +102,11 @@ pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
tmp
};
if tokio::fs::metadata(&conf_dir).await.is_ok() {
tokio::fs::rename(&conf_dir, &conf_dir_tmp).await?;
Command::new("mv")
.arg(&conf_dir)
.arg(&conf_dir_tmp)
.invoke(ErrorKind::Filesystem)
.await?;
}
let mut old_version = pg_version;
while old_version > 13
@@ -154,7 +127,11 @@ pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
if tokio::fs::metadata(&conf_dir).await.is_ok() {
tokio::fs::remove_dir_all(&conf_dir).await?;
}
tokio::fs::rename(&conf_dir_tmp, &conf_dir).await?;
Command::new("mv")
.arg(&conf_dir_tmp)
.arg(&conf_dir)
.invoke(ErrorKind::Filesystem)
.await?;
}
}
@@ -190,6 +167,7 @@ pub struct InitResult {
pub db: patch_db::PatchDb,
}
#[instrument(skip_all)]
pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
tokio::fs::create_dir_all("/run/embassy")
.await
@@ -222,13 +200,14 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
let account = AccountInfo::load(&secret_store).await?;
let db = cfg.db(&account).await?;
db.mutate(|d| {
let model = d.de()?;
d.ser(&model)
})
.await?;
tracing::info!("Opened PatchDB");
let mut handle = db.handle();
let mut server_info = crate::db::DatabaseModel::new()
.server_info()
.get_mut(&mut handle)
.await?;
let receipts = InitReceipts::new(&mut handle).await?;
let peek = db.peek().await?;
let mut server_info = peek.as_server_info().de()?;
// write to ca cert store
tokio::fs::write(
@@ -289,51 +268,42 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
if tokio::fs::metadata(&tmp_dir).await.is_err() {
tokio::fs::create_dir_all(&tmp_dir).await?;
}
let tmp_docker = cfg.datadir().join("package-data/tmp/docker");
let tmp_docker = cfg
.datadir()
.join(format!("package-data/tmp/{CONTAINER_TOOL}"));
let tmp_docker_exists = tokio::fs::metadata(&tmp_docker).await.is_ok();
if should_rebuild && tmp_docker_exists {
tokio::fs::remove_dir_all(&tmp_docker).await?;
}
Command::new("systemctl")
.arg("stop")
.arg("docker")
.invoke(crate::ErrorKind::Docker)
.await?;
crate::disk::mount::util::bind(&tmp_docker, "/var/lib/docker", false).await?;
Command::new("systemctl")
.arg("reset-failed")
.arg("docker")
.invoke(crate::ErrorKind::Docker)
.await?;
Command::new("systemctl")
.arg("start")
.arg("docker")
.invoke(crate::ErrorKind::Docker)
.await?;
if CONTAINER_TOOL == "docker" {
Command::new("systemctl")
.arg("stop")
.arg("docker")
.invoke(crate::ErrorKind::Docker)
.await?;
}
crate::disk::mount::util::bind(&tmp_docker, CONTAINER_DATADIR, false).await?;
if CONTAINER_TOOL == "docker" {
Command::new("systemctl")
.arg("reset-failed")
.arg("docker")
.invoke(crate::ErrorKind::Docker)
.await?;
Command::new("systemctl")
.arg("start")
.arg("docker")
.invoke(crate::ErrorKind::Docker)
.await?;
}
tracing::info!("Mounted Docker Data");
if should_rebuild || !tmp_docker_exists {
tracing::info!("Creating Docker Network");
bollard::Docker::connect_with_unix_defaults()?
.create_network(bollard::network::CreateNetworkOptions {
name: "start9",
driver: "bridge",
ipam: bollard::models::Ipam {
config: Some(vec![bollard::models::IpamConfig {
subnet: Some("172.18.0.1/24".into()),
..Default::default()
}]),
..Default::default()
},
options: {
let mut m = HashMap::new();
m.insert("com.docker.network.bridge.name", "br-start9");
m
},
..Default::default()
})
.await?;
tracing::info!("Created Docker Network");
if CONTAINER_TOOL == "docker" {
tracing::info!("Creating Docker Network");
create_bridge_network("start9", "172.18.0.1/24", "br-start9").await?;
tracing::info!("Created Docker Network");
}
tracing::info!("Loading System Docker Images");
crate::install::load_images("/usr/lib/embassy/system-images").await?;
@@ -344,8 +314,23 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
tracing::info!("Loaded Package Docker Images");
}
if CONTAINER_TOOL == "podman" {
crate::util::docker::remove_container("netdummy", true).await?;
Command::new("podman")
.arg("run")
.arg("-d")
.arg("--rm")
.arg("--network=start9")
.arg("--name=netdummy")
.arg("start9/x_system/utils:latest")
.arg("sleep")
.arg("infinity")
.invoke(crate::ErrorKind::Docker)
.await?;
}
tracing::info!("Enabling Docker QEMU Emulation");
Command::new("docker")
Command::new(CONTAINER_TOOL)
.arg("run")
.arg("--privileged")
.arg("--rm")
@@ -382,9 +367,13 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
server_info.system_start_time = time().await?;
server_info.save(&mut handle).await?;
db.mutate(|v| {
v.as_server_info_mut().ser(&server_info)?;
Ok(())
})
.await?;
crate::version::init(&mut handle, &secret_store, &receipts).await?;
crate::version::init(&db, &secret_store).await?;
if should_rebuild {
match tokio::fs::remove_file(SYSTEM_REBUILD_PATH).await {

428
backend/src/install/cleanup.rs
View File

@@ -1,149 +1,34 @@
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use bollard::image::{ListImagesOptions, RemoveImageOptions};
use patch_db::{DbHandle, LockReceipt, LockTargetId, LockType, PatchDbHandle, Verifier};
use models::OptionExt;
use sqlx::{Executor, Postgres};
use tracing::instrument;
use super::PKG_ARCHIVE_DIR;
use crate::config::{not_found, ConfigReceipts};
use crate::context::RpcContext;
use crate::db::model::{
AllPackageData, CurrentDependencies, CurrentDependents, InstalledPackageDataEntry,
PackageDataEntry,
};
use crate::dependencies::{
reconfigure_dependents_with_live_pointers, DependencyErrors, TryHealReceipts,
CurrentDependencies, Database, PackageDataEntry, PackageDataEntryInstalled,
PackageDataEntryMatchModelRef,
};
use crate::error::ErrorCollection;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::util::{Apply, Version};
use crate::volume::{asset_dir, script_dir};
use crate::Error;
pub struct UpdateDependencyReceipts {
try_heal: TryHealReceipts,
dependency_errors: LockReceipt<DependencyErrors, String>,
manifest: LockReceipt<Manifest, String>,
}
impl UpdateDependencyReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
let dependency_errors = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.status().dependency_errors())
.make_locker(LockType::Write)
.add_to_keys(locks);
let manifest = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest())
.make_locker(LockType::Write)
.add_to_keys(locks);
let try_heal = TryHealReceipts::setup(locks);
move |skeleton_key| {
Ok(Self {
dependency_errors: dependency_errors.verify(skeleton_key)?,
manifest: manifest.verify(skeleton_key)?,
try_heal: try_heal(skeleton_key)?,
})
}
}
}
#[instrument(skip_all)]
pub async fn update_dependency_errors_of_dependents<'a, Db: DbHandle>(
ctx: &RpcContext,
db: &mut Db,
id: &PackageId,
deps: &CurrentDependents,
receipts: &UpdateDependencyReceipts,
) -> Result<(), Error> {
for dep in deps.0.keys() {
if let Some(man) = receipts.manifest.get(db, dep).await? {
if let Err(e) = if let Some(info) = man.dependencies.0.get(id) {
info.satisfied(ctx, db, id, None, dep, &receipts.try_heal)
.await?
} else {
Ok(())
} {
let mut errs = receipts
.dependency_errors
.get(db, dep)
.await?
.ok_or_else(|| not_found!(dep))?;
errs.0.insert(id.clone(), e);
receipts.dependency_errors.set(db, errs, dep).await?
} else {
let mut errs = receipts
.dependency_errors
.get(db, dep)
.await?
.ok_or_else(|| not_found!(dep))?;
errs.0.remove(id);
receipts.dependency_errors.set(db, errs, dep).await?
}
}
}
Ok(())
}
#[instrument(skip_all)]
pub async fn cleanup(ctx: &RpcContext, id: &PackageId, version: &Version) -> Result<(), Error> {
let mut errors = ErrorCollection::new();
ctx.managers.remove(&(id.clone(), version.clone())).await;
// docker images start9/$APP_ID/*:$VERSION -q | xargs docker rmi
let images = ctx
.docker
.list_images(Some(ListImagesOptions {
all: false,
filters: {
let mut f = HashMap::new();
f.insert(
"reference".to_owned(),
vec![format!("start9/{}/*:{}", id, version)],
);
f
},
digests: false,
}))
.await
.apply(|res| errors.handle(res));
let images = crate::util::docker::images_for(id, version).await?;
errors.extend(
futures::future::join_all(
images
.into_iter()
.flatten()
.flat_map(|image| image.repo_tags)
.filter(|tag| {
tag.starts_with(&format!("start9/{}/", id))
&& tag.ends_with(&format!(":{}", version))
})
.map(|tag| async {
let tag = tag; // move into future
ctx.docker
.remove_image(
&tag,
Some(RemoveImageOptions {
force: true,
noprune: false,
}),
None,
)
.await
}),
)
futures::future::join_all(images.into_iter().map(|sha| async {
let sha = sha; // move into future
crate::util::docker::remove_image(&sha).await
}))
.await,
);
let pkg_archive_dir = ctx
@@ -172,66 +57,26 @@ pub async fn cleanup(ctx: &RpcContext, id: &PackageId, version: &Version) -> Res
errors.into_result()
}
pub struct CleanupFailedReceipts {
package_data_entry: LockReceipt<PackageDataEntry, String>,
package_entries: LockReceipt<AllPackageData, ()>,
}
impl CleanupFailedReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
let package_data_entry = crate::db::DatabaseModel::new()
.package_data()
.star()
.make_locker(LockType::Write)
.add_to_keys(locks);
let package_entries = crate::db::DatabaseModel::new()
.package_data()
.make_locker(LockType::Write)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
package_data_entry: package_data_entry.verify(skeleton_key).unwrap(),
package_entries: package_entries.verify(skeleton_key).unwrap(),
})
}
}
}
#[instrument(skip_all)]
pub async fn cleanup_failed<Db: DbHandle>(
ctx: &RpcContext,
db: &mut Db,
id: &PackageId,
receipts: &CleanupFailedReceipts,
) -> Result<(), Error> {
let pde = receipts
.package_data_entry
.get(db, id)
pub async fn cleanup_failed(ctx: &RpcContext, id: &PackageId) -> Result<(), Error> {
if let Some(version) = match ctx
.db
.peek()
.await?
.ok_or_else(|| not_found!(id))?;
if let Some(manifest) = match &pde {
PackageDataEntry::Installing { manifest, .. }
| PackageDataEntry::Restoring { manifest, .. } => Some(manifest),
PackageDataEntry::Updating {
manifest,
installed:
InstalledPackageDataEntry {
manifest: installed_manifest,
..
},
..
} => {
if &manifest.version != &installed_manifest.version {
Some(manifest)
.as_package_data()
.as_idx(id)
.or_not_found(id)?
.as_match()
{
PackageDataEntryMatchModelRef::Installing(m) => Some(m.as_manifest().as_version().de()?),
PackageDataEntryMatchModelRef::Restoring(m) => Some(m.as_manifest().as_version().de()?),
PackageDataEntryMatchModelRef::Updating(m) => {
let manifest_version = m.as_manifest().as_version().de()?;
let installed = m.as_installed().as_manifest().as_version().de()?;
if manifest_version != installed {
Some(manifest_version)
} else {
None
None // do not remove existing data
}
}
_ => {
@@ -239,169 +84,107 @@ pub async fn cleanup_failed<Db: DbHandle>(
None
}
} {
cleanup(ctx, id, &manifest.version).await?;
cleanup(ctx, id, &version).await?;
}
match pde {
PackageDataEntry::Installing { .. } | PackageDataEntry::Restoring { .. } => {
let mut entries = receipts.package_entries.get(db).await?;
entries.0.remove(id);
receipts.package_entries.set(db, entries).await?;
}
PackageDataEntry::Updating {
installed,
static_files,
..
} => {
receipts
.package_data_entry
.set(
db,
PackageDataEntry::Installed {
manifest: installed.manifest.clone(),
installed,
static_files,
},
id,
)
.await?;
}
_ => (),
}
Ok(())
ctx.db
.mutate(|v| {
match v
.clone()
.into_package_data()
.into_idx(id)
.or_not_found(id)?
.as_match()
{
PackageDataEntryMatchModelRef::Installing(_)
| PackageDataEntryMatchModelRef::Restoring(_) => {
v.as_package_data_mut().remove(id)?;
}
PackageDataEntryMatchModelRef::Updating(pde) => {
v.as_package_data_mut()
.as_idx_mut(id)
.or_not_found(id)?
.ser(&PackageDataEntry::Installed(PackageDataEntryInstalled {
manifest: pde.as_installed().as_manifest().de()?,
static_files: pde.as_static_files().de()?,
installed: pde.as_installed().de()?,
}))?;
}
_ => (),
}
Ok(())
})
.await
}
#[instrument(skip_all)]
pub async fn remove_from_current_dependents_lists<'a, Db: DbHandle>(
db: &mut Db,
id: &'a PackageId,
current_dependencies: &'a CurrentDependencies,
current_dependent_receipt: &LockReceipt<CurrentDependents, String>,
pub fn remove_from_current_dependents_lists(
db: &mut Model<Database>,
id: &PackageId,
current_dependencies: &CurrentDependencies,
) -> Result<(), Error> {
for dep in current_dependencies.0.keys().chain(std::iter::once(id)) {
if let Some(mut current_dependents) = current_dependent_receipt.get(db, dep).await? {
if current_dependents.0.remove(id).is_some() {
current_dependent_receipt
.set(db, current_dependents, dep)
.await?;
}
if let Some(current_dependents) = db
.as_package_data_mut()
.as_idx_mut(dep)
.and_then(|d| d.as_installed_mut())
.map(|i| i.as_current_dependents_mut())
{
current_dependents.remove(id)?;
}
}
Ok(())
}
pub struct UninstallReceipts {
config: ConfigReceipts,
removing: LockReceipt<InstalledPackageDataEntry, ()>,
packages: LockReceipt<AllPackageData, ()>,
current_dependents: LockReceipt<CurrentDependents, String>,
update_depenency_receipts: UpdateDependencyReceipts,
}
impl UninstallReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks, id);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(
locks: &mut Vec<LockTargetId>,
id: &PackageId,
) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
let config = ConfigReceipts::setup(locks);
let removing = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|pde| pde.removing())
.make_locker(LockType::Write)
.add_to_keys(locks);
let current_dependents = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.current_dependents())
.make_locker(LockType::Write)
.add_to_keys(locks);
let packages = crate::db::DatabaseModel::new()
.package_data()
.make_locker(LockType::Write)
.add_to_keys(locks);
let update_depenency_receipts = UpdateDependencyReceipts::setup(locks);
move |skeleton_key| {
Ok(Self {
config: config(skeleton_key)?,
removing: removing.verify(skeleton_key)?,
current_dependents: current_dependents.verify(skeleton_key)?,
update_depenency_receipts: update_depenency_receipts(skeleton_key)?,
packages: packages.verify(skeleton_key)?,
})
}
}
}
#[instrument(skip_all)]
pub async fn uninstall<Ex>(
ctx: &RpcContext,
db: &mut PatchDbHandle,
secrets: &mut Ex,
id: &PackageId,
) -> Result<(), Error>
pub async fn uninstall<Ex>(ctx: &RpcContext, secrets: &mut Ex, id: &PackageId) -> Result<(), Error>
where
for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
{
let mut tx = db.begin().await?;
crate::db::DatabaseModel::new()
.package_data()
.idx_model(&id)
.lock(&mut tx, LockType::Write)
.await?;
let receipts = UninstallReceipts::new(&mut tx, id).await?;
let entry = receipts.removing.get(&mut tx).await?;
cleanup(ctx, &entry.manifest.id, &entry.manifest.version).await?;
let db = ctx.db.peek().await?;
let entry = db
.as_package_data()
.as_idx(id)
.or_not_found(id)?
.expect_as_removing()?;
let packages = {
let mut packages = receipts.packages.get(&mut tx).await?;
packages.0.remove(id);
packages
};
let dependents_paths: Vec<PathBuf> = entry
.current_dependents
.0
.keys()
.flat_map(|x| packages.0.get(x))
.flat_map(|x| x.manifest_borrow().volumes.values())
.as_removing()
.as_current_dependents()
.keys()?
.into_iter()
.filter(|x| x != id)
.flat_map(|x| db.as_package_data().as_idx(&x))
.flat_map(|x| x.as_installed())
.flat_map(|x| x.as_manifest().as_volumes().de())
.flat_map(|x| x.values().cloned().collect::<Vec<_>>())
.flat_map(|x| x.pointer_path(&ctx.datadir))
.collect();
receipts.packages.set(&mut tx, packages).await?;
// once we have removed the package entry, we can change all the dependent pointers to null
reconfigure_dependents_with_live_pointers(ctx, &mut tx, &receipts.config, &entry).await?;
remove_from_current_dependents_lists(
&mut tx,
&entry.manifest.id,
&entry.current_dependencies,
&receipts.current_dependents,
)
.await?;
update_dependency_errors_of_dependents(
ctx,
&mut tx,
&entry.manifest.id,
&entry.current_dependents,
&receipts.update_depenency_receipts,
)
.await?;
let volumes = ctx
let volume_dir = ctx
.datadir
.join(crate::volume::PKG_VOLUME_DIR)
.join(&entry.manifest.id);
.join(&*entry.as_manifest().as_id().de()?);
let version = entry.as_removing().as_manifest().as_version().de()?;
tracing::debug!(
"Cleaning up {:?} except for {:?}",
volume_dir,
dependents_paths
);
cleanup(ctx, id, &version).await?;
cleanup_folder(volume_dir, Arc::new(dependents_paths)).await;
remove_tor_keys(secrets, id).await?;
tracing::debug!("Cleaning up {:?} at {:?}", volumes, dependents_paths);
cleanup_folder(volumes, Arc::new(dependents_paths)).await;
remove_tor_keys(secrets, &entry.manifest.id).await?;
tx.commit().await?;
Ok(())
ctx.db
.mutate(|d| {
d.as_package_data_mut().remove(id)?;
remove_from_current_dependents_lists(
d,
id,
&entry.as_removing().as_current_dependencies().de()?,
)
})
.await
}
#[instrument(skip_all)]
@@ -409,8 +192,7 @@ pub async fn remove_tor_keys<Ex>(secrets: &mut Ex, id: &PackageId) -> Result<(),
where
for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
{
let id_str = id.as_str();
sqlx::query!("DELETE FROM tor WHERE package = $1", id_str)
sqlx::query!("DELETE FROM tor WHERE package = $1", &*id)
.execute(secrets)
.await?;
Ok(())

1087
backend/src/install/mod.rs
View File

File diff suppressed because it is too large Load Diff

BIN
backend/src/install/package-icon.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 280 KiB

78
backend/src/install/progress.rs
View File

@@ -6,14 +6,16 @@ use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Duration;
use patch_db::{DbHandle, HasModel, OptionModel, PatchDb};
use models::{OptionExt, PackageId};
use serde::{Deserialize, Serialize};
use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite};
use crate::Error;
use crate::db::model::Database;
use crate::prelude::*;
#[derive(Debug, Deserialize, Serialize, HasModel, Default)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct InstallProgress {
pub size: Option<u64>,
pub downloaded: AtomicU64,
@@ -24,8 +26,8 @@ pub struct InstallProgress {
pub unpack_complete: AtomicBool,
}
impl InstallProgress {
pub fn new(size: Option<u64>) -> Arc<Self> {
Arc::new(InstallProgress {
pub fn new(size: Option<u64>) -> Self {
InstallProgress {
size,
downloaded: AtomicU64::new(0),
download_complete: AtomicBool::new(false),
@@ -33,26 +35,25 @@ impl InstallProgress {
validation_complete: AtomicBool::new(false),
unpacked: AtomicU64::new(0),
unpack_complete: AtomicBool::new(false),
})
}
}
pub fn download_complete(&self) {
self.download_complete.store(true, Ordering::SeqCst)
}
pub async fn track_download<Db: DbHandle>(
self: Arc<Self>,
model: OptionModel<InstallProgress>,
mut db: Db,
) -> Result<(), Error> {
pub async fn track_download(self: Arc<Self>, db: PatchDb, id: PackageId) -> Result<(), Error> {
let update = |d: &mut Model<Database>| {
d.as_package_data_mut()
.as_idx_mut(&id)
.or_not_found(&id)?
.as_install_progress_mut()
.or_not_found("install-progress")?
.ser(&self)
};
while !self.download_complete.load(Ordering::SeqCst) {
let mut tx = db.begin().await?;
model.put(&mut tx, &self).await?;
tx.save().await?;
tokio::time::sleep(Duration::from_secs(1)).await;
db.mutate(&update).await?;
tokio::time::sleep(Duration::from_millis(300)).await;
}
let mut tx = db.begin().await?;
model.put(&mut tx, &self).await?;
tx.save().await?;
Ok(())
db.mutate(&update).await
}
pub async fn track_download_during<
F: FnOnce() -> Fut,
@@ -60,33 +61,35 @@ impl InstallProgress {
T,
>(
self: &Arc<Self>,
model: OptionModel<InstallProgress>,
db: &PatchDb,
db: PatchDb,
id: &PackageId,
f: F,
) -> Result<T, Error> {
let local_db = db.handle();
let tracker = tokio::spawn(self.clone().track_download(model.clone(), local_db));
let tracker = tokio::spawn(self.clone().track_download(db.clone(), id.clone()));
let res = f().await;
self.download_complete.store(true, Ordering::SeqCst);
tracker.await.unwrap()?;
res
}
pub async fn track_read<Db: DbHandle>(
pub async fn track_read(
self: Arc<Self>,
model: OptionModel<InstallProgress>,
mut db: Db,
db: PatchDb,
id: PackageId,
complete: Arc<AtomicBool>,
) -> Result<(), Error> {
let update = |d: &mut Model<Database>| {
d.as_package_data_mut()
.as_idx_mut(&id)
.or_not_found(&id)?
.as_install_progress_mut()
.or_not_found("install-progress")?
.ser(&self)
};
while !complete.load(Ordering::SeqCst) {
let mut tx = db.begin().await?;
model.put(&mut tx, &self).await?;
tx.save().await?;
tokio::time::sleep(Duration::from_secs(1)).await;
db.mutate(&update).await?;
tokio::time::sleep(Duration::from_millis(300)).await;
}
let mut tx = db.begin().await?;
model.put(&mut tx, &self).await?;
tx.save().await?;
Ok(())
db.mutate(&update).await
}
pub async fn track_read_during<
F: FnOnce() -> Fut,
@@ -94,15 +97,14 @@ impl InstallProgress {
T,
>(
self: &Arc<Self>,
model: OptionModel<InstallProgress>,
db: &PatchDb,
db: PatchDb,
id: &PackageId,
f: F,
) -> Result<T, Error> {
let local_db = db.handle();
let complete = Arc::new(AtomicBool::new(false));
let tracker = tokio::spawn(self.clone().track_read(
model.clone(),
local_db,
db.clone(),
id.clone(),
complete.clone(),
));
let res = f().await;

89
backend/src/install/update.rs
View File

@@ -1,105 +1,18 @@
use std::collections::BTreeMap;
use patch_db::{DbHandle, LockReceipt, LockTargetId, LockType, Verifier};
use rpc_toolkit::command;
use tracing::instrument;
use crate::config::not_found;
use crate::context::RpcContext;
use crate::db::model::CurrentDependents;
use crate::dependencies::{
break_transitive, BreakTransitiveReceipts, BreakageRes, DependencyError,
};
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::util::serde::display_serializable;
use crate::util::Version;
use crate::Error;
pub struct UpdateReceipts {
break_receipts: BreakTransitiveReceipts,
current_dependents: LockReceipt<CurrentDependents, String>,
dependency: LockReceipt<crate::dependencies::DepInfo, (String, String)>,
}
impl UpdateReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
let break_receipts = BreakTransitiveReceipts::setup(locks);
let current_dependents = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.current_dependents())
.make_locker(LockType::Write)
.add_to_keys(locks);
let dependency = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest().dependencies().star())
.make_locker(LockType::Write)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
break_receipts: break_receipts(skeleton_key)?,
current_dependents: current_dependents.verify(skeleton_key)?,
dependency: dependency.verify(skeleton_key)?,
})
}
}
}
#[command(subcommands(dry))]
pub async fn update() -> Result<(), Error> {
Ok(())
}
#[instrument(skip_all)]
#[command(display(display_serializable))]
pub async fn dry(
#[context] ctx: RpcContext,
#[arg] id: PackageId,
#[arg] version: Version,
) -> Result<BreakageRes, Error> {
let mut db = ctx.db.handle();
let mut tx = db.begin().await?;
let mut breakages = BTreeMap::new();
let receipts = UpdateReceipts::new(&mut tx).await?;
for dependent in receipts
.current_dependents
.get(&mut tx, &id)
.await?
.ok_or_else(|| not_found!(id))?
.0
.keys()
.into_iter()
.filter(|dependent| &&id != dependent)
{
if let Some(dep_info) = receipts.dependency.get(&mut tx, (&dependent, &id)).await? {
let version_req = dep_info.version;
if !version.satisfies(&version_req) {
break_transitive(
&mut tx,
&dependent,
&id,
DependencyError::IncorrectVersion {
expected: version_req,
received: version.clone(),
},
&mut breakages,
&receipts.break_receipts,
)
.await?;
}
}
}
tx.abort().await?;
Ok(BreakageRes(breakages))
}

9
backend/src/lib.rs
View File

@@ -34,14 +34,15 @@ pub mod inspect;
pub mod install;
pub mod logs;
pub mod manager;
pub mod marketplace;
pub mod middleware;
pub mod migration;
pub mod net;
pub mod notifications;
pub mod os_install;
pub mod prelude;
pub mod procedure;
pub mod properties;
pub mod registry;
pub mod s9pk;
pub mod setup;
pub mod shutdown;
@@ -78,7 +79,7 @@ pub fn echo(#[arg] message: String) -> Result<String, RpcError> {
disk::disk,
notifications::notification,
backup::backup,
marketplace::marketplace,
registry::marketplace::marketplace,
))]
pub fn main_api() -> Result<(), RpcError> {
Ok(())
@@ -105,7 +106,6 @@ pub fn server() -> Result<(), RpcError> {
install::sideload,
install::uninstall,
install::list,
install::update::update,
config::config,
control::start,
control::stop,
@@ -124,7 +124,8 @@ pub fn package() -> Result<(), RpcError> {
s9pk::pack,
developer::verify,
developer::init,
inspect::inspect
inspect::inspect,
registry::admin::publish,
))]
pub fn portable_api() -> Result<(), RpcError> {
Ok(())

187
backend/src/manager/health.rs
View File

@@ -1,111 +1,27 @@
use std::collections::BTreeMap;
use itertools::Itertools;
use patch_db::{DbHandle, LockReceipt, LockType};
use models::OptionExt;
use tracing::instrument;
use crate::context::RpcContext;
use crate::db::model::CurrentDependents;
use crate::dependencies::{break_transitive, heal_transitive, DependencyError};
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::status::health_check::{HealthCheckId, HealthCheckResult};
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::status::MainStatus;
use crate::Error;
struct HealthCheckPreInformationReceipt {
status_model: LockReceipt<MainStatus, ()>,
manifest: LockReceipt<Manifest, ()>,
}
impl HealthCheckPreInformationReceipt {
pub async fn new(db: &'_ mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks, id);
setup(&db.lock_all(locks).await?)
}
pub fn setup(
locks: &mut Vec<patch_db::LockTargetId>,
id: &PackageId,
) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
let status_model = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.status().main())
.make_locker(LockType::Read)
.add_to_keys(locks);
let manifest = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.manifest())
.make_locker(LockType::Read)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
status_model: status_model.verify(skeleton_key)?,
manifest: manifest.verify(skeleton_key)?,
})
}
}
}
struct HealthCheckStatusReceipt {
status: LockReceipt<MainStatus, ()>,
current_dependents: LockReceipt<CurrentDependents, ()>,
}
impl HealthCheckStatusReceipt {
pub async fn new(db: &'_ mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks, id);
setup(&db.lock_all(locks).await?)
}
pub fn setup(
locks: &mut Vec<patch_db::LockTargetId>,
id: &PackageId,
) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
let status = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.status().main())
.make_locker(LockType::Write)
.add_to_keys(locks);
let current_dependents = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.current_dependents())
.make_locker(LockType::Read)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
status: status.verify(skeleton_key)?,
current_dependents: current_dependents.verify(skeleton_key)?,
})
}
}
}
/// So, this is used for a service to run a health check cycle, go out and run the health checks, and store those in the db
#[instrument(skip_all)]
pub async fn check<Db: DbHandle>(
ctx: &RpcContext,
db: &mut Db,
id: &PackageId,
) -> Result<(), Error> {
let mut tx = db.begin().await?;
pub async fn check(ctx: &RpcContext, id: &PackageId) -> Result<(), Error> {
let (manifest, started) = {
let mut checkpoint = tx.begin().await?;
let receipts = HealthCheckPreInformationReceipt::new(&mut checkpoint, id).await?;
let peeked = ctx.db.peek().await?;
let pde = peeked
.as_package_data()
.as_idx(id)
.or_not_found(id)?
.expect_as_installed()?;
let manifest = receipts.manifest.get(&mut checkpoint).await?;
let manifest = pde.as_installed().as_manifest().de()?;
let started = receipts.status_model.get(&mut checkpoint).await?.started();
let started = pde.as_installed().as_status().as_main().de()?.started();
checkpoint.save().await?;
(manifest, started)
};
@@ -113,69 +29,28 @@ pub async fn check<Db: DbHandle>(
tracing::debug!("Checking health of {}", id);
manifest
.health_checks
.check_all(
ctx,
&manifest.containers,
started,
id,
&manifest.version,
&manifest.volumes,
)
.check_all(ctx, started, id, &manifest.version, &manifest.volumes)
.await?
} else {
return Ok(());
};
let current_dependents = {
let mut checkpoint = tx.begin().await?;
let receipts = HealthCheckStatusReceipt::new(&mut checkpoint, id).await?;
ctx.db
.mutate(|v| {
let pde = v
.as_package_data_mut()
.as_idx_mut(id)
.or_not_found(id)?
.expect_as_installed_mut()?;
let status = pde.as_installed_mut().as_status_mut().as_main_mut();
let status = receipts.status.get(&mut checkpoint).await?;
if let MainStatus::Running { health: _, started } = status {
receipts
.status
.set(
&mut checkpoint,
MainStatus::Running {
health: health_results.clone(),
started,
},
)
.await?;
}
let current_dependents = receipts.current_dependents.get(&mut checkpoint).await?;
checkpoint.save().await?;
current_dependents
};
let receipts = crate::dependencies::BreakTransitiveReceipts::new(&mut tx).await?;
for (dependent, info) in (current_dependents).0.iter() {
let failures: BTreeMap<HealthCheckId, HealthCheckResult> = health_results
.iter()
.filter(|(_, hc_res)| !matches!(hc_res, HealthCheckResult::Success { .. }))
.filter(|(hc_id, _)| info.health_checks.contains(hc_id))
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
if !failures.is_empty() {
break_transitive(
&mut tx,
&dependent,
id,
DependencyError::HealthChecksFailed { failures },
&mut BTreeMap::new(),
&receipts,
)
.await?;
} else {
heal_transitive(ctx, &mut tx, &dependent, id, &receipts.dependency_receipt).await?;
}
}
tx.save().await?;
Ok(())
if let MainStatus::Running { health: _, started } = status.de()? {
status.ser(&MainStatus::Running {
health: health_results.clone(),
started,
})?;
}
Ok(())
})
.await
}

217
backend/src/manager/manager_container.rs
View File

@@ -1,22 +1,42 @@
use std::sync::Arc;
use std::time::Duration;
use futures::FutureExt;
use patch_db::PatchDbHandle;
use models::OptionExt;
use tokio::sync::watch;
use tokio::sync::watch::Sender;
use tracing::instrument;
use super::start_stop::StartStop;
use super::{manager_seed, run_main, ManagerPersistentContainer, RunMainResult};
use crate::prelude::*;
use crate::procedure::NoOutput;
use crate::s9pk::manifest::Manifest;
use crate::status::MainStatus;
use crate::util::{GeneralBoxedGuard, NonDetachingJoinHandle};
use crate::util::NonDetachingJoinHandle;
use crate::Error;
pub type ManageContainerOverride = Arc<watch::Sender<Option<MainStatus>>>;
pub type ManageContainerOverride = Arc<watch::Sender<Option<Override>>>;
pub type Override = MainStatus;
pub struct OverrideGuard {
override_main_status: Option<ManageContainerOverride>,
}
impl OverrideGuard {
pub fn drop(self) {}
}
impl Drop for OverrideGuard {
fn drop(&mut self) {
if let Some(override_main_status) = self.override_main_status.take() {
override_main_status.send_modify(|x| {
*x = None;
});
}
}
}
/// This is the thing describing the state machine actor for a service
/// state and current running/ desired states.
pub struct ManageContainer {
pub(super) current_state: Arc<watch::Sender<StartStop>>,
pub(super) desired_state: Arc<watch::Sender<StartStop>>,
@@ -30,10 +50,12 @@ impl ManageContainer {
seed: Arc<manager_seed::ManagerSeed>,
persistent_container: ManagerPersistentContainer,
) -> Result<Self, Error> {
let mut db = seed.ctx.db.handle();
let current_state = Arc::new(watch::channel(StartStop::Stop).0);
let desired_state = Arc::new(
watch::channel::<StartStop>(get_status(&mut db, &seed.manifest).await.into()).0,
watch::channel::<StartStop>(
get_status(seed.ctx.db.peek().await?, &seed.manifest).into(),
)
.0,
);
let override_main_status: ManageContainerOverride = Arc::new(watch::channel(None).0);
let service = tokio::spawn(create_service_manager(
@@ -59,20 +81,40 @@ impl ManageContainer {
})
}
pub fn set_override(&self, override_status: Option<MainStatus>) -> GeneralBoxedGuard {
/// Set override is used during something like a restart of a service. We want to show certain statuses be different
/// from the actual status of the service.
pub fn set_override(&self, override_status: Override) -> Result<OverrideGuard, Error> {
let status = Some(override_status);
if self.override_main_status.borrow().is_some() {
return Err(Error::new(
eyre!("Already have an override"),
ErrorKind::InvalidRequest,
));
}
self.override_main_status
.send_modify(|x| *x = override_status);
let override_main_status = self.override_main_status.clone();
let guard = GeneralBoxedGuard::new(move || {
override_main_status.send_modify(|x| *x = None);
});
guard
.send_modify(|x| *x = status.clone());
Ok(OverrideGuard {
override_main_status: Some(self.override_main_status.clone()),
})
}
/// Set the override, but don't have a guard to revert it. Used only on the mananger to do a shutdown.
pub(super) async fn lock_state_forever(
&self,
seed: &manager_seed::ManagerSeed,
) -> Result<(), Error> {
let current_state = get_status(seed.ctx.db.peek().await?, &seed.manifest);
self.override_main_status
.send_modify(|x| *x = Some(current_state));
Ok(())
}
/// We want to set the state of the service, like to start or stop
pub fn to_desired(&self, new_state: StartStop) {
self.desired_state.send_modify(|x| *x = new_state);
}
/// This is a tool to say wait for the service to be in a certain state.
pub async fn wait_for_desired(&self, new_state: StartStop) {
let mut current_state = self.current_state();
self.to_desired(new_state);
@@ -81,10 +123,12 @@ impl ManageContainer {
}
}
/// Getter
pub fn current_state(&self) -> watch::Receiver<StartStop> {
self.current_state.subscribe()
}
/// Getter
pub fn desired_state(&self) -> watch::Receiver<StartStop> {
self.desired_state.subscribe()
}
@@ -144,41 +188,34 @@ async fn create_service_manager(
async fn save_state(
desired_state: Arc<Sender<StartStop>>,
current_state: Arc<Sender<StartStop>>,
override_main_status: Arc<Sender<Option<MainStatus>>>,
override_main_status: ManageContainerOverride,
seed: Arc<manager_seed::ManagerSeed>,
) {
let mut desired_state_receiver = desired_state.subscribe();
let mut current_state_receiver = current_state.subscribe();
let mut override_main_status_receiver = override_main_status.subscribe();
loop {
let current: StartStop = current_state_receiver.borrow().clone();
let desired: StartStop = desired_state_receiver.borrow().clone();
let current: StartStop = *current_state_receiver.borrow();
let desired: StartStop = *desired_state_receiver.borrow();
let override_status = override_main_status_receiver.borrow().clone();
let mut db = seed.ctx.db.handle();
let res = match (override_status, current, desired) {
(Some(status), _, _) => set_status(&mut db, &seed.manifest, &status).await,
(None, StartStop::Start, StartStop::Start) => {
set_status(
&mut db,
&seed.manifest,
&MainStatus::Running {
started: chrono::Utc::now(),
health: Default::default(),
},
)
.await
}
(None, StartStop::Start, StartStop::Stop) => {
set_status(&mut db, &seed.manifest, &MainStatus::Stopping).await
}
(None, StartStop::Stop, StartStop::Start) => {
set_status(&mut db, &seed.manifest, &MainStatus::Starting).await
}
(None, StartStop::Stop, StartStop::Stop) => {
set_status(&mut db, &seed.manifest, &MainStatus::Stopped).await
}
let status = match (override_status.clone(), current, desired) {
(Some(status), _, _) => status,
(_, StartStop::Start, StartStop::Start) => MainStatus::Running {
started: chrono::Utc::now(),
health: Default::default(),
},
(_, StartStop::Start, StartStop::Stop) => MainStatus::Stopping,
(_, StartStop::Stop, StartStop::Start) => MainStatus::Starting,
(_, StartStop::Stop, StartStop::Stop) => MainStatus::Stopped,
};
if let Err(err) = res {
let manifest = &seed.manifest;
if let Err(err) = seed
.ctx
.db
.mutate(|db| set_status(db, manifest, &status))
.await
{
tracing::error!("Did not set status for {}", seed.container_name);
tracing::debug!("{:?}", err);
}
@@ -223,40 +260,6 @@ async fn run_main_log_result(result: RunMainResult, seed: Arc<manager_seed::Mana
match result {
Ok(Ok(NoOutput)) => (), // restart
Ok(Err(e)) => {
#[cfg(feature = "unstable")]
{
use crate::notifications::NotificationLevel;
let mut db = seed.ctx.db.handle();
let started = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&seed.manifest.id)
.and_then(|pde| pde.installed())
.map::<_, MainStatus>(|i| i.status().main())
.get(&mut db)
.await;
match started.as_deref() {
Ok(Some(MainStatus::Running { .. })) => {
let res = seed.ctx.notification_manager
.notify(
&mut db,
Some(seed.manifest.id.clone()),
NotificationLevel::Warning,
String::from("Service Crashed"),
format!("The service {} has crashed with the following exit code: {}\nDetails: {}", seed.manifest.id.clone(), e.0, e.1),
(),
Some(3600) // 1 hour
)
.await;
if let Err(e) = res {
tracing::error!("Failed to issue notification: {}", e);
tracing::debug!("{:?}", e);
}
}
_ => {
tracing::error!("service just started. not issuing crash notification")
}
}
}
tracing::error!(
"The service {} has crashed with the following exit code: {}",
seed.manifest.id.clone(),
@@ -272,56 +275,26 @@ async fn run_main_log_result(result: RunMainResult, seed: Arc<manager_seed::Mana
}
}
/// Used only in the mod where we are doing a backup
#[instrument(skip(db, manifest))]
pub(super) async fn get_status(db: &mut PatchDbHandle, manifest: &Manifest) -> MainStatus {
async move {
Ok::<_, Error>(
crate::db::DatabaseModel::new()
.package_data()
.idx_model(&manifest.id)
.expect(db)
.await?
.installed()
.expect(db)
.await?
.status()
.main()
.get(db)
.await?
.clone(),
)
}
.map(|x| x.unwrap_or_else(|e| MainStatus::Stopped))
.await
pub(super) fn get_status(db: Peeked, manifest: &Manifest) -> MainStatus {
db.as_package_data()
.as_idx(&manifest.id)
.and_then(|x| x.as_installed())
.filter(|x| x.as_manifest().as_version().de().ok() == Some(manifest.version.clone()))
.and_then(|x| x.as_status().as_main().de().ok())
.unwrap_or(MainStatus::Stopped)
}
#[instrument(skip(db, manifest))]
async fn set_status(
db: &mut PatchDbHandle,
manifest: &Manifest,
main_status: &MainStatus,
) -> Result<(), Error> {
if crate::db::DatabaseModel::new()
.package_data()
.idx_model(&manifest.id)
.expect(db)
.await?
.installed()
.exists(db)
.await?
{
crate::db::DatabaseModel::new()
.package_data()
.idx_model(&manifest.id)
.expect(db)
.await?
.installed()
.expect(db)
.await?
.status()
.main()
.put(db, main_status)
.await?;
}
Ok(())
fn set_status(db: &mut Peeked, manifest: &Manifest, main_status: &MainStatus) -> Result<(), Error> {
let Some(installed) = db
.as_package_data_mut()
.as_idx_mut(&manifest.id)
.or_not_found(&manifest.id)?
.as_installed_mut()
else {
return Ok(());
};
installed.as_status_mut().as_main_mut().ser(main_status)
}

47
backend/src/manager/manager_map.rs
View File

@@ -2,46 +2,31 @@ use std::collections::BTreeMap;
use std::sync::Arc;
use color_eyre::eyre::eyre;
use patch_db::DbHandle;
use sqlx::{Executor, Postgres};
use tokio::sync::RwLock;
use tracing::instrument;
use super::Manager;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::util::Version;
use crate::Error;
/// This is the structure to contain all the service managers
#[derive(Default)]
pub struct ManagerMap(RwLock<BTreeMap<(PackageId, Version), Arc<Manager>>>);
impl ManagerMap {
#[instrument(skip_all)]
pub async fn init<Db: DbHandle, Ex>(
&self,
ctx: &RpcContext,
db: &mut Db,
secrets: &mut Ex,
) -> Result<(), Error>
where
for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
{
pub async fn init(&self, ctx: RpcContext, peeked: Peeked) -> Result<(), Error> {
let mut res = BTreeMap::new();
for package in crate::db::DatabaseModel::new()
.package_data()
.keys(db)
.await?
{
let man: Manifest = if let Some(manifest) = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&package)
.and_then(|pkg| pkg.installed())
.map(|m| m.manifest())
.get(db)
.await?
.to_owned()
for package in peeked.as_package_data().keys()? {
let man: Manifest = if let Some(manifest) = peeked
.as_package_data()
.as_idx(&package)
.and_then(|x| x.as_installed())
.map(|x| x.as_manifest().de())
{
manifest
manifest?
} else {
continue;
};
@@ -55,17 +40,20 @@ impl ManagerMap {
Ok(())
}
/// Used during the install process
#[instrument(skip_all)]
pub async fn add(&self, ctx: RpcContext, manifest: Manifest) -> Result<(), Error> {
pub async fn add(&self, ctx: RpcContext, manifest: Manifest) -> Result<Arc<Manager>, Error> {
let mut lock = self.0.write().await;
let id = (manifest.id.clone(), manifest.version.clone());
if let Some(man) = lock.remove(&id) {
man.exit().await;
}
lock.insert(id, Arc::new(Manager::new(ctx, manifest).await?));
Ok(())
let manager = Arc::new(Manager::new(ctx.clone(), manifest).await?);
lock.insert(id, manager.clone());
Ok(manager)
}
/// This is ran during the cleanup, so when we are uninstalling the service
#[instrument(skip_all)]
pub async fn remove(&self, id: &(PackageId, Version)) {
if let Some(man) = self.0.write().await.remove(id) {
@@ -73,13 +61,14 @@ impl ManagerMap {
}
}
/// Used during a shutdown
#[instrument(skip_all)]
pub async fn empty(&self) -> Result<(), Error> {
let res =
futures::future::join_all(std::mem::take(&mut *self.0.write().await).into_iter().map(
|((id, version), man)| async move {
tracing::debug!("Manager for {}@{} shutting down", id, version);
man.exit().await;
man.shutdown().await?;
tracing::debug!("Manager for {}@{} is shutdown", id, version);
if let Err(e) = Arc::try_unwrap(man) {
tracing::trace!(

43
backend/src/manager/manager_seed.rs
View File

@@ -1,9 +1,11 @@
use bollard::container::StopContainerOptions;
use models::ErrorKind;
use crate::context::RpcContext;
use crate::s9pk::manifest::Manifest;
use crate::util::docker::stop_container;
use crate::Error;
/// This is helper structure for a service, the seed of the data that is needed for the manager_container
pub struct ManagerSeed {
pub ctx: RpcContext,
pub manifest: Manifest,
@@ -12,35 +14,18 @@ pub struct ManagerSeed {
impl ManagerSeed {
pub async fn stop_container(&self) -> Result<(), Error> {
match self
.ctx
.docker
.stop_container(
&self.container_name,
Some(StopContainerOptions {
t: self
.manifest
.containers
.as_ref()
.and_then(|c| c.main.sigterm_timeout)
.map(|d| d.as_secs())
.unwrap_or(30) as i64,
}),
)
.await
match stop_container(
&self.container_name,
self.manifest
.containers
.as_ref()
.and_then(|c| c.main.sigterm_timeout)
.map(|d| *d),
None,
)
.await
{
Err(bollard::errors::Error::DockerResponseServerError {
status_code: 404, // NOT FOUND
..
})
| Err(bollard::errors::Error::DockerResponseServerError {
status_code: 409, // CONFLICT
..
})
| Err(bollard::errors::Error::DockerResponseServerError {
status_code: 304, // NOT MODIFIED
..
}) => (), // Already stopped
Err(e) if e.kind == ErrorKind::NotFound => (), // Already stopped
a => a?,
}
Ok(())

676
backend/src/manager/mod.rs
View File

@@ -4,23 +4,19 @@ use std::sync::Arc;
use std::task::Poll;
use std::time::Duration;
use color_eyre::{eyre::eyre, Report};
use color_eyre::eyre::eyre;
use embassy_container_init::ProcessGroupId;
use futures::future::BoxFuture;
use futures::{Future, FutureExt, TryFutureExt};
use helpers::UnixRpcClient;
use models::{ErrorKind, PackageId};
use models::{ErrorKind, OptionExt, PackageId};
use nix::sys::signal::Signal;
use patch_db::DbHandle;
use persistent_container::PersistentContainer;
use rand::SeedableRng;
use sqlx::Connection;
use start_stop::StartStop;
use tokio::sync::oneshot;
use tokio::sync::{
watch::{self, Sender},
Mutex,
};
use tokio::sync::watch::{self, Sender};
use tokio::sync::{oneshot, Mutex};
use tracing::instrument;
use transition_state::TransitionState;
@@ -28,28 +24,28 @@ use crate::backup::target::PackageBackupInfo;
use crate::backup::PackageBackupReport;
use crate::config::action::ConfigRes;
use crate::config::spec::ValueSpecPointer;
use crate::config::{not_found, ConfigReceipts, ConfigureContext};
use crate::config::ConfigureContext;
use crate::context::RpcContext;
use crate::db::model::{CurrentDependencies, CurrentDependencyInfo};
use crate::dependencies::{
add_dependent_to_current_dependents_lists, break_transitive, heal_all_dependents_transitive,
DependencyError, DependencyErrors, TaggedDependencyError,
add_dependent_to_current_dependents_lists, compute_dependency_config_errs,
};
use crate::disk::mount::backup::BackupMountGuard;
use crate::disk::mount::guard::TmpMountGuard;
use crate::install::cleanup::remove_from_current_dependents_lists;
use crate::net::net_controller::NetService;
use crate::net::vhost::AlpnInfo;
use crate::prelude::*;
use crate::procedure::docker::{DockerContainer, DockerProcedure, LongRunning};
use crate::procedure::{NoOutput, ProcedureName};
use crate::s9pk::manifest::Manifest;
use crate::status::MainStatus;
use crate::util::docker::{get_container_ip, kill_container};
use crate::util::NonDetachingJoinHandle;
use crate::volume::Volume;
use crate::Error;
pub mod health;
mod js_api;
mod manager_container;
mod manager_map;
pub mod manager_seed;
@@ -109,6 +105,7 @@ impl Gid {
}
}
/// This is the controller of the services. Here is where we can control a service with a start, stop, restart, etc.
#[derive(Clone)]
pub struct Manager {
seed: Arc<ManagerSeed>,
@@ -151,7 +148,7 @@ impl Manager {
self._transition_abort();
self.manage_container.to_desired(StartStop::Stop);
}
pub async fn restart(&self) {
pub fn restart(&self) {
if self._is_transition_restart() {
return;
}
@@ -160,35 +157,30 @@ impl Manager {
pub async fn configure(
&self,
configure_context: ConfigureContext,
) -> Result<BTreeMap<PackageId, TaggedDependencyError>, Error> {
) -> Result<BTreeMap<PackageId, String>, Error> {
if self._is_transition_configure() {
return Ok(configure_context.breakages);
}
let context = self.seed.ctx.clone();
let id = self.seed.manifest.id.clone();
let (transition_state, done) = configure(context, id, configure_context).remote_handle();
let breakages = configure(context, id, configure_context).await?;
self._transition_replace({
let manage_container = self.manage_container.clone();
let state_reverter = DesiredStateReverter::new(manage_container.clone());
let transition = self.transition.clone();
TransitionState::Configuring(
tokio::spawn(async move {
let desired_state = manage_container.desired_state();
let state_reverter = DesiredStateReverter::new(manage_container.clone());
let mut current_state = manage_container.current_state();
manage_container.to_desired(StartStop::Stop);
while current_state.borrow().is_start() {
current_state.changed().await.unwrap();
}
transition_state.await;
manage_container.wait_for_desired(StartStop::Stop).await;
state_reverter.revert().await;
transition.send_replace(Default::default());
})
.into(),
)
});
done.await
Ok(breakages)
}
pub async fn backup(&self, backup_guard: BackupGuard) -> BackupReturn {
if self._is_transition_backup() {
@@ -201,21 +193,27 @@ impl Manager {
done.await
}
pub async fn exit(&self) {
self.stop();
let mut current_status = self.manage_container.current_state();
while current_status.borrow().is_start() {
current_status.changed().await.unwrap();
}
self._transition_abort();
self.manage_container
.wait_for_desired(StartStop::Stop)
.await;
}
/// A special exit that is overridden the start state, should only be called in the shutdown, where we remove other containers
async fn shutdown(&self) -> Result<(), Error> {
self.manage_container.lock_state_forever(&self.seed).await?;
self.exit().await;
Ok(())
}
/// Used when we want to shutdown the service
pub async fn signal(&self, signal: Signal) -> Result<(), Error> {
let rpc_client = self.rpc_client();
let seed = self.seed.clone();
let gid = self.gid.clone();
send_signal(self, gid, signal).await
}
/// Used as a getter, but also used in procedure
pub fn rpc_client(&self) -> Option<Arc<UnixRpcClient>> {
(*self.persistent_container)
.as_ref()
@@ -228,7 +226,7 @@ impl Manager {
.send_replace(Default::default())
.join_handle()
{
transition.abort();
(**transition).abort();
}
}
fn _transition_replace(&self, transition_state: TransitionState) {
@@ -237,12 +235,14 @@ impl Manager {
.abort();
}
pub(super) fn perform_restart(&self) -> impl Future<Output = ()> + 'static {
pub(super) fn perform_restart(&self) -> impl Future<Output = Result<(), Error>> + 'static {
let manage_container = self.manage_container.clone();
async move {
let _ = manage_container.set_override(Some(MainStatus::Restarting));
let restart_override = manage_container.set_override(MainStatus::Restarting)?;
manage_container.wait_for_desired(StartStop::Stop).await;
manage_container.wait_for_desired(StartStop::Start).await;
restart_override.drop();
Ok(())
}
}
fn _transition_restart(&self) -> TransitionState {
@@ -250,7 +250,9 @@ impl Manager {
let restart = self.perform_restart();
TransitionState::Restarting(
tokio::spawn(async move {
restart.await;
if let Err(err) = restart.await {
tracing::error!("Error restarting service: {}", err);
}
transition.send_replace(Default::default());
})
.into(),
@@ -259,36 +261,42 @@ impl Manager {
fn perform_backup(
&self,
backup_guard: BackupGuard,
) -> impl Future<Output = Result<Result<PackageBackupInfo, Error>, Error>> + 'static {
) -> impl Future<Output = Result<Result<PackageBackupInfo, Error>, Error>> {
let manage_container = self.manage_container.clone();
let seed = self.seed.clone();
async move {
let peek = seed.ctx.db.peek().await?;
let state_reverter = DesiredStateReverter::new(manage_container.clone());
let mut tx = seed.ctx.db.handle();
let _ = manage_container
.set_override(Some(get_status(&mut tx, &seed.manifest).await.backing_up()));
let override_guard =
manage_container.set_override(get_status(peek, &seed.manifest).backing_up())?;
manage_container.wait_for_desired(StartStop::Stop).await;
let backup_guard = backup_guard.lock().await;
let guard = backup_guard.mount_package_backup(&seed.manifest.id).await?;
let res = seed
.manifest
.backup
.create(
&seed.ctx,
&mut tx,
&seed.manifest.id,
&seed.manifest.title,
&seed.manifest.version,
&seed.manifest.interfaces,
&seed.manifest.volumes,
)
.await;
let return_value = seed.manifest.backup.create(seed.clone()).await;
guard.unmount().await?;
drop(backup_guard);
let return_value = res;
let manifest_id = seed.manifest.id.clone();
seed.ctx
.db
.mutate(|db| {
if let Some(progress) = db
.as_server_info_mut()
.as_status_info_mut()
.as_backup_progress_mut()
.transpose_mut()
.and_then(|p| p.as_idx_mut(&manifest_id))
{
progress.as_complete_mut().ser(&true)?;
}
Ok(())
})
.await?;
state_reverter.revert().await;
override_guard.drop();
Ok::<_, Error>(return_value)
}
}
@@ -297,11 +305,13 @@ impl Manager {
backup_guard: BackupGuard,
) -> (TransitionState, BoxFuture<BackupReturn>) {
let (send, done) = oneshot::channel();
let transition_state = self.transition.clone();
(
TransitionState::BackingUp(
tokio::spawn(
self.perform_backup(backup_guard)
.then(finnish_up_backup_task(self.transition.clone(), send)),
.then(finish_up_backup_task(transition_state, send)),
)
.into(),
),
@@ -325,282 +335,233 @@ impl Manager {
}
#[instrument(skip_all)]
fn configure(
async fn configure(
ctx: RpcContext,
id: PackageId,
mut configure_context: ConfigureContext,
) -> BoxFuture<'static, Result<BTreeMap<PackageId, TaggedDependencyError>, Error>> {
async move {
let mut db = ctx.db.handle();
let mut tx = db.begin().await?;
let db = &mut tx;
) -> Result<BTreeMap<PackageId, String>, Error> {
let db = ctx.db.peek().await?;
let id = &id;
let ctx = &ctx;
let overrides = &mut configure_context.overrides;
// fetch data from db
let manifest = db
.as_package_data()
.as_idx(id)
.or_not_found(id)?
.as_manifest()
.de()?;
let receipts = ConfigReceipts::new(db).await?;
let id = &id;
let ctx = &ctx;
let overrides = &mut configure_context.overrides;
// fetch data from db
let action = receipts
.config_actions
.get(db, id)
.await?
.ok_or_else(|| not_found!(&*id))?;
let dependencies = receipts
.dependencies
.get(db, id)
.await?
.ok_or_else(|| not_found!(&*id))?;
let volumes = receipts
.volumes
.get(db, id)
.await?
.ok_or_else(|| not_found!(&*id))?;
let version = receipts
.version
.get(db, id)
.await?
.ok_or_else(|| not_found!(&*id))?;
// get current config and current spec
let ConfigRes {
config: old_config,
spec,
} = manifest
.config
.as_ref()
.or_not_found("Manifest config")?
.get(ctx, id, &manifest.version, &manifest.volumes)
.await?;
// get current config and current spec
let ConfigRes {
config: old_config,
spec,
} = action.get(ctx, id, &version, &volumes).await?;
// determine new config to use
let mut config = if let Some(config) = configure_context.config.or_else(|| old_config.clone()) {
config
} else {
spec.gen(
&mut rand::rngs::StdRng::from_entropy(),
&configure_context.timeout,
)?
};
// determine new config to use
let mut config =
if let Some(config) = configure_context.config.or_else(|| old_config.clone()) {
config
} else {
spec.gen(
&mut rand::rngs::StdRng::from_entropy(),
&configure_context.timeout,
)?
};
spec.validate(&manifest)?;
spec.matches(&config)?; // check that new config matches spec
let manifest = receipts
.manifest
.get(db, id)
.await?
.ok_or_else(|| not_found!(&*id))?;
// TODO Commit or not?
spec.update(ctx, &manifest, overrides, &mut config).await?; // dereference pointers in the new config
spec.validate(&manifest)?;
spec.matches(&config)?; // check that new config matches spec
spec.update(
ctx,
db,
&manifest,
overrides,
&mut config,
&receipts.config_receipts,
)
.await?; // dereference pointers in the new config
let manifest = db
.as_package_data()
.as_idx(id)
.or_not_found(id)?
.as_installed()
.or_not_found(id)?
.as_manifest()
.de()?;
// create backreferences to pointers
let mut sys = receipts
.system_pointers
.get(db, id)
.await?
.ok_or_else(|| not_found!(&*id))?;
sys.truncate(0);
let mut current_dependencies: CurrentDependencies = CurrentDependencies(
dependencies
.0
.iter()
.filter_map(|(id, info)| {
if info.requirement.required() {
Some((id.clone(), CurrentDependencyInfo::default()))
} else {
None
}
})
.collect(),
);
for ptr in spec.pointers(&config)? {
match ptr {
ValueSpecPointer::Package(pkg_ptr) => {
if let Some(current_dependency) =
current_dependencies.0.get_mut(pkg_ptr.package_id())
{
current_dependency.pointers.push(pkg_ptr);
} else {
current_dependencies.0.insert(
pkg_ptr.package_id().to_owned(),
CurrentDependencyInfo {
pointers: vec![pkg_ptr],
health_checks: BTreeSet::new(),
},
);
}
}
ValueSpecPointer::System(s) => sys.push(s),
}
}
receipts.system_pointers.set(db, sys, id).await?;
let signal = if !configure_context.dry_run {
// run config action
let res = action
.set(ctx, id, &version, &dependencies, &volumes, &config)
.await?;
ctx.call_config_hooks(id.clone(), &serde_json::Value::Object(config.clone()))
.await;
// track dependencies with no pointers
for (package_id, health_checks) in res.depends_on.into_iter() {
if let Some(current_dependency) = current_dependencies.0.get_mut(&package_id) {
current_dependency.health_checks.extend(health_checks);
let dependencies = &manifest.dependencies;
let mut current_dependencies: CurrentDependencies = CurrentDependencies(
dependencies
.0
.iter()
.filter_map(|(id, info)| {
if info.requirement.required() {
Some((id.clone(), CurrentDependencyInfo::default()))
} else {
None
}
})
.collect(),
);
for ptr in spec.pointers(&config)? {
match ptr {
ValueSpecPointer::Package(pkg_ptr) => {
if let Some(info) = current_dependencies.0.get_mut(pkg_ptr.package_id()) {
info.pointers.insert(pkg_ptr);
} else {
let id = pkg_ptr.package_id().to_owned();
let mut pointers = BTreeSet::new();
pointers.insert(pkg_ptr);
current_dependencies.0.insert(
package_id,
id,
CurrentDependencyInfo {
pointers: Vec::new(),
health_checks,
pointers,
health_checks: BTreeSet::new(),
},
);
}
}
ValueSpecPointer::System(_) => (),
}
}
// track dependency health checks
current_dependencies = current_dependencies.map(|x| {
x.into_iter()
.filter(|(dep_id, _)| {
if dep_id != id && !manifest.dependencies.0.contains_key(dep_id) {
tracing::warn!("Illegal dependency specified: {}", dep_id);
false
} else {
true
}
})
.collect()
});
res.signal
} else {
None
};
// update dependencies
let prev_current_dependencies = receipts
.current_dependencies
.get(db, id)
.await?
.unwrap_or_default();
remove_from_current_dependents_lists(
db,
id,
&prev_current_dependencies,
&receipts.current_dependents,
)
.await?; // remove previous
add_dependent_to_current_dependents_lists(
db,
id,
&current_dependencies,
&receipts.current_dependents,
)
.await?; // add new
current_dependencies.0.remove(id);
receipts
.current_dependencies
.set(db, current_dependencies.clone(), id)
let action = manifest.config.as_ref().or_not_found(id)?;
let version = &manifest.version;
let volumes = &manifest.volumes;
if !configure_context.dry_run {
// run config action
let res = action
.set(ctx, id, version, &dependencies, volumes, &config)
.await?;
let errs = receipts
.dependency_errors
.get(db, &id)
.await?
.ok_or_else(|| not_found!(&*id))?;
tracing::warn!("Dependency Errors: {:?}", errs);
let errs = DependencyErrors::init(
ctx,
db,
&manifest,
&current_dependencies,
&receipts.dependency_receipt.try_heal,
)
.await?;
receipts.dependency_errors.set(db, errs, id).await?;
// track dependencies with no pointers
for (package_id, health_checks) in res.depends_on.into_iter() {
if let Some(current_dependency) = current_dependencies.0.get_mut(&package_id) {
current_dependency.health_checks.extend(health_checks);
} else {
current_dependencies.0.insert(
package_id,
CurrentDependencyInfo {
pointers: BTreeSet::new(),
health_checks,
},
);
}
}
// cache current config for dependents
configure_context
.overrides
.insert(id.clone(), config.clone());
// track dependency health checks
current_dependencies = current_dependencies.map(|x| {
x.into_iter()
.filter(|(dep_id, _)| {
if dep_id != id && !manifest.dependencies.0.contains_key(dep_id) {
tracing::warn!("Illegal dependency specified: {}", dep_id);
false
} else {
true
}
})
.collect()
});
}
// handle dependents
let dependents = receipts
.current_dependents
.get(db, id)
.await?
.ok_or_else(|| not_found!(&*id))?;
for (dependent, _dep_info) in dependents.0.iter().filter(|(dep_id, _)| dep_id != &id) {
let dependent_container = receipts.docker_containers.get(db, dependent).await?;
let dependent_container = &dependent_container;
// check if config passes dependent check
if let Some(cfg) = receipts
.manifest_dependencies_config
.get(db, (dependent, id))
let dependency_config_errs =
compute_dependency_config_errs(&ctx, &db, &manifest, &current_dependencies, overrides)
.await?;
// cache current config for dependents
configure_context
.overrides
.insert(id.clone(), config.clone());
// handle dependents
let dependents = db
.as_package_data()
.as_idx(id)
.or_not_found(id)?
.as_installed()
.or_not_found(id)?
.as_current_dependents()
.de()?;
for (dependent, _dep_info) in dependents.0.iter().filter(|(dep_id, _)| dep_id != &id) {
// check if config passes dependent check
if let Some(cfg) = db
.as_package_data()
.as_idx(dependent)
.or_not_found(dependent)?
.as_installed()
.or_not_found(dependent)?
.as_manifest()
.as_dependencies()
.as_idx(id)
.or_not_found(id)?
.as_config()
.de()?
{
let manifest = db
.as_package_data()
.as_idx(dependent)
.or_not_found(dependent)?
.as_installed()
.or_not_found(dependent)?
.as_manifest()
.de()?;
if let Err(error) = cfg
.check(
ctx,
dependent,
&manifest.version,
&manifest.volumes,
id,
&config,
)
.await?
{
let manifest = receipts
.manifest
.get(db, dependent)
.await?
.ok_or_else(|| not_found!(&*id))?;
if let Err(error) = cfg
.check(
ctx,
dependent_container,
dependent,
&manifest.version,
&manifest.volumes,
id,
&config,
)
.await?
{
let dep_err = DependencyError::ConfigUnsatisfied { error };
break_transitive(
db,
dependent,
id,
dep_err,
&mut configure_context.breakages,
&receipts.break_transitive_receipts,
)
.await?;
}
heal_all_dependents_transitive(ctx, db, id, &receipts.dependency_receipt).await?;
configure_context.breakages.insert(dependent.clone(), error);
}
}
if let Some(signal) = signal {
match ctx.managers.get(&(id.clone(), version.clone())).await {
None => {
// in theory this should never happen, which indicates this function should be moved behind the
// Manager interface
return Err(Error::new(
eyre!("Manager Not Found for package being configured"),
crate::ErrorKind::Incoherent,
));
}
Some(m) => {
async move { m.signal(signal).await }.await?;
}
}
}
receipts.configured.set(db, true, &id).await?;
if configure_context.dry_run {
tx.abort().await?;
} else {
tx.commit().await?;
}
Ok(configure_context.breakages)
}
.boxed()
if !configure_context.dry_run {
return ctx
.db
.mutate(move |db| {
remove_from_current_dependents_lists(db, id, &current_dependencies)?;
add_dependent_to_current_dependents_lists(db, id, &current_dependencies)?;
current_dependencies.0.remove(id);
for (dep, errs) in db
.as_package_data_mut()
.as_entries_mut()?
.into_iter()
.filter_map(|(id, pde)| {
pde.as_installed_mut()
.map(|i| (id, i.as_status_mut().as_dependency_config_errors_mut()))
})
{
errs.remove(id)?;
if let Some(err) = configure_context.breakages.get(&dep) {
errs.insert(id, err)?;
}
}
let installed = db
.as_package_data_mut()
.as_idx_mut(id)
.or_not_found(id)?
.as_installed_mut()
.or_not_found(id)?;
installed
.as_current_dependencies_mut()
.ser(&current_dependencies)?;
let status = installed.as_status_mut();
status.as_configured_mut().ser(&true)?;
status
.as_dependency_config_errors_mut()
.ser(&dependency_config_errs)?;
Ok(configure_context.breakages)
})
.await; // add new
}
Ok(configure_context.breakages)
}
struct DesiredStateReverter {
@@ -640,7 +601,7 @@ impl Drop for DesiredStateReverter {
type BackupDoneSender = oneshot::Sender<Result<PackageBackupInfo, Error>>;
fn finnish_up_backup_task(
fn finish_up_backup_task(
transition: Arc<Sender<Arc<TransitionState>>>,
send: BackupDoneSender,
) -> impl FnOnce(Result<Result<PackageBackupInfo, Error>, Error>) -> BoxFuture<'static, ()> {
@@ -764,27 +725,11 @@ enum GetRunningIp {
async fn get_long_running_ip(seed: &ManagerSeed, runtime: &mut LongRunning) -> GetRunningIp {
loop {
match container_inspect(seed).await {
Ok(res) => {
match res
.network_settings
.and_then(|ns| ns.networks)
.and_then(|mut n| n.remove("start9"))
.and_then(|es| es.ip_address)
.filter(|ip| !ip.is_empty())
.map(|ip| ip.parse())
.transpose()
{
Ok(Some(ip_addr)) => return GetRunningIp::Ip(ip_addr),
Ok(None) => (),
Err(e) => return GetRunningIp::Error(e.into()),
}
}
Err(bollard::errors::Error::DockerResponseServerError {
status_code: 404, // NOT FOUND
..
}) => (),
Err(e) => return GetRunningIp::Error(e.into()),
match get_container_ip(&seed.container_name).await {
Ok(Some(ip_addr)) => return GetRunningIp::Ip(ip_addr),
Ok(None) => (),
Err(e) if e.kind == ErrorKind::NotFound => (),
Err(e) => return GetRunningIp::Error(e),
}
if let Poll::Ready(res) = futures::poll!(&mut runtime.running_output) {
match res {
@@ -800,16 +745,6 @@ async fn get_long_running_ip(seed: &ManagerSeed, runtime: &mut LongRunning) -> G
}
}
#[instrument(skip(seed))]
async fn container_inspect(
seed: &ManagerSeed,
) -> Result<bollard::models::ContainerInspectResponse, bollard::errors::Error> {
seed.ctx
.docker
.inspect_container(&seed.container_name, None)
.await
}
#[instrument(skip(seed))]
async fn add_network_for_main(
seed: &ManagerSeed,
@@ -856,8 +791,7 @@ async fn remove_network_for_main(svc: NetService) -> Result<(), Error> {
async fn main_health_check_daemon(seed: Arc<ManagerSeed>) {
tokio::time::sleep(Duration::from_secs(HEALTH_CHECK_GRACE_PERIOD_SECONDS)).await;
loop {
let mut db = seed.ctx.db.handle();
if let Err(e) = health::check(&seed.ctx, &mut db, &seed.manifest.id).await {
if let Err(e) = health::check(&seed.ctx, &seed.manifest.id).await {
tracing::error!(
"Failed to run health check for {}: {}",
&seed.manifest.id,
@@ -871,42 +805,14 @@ async fn main_health_check_daemon(seed: Arc<ManagerSeed>) {
type RuntimeOfCommand = NonDetachingJoinHandle<Result<Result<NoOutput, (i32, String)>, Error>>;
async fn try_get_running_ip(seed: &ManagerSeed) -> Result<Option<Ipv4Addr>, Report> {
Ok(container_inspect(seed)
.await
.map(|x| x.network_settings)?
.and_then(|ns| ns.networks)
.and_then(|mut n| n.remove("start9"))
.and_then(|es| es.ip_address)
.filter(|ip| !ip.is_empty())
.map(|ip| ip.parse())
.transpose()?)
}
#[instrument(skip(seed, runtime))]
async fn get_running_ip(seed: &ManagerSeed, mut runtime: &mut RuntimeOfCommand) -> GetRunningIp {
loop {
match container_inspect(seed).await {
Ok(res) => {
match res
.network_settings
.and_then(|ns| ns.networks)
.and_then(|mut n| n.remove("start9"))
.and_then(|es| es.ip_address)
.filter(|ip| !ip.is_empty())
.map(|ip| ip.parse())
.transpose()
{
Ok(Some(ip_addr)) => return GetRunningIp::Ip(ip_addr),
Ok(None) => (),
Err(e) => return GetRunningIp::Error(e.into()),
}
}
Err(bollard::errors::Error::DockerResponseServerError {
status_code: 404, // NOT FOUND
..
}) => (),
Err(e) => return GetRunningIp::Error(e.into()),
match get_container_ip(&seed.container_name).await {
Ok(Some(ip_addr)) => return GetRunningIp::Ip(ip_addr),
Ok(None) => (),
Err(e) if e.kind == ErrorKind::NotFound => (),
Err(e) => return GetRunningIp::Error(e),
}
if let Poll::Ready(res) = futures::poll!(&mut runtime) {
match res {
@@ -960,7 +866,7 @@ async fn send_signal(manager: &Manager, gid: Arc<Gid>, signal: Signal) -> Result
None, // TODO
next_gid,
Some(rpc_client),
Arc::new(manager.clone()),
todo!(),
)
.await?
{
@@ -969,28 +875,10 @@ async fn send_signal(manager: &Manager, gid: Arc<Gid>, signal: Signal) -> Result
}
} else {
// send signal to container
manager
.seed
.ctx
.docker
.kill_container(
&manager.seed.container_name,
Some(bollard::container::KillContainerOptions {
signal: signal.to_string(),
}),
)
kill_container(&manager.seed.container_name, Some(signal))
.await
.or_else(|e| {
if matches!(
e,
bollard::errors::Error::DockerResponseServerError {
status_code: 409, // CONFLICT
..
} | bollard::errors::Error::DockerResponseServerError {
status_code: 404, // NOT FOUND
..
}
) {
if e.kind == ErrorKind::NotFound {
Ok(())
} else {
Err(e)

2
backend/src/manager/persistent_container.rs
View File

@@ -16,6 +16,8 @@ use crate::procedure::docker::DockerContainer;
use crate::util::NonDetachingJoinHandle;
use crate::Error;
/// Persistant container are the old containers that need to run all the time
/// The goal is that all services will be persistent containers, waiting to run the main system.
pub struct PersistentContainer {
_running_docker: NonDetachingJoinHandle<()>,
pub rpc_client: Receiver<Arc<UnixRpcClient>>,

15
backend/src/manager/start_stop.rs
View File

@@ -10,9 +10,6 @@ impl StartStop {
pub(crate) fn is_start(&self) -> bool {
matches!(self, StartStop::Start)
}
pub(crate) fn is_stop(&self) -> bool {
matches!(self, StartStop::Stop)
}
}
impl From<MainStatus> for StartStop {
fn from(value: MainStatus) -> Self {
@@ -21,9 +18,15 @@ impl From<MainStatus> for StartStop {
MainStatus::Restarting => StartStop::Start,
MainStatus::Stopping => StartStop::Stop,
MainStatus::Starting => StartStop::Start,
MainStatus::Running { started, health } => StartStop::Start,
MainStatus::BackingUp { started, health } if started.is_some() => StartStop::Start,
MainStatus::BackingUp { started, health } => StartStop::Stop,
MainStatus::Running {
started: _,
health: _,
} => StartStop::Start,
MainStatus::BackingUp { started, health: _ } if started.is_some() => StartStop::Start,
MainStatus::BackingUp {
started: _,
health: _,
} => StartStop::Stop,
}
}
}

8
backend/src/manager/transition_state.rs
View File

@@ -1,6 +1,8 @@
use helpers::NonDetachingJoinHandle;
pub(crate) enum TransitionState {
/// Used only in the manager/mod and is used to keep track of the state of the manager during the
/// transitional states
pub(super) enum TransitionState {
BackingUp(NonDetachingJoinHandle<()>),
Restarting(NonDetachingJoinHandle<()>),
Configuring(NonDetachingJoinHandle<()>),
@@ -8,7 +10,7 @@ pub(crate) enum TransitionState {
}
impl TransitionState {
pub(crate) fn join_handle(&self) -> Option<&NonDetachingJoinHandle<()>> {
pub(super) fn join_handle(&self) -> Option<&NonDetachingJoinHandle<()>> {
Some(match self {
TransitionState::BackingUp(a) => a,
TransitionState::Restarting(a) => a,
@@ -16,7 +18,7 @@ impl TransitionState {
TransitionState::None => return None,
})
}
pub(crate) fn abort(&self) {
pub(super) fn abort(&self) {
self.join_handle().map(|transition| transition.abort());
}
}

11
backend/src/migration.rs
View File

@@ -10,6 +10,7 @@ use serde::{Deserialize, Serialize};
use tracing::instrument;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::procedure::docker::DockerContainers;
use crate::procedure::{PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
@@ -19,6 +20,7 @@ use crate::{Error, ResultExt};
#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct Migrations {
pub from: IndexMap<VersionRange, PackageProcedure>,
pub to: IndexMap<VersionRange, PackageProcedure>,
@@ -27,14 +29,14 @@ impl Migrations {
#[instrument(skip_all)]
pub fn validate(
&self,
container: &Option<DockerContainers>,
_container: &Option<DockerContainers>,
eos_version: &Version,
volumes: &Volumes,
image_ids: &BTreeSet<ImageId>,
) -> Result<(), Error> {
for (version, migration) in &self.from {
migration
.validate(container, eos_version, volumes, image_ids, true)
.validate(eos_version, volumes, image_ids, true)
.with_ctx(|_| {
(
crate::ErrorKind::ValidateS9pk,
@@ -44,7 +46,7 @@ impl Migrations {
}
for (version, migration) in &self.to {
migration
.validate(container, eos_version, volumes, image_ids, true)
.validate(eos_version, volumes, image_ids, true)
.with_ctx(|_| {
(
crate::ErrorKind::ValidateS9pk,
@@ -58,7 +60,7 @@ impl Migrations {
#[instrument(skip_all)]
pub fn from<'a>(
&'a self,
container: &'a Option<DockerContainers>,
_container: &'a Option<DockerContainers>,
ctx: &'a RpcContext,
version: &'a Version,
pkg_id: &'a PackageId,
@@ -133,6 +135,7 @@ impl Migrations {
#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct MigrationRes {
pub configured: bool,
}

13
backend/src/net/dhcp.rs
View File

@@ -8,6 +8,7 @@ use tokio::sync::RwLock;
use crate::context::RpcContext;
use crate::db::model::IpInfo;
use crate::net::utils::{iface_is_physical, list_interfaces};
use crate::prelude::*;
use crate::util::display_none;
use crate::Error;
@@ -58,12 +59,14 @@ pub async fn dhcp() -> Result<(), Error> {
pub async fn update(#[context] ctx: RpcContext, #[arg] interface: String) -> Result<(), Error> {
if iface_is_physical(&interface).await {
let ip_info = IpInfo::for_interface(&interface).await?;
crate::db::DatabaseModel::new()
.server_info()
.ip_info()
.idx_model(&interface)
.put(&mut ctx.db.handle(), &ip_info)
ctx.db
.mutate(|db| {
db.as_server_info_mut()
.as_ip_info_mut()
.insert(&interface, &ip_info)
})
.await?;
let mut cached = CACHED_IPS.write().await;
if cached.is_empty() {
*cached = _ips().await?;

18
backend/src/net/dns.rs
View File

@@ -50,17 +50,15 @@ impl Resolver {
} else {
None
}
} else if let Some(ip) = self.services.read().await.get(&None) {
Some(
ip.iter()
.filter(|(_, rc)| rc.strong_count() > 0)
.map(|(ip, _)| *ip)
.collect(),
)
} else {
if let Some(ip) = self.services.read().await.get(&None) {
Some(
ip.iter()
.filter(|(_, rc)| rc.strong_count() > 0)
.map(|(ip, _)| *ip)
.collect(),
)
} else {
None
}
None
}
}
_ => None,

30
backend/src/net/interface.rs
View File

@@ -4,10 +4,10 @@ use indexmap::IndexSet;
pub use models::InterfaceId;
use serde::{Deserialize, Deserializer, Serialize};
use sqlx::{Executor, Postgres};
use torut::onion::TorSecretKeyV3;
use tracing::instrument;
use crate::db::model::{InterfaceAddressMap, InterfaceAddresses};
use crate::net::keys::Key;
use crate::s9pk::manifest::PackageId;
use crate::util::serde::Port;
use crate::{Error, ResultExt};
@@ -44,33 +44,13 @@ impl Interfaces {
lan_address: None,
};
if iface.tor_config.is_some() || iface.lan_config.is_some() {
let key = TorSecretKeyV3::generate();
let key_vec = key.as_bytes().to_vec();
sqlx::query!(
"INSERT INTO tor (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
**package_id,
**id,
key_vec,
)
.execute(&mut *secrets)
.await?;
let key_row = sqlx::query!(
"SELECT key FROM tor WHERE package = $1 AND interface = $2",
**package_id,
**id,
)
.fetch_one(&mut *secrets)
.await?;
let mut key = [0_u8; 64];
key.clone_from_slice(&key_row.key);
let key = TorSecretKeyV3::from(key);
let onion = key.public().get_onion_address();
let key =
Key::for_interface(secrets, Some((package_id.clone(), id.clone()))).await?;
if iface.tor_config.is_some() {
addrs.tor_address = Some(onion.to_string());
addrs.tor_address = Some(key.tor_address().to_string());
}
if iface.lan_config.is_some() {
addrs.lan_address =
Some(format!("{}.local", onion.get_address_without_dot_onion()));
addrs.lan_address = Some(key.local_address());
}
}
interface_addresses.0.insert(id.clone(), addrs);

26
backend/src/net/keys.rs
View File

@@ -23,8 +23,8 @@ async fn compat(
if let Some((package, interface)) = interface {
if let Some(r) = sqlx::query!(
"SELECT key FROM tor WHERE package = $1 AND interface = $2",
**package,
**interface
package,
interface
)
.fetch_optional(secrets)
.await?
@@ -33,16 +33,14 @@ async fn compat(
} else {
Ok(None)
}
} else if let Some(key) = sqlx::query!("SELECT tor_key FROM account WHERE id = 0")
.fetch_one(secrets)
.await?
.tor_key
{
Ok(Some(ExpandedSecretKey::from_bytes(&key)?))
} else {
if let Some(key) = sqlx::query!("SELECT tor_key FROM account WHERE id = 0")
.fetch_one(secrets)
.await?
.tor_key
{
Ok(Some(ExpandedSecretKey::from_bytes(&key)?))
} else {
Ok(None)
}
Ok(None)
}
}
@@ -152,7 +150,7 @@ impl Key {
WHERE
network_keys.package = $1
"#,
**package
package
)
.fetch_all(secrets)
.await?
@@ -197,8 +195,8 @@ impl Key {
let k = tentative.as_slice();
let actual = sqlx::query!(
"INSERT INTO network_keys (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO UPDATE SET package = EXCLUDED.package RETURNING key",
**pkg,
**iface,
pkg,
iface,
k,
)
.fetch_one(&mut *secrets)

1
backend/src/net/mod.rs
View File

@@ -11,7 +11,6 @@ pub mod dns;
pub mod forward;
pub mod interface;
pub mod keys;
#[cfg(feature = "avahi")]
pub mod mdns;
pub mod net_controller;
pub mod ssl;

33
backend/src/net/net_controller.rs
View File

@@ -4,16 +4,16 @@ use std::sync::{Arc, Weak};
use color_eyre::eyre::eyre;
use models::InterfaceId;
use patch_db::{DbHandle, LockType, PatchDb};
use patch_db::PatchDb;
use sqlx::PgExecutor;
use tracing::instrument;
use crate::db::prelude::PatchDbExt;
use crate::error::ErrorCollection;
use crate::hostname::Hostname;
use crate::net::dns::DnsController;
use crate::net::forward::LpfController;
use crate::net::keys::Key;
#[cfg(feature = "avahi")]
use crate::net::mdns::MdnsController;
use crate::net::ssl::{export_cert, export_key, SslManager};
use crate::net::tor::TorController;
@@ -24,7 +24,6 @@ use crate::{Error, HOST_IP};
pub struct NetController {
pub(super) tor: TorController,
#[cfg(feature = "avahi")]
pub(super) mdns: MdnsController,
pub(super) vhost: VHostController,
pub(super) dns: DnsController,
@@ -46,7 +45,6 @@ impl NetController {
let ssl = Arc::new(ssl);
let mut res = Self {
tor: TorController::new(tor_control, tor_socks),
#[cfg(feature = "avahi")]
mdns: MdnsController::init().await?,
vhost: VHostController::new(ssl.clone()),
dns: DnsController::init(dns_bind).await?,
@@ -206,14 +204,12 @@ impl NetController {
)
.await?,
);
#[cfg(feature = "avahi")]
rcs.push(self.mdns.add(key.base_address()).await?);
Ok(rcs)
}
async fn remove_lan(&self, key: &Key, external: u16, rcs: Vec<Arc<()>>) -> Result<(), Error> {
drop(rcs);
#[cfg(feature = "avahi")]
self.mdns.gc(key.base_address()).await?;
self.vhost.gc(Some(key.local_address()), external).await
}
@@ -321,18 +317,19 @@ impl NetService {
}
pub async fn add_lpf(&mut self, db: &PatchDb, internal: u16) -> Result<u16, Error> {
let ctrl = self.net_controller()?;
let mut db = db.handle();
let lpf_model = crate::db::DatabaseModel::new().lan_port_forwards();
lpf_model.lock(&mut db, LockType::Write).await?; // TODO: replace all this with an RMW
let mut lpf = lpf_model.get_mut(&mut db).await?;
let external = lpf.alloc(self.id.clone(), internal).ok_or_else(|| {
Error::new(
eyre!("No ephemeral ports available"),
crate::ErrorKind::Network,
)
})?;
lpf.save(&mut db).await?;
drop(db);
let external = db
.mutate(|db| {
let mut lpf = db.as_lan_port_forwards().de()?;
let external = lpf.alloc(self.id.clone(), internal).ok_or_else(|| {
Error::new(
eyre!("No ephemeral ports available"),
crate::ErrorKind::Network,
)
})?;
db.as_lan_port_forwards_mut().ser(&lpf)?;
Ok(external)
})
.await?;
let rc = ctrl.add_lpf(external, (self.ip, internal).into()).await?;
let (_, mut lpfs) = self.lpf.remove(&internal).unwrap_or_default();
lpfs.push(rc);

117
backend/src/net/static_server.rs
View File

@@ -1,5 +1,5 @@
use std::borrow::Cow;
use std::fs::Metadata;
use std::future::Future;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::UNIX_EPOCH;
@@ -10,7 +10,6 @@ use digest::Digest;
use futures::FutureExt;
use http::header::ACCEPT_ENCODING;
use http::request::Parts as RequestParts;
use http::response::Builder;
use hyper::{Body, Method, Request, Response, StatusCode};
use include_dir::{include_dir, Dir};
use new_mime_guess::MimeGuess;
@@ -264,6 +263,21 @@ async fn alt_ui(req: Request<Body>, ui_mode: UiMode) -> Result<Response<Body>, E
}
}
async fn if_authorized<
F: FnOnce() -> Fut,
Fut: Future<Output = Result<Response<Body>, Error>> + Send + Sync,
>(
ctx: &RpcContext,
parts: &RequestParts,
f: F,
) -> Result<Response<Body>, Error> {
if let Err(e) = HasValidSession::from_request_parts(parts, ctx).await {
un_authorized(e, parts.uri.path())
} else {
f().await
}
}
async fn main_embassy_ui(req: Request<Body>, ctx: RpcContext) -> Result<Response<Body>, Error> {
let (request_parts, _body) = req.into_parts();
match (
@@ -276,69 +290,56 @@ async fn main_embassy_ui(req: Request<Body>, ctx: RpcContext) -> Result<Response
.split_once('/'),
) {
(&Method::GET, Some(("public", path))) => {
match HasValidSession::from_request_parts(&request_parts, &ctx).await {
Ok(_) => {
let sub_path = Path::new(path);
if let Ok(rest) = sub_path.strip_prefix("package-data") {
FileData::from_path(
&request_parts,
&ctx.datadir.join(PKG_PUBLIC_DIR).join(rest),
)
.await?
.into_response(&request_parts)
.await
} else if let Ok(rest) = sub_path.strip_prefix("eos") {
match rest.to_str() {
Some("local.crt") => cert_send(&ctx.account.read().await.root_ca_cert),
None => Ok(bad_request()),
_ => Ok(not_found()),
}
} else {
Ok(not_found())
}
if_authorized(&ctx, &request_parts, || async {
let sub_path = Path::new(path);
if let Ok(rest) = sub_path.strip_prefix("package-data") {
FileData::from_path(
&request_parts,
&ctx.datadir.join(PKG_PUBLIC_DIR).join(rest),
)
.await?
.into_response(&request_parts)
.await
} else {
Ok(not_found())
}
Err(e) => un_authorized(e, &format!("public/{path}")),
}
})
.await
}
(&Method::GET, Some(("proxy", target))) => {
match HasValidSession::from_request_parts(&request_parts, &ctx).await {
Ok(_) => {
let target = urlencoding::decode(target)?;
let res = ctx
.client
.get(target.as_ref())
.headers(
request_parts
.headers
.iter()
.filter(|(h, _)| {
!PROXY_STRIP_HEADERS
.iter()
.any(|bad| h.as_str().eq_ignore_ascii_case(bad))
})
.map(|(h, v)| (h.clone(), v.clone()))
.collect(),
)
.send()
.await
.with_kind(crate::ErrorKind::Network)?;
let mut hres = Response::builder().status(res.status());
for (h, v) in res.headers().clone() {
if let Some(h) = h {
hres = hres.header(h, v);
}
if_authorized(&ctx, &request_parts, || async {
let target = urlencoding::decode(target)?;
let res = ctx
.client
.get(target.as_ref())
.headers(
request_parts
.headers
.iter()
.filter(|(h, _)| {
!PROXY_STRIP_HEADERS
.iter()
.any(|bad| h.as_str().eq_ignore_ascii_case(bad))
})
.map(|(h, v)| (h.clone(), v.clone()))
.collect(),
)
.send()
.await
.with_kind(crate::ErrorKind::Network)?;
let mut hres = Response::builder().status(res.status());
for (h, v) in res.headers().clone() {
if let Some(h) = h {
hres = hres.header(h, v);
}
hres.body(Body::wrap_stream(res.bytes_stream()))
.with_kind(crate::ErrorKind::Network)
}
Err(e) => un_authorized(e, &format!("proxy/{target}")),
}
hres.body(Body::wrap_stream(res.bytes_stream()))
.with_kind(crate::ErrorKind::Network)
})
.await
}
(&Method::GET, Some(("eos", "local.crt"))) => {
match HasValidSession::from_request_parts(&request_parts, &ctx).await {
Ok(_) => cert_send(&ctx.account.read().await.root_ca_cert),
Err(e) => un_authorized(e, "eos/local.crt"),
}
cert_send(&ctx.account.read().await.root_ca_cert)
}
(&Method::GET, _) => {
let uri_path = UiMode::Main.path(

3
backend/src/net/tor.rs
View File

@@ -617,7 +617,7 @@ async fn torctl(
let mut last_success = Instant::now();
loop {
tokio::time::sleep(Duration::from_secs(30)).await;
if let Err(e) = tokio::time::timeout(
if tokio::time::timeout(
Duration::from_secs(30),
tokio_socks::tcp::Socks5Stream::connect(
tor_socks,
@@ -627,6 +627,7 @@ async fn torctl(
.await
.map_err(|e| e.to_string())
.and_then(|e| e.map_err(|e| e.to_string()))
.is_err()
{
if last_success.elapsed() > *health_timeout {
let err = Error::new(eyre!("Tor health check failed for longer than current timeout ({health_timeout:?})"), crate::ErrorKind::Tor);

2
backend/src/net/utils.rs
View File

@@ -152,7 +152,7 @@ impl hyper::server::accept::Accept for TcpListeners {
type Error = std::io::Error;
fn poll_accept(
mut self: std::pin::Pin<&mut Self>,
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Result<Self::Conn, Self::Error>>> {
for listener in self.listeners.iter() {

64
backend/src/net/wifi.rs
View File

@@ -6,7 +6,6 @@ use std::time::Duration;
use clap::ArgMatches;
use isocountry::CountryCode;
use lazy_static::lazy_static;
use patch_db::DbHandle;
use regex::Regex;
use rpc_toolkit::command;
use tokio::process::Command;
@@ -14,6 +13,7 @@ use tokio::sync::RwLock;
use tracing::instrument;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::util::serde::{display_serializable, IoFormat};
use crate::util::{display_none, Invoke};
use crate::{Error, ErrorKind};
@@ -52,8 +52,6 @@ pub async fn add(
#[context] ctx: RpcContext,
#[arg] ssid: String,
#[arg] password: String,
#[arg] priority: isize,
#[arg] connect: bool,
) -> Result<(), Error> {
let wifi_manager = wifi_manager(&ctx)?;
if !ssid.is_ascii() {
@@ -69,26 +67,22 @@ pub async fn add(
));
}
async fn add_procedure(
db: impl DbHandle,
db: PatchDb,
wifi_manager: WifiManager,
ssid: &Ssid,
password: &Psk,
priority: isize,
) -> Result<(), Error> {
tracing::info!("Adding new WiFi network: '{}'", ssid.0);
let mut wpa_supplicant = wifi_manager.write().await;
wpa_supplicant
.add_network(db, ssid, password, priority)
.await?;
wpa_supplicant.add_network(db, ssid, password).await?;
drop(wpa_supplicant);
Ok(())
}
if let Err(err) = add_procedure(
&mut ctx.db.handle(),
ctx.db.clone(),
wifi_manager.clone(),
&Ssid(ssid.clone()),
&Psk(password.clone()),
priority,
)
.await
{
@@ -113,7 +107,7 @@ pub async fn connect(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result<
));
}
async fn connect_procedure(
mut db: impl DbHandle,
db: PatchDb,
wifi_manager: WifiManager,
ssid: &Ssid,
) -> Result<(), Error> {
@@ -121,7 +115,7 @@ pub async fn connect(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result<
let current = wpa_supplicant.get_current_network().await?;
drop(wpa_supplicant);
let mut wpa_supplicant = wifi_manager.write().await;
let connected = wpa_supplicant.select_network(&mut db, ssid).await?;
let connected = wpa_supplicant.select_network(db.clone(), ssid).await?;
if connected {
tracing::info!("Successfully connected to WiFi: '{}'", ssid.0);
} else {
@@ -131,19 +125,15 @@ pub async fn connect(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result<
tracing::info!("No WiFi to revert to!");
}
Some(current) => {
wpa_supplicant.select_network(&mut db, &current).await?;
wpa_supplicant.select_network(db, &current).await?;
}
}
}
Ok(())
}
if let Err(err) = connect_procedure(
&mut ctx.db.handle(),
wifi_manager.clone(),
&Ssid(ssid.clone()),
)
.await
if let Err(err) =
connect_procedure(ctx.db.clone(), wifi_manager.clone(), &Ssid(ssid.clone())).await
{
tracing::error!("Failed to connect to WiFi network '{}': {}", &ssid, err);
return Err(Error::new(
@@ -176,9 +166,7 @@ pub async fn delete(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result<(
return Err(Error::new(color_eyre::eyre::eyre!("Forbidden: Deleting this network would make your server unreachable. Either connect to ethernet or connect to a different WiFi network to remedy this."), ErrorKind::Wifi));
}
wpa_supplicant
.remove_network(&mut ctx.db.handle(), &ssid)
.await?;
wpa_supplicant.remove_network(ctx.db.clone(), &ssid).await?;
Ok(())
}
#[derive(serde::Serialize, serde::Deserialize)]
@@ -397,7 +385,7 @@ pub async fn set_country(
}
wpa_supplicant.remove_all_connections().await?;
wpa_supplicant.save_config(&mut ctx.db.handle()).await?;
wpa_supplicant.save_config(ctx.db.clone()).await?;
Ok(())
}
@@ -645,13 +633,14 @@ impl WpaCli {
Ok(())
}
pub async fn save_config(&mut self, mut db: impl DbHandle) -> Result<(), Error> {
crate::db::DatabaseModel::new()
.server_info()
.last_wifi_region()
.put(&mut db, &Some(self.get_country_low().await?))
.await?;
Ok(())
pub async fn save_config(&mut self, db: PatchDb) -> Result<(), Error> {
let new_country = Some(self.get_country_low().await?);
db.mutate(|d| {
d.as_server_info_mut()
.as_last_wifi_region_mut()
.ser(&new_country)
})
.await
}
async fn check_active_network(&self, ssid: &Ssid) -> Result<Option<NetworkId>, Error> {
Ok(self
@@ -682,7 +671,7 @@ impl WpaCli {
.collect())
}
#[instrument(skip_all)]
pub async fn select_network(&mut self, db: impl DbHandle, ssid: &Ssid) -> Result<bool, Error> {
pub async fn select_network(&mut self, db: PatchDb, ssid: &Ssid) -> Result<bool, Error> {
let m_id = self.check_active_network(ssid).await?;
match m_id {
None => Err(Error::new(
@@ -734,7 +723,7 @@ impl WpaCli {
}
}
#[instrument(skip_all)]
pub async fn remove_network(&mut self, db: impl DbHandle, ssid: &Ssid) -> Result<bool, Error> {
pub async fn remove_network(&mut self, db: PatchDb, ssid: &Ssid) -> Result<bool, Error> {
let found_networks = self.find_networks(ssid).await?;
if found_networks.is_empty() {
return Ok(true);
@@ -748,23 +737,16 @@ impl WpaCli {
#[instrument(skip_all)]
pub async fn set_add_network(
&mut self,
db: impl DbHandle,
db: PatchDb,
ssid: &Ssid,
psk: &Psk,
priority: isize,
) -> Result<(), Error> {
self.set_add_network_low(ssid, psk).await?;
self.save_config(db).await?;
Ok(())
}
#[instrument(skip_all)]
pub async fn add_network(
&mut self,
db: impl DbHandle,
ssid: &Ssid,
psk: &Psk,
priority: isize,
) -> Result<(), Error> {
pub async fn add_network(&mut self, db: PatchDb, ssid: &Ssid, psk: &Psk) -> Result<(), Error> {
self.add_network_low(ssid, psk).await?;
self.save_config(db).await?;
Ok(())

50
backend/src/notifications.rs
View File

@@ -4,7 +4,6 @@ use std::str::FromStr;
use chrono::{DateTime, Utc};
use color_eyre::eyre::eyre;
use patch_db::{DbHandle, LockType};
use rpc_toolkit::command;
use sqlx::PgPool;
use tokio::sync::Mutex;
@@ -12,6 +11,7 @@ use tracing::instrument;
use crate::backup::BackupReport;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::util::display_none;
use crate::util::serde::display_serializable;
@@ -30,13 +30,8 @@ pub async fn list(
#[arg] limit: Option<u32>,
) -> Result<Vec<Notification>, Error> {
let limit = limit.unwrap_or(40);
let mut handle = ctx.db.handle();
match before {
None => {
let model = crate::db::DatabaseModel::new()
.server_info()
.unread_notification_count();
model.lock(&mut handle, LockType::Write).await?;
let records = sqlx::query!(
"SELECT id, package_id, created_at, code, level, title, message, data FROM notifications ORDER BY id DESC LIMIT $1",
limit as i64
@@ -70,8 +65,14 @@ pub async fn list(
})
})
.collect::<Result<Vec<Notification>, Error>>()?;
// set notification count to zero
model.put(&mut handle, &0).await?;
ctx.db
.mutate(|d| {
d.as_server_info_mut()
.as_unread_notification_count_mut()
.ser(&0)
})
.await?;
Ok(notifs)
}
Some(before) => {
@@ -139,15 +140,7 @@ pub async fn create(
#[arg] message: String,
) -> Result<(), Error> {
ctx.notification_manager
.notify(
&mut ctx.db.handle(),
package,
level,
title,
message,
(),
None,
)
.notify(ctx.db.clone(), package, level, title, message, (), None)
.await
}
@@ -232,10 +225,10 @@ impl NotificationManager {
cache: Mutex::new(HashMap::new()),
}
}
#[instrument(skip_all)]
pub async fn notify<Db: DbHandle, T: NotificationType>(
#[instrument(skip(db, subtype, self))]
pub async fn notify<T: NotificationType>(
&self,
db: &mut Db,
db: PatchDb,
package_id: Option<PackageId>,
level: NotificationLevel,
title: String,
@@ -243,17 +236,14 @@ impl NotificationManager {
subtype: T,
debounce_interval: Option<u32>,
) -> Result<(), Error> {
let peek = db.peek().await?;
if !self
.should_notify(&package_id, &level, &title, debounce_interval)
.await
{
return Ok(());
}
let mut count = crate::db::DatabaseModel::new()
.server_info()
.unread_notification_count()
.get_mut(db)
.await?;
let mut count = peek.as_server_info().as_unread_notification_count().de()?;
let sql_package_id = package_id.as_ref().map(|p| &**p);
let sql_code = T::CODE;
let sql_level = format!("{}", level);
@@ -268,9 +258,13 @@ impl NotificationManager {
message,
sql_data
).execute(&self.sqlite).await?;
*count += 1;
count.save(db).await?;
Ok(())
count += 1;
db.mutate(|db| {
db.as_server_info_mut()
.as_unread_notification_count_mut()
.ser(&count)
})
.await
}
async fn should_notify(
&self,

5
backend/src/os_install/mod.rs
View File

@@ -272,7 +272,7 @@ pub async fn execute(
.invoke(crate::ErrorKind::OpenSsh)
.await?;
let dev = MountGuard::mount(
let embassy_fs = MountGuard::mount(
&Bind::new(rootfs.as_ref()),
current.join("media/embassy/embassyfs"),
MountType::ReadOnly,
@@ -315,19 +315,18 @@ pub async fn execute(
.arg("update-grub2")
.invoke(crate::ErrorKind::Grub)
.await?;
dev.unmount(false).await?;
if let Some(efivarfs) = efivarfs {
efivarfs.unmount(false).await?;
}
sys.unmount(false).await?;
proc.unmount(false).await?;
embassy_fs.unmount(false).await?;
if let Some(efi) = efi {
efi.unmount(false).await?;
}
boot.unmount(false).await?;
rootfs.unmount().await?;
Ok(())
}

6
backend/src/prelude.rs Normal file
View File

@@ -0,0 +1,6 @@
pub use color_eyre::eyre::eyre;
pub use models::OptionExt;
pub use crate::db::prelude::*;
pub use crate::ensure_code;
pub use crate::error::{Error, ErrorCollection, ErrorKind, ResultExt};

104
backend/src/procedure/docker.rs
View File

@@ -7,14 +7,12 @@ use std::path::{Path, PathBuf};
use std::time::Duration;
use async_stream::stream;
use bollard::container::RemoveContainerOptions;
use chrono::format::Item;
use color_eyre::eyre::eyre;
use color_eyre::Report;
use futures::future::{BoxFuture, Either as EitherFuture};
use futures::{FutureExt, TryStreamExt};
use helpers::{NonDetachingJoinHandle, UnixRpcClient};
use models::{Id, ImageId};
use models::{Id, ImageId, SYSTEM_PACKAGE_ID};
use nix::sys::signal;
use nix::unistd::Pid;
use serde::de::DeserializeOwned;
@@ -26,7 +24,9 @@ use tracing::instrument;
use super::ProcedureName;
use crate::context::RpcContext;
use crate::s9pk::manifest::{PackageId, SYSTEM_PACKAGE_ID};
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::util::docker::{remove_container, CONTAINER_TOOL};
use crate::util::serde::{Duration as SerdeDuration, IoFormat};
use crate::util::Version;
use crate::volume::{VolumeId, Volumes};
@@ -45,8 +45,9 @@ lazy_static::lazy_static! {
};
}
#[derive(Clone, Debug, Deserialize, Serialize, patch_db::HasModel)]
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct DockerContainers {
pub main: DockerContainer,
// #[serde(default)]
@@ -58,6 +59,7 @@ pub struct DockerContainers {
/// part of this struct by choice. Used for the times that we are creating our own entry points
#[derive(Clone, Debug, Deserialize, Serialize, patch_db::HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct DockerContainer {
pub image: ImageId,
#[serde(default)]
@@ -199,7 +201,7 @@ impl DockerProcedure {
image_ids: &BTreeSet<ImageId>,
expected_io: bool,
) -> Result<(), color_eyre::eyre::Report> {
for (volume, _) in &self.mounts {
for volume in self.mounts.keys() {
if !volumes.contains_key(volume) && !matches!(&volume, &VolumeId::Backup) {
color_eyre::eyre::bail!("unknown volume: {}", volume);
}
@@ -229,8 +231,8 @@ impl DockerProcedure {
timeout: Option<Duration>,
) -> Result<Result<O, (i32, String)>, Error> {
let name = name.docker_name();
let name: Option<&str> = name.as_ref().map(|x| &**x);
let mut cmd = tokio::process::Command::new("docker");
let name: Option<&str> = name.as_deref();
let mut cmd = tokio::process::Command::new(CONTAINER_TOOL);
let container_name = Self::container_name(pkg_id, name);
cmd.arg("run")
.arg("--rm")
@@ -241,25 +243,7 @@ impl DockerProcedure {
.arg(format!("--hostname={}", &container_name))
.arg("--no-healthcheck")
.kill_on_drop(true);
match ctx
.docker
.remove_container(
&container_name,
Some(RemoveContainerOptions {
v: false,
force: true,
link: false,
}),
)
.await
{
Ok(())
| Err(bollard::errors::Error::DockerResponseServerError {
status_code: 404, // NOT FOUND
..
}) => Ok(()),
Err(e) => Err(e),
}?;
remove_container(&container_name, true).await?;
cmd.args(self.docker_args(ctx, pkg_id, pkg_version, volumes).await?);
let input_buf = if let (Some(input), Some(format)) = (&input, &self.io_format) {
cmd.stdin(std::process::Stdio::piped());
@@ -402,15 +386,13 @@ impl DockerProcedure {
&self,
_ctx: &RpcContext,
pkg_id: &PackageId,
pkg_version: &Version,
name: ProcedureName,
volumes: &Volumes,
_pkg_version: &Version,
_name: ProcedureName,
_volumes: &Volumes,
input: Option<I>,
timeout: Option<Duration>,
) -> Result<Result<O, (i32, String)>, Error> {
let name = name.docker_name();
let name: Option<&str> = name.as_deref();
let mut cmd = tokio::process::Command::new("docker");
let mut cmd = tokio::process::Command::new(CONTAINER_TOOL);
cmd.arg("exec");
@@ -561,7 +543,7 @@ impl DockerProcedure {
input: Option<I>,
timeout: Option<Duration>,
) -> Result<Result<O, (i32, String)>, Error> {
let mut cmd = tokio::process::Command::new("docker");
let mut cmd = tokio::process::Command::new(CONTAINER_TOOL);
cmd.arg("run").arg("--rm").arg("--network=none");
cmd.args(
self.docker_args(ctx, pkg_id, pkg_version, &volumes.to_readonly())
@@ -642,7 +624,18 @@ impl DockerProcedure {
}
}));
let exit_status = handle.wait().await.with_kind(crate::ErrorKind::Docker)?;
let handle = if let Some(dur) = timeout {
async move {
tokio::time::timeout(dur, handle.wait())
.await
.with_kind(crate::ErrorKind::Docker)?
.with_kind(crate::ErrorKind::Docker)
}
.boxed()
} else {
async { handle.wait().await.with_kind(crate::ErrorKind::Docker) }.boxed()
};
let exit_status = handle.await?;
Ok(
if exit_status.success() || exit_status.code() == Some(143) {
Ok(serde_json::from_value(
@@ -726,7 +719,7 @@ impl DockerProcedure {
if fty.is_block_device() || fty.is_char_device() {
res.push(entry.path());
} else if fty.is_dir() {
get_devices(&*entry.path(), res).await?;
get_devices(&entry.path(), res).await?;
}
}
Ok(())
@@ -745,7 +738,7 @@ impl DockerProcedure {
res.push(OsStr::new("--entrypoint").into());
res.push(OsStr::new(&self.entrypoint).into());
if self.system {
res.push(OsString::from(self.image.for_package(&*SYSTEM_PACKAGE_ID, None)).into());
res.push(OsString::from(self.image.for_package(&SYSTEM_PACKAGE_ID, None)).into());
} else {
res.push(OsString::from(self.image.for_package(pkg_id, Some(pkg_version))).into());
}
@@ -823,17 +816,17 @@ impl LongRunning {
const BIND_LOCATION: &str = "/usr/lib/embassy/container/";
tracing::trace!("setup_long_running_docker_cmd");
LongRunning::cleanup_previous_container(ctx, container_name).await?;
remove_container(container_name, true).await?;
let image_architecture = {
let mut cmd = tokio::process::Command::new("docker");
let mut cmd = tokio::process::Command::new(CONTAINER_TOOL);
cmd.arg("image")
.arg("inspect")
.arg("--format")
.arg("'{{.Architecture}}'");
if docker.system {
cmd.arg(docker.image.for_package(&*SYSTEM_PACKAGE_ID, None));
cmd.arg(docker.image.for_package(&SYSTEM_PACKAGE_ID, None));
} else {
cmd.arg(docker.image.for_package(pkg_id, Some(pkg_version)));
}
@@ -841,7 +834,7 @@ impl LongRunning {
arch.replace('\'', "").trim().to_string()
};
let mut cmd = tokio::process::Command::new("docker");
let mut cmd = tokio::process::Command::new(CONTAINER_TOOL);
cmd.arg("run")
.arg("--network=start9")
.arg(format!("--add-host=embassy:{}", Ipv4Addr::from(HOST_IP)))
@@ -855,7 +848,7 @@ impl LongRunning {
input = socket_path.display()
))
.arg("--name")
.arg(&container_name)
.arg(container_name)
.arg(format!("--hostname={}", &container_name))
.arg("--entrypoint")
.arg(format!("{INIT_EXEC}.{image_architecture}"))
@@ -885,7 +878,7 @@ impl LongRunning {
}
cmd.arg("--log-driver=journald");
if docker.system {
cmd.arg(docker.image.for_package(&*SYSTEM_PACKAGE_ID, None));
cmd.arg(docker.image.for_package(&SYSTEM_PACKAGE_ID, None));
} else {
cmd.arg(docker.image.for_package(pkg_id, Some(pkg_version)));
}
@@ -894,31 +887,6 @@ impl LongRunning {
cmd.stdin(std::process::Stdio::piped());
Ok(cmd)
}
async fn cleanup_previous_container(
ctx: &RpcContext,
container_name: &str,
) -> Result<(), Error> {
match ctx
.docker
.remove_container(
container_name,
Some(RemoveContainerOptions {
v: false,
force: true,
link: false,
}),
)
.await
{
Ok(())
| Err(bollard::errors::Error::DockerResponseServerError {
status_code: 404, // NOT FOUND
..
}) => Ok(()),
Err(e) => Err(e)?,
}
}
}
async fn buf_reader_to_lines(
reader: impl AsyncBufRead + Unpin,

6
backend/src/procedure/mod.rs
View File

@@ -8,8 +8,9 @@ use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use tracing::instrument;
use self::docker::{DockerContainers, DockerProcedure};
use self::docker::DockerProcedure;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::util::Version;
use crate::volume::Volumes;
@@ -25,6 +26,7 @@ pub use models::ProcedureName;
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[serde(tag = "type")]
#[model = "Model<Self>"]
pub enum PackageProcedure {
Docker(DockerProcedure),
@@ -43,7 +45,6 @@ impl PackageProcedure {
#[instrument(skip_all)]
pub fn validate(
&self,
container: &Option<DockerContainers>,
eos_version: &Version,
volumes: &Volumes,
image_ids: &BTreeSet<ImageId>,
@@ -121,7 +122,6 @@ impl PackageProcedure {
#[instrument(skip_all)]
pub async fn sandboxed<I: Serialize, O: DeserializeOwned>(
&self,
container: &Option<DockerContainers>,
ctx: &RpcContext,
pkg_id: &PackageId,
pkg_version: &Version,

21
backend/src/properties.rs
View File

@@ -5,8 +5,9 @@ use serde_json::Value;
use tracing::instrument;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::procedure::ProcedureName;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::s9pk::manifest::PackageId;
use crate::{Error, ErrorKind};
pub fn display_properties(response: Value, _: &ArgMatches) {
@@ -20,17 +21,15 @@ pub async fn properties(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Res
#[instrument(skip_all)]
pub async fn fetch_properties(ctx: RpcContext, id: PackageId) -> Result<Value, Error> {
let mut db = ctx.db.handle();
let peek = ctx.db.peek().await?;
let manifest: Manifest = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&id)
.and_then(|p| p.installed())
.map(|m| m.manifest())
.get(&mut db)
.await?
.to_owned()
.ok_or_else(|| Error::new(eyre!("{} is not installed", id), ErrorKind::NotFound))?;
let manifest = peek
.as_package_data()
.as_idx(&id)
.ok_or_else(|| Error::new(eyre!("{} is not installed", id), ErrorKind::NotFound))?
.expect_as_installed()?
.as_manifest()
.de()?;
if let Some(props) = manifest.properties {
props
.execute::<(), Value>(

211
backend/src/registry/admin.rs Normal file
View File

@@ -0,0 +1,211 @@
use std::path::PathBuf;
use std::time::Duration;
use color_eyre::eyre::eyre;
use console::style;
use futures::StreamExt;
use indicatif::{ProgressBar, ProgressStyle};
use reqwest::{header, Body, Client, Url};
use rpc_toolkit::command;
use crate::s9pk::reader::S9pkReader;
use crate::util::display_none;
use crate::{Error, ErrorKind};
async fn registry_user_pass(location: &str) -> Result<(Url, String, String), Error> {
let mut url = Url::parse(location)?;
let user = url.username().to_string();
let pass = url.password().map(str::to_string);
if user.is_empty() || url.path() != "/" {
return Err(Error::new(
eyre!("{location:?} is not like \"https://user@registry.example.com/\""),
ErrorKind::ParseUrl,
));
}
let _ = url.set_username("");
let _ = url.set_password(None);
let pass = match pass {
Some(p) => p,
None => {
let pass_prompt = format!("{} Password for {user}: ", style("?").yellow());
tokio::task::spawn_blocking(move || rpassword::prompt_password(pass_prompt))
.await
.unwrap()?
}
};
Ok((url, user.to_string(), pass.to_string()))
}
#[derive(serde::Serialize, Debug)]
struct Package {
id: String,
version: String,
arches: Option<Vec<String>>,
}
async fn do_index(
httpc: &Client,
mut url: Url,
user: &str,
pass: &str,
pkg: &Package,
) -> Result<(), Error> {
url.set_path("/admin/v0/index");
let mut req = httpc
.post(url)
.header(header::ACCEPT, "text/plain")
.basic_auth(user, Some(pass))
.json(pkg)
.build()?;
let res = httpc.execute(req).await?;
if !res.status().is_success() {
let info = res.text().await?;
return Err(Error::new(eyre!("{}", info), ErrorKind::Registry));
}
Ok(())
}
async fn do_upload(
httpc: &Client,
mut url: Url,
user: &str,
pass: &str,
body: Body,
) -> Result<(), Error> {
url.set_path("/admin/v0/upload");
let mut req = httpc
.post(url)
.header(header::ACCEPT, "text/plain")
.basic_auth(user, Some(pass))
.body(body)
.build()?;
let res = httpc.execute(req).await?;
if !res.status().is_success() {
let info = res.text().await?;
return Err(Error::new(eyre!("{}", info), ErrorKind::Registry));
}
Ok(())
}
#[command(cli_only, display(display_none))]
pub async fn publish(
#[arg] location: String,
#[arg] path: PathBuf,
#[arg(rename = "no-verify", long = "no-verify")] no_verify: bool,
#[arg(rename = "no-upload", long = "no-upload")] no_upload: bool,
#[arg(rename = "no-index", long = "no-index")] no_index: bool,
) -> Result<(), Error> {
// Prepare for progress bars.
let bytes_bar_style =
ProgressStyle::with_template("{percent}% {wide_bar} [{bytes}/{total_bytes}] [{eta}]")
.unwrap();
let plain_line_style =
ProgressStyle::with_template("{prefix:.bold.dim} {wide_msg}...").unwrap();
let spinner_line_style =
ProgressStyle::with_template("{prefix:.bold.dim} {spinner} {wide_msg}...").unwrap();
// Read the file to get manifest information and check validity..
// Open file right away so it can not change out from under us.
let file = tokio::fs::File::open(&path).await?;
let manifest = if no_verify {
let pb = ProgressBar::new(1)
.with_style(spinner_line_style.clone())
.with_prefix("[1/3]")
.with_message("Querying s9pk");
pb.enable_steady_tick(Duration::from_millis(200));
let mut s9pk = S9pkReader::open(&path, false).await?;
let m = s9pk.manifest().await?.clone();
pb.set_style(plain_line_style.clone());
pb.abandon();
m
} else {
let pb = ProgressBar::new(1)
.with_style(spinner_line_style.clone())
.with_prefix("[1/3]")
.with_message("Verifying s9pk");
pb.enable_steady_tick(Duration::from_millis(200));
let mut s9pk = S9pkReader::open(&path, true).await?;
s9pk.validate().await?;
let m = s9pk.manifest().await?.clone();
pb.set_style(plain_line_style.clone());
pb.abandon();
m
};
let pkg = Package {
id: manifest.id.to_string(),
version: manifest.version.to_string(),
arches: manifest.hardware_requirements.arch.clone(),
};
println!("{} id = {}", style(">").green(), pkg.id);
println!("{} version = {}", style(">").green(), pkg.version);
if let Some(arches) = &pkg.arches {
println!("{} arches = {:?}", style(">").green(), arches);
} else {
println!(
"{} No architecture listed in hardware_requirements",
style(">").red()
);
}
// Process the url and get the user's password.
let (registry, user, pass) = registry_user_pass(&location).await?;
// Now prepare a stream of the file which will show a progress bar as it is consumed.
let file_size = file.metadata().await?.len();
let file_stream = tokio_util::io::ReaderStream::new(file);
ProgressBar::new(0)
.with_style(plain_line_style.clone())
.with_prefix("[2/3]")
.with_message("Uploading s9pk")
.abandon();
let pb = ProgressBar::new(file_size).with_style(bytes_bar_style.clone());
let stream_pb = pb.clone();
let file_stream = file_stream.inspect(move |bytes| {
if let Ok(bytes) = bytes {
stream_pb.inc(bytes.len() as u64);
}
});
let httpc = Client::builder().build().unwrap();
// And upload!
if no_upload {
println!("{} Skipping upload", style(">").yellow());
} else {
do_upload(
&httpc,
registry.clone(),
&user,
&pass,
Body::wrap_stream(file_stream),
)
.await?;
}
pb.finish_and_clear();
// Also index, so it will show up in the registry.
let pb = ProgressBar::new(0)
.with_style(spinner_line_style.clone())
.with_prefix("[3/3]")
.with_message("Indexing registry");
pb.enable_steady_tick(Duration::from_millis(200));
if no_index {
println!("{} Skipping index", style(">").yellow());
} else {
do_index(&httpc, registry.clone(), &user, &pass, &pkg).await?;
}
pb.set_style(plain_line_style.clone());
pb.abandon();
// All done
if !no_index {
println!(
"{} Package {} is now published to {}",
style(">").green(),
pkg.id,
registry
);
}
Ok(())
}

4
backend/src/marketplace.rs → backend/src/registry/marketplace.rs
View File

@@ -12,7 +12,7 @@ pub fn marketplace() -> Result<(), Error> {
Ok(())
}
pub fn with_query_params(ctx: &RpcContext, mut url: Url) -> Url {
pub fn with_query_params(ctx: RpcContext, mut url: Url) -> Url {
url.query_pairs_mut()
.append_pair(
"os.version",
@@ -38,7 +38,7 @@ pub fn with_query_params(ctx: &RpcContext, mut url: Url) -> Url {
pub async fn get(#[context] ctx: RpcContext, #[arg] url: Url) -> Result<Value, Error> {
let mut response = ctx
.client
.get(with_query_params(&ctx, url))
.get(with_query_params(ctx.clone(), url))
.send()
.await
.with_kind(crate::ErrorKind::Network)?;

2
backend/src/registry/mod.rs Normal file
View File

@@ -0,0 +1,2 @@
pub mod admin;
pub mod marketplace;

16
backend/src/s9pk/manifest.rs
View File

@@ -2,8 +2,7 @@ use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use color_eyre::eyre::eyre;
pub use models::{PackageId, SYSTEM_PACKAGE_ID};
use patch_db::HasModel;
pub use models::PackageId;
use serde::{Deserialize, Serialize};
use url::Url;
@@ -14,6 +13,7 @@ use crate::config::action::ConfigActions;
use crate::dependencies::Dependencies;
use crate::migration::Migrations;
use crate::net::interface::Interfaces;
use crate::prelude::*;
use crate::procedure::docker::DockerContainers;
use crate::procedure::PackageProcedure;
use crate::status::health_check::HealthChecks;
@@ -29,6 +29,7 @@ fn current_version() -> Version {
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct Manifest {
#[serde(default = "current_version")]
pub eos_version: Version,
@@ -36,7 +37,6 @@ pub struct Manifest {
#[serde(default)]
pub git_hash: Option<GitHash>,
pub title: String,
#[model]
pub version: Version,
pub description: Description,
#[serde(default)]
@@ -52,31 +52,23 @@ pub struct Manifest {
pub donation_url: Option<Url>,
#[serde(default)]
pub alerts: Alerts,
#[model]
pub main: PackageProcedure,
pub health_checks: HealthChecks,
#[model]
pub config: Option<ConfigActions>,
#[model]
pub properties: Option<PackageProcedure>,
#[model]
pub volumes: Volumes,
// #[serde(default)]
pub interfaces: Interfaces,
// #[serde(default)]
#[model]
pub backup: BackupActions,
#[serde(default)]
#[model]
pub migrations: Migrations,
#[serde(default)]
pub actions: Actions,
// #[serde(default)]
// pub permissions: Permissions,
#[serde(default)]
#[model]
pub dependencies: Dependencies,
#[model]
pub containers: Option<DockerContainers>,
#[serde(default)]
@@ -120,7 +112,7 @@ pub struct HardwareRequirements {
#[serde(default)]
device: BTreeMap<String, Regex>,
ram: Option<u64>,
arch: Option<Vec<String>>,
pub arch: Option<Vec<String>>,
}
#[derive(Clone, Debug, Default, Deserialize, Serialize)]

2
backend/src/s9pk/mod.rs
View File

@@ -28,7 +28,7 @@ pub mod header;
pub mod manifest;
pub mod reader;
pub const SIG_CONTEXT: &'static [u8] = b"s9pk";
pub const SIG_CONTEXT: &[u8] = b"s9pk";
#[command(cli_only, display(display_none))]
#[instrument(skip_all)]

66
backend/src/s9pk/reader.rs
View File

@@ -185,18 +185,14 @@ impl<R: AsyncRead + AsyncSeek + Unpin + Send + Sync> S9pkReader<R> {
.map(|i| i.validate(&man.id, &man.version).map(|_| i.image_id))
.collect::<Result<BTreeSet<ImageId>, _>>()?;
man.description.validate()?;
man.actions
.0
.iter()
.map(|(_, action)| {
action.validate(
containers,
&man.eos_version,
&man.volumes,
&validated_image_ids,
)
})
.collect::<Result<(), Error>>()?;
man.actions.0.iter().try_for_each(|(_, action)| {
action.validate(
containers,
&man.eos_version,
&man.volumes,
&validated_image_ids,
)
})?;
man.backup.validate(
containers,
&man.eos_version,
@@ -211,21 +207,11 @@ impl<R: AsyncRead + AsyncSeek + Unpin + Send + Sync> S9pkReader<R> {
&validated_image_ids,
)?;
}
man.health_checks.validate(
containers,
&man.eos_version,
&man.volumes,
&validated_image_ids,
)?;
man.health_checks
.validate(&man.eos_version, &man.volumes, &validated_image_ids)?;
man.interfaces.validate()?;
man.main
.validate(
containers,
&man.eos_version,
&man.volumes,
&validated_image_ids,
false,
)
.validate(&man.eos_version, &man.volumes, &validated_image_ids, false)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Main"))?;
man.migrations.validate(
containers,
@@ -263,13 +249,7 @@ impl<R: AsyncRead + AsyncSeek + Unpin + Send + Sync> S9pkReader<R> {
}
if let Some(props) = &man.properties {
props
.validate(
containers,
&man.eos_version,
&man.volumes,
&validated_image_ids,
true,
)
.validate(&man.eos_version, &man.volumes, &validated_image_ids, true)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Properties"))?;
}
man.volumes.validate(&man.interfaces)?;
@@ -377,7 +357,7 @@ impl<R: AsyncRead + AsyncSeek + Unpin + Send + Sync> S9pkReader<R> {
})
}
pub async fn manifest_raw<'a>(&'a mut self) -> Result<ReadHandle<'a, R>, Error> {
pub async fn manifest_raw(&mut self) -> Result<ReadHandle<'_, R>, Error> {
self.read_handle(self.toc.manifest).await
}
@@ -387,27 +367,27 @@ impl<R: AsyncRead + AsyncSeek + Unpin + Send + Sync> S9pkReader<R> {
.with_ctx(|_| (crate::ErrorKind::ParseS9pk, "Deserializing Manifest (CBOR)"))
}
pub async fn license<'a>(&'a mut self) -> Result<ReadHandle<'a, R>, Error> {
Ok(self.read_handle(self.toc.license).await?)
pub async fn license(&mut self) -> Result<ReadHandle<'_, R>, Error> {
self.read_handle(self.toc.license).await
}
pub async fn instructions<'a>(&'a mut self) -> Result<ReadHandle<'a, R>, Error> {
Ok(self.read_handle(self.toc.instructions).await?)
pub async fn instructions(&mut self) -> Result<ReadHandle<'_, R>, Error> {
self.read_handle(self.toc.instructions).await
}
pub async fn icon<'a>(&'a mut self) -> Result<ReadHandle<'a, R>, Error> {
Ok(self.read_handle(self.toc.icon).await?)
pub async fn icon(&mut self) -> Result<ReadHandle<'_, R>, Error> {
self.read_handle(self.toc.icon).await
}
pub async fn docker_images<'a>(&'a mut self) -> Result<DockerReader<ReadHandle<'a, R>>, Error> {
pub async fn docker_images(&mut self) -> Result<DockerReader<ReadHandle<'_, R>>, Error> {
DockerReader::new(self.read_handle(self.toc.docker_images).await?).await
}
pub async fn assets<'a>(&'a mut self) -> Result<ReadHandle<'a, R>, Error> {
Ok(self.read_handle(self.toc.assets).await?)
pub async fn assets(&mut self) -> Result<ReadHandle<'_, R>, Error> {
self.read_handle(self.toc.assets).await
}
pub async fn scripts<'a>(&'a mut self) -> Result<Option<ReadHandle<'a, R>>, Error> {
pub async fn scripts(&mut self) -> Result<Option<ReadHandle<'_, R>>, Error> {
Ok(match self.toc.scripts {
None => None,
Some(a) => Some(self.read_handle(a).await?),

28
backend/src/s9pk/specv2.md Normal file
View File

@@ -0,0 +1,28 @@
## Header
### Magic
2B: `0x3b3b`
### Version
varint: `0x02`
### Pubkey
32B: ed25519 pubkey
### TOC
- number of sections (varint)
- FOREACH section
- sig (32B: ed25519 signature of BLAKE-3 of rest of section)
- name (varstring)
- TYPE (varint)
- TYPE=FILE (`0x01`)
- mime (varstring)
- pos (32B: u64 BE)
- len (32B: u64 BE)
- hash (32B: BLAKE-3 of file contents)
- TYPE=TOC (`0x02`)
- recursively defined

93
backend/src/setup.rs
View File

@@ -3,10 +3,8 @@ use std::sync::Arc;
use std::time::Duration;
use color_eyre::eyre::eyre;
use futures::StreamExt;
use josekit::jwk::Jwk;
use openssl::x509::X509;
use patch_db::DbHandle;
use rpc_toolkit::command;
use rpc_toolkit::yajrc::RpcError;
use serde::{Deserialize, Serialize};
@@ -33,6 +31,7 @@ use crate::disk::REPAIR_DISK_PATH;
use crate::hostname::Hostname;
use crate::init::{init, InitResult};
use crate::middleware::encrypt::EncryptedWire;
use crate::prelude::*;
use crate::util::io::{dir_copy, dir_size, Counter};
use crate::{Error, ErrorKind, ResultExt};
@@ -58,23 +57,21 @@ async fn setup_init(
let InitResult { secret_store, db } =
init(&RpcContextConfig::load(ctx.config_path.clone()).await?).await?;
let mut secrets_handle = secret_store.acquire().await?;
let mut db_handle = db.handle();
let mut secrets_tx = secrets_handle.begin().await?;
let mut db_tx = db_handle.begin().await?;
let mut account = AccountInfo::load(&mut secrets_tx).await?;
if let Some(password) = password {
account.set_password(&password)?;
account.save(&mut secrets_tx).await?;
crate::db::DatabaseModel::new()
.server_info()
.password_hash()
.put(&mut db_tx, &account.password)
.await?;
db.mutate(|m| {
m.as_server_info_mut()
.as_password_hash_mut()
.ser(&account.password)
})
.await?;
}
db_tx.commit().await?;
secrets_tx.commit().await?;
Ok((
@@ -266,39 +263,47 @@ pub async fn execute(
complete: false,
}));
drop(status);
tokio::task::spawn(async move {
match execute_inner(
ctx.clone(),
embassy_logicalname,
embassy_password,
recovery_source,
recovery_password,
)
.await
{
Ok((guid, hostname, tor_addr, root_ca)) => {
tracing::info!("Setup Complete!");
*ctx.setup_result.write().await = Some((
guid,
SetupResult {
tor_address: format!("https://{}", tor_addr),
lan_address: hostname.lan_address(),
root_ca: String::from_utf8(
root_ca.to_pem().expect("failed to serialize root ca"),
)
.expect("invalid pem string"),
},
));
*ctx.setup_status.write().await = Some(Ok(SetupStatus {
bytes_transferred: 0,
total_bytes: None,
complete: true,
}));
}
Err(e) => {
tracing::error!("Error Setting Up Server: {}", e);
tracing::debug!("{:?}", e);
*ctx.setup_status.write().await = Some(Err(e.into()));
tokio::task::spawn({
async move {
let ctx = ctx.clone();
let recovery_source = recovery_source;
let embassy_password = embassy_password;
let recovery_source = recovery_source;
let recovery_password = recovery_password;
match execute_inner(
ctx.clone(),
embassy_logicalname,
embassy_password,
recovery_source,
recovery_password,
)
.await
{
Ok((guid, hostname, tor_addr, root_ca)) => {
tracing::info!("Setup Complete!");
*ctx.setup_result.write().await = Some((
guid,
SetupResult {
tor_address: format!("https://{}", tor_addr),
lan_address: hostname.lan_address(),
root_ca: String::from_utf8(
root_ca.to_pem().expect("failed to serialize root ca"),
)
.expect("invalid pem string"),
},
));
*ctx.setup_status.write().await = Some(Ok(SetupStatus {
bytes_transferred: 0,
total_bytes: None,
complete: true,
}));
}
Err(e) => {
tracing::error!("Error Setting Up Server: {}", e);
tracing::debug!("{:?}", e);
*ctx.setup_status.write().await = Some(Err(e.into()));
}
}
}
});
@@ -397,7 +402,7 @@ async fn recover(
) -> Result<(Arc<String>, Hostname, OnionAddressV3, X509), Error> {
let recovery_source = TmpMountGuard::mount(&recovery_source, ReadWrite).await?;
recover_full_embassy(
ctx.clone(),
ctx,
guid.clone(),
embassy_password,
recovery_source,

41
backend/src/shutdown.rs
View File

@@ -7,8 +7,9 @@ use crate::context::RpcContext;
use crate::disk::main::export;
use crate::init::{STANDBY_MODE_PATH, SYSTEM_REBUILD_PATH};
use crate::sound::SHUTDOWN;
use crate::util::docker::CONTAINER_TOOL;
use crate::util::{display_none, Invoke};
use crate::{Error, ErrorKind, OS_ARCH};
use crate::{Error, OS_ARCH};
#[derive(Debug, Clone)]
pub struct Shutdown {
@@ -43,14 +44,16 @@ impl Shutdown {
tracing::error!("Error Stopping Journald: {}", e);
tracing::debug!("{:?}", e);
}
if let Err(e) = Command::new("systemctl")
.arg("stop")
.arg("docker")
.invoke(crate::ErrorKind::Docker)
.await
{
tracing::error!("Error Stopping Docker: {}", e);
tracing::debug!("{:?}", e);
if CONTAINER_TOOL == "docker" {
if let Err(e) = Command::new("systemctl")
.arg("stop")
.arg("docker")
.invoke(crate::ErrorKind::Docker)
.await
{
tracing::error!("Error Stopping Docker: {}", e);
tracing::debug!("{:?}", e);
}
}
if let Some(guid) = &self.disk_guid {
if let Err(e) = export(guid, &self.datadir).await {
@@ -72,18 +75,16 @@ impl Shutdown {
Command::new("sync").spawn().unwrap().wait().unwrap();
}
Command::new("reboot").spawn().unwrap().wait().unwrap();
} else if self.restart {
Command::new("reboot").spawn().unwrap().wait().unwrap();
} else {
if self.restart {
Command::new("reboot").spawn().unwrap().wait().unwrap();
} else {
Command::new("shutdown")
.arg("-h")
.arg("now")
.spawn()
.unwrap()
.wait()
.unwrap();
}
Command::new("shutdown")
.arg("-h")
.arg("now")
.spawn()
.unwrap()
.wait()
.unwrap();
}
}
}

2
backend/src/sound.rs
View File

@@ -15,7 +15,7 @@ lazy_static::lazy_static! {
static ref C_0: f64 = *A_4 / SEMITONE_K.powf(9f64) / 2f64.powf(4f64);
}
pub const SOUND_LOCK_FILE: &'static str = "/etc/embassy/sound.lock";
pub const SOUND_LOCK_FILE: &str = "/etc/embassy/sound.lock";
struct SoundInterface {
guard: Option<FileLock>,

2
backend/src/ssh.rs
View File

@@ -4,7 +4,7 @@ use chrono::Utc;
use clap::ArgMatches;
use color_eyre::eyre::eyre;
use rpc_toolkit::command;
use sqlx::{Executor, Pool, Postgres};
use sqlx::{Pool, Postgres};
use tracing::instrument;
use crate::context::RpcContext;

12
backend/src/status/health_check.rs
View File

@@ -7,7 +7,6 @@ use serde::{Deserialize, Serialize};
use tracing::instrument;
use crate::context::RpcContext;
use crate::procedure::docker::DockerContainers;
use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
use crate::util::serde::Duration;
@@ -21,15 +20,14 @@ impl HealthChecks {
#[instrument(skip_all)]
pub fn validate(
&self,
container: &Option<DockerContainers>,
eos_version: &Version,
volumes: &Volumes,
image_ids: &BTreeSet<ImageId>,
) -> Result<(), Error> {
for (_, check) in &self.0 {
for check in self.0.values() {
check
.implementation
.validate(container, eos_version, &volumes, image_ids, false)
.validate(eos_version, volumes, image_ids, false)
.with_ctx(|_| {
(
crate::ErrorKind::ValidateS9pk,
@@ -42,7 +40,6 @@ impl HealthChecks {
pub async fn check_all(
&self,
ctx: &RpcContext,
container: &Option<DockerContainers>,
started: DateTime<Utc>,
pkg_id: &PackageId,
pkg_version: &Version,
@@ -52,7 +49,7 @@ impl HealthChecks {
Ok::<_, Error>((
id.clone(),
check
.check(ctx, container, id, started, pkg_id, pkg_version, volumes)
.check(ctx, id, started, pkg_id, pkg_version, volumes)
.await?,
))
}))
@@ -75,7 +72,6 @@ impl HealthCheck {
pub async fn check(
&self,
ctx: &RpcContext,
container: &Option<DockerContainers>,
id: &HealthCheckId,
started: DateTime<Utc>,
pkg_id: &PackageId,
@@ -107,7 +103,7 @@ impl HealthCheck {
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
#[serde(tag = "result")]
pub enum HealthCheckResult {

29
backend/src/status/mod.rs
View File

@@ -1,25 +1,36 @@
use std::collections::BTreeMap;
use chrono::{DateTime, Utc};
use patch_db::{HasModel, Model};
use models::PackageId;
use serde::{Deserialize, Serialize};
use self::health_check::HealthCheckId;
use crate::dependencies::DependencyErrors;
use crate::prelude::*;
use crate::status::health_check::HealthCheckResult;
pub mod health_check;
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct Status {
pub configured: bool,
#[model]
pub main: MainStatus,
#[model]
pub dependency_errors: DependencyErrors,
#[serde(default)]
pub dependency_errors: BTreeMap<(), ()>, // TODO: remove
#[serde(default)]
pub dependency_config_errors: DependencyConfigErrors,
}
#[derive(Debug, Clone, Deserialize, Serialize, HasModel)]
#[derive(Clone, Debug, Deserialize, Serialize, HasModel, Default)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct DependencyConfigErrors(pub BTreeMap<PackageId, String>);
impl Map for DependencyConfigErrors {
type Key = PackageId;
type Value = String;
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
#[serde(tag = "status")]
#[serde(rename_all = "kebab-case")]
pub enum MainStatus {
@@ -71,7 +82,6 @@ impl MainStatus {
MainStatus::Starting { .. } => None,
}
}
pub fn backing_up(&self) -> Self {
let (started, health) = match self {
MainStatus::Starting { .. } => (Some(Utc::now()), Default::default()),
@@ -84,8 +94,3 @@ impl MainStatus {
MainStatus::BackingUp { started, health }
}
}
impl MainStatusModel {
pub fn started(self) -> Model<Option<DateTime<Utc>>> {
self.0.child("started")
}
}

20
backend/src/system.rs
View File

@@ -17,6 +17,7 @@ use crate::logs::{
cli_logs_generic_follow, cli_logs_generic_nofollow, fetch_logs, follow_logs, LogFollowResponse,
LogResponse, LogSource,
};
use crate::prelude::*;
use crate::shutdown::Shutdown;
use crate::util::serde::{display_serializable, IoFormat};
use crate::util::{display_none, Invoke};
@@ -59,16 +60,12 @@ pub async fn enable_zram() -> Result<(), Error> {
#[command(display(display_none))]
pub async fn zram(#[context] ctx: RpcContext, #[arg] enable: bool) -> Result<(), Error> {
let mut db = ctx.db.handle();
let mut zram = crate::db::DatabaseModel::new()
.server_info()
.zram()
.get_mut(&mut db)
.await?;
if enable == *zram {
let db = ctx.db.peek().await?;
let zram = db.as_server_info().as_zram().de()?;
if enable == zram {
return Ok(());
}
*zram = enable;
if enable {
enable_zram().await?;
} else {
@@ -80,7 +77,12 @@ pub async fn zram(#[context] ctx: RpcContext, #[arg] enable: bool) -> Result<(),
.await
.with_kind(ErrorKind::Zram)?;
}
zram.save(&mut db).await?;
ctx.db
.mutate(|v| {
v.as_server_info_mut().as_zram_mut().ser(&enable)?;
Ok(())
})
.await?;
Ok(())
}

123
backend/src/update/mod.rs
View File

@@ -1,13 +1,11 @@
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use clap::ArgMatches;
use color_eyre::eyre::{eyre, Result};
use emver::Version;
use helpers::{Rsync, RsyncOptions};
use lazy_static::lazy_static;
use patch_db::{DbHandle, LockType, Revision};
use reqwest::Url;
use rpc_toolkit::command;
use tokio::process::Command;
@@ -19,14 +17,14 @@ use crate::db::model::UpdateProgress;
use crate::disk::mount::filesystem::bind::Bind;
use crate::disk::mount::filesystem::ReadWrite;
use crate::disk::mount::guard::MountGuard;
use crate::marketplace::with_query_params;
use crate::notifications::NotificationLevel;
use crate::prelude::*;
use crate::registry::marketplace::with_query_params;
use crate::sound::{
CIRCLE_OF_5THS_SHORT, UPDATE_FAILED_1, UPDATE_FAILED_2, UPDATE_FAILED_3, UPDATE_FAILED_4,
};
use crate::update::latest_information::LatestInformation;
use crate::util::Invoke;
use crate::version::{Current, VersionT};
use crate::{Error, ErrorKind, ResultExt, OS_ARCH};
mod latest_information;
@@ -77,15 +75,12 @@ fn display_update_result(status: UpdateResult, _: &ArgMatches) {
}
#[instrument(skip_all)]
async fn maybe_do_update(
ctx: RpcContext,
marketplace_url: Url,
) -> Result<Option<Arc<Revision>>, Error> {
let mut db = ctx.db.handle();
async fn maybe_do_update(ctx: RpcContext, marketplace_url: Url) -> Result<Option<()>, Error> {
let peeked = ctx.db.peek().await?;
let latest_version: Version = ctx
.client
.get(with_query_params(
&ctx,
ctx.clone(),
format!("{}/eos/v0/latest", marketplace_url,).parse()?,
))
.send()
@@ -95,31 +90,8 @@ async fn maybe_do_update(
.await
.with_kind(ErrorKind::Network)?
.version;
crate::db::DatabaseModel::new()
.server_info()
.lock(&mut db, LockType::Write)
.await?;
let current_version = crate::db::DatabaseModel::new()
.server_info()
.version()
.get_mut(&mut db)
.await?;
if &latest_version < &current_version {
return Ok(None);
}
let mut tx = db.begin().await?;
let mut status = crate::db::DatabaseModel::new()
.server_info()
.status_info()
.get_mut(&mut tx)
.await?;
if status.update_progress.is_some() {
return Err(Error::new(
eyre!("Server is already updating!"),
crate::ErrorKind::InvalidRequest,
));
}
if status.updated {
let current_version = peeked.as_server_info().as_version().de()?;
if latest_version < *current_version {
return Ok(None);
}
@@ -127,38 +99,59 @@ async fn maybe_do_update(
base: marketplace_url,
version: latest_version,
};
let status = ctx
.db
.mutate(|db| {
let mut status = peeked.as_server_info().as_status_info().de()?;
if status.update_progress.is_some() {
return Err(Error::new(
eyre!("Server is already updating!"),
crate::ErrorKind::InvalidRequest,
));
}
status.update_progress = Some(UpdateProgress {
size: None,
downloaded: 0,
});
status.save(&mut tx).await?;
let rev = tx.commit().await?;
status.update_progress = Some(UpdateProgress {
size: None,
downloaded: 0,
});
db.as_server_info_mut().as_status_info_mut().ser(&status)?;
Ok(status)
})
.await?;
if status.updated {
return Ok(None);
}
tokio::spawn(async move {
let res = do_update(ctx.clone(), eos_url).await;
let mut db = ctx.db.handle();
let mut status = crate::db::DatabaseModel::new()
.server_info()
.status_info()
.get_mut(&mut db)
.await
.expect("could not access status");
status.update_progress = None;
ctx.db
.mutate(|db| {
db.as_server_info_mut()
.as_status_info_mut()
.as_update_progress_mut()
.ser(&None)
})
.await?;
match res {
Ok(()) => {
status.updated = true;
status.save(&mut db).await.expect("could not save status");
ctx.db
.mutate(|db| {
db.as_server_info_mut()
.as_status_info_mut()
.as_updated_mut()
.ser(&true)
})
.await?;
CIRCLE_OF_5THS_SHORT
.play()
.await
.expect("could not play sound");
}
Err(e) => {
status.save(&mut db).await.expect("could not save status");
ctx.notification_manager
.notify(
&mut db,
ctx.db.clone(),
None,
NotificationLevel::Error,
"embassyOS Update Failed".to_owned(),
@@ -187,8 +180,9 @@ async fn maybe_do_update(
.expect("could not play song: update failed 4");
}
}
Ok::<(), Error>(())
});
Ok(rev)
Ok(Some(()))
}
#[instrument(skip_all)]
@@ -200,17 +194,16 @@ async fn do_update(ctx: RpcContext, eos_url: EosUrl) -> Result<(), Error> {
)
.await?;
while let Some(progress) = rsync.progress.next().await {
crate::db::DatabaseModel::new()
.server_info()
.status_info()
.update_progress()
.put(
&mut ctx.db.handle(),
&UpdateProgress {
size: Some(100),
downloaded: (100.0 * progress) as u64,
},
)
ctx.db
.mutate(|db| {
db.as_server_info_mut()
.as_status_info_mut()
.as_update_progress_mut()
.ser(&Some(UpdateProgress {
size: Some(100),
downloaded: (100.0 * progress) as u64,
}))
})
.await?;
}
rsync.wait().await?;

7
backend/src/util/config.rs
View File

@@ -1,11 +1,12 @@
use std::fs::File;
use std::path::{Path, PathBuf};
use patch_db::Value;
use serde::Deserialize;
use serde_json::Value;
use crate::prelude::*;
use crate::util::serde::IoFormat;
use crate::{Config, Error, ResultExt};
use crate::{Config, Error};
pub const DEVICE_CONFIG_PATH: &str = "/media/embassy/config/config.yaml";
pub const CONFIG_PATH: &str = "/etc/embassy/config.yaml";
@@ -37,7 +38,7 @@ pub fn load_config_from_paths<'a, T: for<'de> Deserialize<'de>>(
config = merge_configs(config, new);
}
}
serde_json::from_value(Value::Object(config)).with_kind(crate::ErrorKind::Deserialization)
from_value(Value::Object(config))
}
pub fn merge_configs(mut first: Config, second: Config) -> Config {

239
backend/src/util/docker.rs Normal file
View File

@@ -0,0 +1,239 @@
use std::net::Ipv4Addr;
use std::time::Duration;
use models::{Error, ErrorKind, PackageId, ResultExt, Version};
use nix::sys::signal::Signal;
use tokio::process::Command;
use crate::util::Invoke;
#[cfg(not(feature = "podman"))]
pub const CONTAINER_TOOL: &str = "docker";
#[cfg(feature = "podman")]
pub const CONTAINER_TOOL: &str = "podman";
#[cfg(not(feature = "podman"))]
pub const CONTAINER_DATADIR: &str = "/var/lib/docker";
#[cfg(feature = "podman")]
pub const CONTAINER_DATADIR: &str = "/var/lib/containers";
pub struct DockerImageSha(String);
// docker images start9/${package}/*:${version} -q --no-trunc
pub async fn images_for(
package: &PackageId,
version: &Version,
) -> Result<Vec<DockerImageSha>, Error> {
Ok(String::from_utf8(
Command::new(CONTAINER_TOOL)
.arg("images")
.arg(format!("start9/{package}/*:{version}"))
.arg("--no-trunc")
.arg("-q")
.invoke(ErrorKind::Docker)
.await?,
)?
.lines()
.map(|l| DockerImageSha(l.trim().to_owned()))
.collect())
}
// docker rmi -f ${sha}
pub async fn remove_image(sha: &DockerImageSha) -> Result<(), Error> {
match Command::new(CONTAINER_TOOL)
.arg("rmi")
.arg("-f")
.arg(&sha.0)
.invoke(ErrorKind::Docker)
.await
.map(|_| ())
{
Err(e)
if e.source
.to_string()
.to_ascii_lowercase()
.contains("no such image") =>
{
Ok(())
}
a => a,
}?;
Ok(())
}
// docker image prune -f
pub async fn prune_images() -> Result<(), Error> {
Command::new(CONTAINER_TOOL)
.arg("image")
.arg("prune")
.arg("-f")
.invoke(ErrorKind::Docker)
.await?;
Ok(())
}
// docker container inspect ${name} --format '{{.NetworkSettings.Networks.start9.IPAddress}}'
pub async fn get_container_ip(name: &str) -> Result<Option<Ipv4Addr>, Error> {
match Command::new(CONTAINER_TOOL)
.arg("container")
.arg("inspect")
.arg(name)
.arg("--format")
.arg("{{.NetworkSettings.Networks.start9.IPAddress}}")
.invoke(ErrorKind::Docker)
.await
{
Err(e)
if e.source
.to_string()
.to_ascii_lowercase()
.contains("no such container") =>
{
Ok(None)
}
Err(e) => Err(e),
Ok(a) => {
let out = std::str::from_utf8(&a)?.trim();
if out.is_empty() {
Ok(None)
} else {
Ok(Some({
out.parse()
.with_ctx(|_| (ErrorKind::ParseNetAddress, out.to_string()))?
}))
}
}
}
}
// docker stop -t ${timeout} -s ${signal} ${name}
pub async fn stop_container(
name: &str,
timeout: Option<Duration>,
signal: Option<Signal>,
) -> Result<(), Error> {
let mut cmd = Command::new(CONTAINER_TOOL);
cmd.arg("stop");
if let Some(dur) = timeout {
cmd.arg("-t").arg(dur.as_secs().to_string());
}
if let Some(sig) = signal {
cmd.arg("-s").arg(sig.to_string());
}
cmd.arg(name);
match cmd.invoke(ErrorKind::Docker).await {
Ok(_) => Ok(()),
Err(mut e)
if e.source
.to_string()
.to_ascii_lowercase()
.contains("no such container") =>
{
e.kind = ErrorKind::NotFound;
Err(e)
}
Err(e) => Err(e),
}
}
// docker kill -s ${signal} ${name}
pub async fn kill_container(name: &str, signal: Option<Signal>) -> Result<(), Error> {
let mut cmd = Command::new(CONTAINER_TOOL);
cmd.arg("kill");
if let Some(sig) = signal {
cmd.arg("-s").arg(sig.to_string());
}
cmd.arg(name);
match cmd.invoke(ErrorKind::Docker).await {
Ok(_) => Ok(()),
Err(mut e)
if e.source
.to_string()
.to_ascii_lowercase()
.contains("no such container") =>
{
e.kind = ErrorKind::NotFound;
Err(e)
}
Err(e) => Err(e),
}
}
// docker pause ${name}
pub async fn pause_container(name: &str) -> Result<(), Error> {
let mut cmd = Command::new(CONTAINER_TOOL);
cmd.arg("pause");
cmd.arg(name);
match cmd.invoke(ErrorKind::Docker).await {
Ok(_) => Ok(()),
Err(mut e)
if e.source
.to_string()
.to_ascii_lowercase()
.contains("no such container") =>
{
e.kind = ErrorKind::NotFound;
Err(e)
}
Err(e) => Err(e),
}
}
// docker unpause ${name}
pub async fn unpause_container(name: &str) -> Result<(), Error> {
let mut cmd = Command::new(CONTAINER_TOOL);
cmd.arg("unpause");
cmd.arg(name);
match cmd.invoke(ErrorKind::Docker).await {
Ok(_) => Ok(()),
Err(mut e)
if e.source
.to_string()
.to_ascii_lowercase()
.contains("no such container") =>
{
e.kind = ErrorKind::NotFound;
Err(e)
}
Err(e) => Err(e),
}
}
// docker rm -f ${name}
pub async fn remove_container(name: &str, force: bool) -> Result<(), Error> {
let mut cmd = Command::new(CONTAINER_TOOL);
cmd.arg("rm");
if force {
cmd.arg("-f");
}
cmd.arg(name);
match cmd.invoke(ErrorKind::Docker).await {
Ok(_) => Ok(()),
Err(e)
if e.source
.to_string()
.to_ascii_lowercase()
.contains("no such container") =>
{
Ok(())
}
Err(e) => Err(e),
}
}
// docker network create -d bridge --subnet ${subnet} --opt com.podman.network.bridge.name=${bridge_name}
pub async fn create_bridge_network(
name: &str,
subnet: &str,
bridge_name: &str,
) -> Result<(), Error> {
let mut cmd = Command::new(CONTAINER_TOOL);
cmd.arg("network").arg("create");
cmd.arg("-d").arg("bridge");
cmd.arg("--subnet").arg(subnet);
cmd.arg("--opt")
.arg(format!("com.docker.network.bridge.name={bridge_name}"));
cmd.arg(name);
cmd.invoke(ErrorKind::Docker).await?;
Ok(())
}

2
backend/src/util/http_reader.rs
View File

@@ -376,5 +376,5 @@ async fn s9pk_test() {
.unwrap();
let manifest = s9pk.manifest().await.unwrap();
assert_eq!(&**manifest.id, "ghost");
assert_eq!(&manifest.id.to_string(), "ghost");
}

7
backend/src/util/io.rs
View File

@@ -522,7 +522,6 @@ pub fn dir_copy<'a, P0: AsRef<Path> + 'a + Send + Sync, P1: AsRef<Path> + 'a + S
let src_path = e.path();
let dst_path = dst_path.join(e.file_name());
if m.is_file() {
let len = m.len();
let mut dst_file = tokio::fs::File::create(&dst_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
@@ -638,7 +637,7 @@ impl<S: AsyncRead + AsyncWrite> AsyncWrite for TimeoutStream<S> {
cx: &mut std::task::Context<'_>,
buf: &[u8],
) -> std::task::Poll<Result<usize, std::io::Error>> {
let mut this = self.project();
let this = self.project();
let res = this.stream.poll_write(cx, buf);
if res.is_ready() {
this.sleep.reset(Instant::now() + *this.timeout);
@@ -649,7 +648,7 @@ impl<S: AsyncRead + AsyncWrite> AsyncWrite for TimeoutStream<S> {
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Result<(), std::io::Error>> {
let mut this = self.project();
let this = self.project();
let res = this.stream.poll_flush(cx);
if res.is_ready() {
this.sleep.reset(Instant::now() + *this.timeout);
@@ -660,7 +659,7 @@ impl<S: AsyncRead + AsyncWrite> AsyncWrite for TimeoutStream<S> {
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Result<(), std::io::Error>> {
let mut this = self.project();
let this = self.project();
let res = this.stream.poll_shutdown(cx);
if res.is_ready() {
this.sleep.reset(Instant::now() + *this.timeout);

18
backend/src/util/lshw.rs
View File

@@ -44,6 +44,20 @@ pub async fn lshw() -> Result<Vec<LshwDevice>, Error> {
for class in KNOWN_CLASSES {
cmd.arg("-class").arg(*class);
}
serde_json::from_slice(&cmd.invoke(crate::ErrorKind::Lshw).await?)
.with_kind(crate::ErrorKind::Deserialization)
Ok(
serde_json::from_slice::<Vec<serde_json::Value>>(
&cmd.invoke(crate::ErrorKind::Lshw).await?,
)
.with_kind(crate::ErrorKind::Deserialization)?
.into_iter()
.filter_map(|v| match serde_json::from_value(v) {
Ok(a) => Some(a),
Err(e) => {
tracing::error!("Failed to parse lshw output: {e}");
tracing::debug!("{e:?}");
None
}
})
.collect(),
)
}

57
backend/src/util/mod.rs
View File

@@ -24,13 +24,14 @@ use tracing::instrument;
use crate::shutdown::Shutdown;
use crate::{Error, ErrorKind, ResultExt as _};
pub mod config;
pub mod docker;
pub mod http_reader;
pub mod io;
pub mod logger;
pub mod lshw;
pub mod serde;
#[derive(Clone, Copy, Debug)]
#[derive(Clone, Copy, Debug, ::serde::Deserialize, ::serde::Serialize)]
pub enum Never {}
impl Never {}
impl Never {
@@ -170,9 +171,7 @@ impl<W: std::fmt::Write> std::io::Write for FmtWriter<W> {
}
}
pub fn display_none<T>(_: T, _: &ArgMatches) {
()
}
pub fn display_none<T>(_: T, _: &ArgMatches) {}
pub struct Container<T>(RwLock<Option<T>>);
impl<T> Container<T> {
@@ -183,7 +182,7 @@ impl<T> Container<T> {
std::mem::replace(&mut *self.0.write().await, Some(value))
}
pub async fn take(&self) -> Option<T> {
std::mem::replace(&mut *self.0.write().await, None)
self.0.write().await.take()
}
pub async fn is_empty(&self) -> bool {
self.0.read().await.is_none()
@@ -220,7 +219,7 @@ impl<H: Digest, W: tokio::io::AsyncWrite> tokio::io::AsyncWrite for HashWriter<H
buf: &[u8],
) -> Poll<std::io::Result<usize>> {
let this = self.project();
let written = tokio::io::AsyncWrite::poll_write(this.writer, cx, &buf);
let written = tokio::io::AsyncWrite::poll_write(this.writer, cx, buf);
match written {
// only update the hasher once
Poll::Ready(res) => {
@@ -256,6 +255,29 @@ where
}
}
pub struct GeneralBoxedGuard(Option<Box<dyn FnOnce() + Send + Sync>>);
impl GeneralBoxedGuard {
pub fn new(f: impl FnOnce() + 'static + Send + Sync) -> Self {
GeneralBoxedGuard(Some(Box::new(f)))
}
pub fn drop(mut self) {
self.0.take().unwrap()()
}
pub fn drop_without_action(mut self) {
self.0 = None;
}
}
impl Drop for GeneralBoxedGuard {
fn drop(&mut self) {
if let Some(destroy) = self.0.take() {
destroy();
}
}
}
pub struct GeneralGuard<F: FnOnce() -> T, T = ()>(Option<F>);
impl<F: FnOnce() -> T, T> GeneralGuard<F, T> {
pub fn new(f: F) -> Self {
@@ -279,29 +301,6 @@ impl<F: FnOnce() -> T, T> Drop for GeneralGuard<F, T> {
}
}
pub struct GeneralBoxedGuard(Option<Box<dyn FnOnce() -> ()>>);
impl GeneralBoxedGuard {
pub fn new(f: impl FnOnce() -> () + 'static) -> Self {
GeneralBoxedGuard(Some(Box::new(f)))
}
pub fn drop(mut self) -> () {
self.0.take().unwrap()()
}
pub fn drop_without_action(mut self) {
self.0 = None;
}
}
impl Drop for GeneralBoxedGuard {
fn drop(&mut self) {
if let Some(destroy) = self.0.take() {
destroy();
}
}
}
pub struct FileLock(OwnedMutexGuard<()>, Option<FdLock<File>>);
impl Drop for FileLock {
fn drop(&mut self) {

101
backend/src/version/mod.rs
View File

@@ -2,14 +2,13 @@ use std::cmp::Ordering;
use async_trait::async_trait;
use color_eyre::eyre::eyre;
use patch_db::DbHandle;
use rpc_toolkit::command;
use sqlx::PgPool;
use crate::init::InitReceipts;
use crate::prelude::*;
use crate::Error;
mod v0_3_4_3;
mod v0_3_5;
mod v0_4_0;
pub type Current = v0_4_0::Version;
@@ -17,8 +16,8 @@ pub type Current = v0_4_0::Version;
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
#[serde(untagged)]
enum Version {
LT0_3_4_3(LTWrapper<v0_3_4_3::Version>),
V0_3_4_3(Wrapper<v0_3_4_3::Version>),
LT0_3_4_3(LTWrapper<v0_3_5::Version>),
V0_3_4_3(Wrapper<v0_3_5::Version>),
V0_4_0(Wrapper<v0_4_0::Version>),
Other(emver::Version),
}
@@ -52,55 +51,43 @@ where
fn new() -> Self;
fn semver(&self) -> emver::Version;
fn compat(&self) -> &'static emver::VersionRange;
async fn up<Db: DbHandle>(&self, db: &mut Db, secrets: &PgPool) -> Result<(), Error>;
async fn down<Db: DbHandle>(&self, db: &mut Db, secrets: &PgPool) -> Result<(), Error>;
async fn commit<Db: DbHandle>(
&self,
db: &mut Db,
receipts: &InitReceipts,
) -> Result<(), Error> {
receipts
.version_range
.set(db, self.compat().clone())
.await?;
receipts
.server_version
.set(db, self.semver().into())
.await?;
async fn up(&self, db: &PatchDb, secrets: &PgPool) -> Result<(), Error>;
async fn down(&self, db: &PatchDb, secrets: &PgPool) -> Result<(), Error>;
async fn commit(&self, db: &PatchDb) -> Result<(), Error> {
let semver = self.semver().into();
let compat = self.compat().clone();
db.mutate(|d| {
d.as_server_info_mut().as_version_mut().ser(&semver)?;
d.as_server_info_mut()
.as_eos_version_compat_mut()
.ser(&compat)?;
Ok(())
})
.await?;
Ok(())
}
async fn migrate_to<V: VersionT, Db: DbHandle>(
async fn migrate_to<V: VersionT>(
&self,
version: &V,
db: &mut Db,
db: &PatchDb,
secrets: &PgPool,
receipts: &InitReceipts,
) -> Result<(), Error> {
match self.semver().cmp(&version.semver()) {
Ordering::Greater => {
self.rollback_to_unchecked(version, db, secrets, receipts)
.await
}
Ordering::Less => {
version
.migrate_from_unchecked(self, db, secrets, receipts)
.await
}
Ordering::Greater => self.rollback_to_unchecked(version, db, secrets).await,
Ordering::Less => version.migrate_from_unchecked(self, db, secrets).await,
Ordering::Equal => Ok(()),
}
}
async fn migrate_from_unchecked<V: VersionT, Db: DbHandle>(
async fn migrate_from_unchecked<V: VersionT>(
&self,
version: &V,
db: &mut Db,
db: &PatchDb,
secrets: &PgPool,
receipts: &InitReceipts,
) -> Result<(), Error> {
let previous = Self::Previous::new();
if version.semver() < previous.semver() {
previous
.migrate_from_unchecked(version, db, secrets, receipts)
.migrate_from_unchecked(version, db, secrets)
.await?;
} else if version.semver() > previous.semver() {
return Err(Error::new(
@@ -113,24 +100,21 @@ where
}
tracing::info!("{} -> {}", previous.semver(), self.semver(),);
self.up(db, secrets).await?;
self.commit(db, receipts).await?;
self.commit(db).await?;
Ok(())
}
async fn rollback_to_unchecked<V: VersionT, Db: DbHandle>(
async fn rollback_to_unchecked<V: VersionT>(
&self,
version: &V,
db: &mut Db,
db: &PatchDb,
secrets: &PgPool,
receipts: &InitReceipts,
) -> Result<(), Error> {
let previous = Self::Previous::new();
tracing::info!("{} -> {}", self.semver(), previous.semver(),);
self.down(db, secrets).await?;
previous.commit(db, receipts).await?;
previous.commit(db).await?;
if version.semver() < previous.semver() {
previous
.rollback_to_unchecked(version, db, secrets, receipts)
.await?;
previous.rollback_to_unchecked(version, db, secrets).await?;
} else if version.semver() > previous.semver() {
return Err(Error::new(
eyre!(
@@ -194,12 +178,9 @@ where
}
}
pub async fn init<Db: DbHandle>(
db: &mut Db,
secrets: &PgPool,
receipts: &crate::init::InitReceipts,
) -> Result<(), Error> {
let version = Version::from_util_version(receipts.server_version.get(db).await?);
pub async fn init(db: &PatchDb, secrets: &PgPool) -> Result<(), Error> {
let version = Version::from_util_version(db.peek().await?.as_server_info().as_version().de()?);
match version {
Version::LT0_3_4_3(_) => {
return Err(Error::new(
@@ -207,14 +188,8 @@ pub async fn init<Db: DbHandle>(
crate::ErrorKind::MigrationFailed,
));
}
Version::V0_3_4_3(v) => {
v.0.migrate_to(&Current::new(), db, secrets, receipts)
.await?
}
Version::V0_4_0(v) => {
v.0.migrate_to(&Current::new(), db, secrets, receipts)
.await?
}
Version::V0_3_4_3(v) => v.0.migrate_to(&Current::new(), db, secrets).await?,
Version::V0_4_0(v) => v.0.migrate_to(&Current::new(), db, secrets).await?,
Version::Other(_) => {
return Err(Error::new(
eyre!("Cannot downgrade"),
@@ -247,15 +222,15 @@ mod tests {
fn versions() -> impl Strategy<Value = Version> {
prop_oneof![
em_version().prop_map(|v| if v < v0_3_4_3::Version::new().semver() {
Version::LT0_3_4_3(LTWrapper(v0_3_4_3::Version::new(), v))
em_version().prop_map(|v| if v < v0_3_5::Version::new().semver() {
Version::LT0_3_4_3(LTWrapper(v0_3_5::Version::new(), v))
} else {
Version::LT0_3_4_3(LTWrapper(
v0_3_4_3::Version::new(),
v0_3_5::Version::new(),
emver::Version::new(0, 3, 0, 0),
))
}),
Just(Version::V0_3_4_3(Wrapper(v0_3_4_3::Version::new()))),
Just(Version::V0_3_4_3(Wrapper(v0_3_5::Version::new()))),
Just(Version::V0_4_0(Wrapper(v0_4_0::Version::new()))),
em_version().prop_map(Version::Other),
]

41
backend/src/version/v0_3_4_4.rs
View File

@@ -1,41 +0,0 @@
use async_trait::async_trait;
use emver::VersionRange;
use models::ResultExt;
use super::v0_3_0::V0_3_0_COMPAT;
use super::*;
const V0_3_4_4: emver::Version = emver::Version::new(0, 3, 4, 4);
#[derive(Clone, Debug)]
pub struct Version;
#[async_trait]
impl VersionT for Version {
type Previous = v0_3_4_3::Version;
fn new() -> Self {
Version
}
fn semver(&self) -> emver::Version {
V0_3_4_4
}
fn compat(&self) -> &'static VersionRange {
&*V0_3_0_COMPAT
}
async fn up<Db: DbHandle>(&self, db: &mut Db, _secrets: &PgPool) -> Result<(), Error> {
let mut tor_addr = crate::db::DatabaseModel::new()
.server_info()
.tor_address()
.get_mut(db)
.await?;
tor_addr
.set_scheme("https")
.map_err(|_| eyre!("unable to update url scheme to https"))
.with_kind(crate::ErrorKind::ParseUrl)?;
tor_addr.save(db).await?;
Ok(())
}
async fn down<Db: DbHandle>(&self, _db: &mut Db, _secrets: &PgPool) -> Result<(), Error> {
Ok(())
}
}

11
backend/src/version/v0_3_4_3.rs → backend/src/version/v0_3_5.rs
View File

@@ -3,8 +3,9 @@ use emver::VersionRange;
use lazy_static::lazy_static;
use super::*;
use crate::prelude::*;
const V0_3_4_3: emver::Version = emver::Version::new(0, 3, 4, 3);
const V0_3_5: emver::Version = emver::Version::new(0, 3, 5, 0);
lazy_static! {
static ref V0_3_0_COMPAT: VersionRange = VersionRange::Conj(
Box::new(VersionRange::Anchor(
@@ -25,15 +26,15 @@ impl VersionT for Version {
Version
}
fn semver(&self) -> emver::Version {
V0_3_4_3
V0_3_5
}
fn compat(&self) -> &'static VersionRange {
&*V0_3_0_COMPAT
&V0_3_0_COMPAT
}
async fn up<Db: DbHandle>(&self, db: &mut Db, _secrets: &PgPool) -> Result<(), Error> {
async fn up(&self, _db: &PatchDb, _secrets: &PgPool) -> Result<(), Error> {
Ok(())
}
async fn down<Db: DbHandle>(&self, _db: &mut Db, _secrets: &PgPool) -> Result<(), Error> {
async fn down(&self, _db: &PatchDb, _secrets: &PgPool) -> Result<(), Error> {
Ok(())
}
}

6
backend/src/version/v0_4_0.rs
View File

@@ -17,7 +17,7 @@ pub struct Version;
#[async_trait]
impl VersionT for Version {
type Previous = v0_3_4_3::Version;
type Previous = v0_3_5::Version;
fn new() -> Self {
Version
}
@@ -27,10 +27,10 @@ impl VersionT for Version {
fn compat(&self) -> &'static VersionRange {
&*V0_4_0_COMPAT
}
async fn up<Db: DbHandle>(&self, db: &mut Db, secrets: &PgPool) -> Result<(), Error> {
async fn up(&self, _db: &PatchDb, _secrets: &PgPool) -> Result<(), Error> {
Ok(())
}
async fn down<Db: DbHandle>(&self, db: &mut Db, secrets: &PgPool) -> Result<(), Error> {
async fn down(&self, _db: &PatchDb, _secrets: &PgPool) -> Result<(), Error> {
Ok(())
}
}

11
backend/src/volume.rs
View File

@@ -4,13 +4,13 @@ use std::path::{Path, PathBuf};
pub use helpers::script_dir;
pub use models::VolumeId;
use patch_db::{HasModel, Map, MapModel};
use serde::{Deserialize, Serialize};
use tracing::instrument;
use crate::context::RpcContext;
use crate::net::interface::{InterfaceId, Interfaces};
use crate::net::PACKAGE_CERT_PATH;
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::util::Version;
use crate::{Error, ResultExt};
@@ -82,13 +82,6 @@ impl DerefMut for Volumes {
impl Map for Volumes {
type Key = VolumeId;
type Value = Volume;
fn get(&self, key: &Self::Key) -> Option<&Self::Value> {
self.0.get(key)
}
}
pub type VolumesModel = MapModel<Volumes>;
impl HasModel for Volumes {
type Model = MapModel<Self>;
}
pub fn data_dir<P: AsRef<Path>>(datadir: P, pkg_id: &PackageId, volume_id: &VolumeId) -> PathBuf {
@@ -117,7 +110,7 @@ pub fn cert_dir(pkg_id: &PackageId, interface_id: &InterfaceId) -> PathBuf {
Path::new(PACKAGE_CERT_PATH).join(pkg_id).join(interface_id)
}
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(tag = "type")]
#[serde(rename_all = "kebab-case")]
pub enum Volume {