Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-04-01 21:13:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feat bulk locking (#1422)

Browse Source 
* Feat: Multi-lock capabilities add to config

* wip: RPC.rs fixes, new combinatoric

* wip: changes

* chore: More things that are bulk

* fix: Saving

* chore: Remove a dyn object

* chore: Add tests + remove unused

* Fix/feat  bulk locking (#1427)

* fix: health check

* fix: start/stop service

* fix: install/uninstall services

* chore: Fix the notifications

* fix: Version

* fix: Version as serde

* chore: Update to latest patch db

* chore: Change the htLock to something that makes more sense

* chore: Fix the rest of the ht

* "chore: More ht_lock":
This commit is contained in:
J M
2022-05-09 14:53:39 -06:00
committed by GitHub
parent 5d3bc8cfa5
commit 864555bcf0
26 changed files with 2080 additions and 743 deletions
Download Patch File Download Diff File Expand all files Collapse all files

521
backend/src/config/mod.rs
View File

@@ -6,7 +6,7 @@ use color_eyre::eyre::eyre;
use futures::future::{BoxFuture, FutureExt};
use indexmap::IndexSet;
use itertools::Itertools;
use patch_db::{DbHandle, LockType};
use patch_db::{DbHandle, LockReceipt, LockTarget, LockTargetId, LockType, Verifier};
use rand::SeedableRng;
use regex::Regex;
use rpc_toolkit::command;
@@ -14,17 +14,20 @@ use serde_json::Value;
use tracing::instrument;
use crate::context::RpcContext;
use crate::db::model::CurrentDependencyInfo;
use crate::db::util::WithRevision;
use crate::dependencies::{
add_dependent_to_current_dependents_lists, break_transitive, heal_all_dependents_transitive,
BreakageRes, DependencyError, DependencyErrors, TaggedDependencyError,
BreakageRes, DependencyConfig, DependencyError, DependencyErrors, TaggedDependencyError,
};
use crate::install::cleanup::remove_from_current_dependents_lists;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::util::display_none;
use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
use crate::{Error, ResultExt as _};
use crate::Error;
use crate::{db::model::CurrentDependencyInfo, dependencies::DependencyReceipt};
use crate::{db::util::WithRevision, dependencies::Dependencies};
use crate::{
dependencies::BreakTransitiveReceipts,
install::cleanup::{remove_from_current_dependents_lists, UpdateDependencyReceipts},
};
use crate::{dependencies::TryHealReceipts, util::display_none};
pub mod action;
pub mod spec;
@@ -33,8 +36,11 @@ pub mod util;
pub use spec::{ConfigSpec, Defaultable};
use util::NumRange;
use self::action::ConfigRes;
use self::spec::{PackagePointerSpec, ValueSpecPointer};
use self::{
action::{ConfigActions, ConfigRes},
spec::ConfigPointerReceipts,
};
pub type Config = serde_json::Map<String, Value>;
pub trait TypeOf {
@@ -163,6 +169,56 @@ pub fn config(#[arg] id: PackageId) -> Result<PackageId, Error> {
Ok(id)
}
pub struct ConfigGetReceipts {
manifest_volumes: LockReceipt<crate::volume::Volumes, ()>,
manifest_version: LockReceipt<crate::util::Version, ()>,
manifest_config: LockReceipt<Option<ConfigActions>, ()>,
}
impl ConfigGetReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks, id);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(
locks: &mut Vec<LockTargetId>,
id: &PackageId,
) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
let manifest_version = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.manifest().version())
.make_locker(LockType::Write)
.add_to_keys(locks);
let manifest_volumes = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.manifest().volumes())
.make_locker(LockType::Write)
.add_to_keys(locks);
let manifest_config = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|x| x.installed())
.map(|x| x.manifest().config())
.make_locker(LockType::Write)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
manifest_volumes: manifest_volumes.verify(skeleton_key)?,
manifest_version: manifest_version.verify(skeleton_key)?,
manifest_config: manifest_config.verify(skeleton_key)?,
})
}
}
}
#[command(display(display_serializable))]
#[instrument(skip(ctx))]
pub async fn get(
@@ -173,29 +229,16 @@ pub async fn get(
format: Option<IoFormat>,
) -> Result<ConfigRes, Error> {
let mut db = ctx.db.handle();
let pkg_model = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&id)
.and_then(|m| m.installed())
.expect(&mut db)
.await
.with_kind(crate::ErrorKind::NotFound)?;
let action = pkg_model
.clone()
.manifest()
.config()
.get(&mut db, true)
let receipts = ConfigGetReceipts::new(&mut db, &id).await?;
let action = receipts
.manifest_config
.get(&mut db)
.await?
.to_owned()
.ok_or_else(|| Error::new(eyre!("{} has no config", id), crate::ErrorKind::NotFound))?;
let version = pkg_model
.clone()
.manifest()
.version()
.get(&mut db, true)
.await?;
let volumes = pkg_model.manifest().volumes().get(&mut db, true).await?;
action.get(&ctx, &id, &*version, &*volumes).await
let volumes = receipts.manifest_volumes.get(&mut db).await?;
let version = receipts.manifest_version.get(&mut db).await?;
action.get(&ctx, &id, &version, &volumes).await
}
#[command(
@@ -215,6 +258,147 @@ pub fn set(
Ok((id, config, timeout.map(|d| *d), expire_id))
}
/// So, the new locking finds all the possible locks and lifts them up into a bundle of locks.
/// Then this bundle will be passed down into the functions that will need to touch the db, and
/// instead of doing the locks down in the system, we have already done the locks and can
/// do the operation on the db.
/// An UnlockedLock has two types, the type of setting and getting from the db, and the second type
/// is the keys that we need to insert on getting/setting because we have included wild cards into the paths.
pub struct ConfigReceipts {
pub dependency_receipt: DependencyReceipt,
pub config_receipts: ConfigPointerReceipts,
pub update_dependency_receipts: UpdateDependencyReceipts,
pub try_heal_receipts: TryHealReceipts,
pub break_transitive_receipts: BreakTransitiveReceipts,
configured: LockReceipt<bool, String>,
config_actions: LockReceipt<ConfigActions, String>,
dependencies: LockReceipt<Dependencies, String>,
volumes: LockReceipt<crate::volume::Volumes, String>,
version: LockReceipt<crate::util::Version, String>,
manifest: LockReceipt<Manifest, String>,
system_pointers: LockReceipt<Vec<spec::SystemPointerSpec>, String>,
pub current_dependents: LockReceipt<BTreeMap<PackageId, CurrentDependencyInfo>, String>,
dependency_errors: LockReceipt<DependencyErrors, String>,
manifest_dependencies_config: LockReceipt<DependencyConfig, (String, String)>,
}
impl ConfigReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(locks: &mut Vec<LockTargetId>) -> impl FnOnce(&Verifier) -> Result<Self, Error> {
let dependency_receipt = DependencyReceipt::setup(locks);
let config_receipts = ConfigPointerReceipts::setup(locks);
let update_dependency_receipts = UpdateDependencyReceipts::setup(locks);
let break_transitive_receipts = BreakTransitiveReceipts::setup(locks);
let try_heal_receipts = TryHealReceipts::setup(locks);
let configured: LockTarget<bool, String> = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.status().configured())
.make_locker(LockType::Write)
.add_to_keys(locks);
let config_actions = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.and_then(|x| x.manifest().config())
.make_locker(LockType::Read)
.add_to_keys(locks);
let dependencies = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest().dependencies())
.make_locker(LockType::Read)
.add_to_keys(locks);
let volumes = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest().volumes())
.make_locker(LockType::Read)
.add_to_keys(locks);
let version = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest().version())
.make_locker(LockType::Read)
.add_to_keys(locks);
let manifest = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest())
.make_locker(LockType::Read)
.add_to_keys(locks);
let system_pointers = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.system_pointers())
.make_locker(LockType::Write)
.add_to_keys(locks);
let current_dependents = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.current_dependents())
.make_locker(LockType::Write)
.add_to_keys(locks);
let dependency_errors = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.status().dependency_errors())
.make_locker(LockType::Write)
.add_to_keys(locks);
let manifest_dependencies_config = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.and_then(|x| x.manifest().dependencies().star().config())
.make_locker(LockType::Write)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
dependency_receipt: dependency_receipt(skeleton_key)?,
config_receipts: config_receipts(skeleton_key)?,
try_heal_receipts: try_heal_receipts(skeleton_key)?,
break_transitive_receipts: break_transitive_receipts(skeleton_key)?,
update_dependency_receipts: update_dependency_receipts(skeleton_key)?,
configured: configured.verify(skeleton_key)?,
config_actions: config_actions.verify(skeleton_key)?,
dependencies: dependencies.verify(skeleton_key)?,
volumes: volumes.verify(skeleton_key)?,
version: version.verify(skeleton_key)?,
manifest: manifest.verify(skeleton_key)?,
system_pointers: system_pointers.verify(skeleton_key)?,
current_dependents: current_dependents.verify(skeleton_key)?,
dependency_errors: dependency_errors.verify(skeleton_key)?,
manifest_dependencies_config: manifest_dependencies_config.verify(skeleton_key)?,
})
}
}
}
#[command(rename = "dry", display(display_serializable))]
#[instrument(skip(ctx))]
pub async fn set_dry(
@@ -229,6 +413,7 @@ pub async fn set_dry(
let mut db = ctx.db.handle();
let mut tx = db.begin().await?;
let mut breakages = BTreeMap::new();
let locks = ConfigReceipts::new(&mut tx).await?;
configure(
&ctx,
&mut tx,
@@ -238,20 +423,11 @@ pub async fn set_dry(
true,
&mut BTreeMap::new(),
&mut breakages,
&locks,
)
.await?;
crate::db::DatabaseModel::new()
.package_data()
.idx_model(&id)
.expect(&mut tx)
.await?
.installed()
.expect(&mut tx)
.await?
.status()
.configured()
.put(&mut tx, &true)
.await?;
locks.configured.set(&mut tx, true, &id).await?;
tx.abort().await?;
Ok(BreakageRes(breakages))
}
@@ -264,6 +440,7 @@ pub async fn set_impl(
let mut db = ctx.db.handle();
let mut tx = db.begin().await?;
let mut breakages = BTreeMap::new();
let locks = ConfigReceipts::new(&mut tx).await?;
configure(
&ctx,
&mut tx,
@@ -273,6 +450,7 @@ pub async fn set_impl(
false,
&mut BTreeMap::new(),
&mut breakages,
&locks,
)
.await?;
Ok(WithRevision {
@@ -281,34 +459,27 @@ pub async fn set_impl(
})
}
#[instrument(skip(ctx, db))]
pub async fn configure<Db: DbHandle>(
#[instrument(skip(ctx, db, receipts))]
pub async fn configure<'a, Db: DbHandle>(
ctx: &RpcContext,
db: &mut Db,
db: &'a mut Db,
id: &PackageId,
config: Option<Config>,
timeout: &Option<Duration>,
dry_run: bool,
overrides: &mut BTreeMap<PackageId, Config>,
breakages: &mut BTreeMap<PackageId, TaggedDependencyError>,
receipts: &ConfigReceipts,
) -> Result<(), Error> {
configure_rec(ctx, db, id, config, timeout, dry_run, overrides, breakages).await?;
crate::db::DatabaseModel::new()
.package_data()
.idx_model(&id)
.expect(db)
.await?
.installed()
.expect(db)
.await?
.status()
.configured()
.put(db, &true)
.await?;
configure_rec(
ctx, db, id, config, timeout, dry_run, overrides, breakages, receipts,
)
.await?;
receipts.configured.set(db, true, &id).await?;
Ok(())
}
#[instrument(skip(ctx, db))]
#[instrument(skip(ctx, db, receipts))]
pub fn configure_rec<'a, Db: DbHandle>(
ctx: &'a RpcContext,
db: &'a mut Db,
@@ -318,48 +489,33 @@ pub fn configure_rec<'a, Db: DbHandle>(
dry_run: bool,
overrides: &'a mut BTreeMap<PackageId, Config>,
breakages: &'a mut BTreeMap<PackageId, TaggedDependencyError>,
receipts: &'a ConfigReceipts,
) -> BoxFuture<'a, Result<(), Error>> {
async move {
crate::db::DatabaseModel::new()
.package_data()
.lock(db, LockType::Write)
.await?;
// fetch data from db
let pkg_model = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|m| m.installed())
.expect(db)
.await
.with_kind(crate::ErrorKind::NotFound)?;
let action = pkg_model
.clone()
.manifest()
.config()
.get(db, true)
let action = receipts
.config_actions
.get(db, id)
.await?
.to_owned()
.ok_or_else(|| Error::new(eyre!("{} has no config", id), crate::ErrorKind::NotFound))?;
let version = pkg_model.clone().manifest().version().get(db, true).await?;
let dependencies = pkg_model
.clone()
.manifest()
.dependencies()
.get(db, true)
.await?;
let volumes = pkg_model.clone().manifest().volumes().get(db, true).await?;
let is_needs_config = !*pkg_model
.clone()
.status()
.configured()
.get(db, true)
.await?;
.ok_or_else(not_found)?;
let dependencies = receipts
.dependencies
.get(db, id)
.await?
.ok_or_else(not_found)?;
let volumes = receipts.volumes.get(db, id).await?.ok_or_else(not_found)?;
let is_needs_config = !receipts
.configured
.get(db, id)
.await?
.ok_or_else(not_found)?;
let version = receipts.version.get(db, id).await?.ok_or_else(not_found)?;
// get current config and current spec
let ConfigRes {
config: old_config,
spec,
} = action.get(ctx, id, &*version, &*volumes).await?;
} = action.get(ctx, id, &version, &volumes).await?;
// determine new config to use
let mut config = if let Some(config) = config.or_else(|| old_config.clone()) {
@@ -368,24 +524,26 @@ pub fn configure_rec<'a, Db: DbHandle>(
spec.gen(&mut rand::rngs::StdRng::from_entropy(), timeout)?
};
let manifest = crate::db::DatabaseModel::new()
.package_data()
.idx_model(id)
.and_then(|m| m.installed())
.map::<_, Manifest>(|i| i.manifest())
.expect(db)
.await?
.get(db, true)
.await
.with_kind(crate::ErrorKind::NotFound)?;
let manifest = receipts.manifest.get(db, id).await?.ok_or_else(not_found)?;
spec.validate(&*manifest)?;
spec.validate(&manifest)?;
spec.matches(&config)?; // check that new config matches spec
spec.update(ctx, db, &*manifest, &*overrides, &mut config)
.await?; // dereference pointers in the new config
spec.update(
ctx,
db,
&manifest,
&*overrides,
&mut config,
&receipts.config_receipts,
)
.await?; // dereference pointers in the new config
// create backreferences to pointers
let mut sys = pkg_model.clone().system_pointers().get_mut(db).await?;
let mut sys = receipts
.system_pointers
.get(db, &id)
.await?
.ok_or_else(not_found)?;
sys.truncate(0);
let mut current_dependencies: BTreeMap<PackageId, CurrentDependencyInfo> = dependencies
.0
@@ -418,12 +576,12 @@ pub fn configure_rec<'a, Db: DbHandle>(
ValueSpecPointer::System(s) => sys.push(s),
}
}
sys.save(db).await?;
receipts.system_pointers.set(db, sys, &id).await?;
let signal = if !dry_run {
// run config action
let res = action
.set(ctx, id, &*version, &*dependencies, &*volumes, &config)
.set(ctx, id, &version, &dependencies, &volumes, &config)
.await?;
// track dependencies with no pointers
@@ -459,50 +617,63 @@ pub fn configure_rec<'a, Db: DbHandle>(
};
// update dependencies
let mut deps = pkg_model.clone().current_dependencies().get_mut(db).await?;
remove_from_current_dependents_lists(db, id, deps.keys()).await?; // remove previous
add_dependent_to_current_dependents_lists(db, id, &current_dependencies).await?; // add new
remove_from_current_dependents_lists(
db,
id,
current_dependencies.keys(),
&receipts.current_dependents,
)
.await?; // remove previous
add_dependent_to_current_dependents_lists(
db,
id,
&current_dependencies,
&receipts.current_dependents,
)
.await?; // add new
current_dependencies.remove(id);
*deps = current_dependencies.clone();
deps.save(db).await?;
let mut errs = pkg_model
.clone()
.status()
.dependency_errors()
.get_mut(db)
receipts
.current_dependents
.set(db, current_dependencies.clone(), &id)
.await?;
*errs = DependencyErrors::init(ctx, db, &*manifest, &current_dependencies).await?;
errs.save(db).await?;
let errs = receipts
.dependency_errors
.get(db, &id)
.await?
.ok_or_else(not_found)?;
tracing::warn!("Dependency Errors: {:?}", errs);
let errs = DependencyErrors::init(
ctx,
db,
&manifest,
&current_dependencies,
&receipts.dependency_receipt.try_heal,
)
.await?;
receipts.dependency_errors.set(db, errs, &id).await?;
// cache current config for dependents
overrides.insert(id.clone(), config.clone());
// handle dependents
let dependents = pkg_model.clone().current_dependents().get(db, true).await?;
let dependents = current_dependencies;
let prev = if is_needs_config { None } else { old_config }
.map(Value::Object)
.unwrap_or_default();
let next = Value::Object(config.clone());
for (dependent, dep_info) in dependents.iter().filter(|(dep_id, _)| dep_id != &id) {
// check if config passes dependent check
let dependent_model = crate::db::DatabaseModel::new()
.package_data()
.idx_model(dependent)
.and_then(|pkg| pkg.installed())
.expect(db)
.await?;
if let Some(cfg) = &*dependent_model
.clone()
.manifest()
.dependencies()
.idx_model(id)
.expect(db)
.await?
.config()
.get(db, true)
if let Some(cfg) = receipts
.manifest_dependencies_config
.get(db, (&dependent, &id))
.await?
{
let manifest = dependent_model.clone().manifest().get(db, true).await?;
let manifest = receipts
.manifest
.get(db, &dependent)
.await?
.ok_or_else(not_found)?;
if let Err(error) = cfg
.check(
ctx,
@@ -514,7 +685,15 @@ pub fn configure_rec<'a, Db: DbHandle>(
.await?
{
let dep_err = DependencyError::ConfigUnsatisfied { error };
break_transitive(db, dependent, id, dep_err, breakages).await?;
break_transitive(
db,
dependent,
id,
dep_err,
breakages,
&receipts.break_transitive_receipts,
)
.await?;
}
// handle backreferences
@@ -523,6 +702,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
if cfg_ptr.select(&next) != cfg_ptr.select(&prev) {
if let Err(e) = configure_rec(
ctx, db, dependent, None, timeout, dry_run, overrides, breakages,
receipts,
)
.await
{
@@ -535,6 +715,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
error: format!("{}", e),
},
breakages,
&receipts.break_transitive_receipts,
)
.await?;
} else {
@@ -544,7 +725,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
}
}
}
heal_all_dependents_transitive(ctx, db, id).await?;
heal_all_dependents_transitive(ctx, db, id, &receipts.dependency_receipt).await?;
}
}
@@ -568,3 +749,67 @@ pub fn configure_rec<'a, Db: DbHandle>(
}
.boxed()
}
#[instrument]
pub fn not_found() -> Error {
Error::new(eyre!("Could not find"), crate::ErrorKind::Incoherent)
}
/// We want to have a double check that the paths are what we expect them to be.
/// Found that earlier the paths where not what we expected them to be.
#[tokio::test]
async fn ensure_creation_of_config_paths_makes_sense() {
let mut fake = patch_db::test_utils::NoOpDb();
let config_locks = ConfigReceipts::new(&mut fake).await.unwrap();
assert_eq!(
&format!("{}", config_locks.configured.lock.glob),
"/package-data/*/installed/status/configured"
);
assert_eq!(
&format!("{}", config_locks.config_actions.lock.glob),
"/package-data/*/installed/manifest/config"
);
assert_eq!(
&format!("{}", config_locks.dependencies.lock.glob),
"/package-data/*/installed/manifest/dependencies"
);
assert_eq!(
&format!("{}", config_locks.volumes.lock.glob),
"/package-data/*/installed/manifest/volumes"
);
assert_eq!(
&format!("{}", config_locks.version.lock.glob),
"/package-data/*/installed/manifest/version"
);
assert_eq!(
&format!("{}", config_locks.volumes.lock.glob),
"/package-data/*/installed/manifest/volumes"
);
assert_eq!(
&format!("{}", config_locks.manifest.lock.glob),
"/package-data/*/installed/manifest"
);
assert_eq!(
&format!("{}", config_locks.manifest.lock.glob),
"/package-data/*/installed/manifest"
);
assert_eq!(
&format!("{}", config_locks.system_pointers.lock.glob),
"/package-data/*/installed/system-pointers"
);
assert_eq!(
&format!("{}", config_locks.current_dependents.lock.glob),
"/package-data/*/installed/current-dependents"
);
assert_eq!(
&format!("{}", config_locks.dependency_errors.lock.glob),
"/package-data/*/installed/status/dependency-errors"
);
assert_eq!(
&format!("{}", config_locks.manifest_dependencies_config.lock.glob),
"/package-data/*/installed/manifest/dependencies/*/config"
);
assert_eq!(
&format!("{}", config_locks.system_pointers.lock.glob),
"/package-data/*/installed/system-pointers"
);
}

317
backend/src/config/spec.rs
View File

@@ -12,7 +12,7 @@ use async_trait::async_trait;
use indexmap::{IndexMap, IndexSet};
use itertools::Itertools;
use jsonpath_lib::Compiled as CompiledJsonPath;
use patch_db::{DbHandle, OptionModel};
use patch_db::{DbHandle, LockReceipt, LockType};
use rand::{CryptoRng, Rng};
use regex::Regex;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -44,6 +44,7 @@ pub trait ValueSpec {
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError>;
// returns all pointers that are live in the provided config
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath>;
@@ -160,9 +161,10 @@ where
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
self.inner
.update(ctx, db, manifest, config_overrides, value)
.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -204,9 +206,10 @@ where
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
self.inner
.update(ctx, db, manifest, config_overrides, value)
.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -281,9 +284,10 @@ where
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
self.inner
.update(ctx, db, manifest, config_overrides, value)
.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -343,7 +347,7 @@ pub enum ValueSpecAny {
Pointer(WithDescription<ValueSpecPointer>),
}
impl ValueSpecAny {
pub fn name<'a>(&'a self) -> &'a str {
pub fn name(&self) -> &'_ str {
match self {
ValueSpecAny::Boolean(b) => b.name.as_str(),
ValueSpecAny::Enum(e) => e.name.as_str(),
@@ -395,16 +399,41 @@ impl ValueSpec for ValueSpecAny {
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
match self {
ValueSpecAny::Boolean(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecAny::Enum(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecAny::List(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecAny::Number(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecAny::Object(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecAny::String(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecAny::Union(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecAny::Pointer(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecAny::Boolean(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::Enum(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::List(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::Number(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::Object(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::String(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::Union(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecAny::Pointer(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
}
}
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -489,6 +518,7 @@ impl ValueSpec for ValueSpecBoolean {
_manifest: &Manifest,
_config_overrides: &BTreeMap<PackageId, Config>,
_value: &mut Value,
_receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
Ok(())
}
@@ -578,6 +608,7 @@ impl ValueSpec for ValueSpecEnum {
_manifest: &Manifest,
_config_overrides: &BTreeMap<PackageId, Config>,
_value: &mut Value,
_receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
Ok(())
}
@@ -664,12 +695,13 @@ where
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
if let Value::Array(ref mut ls) = value {
for (i, val) in ls.into_iter().enumerate() {
match self
.spec
.update(ctx, db, manifest, config_overrides, val)
.update(ctx, db, manifest, config_overrides, val, receipts)
.await
{
Err(ConfigurationError::NoMatch(e)) => {
@@ -771,13 +803,29 @@ impl ValueSpec for ValueSpecList {
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
match self {
ValueSpecList::Enum(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecList::Number(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecList::Object(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecList::String(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecList::Union(a) => a.update(ctx, db, manifest, config_overrides, value).await,
ValueSpecList::Enum(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecList::Number(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecList::Object(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecList::String(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecList::Union(a) => {
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
}
}
fn pointers(&self, value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -898,6 +946,7 @@ impl ValueSpec for ValueSpecNumber {
_manifest: &Manifest,
_config_overrides: &BTreeMap<PackageId, Config>,
_value: &mut Value,
_receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
Ok(())
}
@@ -961,10 +1010,11 @@ impl ValueSpec for ValueSpecObject {
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
if let Value::Object(o) = value {
self.spec
.update(ctx, db, manifest, config_overrides, o)
.update(ctx, db, manifest, config_overrides, o, receipts)
.await
} else {
Err(ConfigurationError::NoMatch(NoMatchWithPath::new(
@@ -1063,16 +1113,20 @@ impl ConfigSpec {
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
cfg: &mut Config,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
for (k, vs) in self.0.iter() {
match cfg.get_mut(k) {
None => {
let mut v = Value::Null;
vs.update(ctx, db, manifest, config_overrides, &mut v)
vs.update(ctx, db, manifest, config_overrides, &mut v, receipts)
.await?;
cfg.insert(k.clone(), v);
}
Some(v) => match vs.update(ctx, db, manifest, config_overrides, v).await {
Some(v) => match vs
.update(ctx, db, manifest, config_overrides, v, receipts)
.await
{
Err(ConfigurationError::NoMatch(e)) => {
Err(ConfigurationError::NoMatch(e.prepend(k.clone())))
}
@@ -1160,6 +1214,7 @@ impl ValueSpec for ValueSpecString {
_manifest: &Manifest,
_config_overrides: &BTreeMap<PackageId, Config>,
_value: &mut Value,
_receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
Ok(())
}
@@ -1192,10 +1247,7 @@ impl DefaultableWith for ValueSpecString {
let candidate = spec.gen(rng);
match (spec, &self.pattern) {
(DefaultString::Entropy(_), Some(pattern))
if !pattern.pattern.is_match(&candidate) =>
{
()
}
if !pattern.pattern.is_match(&candidate) => {}
_ => {
return Ok(Value::String(candidate));
}
@@ -1371,6 +1423,7 @@ impl ValueSpec for ValueSpecUnion {
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
if let Value::Object(o) = value {
match o.get(&self.tag.id) {
@@ -1381,7 +1434,10 @@ impl ValueSpec for ValueSpecUnion {
None => Err(ConfigurationError::NoMatch(NoMatchWithPath::new(
MatchError::Union(tag.clone(), self.variants.keys().cloned().collect()),
))),
Some(spec) => spec.update(ctx, db, manifest, config_overrides, o).await,
Some(spec) => {
spec.update(ctx, db, manifest, config_overrides, o, receipts)
.await
}
},
Some(other) => Err(ConfigurationError::NoMatch(
NoMatchWithPath::new(MatchError::InvalidType("string", other.type_of()))
@@ -1513,13 +1569,16 @@ impl ValueSpec for ValueSpecPointer {
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
match self {
ValueSpecPointer::Package(a) => {
a.update(ctx, db, manifest, config_overrides, value).await
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
ValueSpecPointer::System(a) => {
a.update(ctx, db, manifest, config_overrides, value).await
a.update(ctx, db, manifest, config_overrides, value, receipts)
.await
}
}
}
@@ -1563,12 +1622,17 @@ impl PackagePointerSpec {
db: &mut Db,
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
receipts: &ConfigPointerReceipts,
) -> Result<Value, ConfigurationError> {
match &self {
PackagePointerSpec::TorKey(key) => key.deref(&manifest.id, &ctx.secret_store).await,
PackagePointerSpec::TorAddress(tor) => tor.deref(db).await,
PackagePointerSpec::LanAddress(lan) => lan.deref(db).await,
PackagePointerSpec::Config(cfg) => cfg.deref(ctx, db, config_overrides).await,
PackagePointerSpec::TorAddress(tor) => {
tor.deref(db, &receipts.interface_addresses_receipt).await
}
PackagePointerSpec::LanAddress(lan) => {
lan.deref(db, &receipts.interface_addresses_receipt).await
}
PackagePointerSpec::Config(cfg) => cfg.deref(ctx, db, config_overrides, receipts).await,
}
}
}
@@ -1616,8 +1680,11 @@ impl ValueSpec for PackagePointerSpec {
manifest: &Manifest,
config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
*value = self.deref(ctx, db, manifest, config_overrides).await?;
*value = self
.deref(ctx, db, manifest, config_overrides, receipts)
.await?;
Ok(())
}
fn pointers(&self, _value: &Value) -> Result<BTreeSet<ValueSpecPointer>, NoMatchWithPath> {
@@ -1640,16 +1707,17 @@ pub struct TorAddressPointer {
interface: InterfaceId,
}
impl TorAddressPointer {
async fn deref<Db: DbHandle>(&self, db: &mut Db) -> Result<Value, ConfigurationError> {
let addr = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&self.package_id)
.and_then(|pde| pde.installed())
.and_then(|installed| installed.interface_addresses().idx_model(&self.interface))
.and_then(|addresses| addresses.tor_address())
.get(db, true)
async fn deref<Db: DbHandle>(
&self,
db: &mut Db,
receipt: &InterfaceAddressesReceipt,
) -> Result<Value, ConfigurationError> {
let addr = receipt
.interface_addresses
.get(db, (&self.package_id, &self.interface))
.await
.map_err(|e| ConfigurationError::SystemError(Error::from(e)))?;
.map_err(|e| ConfigurationError::SystemError(Error::from(e)))?
.and_then(|addresses| addresses.tor_address);
Ok(addr.to_owned().map(Value::String).unwrap_or(Value::Null))
}
}
@@ -1664,6 +1732,39 @@ impl fmt::Display for TorAddressPointer {
}
}
pub struct InterfaceAddressesReceipt {
interface_addresses: LockReceipt<crate::db::model::InterfaceAddresses, (String, String)>,
}
impl InterfaceAddressesReceipt {
pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(
locks: &mut Vec<patch_db::LockTargetId>,
) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
// let cleanup_receipts = CleanupFailedReceipts::setup(locks);
let interface_addresses = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.interface_addresses().star())
.make_locker(LockType::Read)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
// cleanup_receipts: cleanup_receipts(skeleton_key)?,
interface_addresses: interface_addresses.verify(skeleton_key)?,
})
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct LanAddressPointer {
@@ -1672,28 +1773,81 @@ pub struct LanAddressPointer {
}
impl fmt::Display for LanAddressPointer {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
LanAddressPointer {
package_id,
interface,
} => write!(f, "{}: lan-address: {}", package_id, interface),
}
let LanAddressPointer {
package_id,
interface,
} = self;
write!(f, "{}: lan-address: {}", package_id, interface)
}
}
impl LanAddressPointer {
async fn deref<Db: DbHandle>(&self, db: &mut Db) -> Result<Value, ConfigurationError> {
let addr = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&self.package_id)
.and_then(|pde| pde.installed())
.and_then(|installed| installed.interface_addresses().idx_model(&self.interface))
.and_then(|addresses| addresses.lan_address())
.get(db, true)
async fn deref<Db: DbHandle>(
&self,
db: &mut Db,
receipts: &InterfaceAddressesReceipt,
) -> Result<Value, ConfigurationError> {
let addr = receipts
.interface_addresses
.get(db, (&self.package_id, &self.interface))
.await
.map_err(|e| ConfigurationError::SystemError(Error::from(e)))?;
.ok()
.flatten()
.and_then(|x| x.lan_address);
Ok(addr.to_owned().map(Value::String).unwrap_or(Value::Null))
}
}
pub struct ConfigPointerReceipts {
interface_addresses_receipt: InterfaceAddressesReceipt,
manifest_volumes: LockReceipt<crate::volume::Volumes, String>,
manifest_version: LockReceipt<crate::util::Version, String>,
config_actions: LockReceipt<super::action::ConfigActions, String>,
}
impl ConfigPointerReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(
locks: &mut Vec<patch_db::LockTargetId>,
) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
let interface_addresses_receipt = InterfaceAddressesReceipt::setup(locks);
let manifest_volumes = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest().volumes())
.make_locker(LockType::Read)
.add_to_keys(locks);
let manifest_version = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.map(|x| x.manifest().version())
.make_locker(LockType::Read)
.add_to_keys(locks);
let config_actions = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.and_then(|x| x.manifest().config())
.make_locker(LockType::Read)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
interface_addresses_receipt: interface_addresses_receipt(skeleton_key)?,
manifest_volumes: manifest_volumes.verify(skeleton_key)?,
config_actions: config_actions.verify(skeleton_key)?,
manifest_version: manifest_version.verify(skeleton_key)?,
})
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct ConfigPointer {
@@ -1710,40 +1864,22 @@ impl ConfigPointer {
ctx: &RpcContext,
db: &mut Db,
config_overrides: &BTreeMap<PackageId, Config>,
receipts: &ConfigPointerReceipts,
) -> Result<Value, ConfigurationError> {
if let Some(cfg) = config_overrides.get(&self.package_id) {
Ok(self.select(&Value::Object(cfg.clone())))
} else {
let manifest_model: OptionModel<Manifest> = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&self.package_id)
.and_then(|pde| pde.installed())
.map(|installed| installed.manifest())
.into();
let version = manifest_model
.clone()
.map(|manifest| manifest.version())
.get(db, true)
.await
.map_err(|e| ConfigurationError::SystemError(Error::from(e)))?;
let cfg_actions = manifest_model
.clone()
.and_then(|manifest| manifest.config())
.get(db, true)
.await
.map_err(|e| ConfigurationError::SystemError(Error::from(e)))?;
let volumes = manifest_model
.map(|manifest| manifest.volumes())
.get(db, true)
.await
.map_err(|e| ConfigurationError::SystemError(Error::from(e)))?;
let id = &self.package_id;
let version = receipts.manifest_version.get(db, id).await.ok().flatten();
let cfg_actions = receipts.config_actions.get(db, id).await.ok().flatten();
let volumes = receipts.manifest_volumes.get(db, id).await.ok().flatten();
if let (Some(version), Some(cfg_actions), Some(volumes)) =
(&*version, &*cfg_actions, &*volumes)
(&version, &cfg_actions, &volumes)
{
let cfg_res = cfg_actions
.get(&ctx, &self.package_id, version, volumes)
.get(ctx, &self.package_id, version, volumes)
.await
.map_err(|e| ConfigurationError::SystemError(Error::from(e)))?;
.map_err(|e| ConfigurationError::SystemError(e))?;
if let Some(cfg) = cfg_res.config {
Ok(self.select(&Value::Object(cfg)))
} else {
@@ -1757,13 +1893,12 @@ impl ConfigPointer {
}
impl fmt::Display for ConfigPointer {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
ConfigPointer {
package_id,
selector,
..
} => write!(f, "{}: config: {}", package_id, selector),
}
let ConfigPointer {
package_id,
selector,
..
} = self;
write!(f, "{}: config: {}", package_id, selector)
}
}
@@ -1909,6 +2044,8 @@ impl ValueSpec for SystemPointerSpec {
_manifest: &Manifest,
_config_overrides: &BTreeMap<PackageId, Config>,
value: &mut Value,
receipts: &ConfigPointerReceipts,
) -> Result<(), ConfigurationError> {
*value = self.deref(db).await?;
Ok(())