Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 12:11:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Feat bulk locking (#1422)

Browse Source 
* Feat: Multi-lock capabilities add to config

* wip: RPC.rs fixes, new combinatoric

* wip: changes

* chore: More things that are bulk

* fix: Saving

* chore: Remove a dyn object

* chore: Add tests + remove unused

* Fix/feat  bulk locking (#1427)

* fix: health check

* fix: start/stop service

* fix: install/uninstall services

* chore: Fix the notifications

* fix: Version

* fix: Version as serde

* chore: Update to latest patch db

* chore: Change the htLock to something that makes more sense

* chore: Fix the rest of the ht

* "chore: More ht_lock":
This commit is contained in:
J M
2022-05-09 14:53:39 -06:00
committed by GitHub
parent 5d3bc8cfa5
commit 864555bcf0
26 changed files with 2080 additions and 743 deletions
Download Patch File Download Diff File Expand all files Collapse all files

238
backend/src/install/mod.rs
View File

@@ -14,7 +14,7 @@ use futures::{FutureExt, StreamExt, TryStreamExt};
use http::header::CONTENT_LENGTH;
use http::{Request, Response, StatusCode};
use hyper::Body;
use patch_db::{DbHandle, LockType};
use patch_db::{DbHandle, LockReceipt, LockType};
use reqwest::Url;
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::{command, Context};
@@ -25,8 +25,6 @@ use tokio_stream::wrappers::ReadDirStream;
use tracing::instrument;
use self::cleanup::{cleanup_failed, remove_from_current_dependents_lists};
use crate::context::{CliContext, RpcContext};
use crate::core::rpc_continuations::{RequestGuid, RpcContinuation};
use crate::db::model::{
CurrentDependencyInfo, InstalledPackageDataEntry, PackageDataEntry, RecoveredPackageInfo,
StaticDependencyInfo, StaticFiles,
@@ -47,16 +45,24 @@ use crate::util::serde::{display_serializable, IoFormat, Port};
use crate::util::{display_none, AsyncFileExt, Version};
use crate::version::{Current, VersionT};
use crate::volume::asset_dir;
use crate::{
config::ConfigReceipts,
context::{CliContext, RpcContext},
};
use crate::{
core::rpc_continuations::{RequestGuid, RpcContinuation},
dependencies::BreakTransitiveReceipts,
};
use crate::{Error, ErrorKind, ResultExt};
pub mod cleanup;
pub mod progress;
pub mod update;
pub const PKG_ARCHIVE_DIR: &'static str = "package-data/archive";
pub const PKG_PUBLIC_DIR: &'static str = "package-data/public";
pub const PKG_DOCKER_DIR: &'static str = "package-data/docker";
pub const PKG_WASM_DIR: &'static str = "package-data/wasm";
pub const PKG_ARCHIVE_DIR: &str = "package-data/archive";
pub const PKG_PUBLIC_DIR: &str = "package-data/public";
pub const PKG_DOCKER_DIR: &str = "package-data/docker";
pub const PKG_WASM_DIR: &str = "package-data/wasm";
#[command(display(display_serializable))]
pub async fn list(#[context] ctx: RpcContext) -> Result<Vec<(PackageId, Version)>, Error> {
@@ -447,7 +453,7 @@ pub async fn sideload(
});
let cont = RpcContinuation {
created_at: Instant::now(), // TODO
handler: handler,
handler,
};
// gc the map
let mut guard = ctx.rpc_stream_continuations.lock().await;
@@ -562,8 +568,15 @@ pub async fn uninstall_dry(
let mut db = ctx.db.handle();
let mut tx = db.begin().await?;
let mut breakages = BTreeMap::new();
break_all_dependents_transitive(&mut tx, &id, DependencyError::NotInstalled, &mut breakages)
.await?;
let receipts = BreakTransitiveReceipts::new(&mut tx).await?;
break_all_dependents_transitive(
&mut tx,
&id,
DependencyError::NotInstalled,
&mut breakages,
&receipts,
)
.await?;
tx.abort().await?;
@@ -678,6 +691,35 @@ pub async fn delete_recovered(
})
}
pub struct DownloadInstallReceipts {
package_receipts: crate::db::package::PackageReceipts,
manifest_receipts: crate::db::package::ManifestReceipts,
}
impl DownloadInstallReceipts {
pub async fn new<'a>(db: &'a mut impl DbHandle, id: &PackageId) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks, id);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(
locks: &mut Vec<patch_db::LockTargetId>,
id: &PackageId,
) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
let package_receipts = crate::db::package::PackageReceipts::setup(locks);
let manifest_receipts = crate::db::package::ManifestReceipts::setup(locks, id);
move |skeleton_key| {
Ok(Self {
package_receipts: package_receipts(skeleton_key)?,
manifest_receipts: manifest_receipts(skeleton_key)?,
})
}
}
}
#[instrument(skip(ctx, temp_manifest, s9pk))]
pub async fn download_install_s9pk(
ctx: &RpcContext,
@@ -692,14 +734,14 @@ pub async fn download_install_s9pk(
if let Err(e) = async {
let mut db_handle = ctx.db.handle();
let mut tx = db_handle.begin().await?;
let receipts = DownloadInstallReceipts::new(&mut tx, &pkg_id).await?;
// Build set of existing manifests
let mut manifests = Vec::new();
for pkg in crate::db::package::get_packages(&mut tx).await? {
match crate::db::package::get_manifest(&mut tx, &pkg).await? {
Some(m) => {
manifests.push(m);
}
None => {}
for pkg in crate::db::package::get_packages(&mut tx, &receipts.package_receipts).await? {
if let Some(m) =
crate::db::package::get_manifest(&mut tx, &pkg, &receipts.manifest_receipts).await?
{
manifests.push(m);
}
}
// Build map of current port -> ssl mappings
@@ -732,6 +774,7 @@ pub async fn download_install_s9pk(
}
}
}
drop(receipts);
tx.save().await?;
drop(db_handle);
@@ -792,8 +835,9 @@ pub async fn download_install_s9pk(
{
let mut handle = ctx.db.handle();
let mut tx = handle.begin().await?;
let receipts = cleanup::CleanupFailedReceipts::new(&mut tx).await?;
if let Err(e) = cleanup_failed(&ctx, &mut tx, pkg_id).await {
if let Err(e) = cleanup_failed(&ctx, &mut tx, pkg_id, &receipts).await {
tracing::error!("Failed to clean up {}@{}: {}", pkg_id, version, e);
tracing::debug!("{:?}", e);
} else {
@@ -805,6 +849,39 @@ pub async fn download_install_s9pk(
}
}
pub struct InstallS9Receipts {
config: ConfigReceipts,
recovered_packages: LockReceipt<BTreeMap<PackageId, RecoveredPackageInfo>, ()>,
}
impl InstallS9Receipts {
pub async fn new<'a>(db: &'a mut impl DbHandle) -> Result<Self, Error> {
let mut locks = Vec::new();
let setup = Self::setup(&mut locks);
Ok(setup(&db.lock_all(locks).await?)?)
}
pub fn setup(
locks: &mut Vec<patch_db::LockTargetId>,
) -> impl FnOnce(&patch_db::Verifier) -> Result<Self, Error> {
let config = ConfigReceipts::setup(locks);
let recovered_packages = crate::db::DatabaseModel::new()
.recovered_packages()
.make_locker(LockType::Write)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
config: config(skeleton_key)?,
recovered_packages: recovered_packages.verify(skeleton_key)?,
})
}
}
}
#[instrument(skip(ctx, rdr))]
pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
ctx: &RpcContext,
@@ -1183,6 +1260,8 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
},
);
pde.save(&mut tx).await?;
let receipts = InstallS9Receipts::new(&mut tx).await?;
// UpdateDependencyReceipts
let mut dep_errs = model
.expect(&mut tx)
.await?
@@ -1193,7 +1272,14 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
.dependency_errors()
.get_mut(&mut tx)
.await?;
*dep_errs = DependencyErrors::init(ctx, &mut tx, &manifest, &current_dependencies).await?;
*dep_errs = DependencyErrors::init(
ctx,
&mut tx,
&manifest,
&current_dependencies,
&receipts.config.try_heal_receipts,
)
.await?;
dep_errs.save(&mut tx).await?;
if let PackageDataEntry::Updating {
@@ -1244,6 +1330,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
false,
&mut BTreeMap::new(),
&mut BTreeMap::new(),
&receipts.config,
)
.await?;
let mut main_status = crate::db::DatabaseModel::new()
@@ -1261,9 +1348,20 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
*main_status = prev.status.main;
main_status.save(&mut tx).await?;
}
remove_from_current_dependents_lists(&mut tx, pkg_id, prev.current_dependencies.keys())
.await?; // remove previous
add_dependent_to_current_dependents_lists(&mut tx, pkg_id, &current_dependencies).await?; // add new
remove_from_current_dependents_lists(
&mut tx,
pkg_id,
prev.current_dependencies.keys(),
&receipts.config.current_dependents,
)
.await?; // remove previous
add_dependent_to_current_dependents_lists(
&mut tx,
pkg_id,
&current_dependencies,
&receipts.config.current_dependents,
)
.await?; // add new
update_dependency_errors_of_dependents(
ctx,
&mut tx,
@@ -1272,6 +1370,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
.keys()
.chain(prev.current_dependents.keys())
.collect::<BTreeSet<_>>(),
&receipts.config.update_dependency_receipts,
)
.await?;
if &prev.manifest.version != version {
@@ -1290,39 +1389,84 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
&manifest.volumes,
)
.await?;
add_dependent_to_current_dependents_lists(&mut tx, pkg_id, &current_dependencies).await?;
update_dependency_errors_of_dependents(ctx, &mut tx, pkg_id, current_dependents.keys())
.await?;
add_dependent_to_current_dependents_lists(
&mut tx,
pkg_id,
&current_dependencies,
&receipts.config.current_dependents,
)
.await?;
update_dependency_errors_of_dependents(
ctx,
&mut tx,
pkg_id,
current_dependents.keys(),
&receipts.config.update_dependency_receipts,
)
.await?;
} else if let Some(recovered) = {
// solve taxonomy escalation
crate::db::DatabaseModel::new()
.recovered_packages()
.lock(&mut tx, LockType::Write)
.await?;
crate::db::DatabaseModel::new()
.recovered_packages()
.idx_model(pkg_id)
.get(&mut tx, true)
receipts
.recovered_packages
.get(&mut tx)
.await?
.into_owned()
.remove(pkg_id)
} {
handle_recovered_package(recovered, manifest, ctx, pkg_id, version, &mut tx).await?;
add_dependent_to_current_dependents_lists(&mut tx, pkg_id, &current_dependencies).await?;
update_dependency_errors_of_dependents(ctx, &mut tx, pkg_id, current_dependents.keys())
.await?;
handle_recovered_package(
recovered,
manifest,
ctx,
pkg_id,
version,
&mut tx,
&receipts.config,
)
.await?;
add_dependent_to_current_dependents_lists(
&mut tx,
pkg_id,
&current_dependencies,
&receipts.config.current_dependents,
)
.await?;
update_dependency_errors_of_dependents(
ctx,
&mut tx,
pkg_id,
current_dependents.keys(),
&receipts.config.update_dependency_receipts,
)
.await?;
} else {
add_dependent_to_current_dependents_lists(&mut tx, pkg_id, &current_dependencies).await?;
update_dependency_errors_of_dependents(ctx, &mut tx, pkg_id, current_dependents.keys())
.await?;
add_dependent_to_current_dependents_lists(
&mut tx,
pkg_id,
&current_dependencies,
&receipts.config.current_dependents,
)
.await?;
update_dependency_errors_of_dependents(
ctx,
&mut tx,
pkg_id,
current_dependents.keys(),
&receipts.config.update_dependency_receipts,
)
.await?;
}
crate::db::DatabaseModel::new()
.recovered_packages()
.remove(&mut tx, pkg_id)
let recovered_packages = {
let mut r = receipts.recovered_packages.get(&mut tx).await?;
r.remove(pkg_id);
r
};
receipts
.recovered_packages
.set(&mut tx, recovered_packages)
.await?;
if let Some(installed) = pde.installed() {
reconfigure_dependents_with_live_pointers(ctx, &mut tx, installed).await?;
reconfigure_dependents_with_live_pointers(ctx, &mut tx, &receipts.config, installed)
.await?;
}
sql_tx.commit().await?;
@@ -1333,7 +1477,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
Ok(())
}
#[instrument(skip(ctx, tx))]
#[instrument(skip(ctx, tx, receipts))]
async fn handle_recovered_package(
recovered: RecoveredPackageInfo,
manifest: Manifest,
@@ -1341,6 +1485,7 @@ async fn handle_recovered_package(
pkg_id: &PackageId,
version: &Version,
tx: &mut patch_db::Transaction<&mut patch_db::PatchDbHandle>,
receipts: &ConfigReceipts,
) -> Result<(), Error> {
let configured = if let Some(migration) =
manifest
@@ -1361,6 +1506,7 @@ async fn handle_recovered_package(
false,
&mut BTreeMap::new(),
&mut BTreeMap::new(),
&receipts,
)
.await?;
}