Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 12:11:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

migrations and driver fixes

Browse Source
This commit is contained in:
Aiden McClelland
2026-01-05 14:11:57 -07:00
parent a4185d47b5
commit 56b7aa07d6
8 changed files with 443 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
core/src/net/gateway.rs
View File

@@ -703,22 +703,22 @@ async fn watch_ip(
.into_iter()
.map(IpNet::try_from)
.try_collect()?;
let tables = ip4_proxy.route_data().await?.into_iter().filter_map(|d|d.table).collect::<Vec<_>>();
if !tables.is_empty() {
let rules = String::from_utf8(Command::new("ip").arg("rule").arg("list").invoke(ErrorKind::Network).await?)?;
for table in tables {
for subnet in subnets.iter().filter(|s| s.addr().is_ipv4()) {
let subnet_string = subnet.trunc().to_string();
let rule = ["from", &subnet_string, "lookup", &table.to_string()];
if !rules.contains(&rule.join(" ")) {
if rules.contains(&rule[..2].join(" ")) {
Command::new("ip").arg("rule").arg("del").args(&rule[..2]).invoke(ErrorKind::Network).await?;
}
Command::new("ip").arg("rule").arg("add").args(rule).invoke(ErrorKind::Network).await?;
}
}
}
}
// let tables = ip4_proxy.route_data().await?.into_iter().filter_map(|d|d.table).collect::<Vec<_>>();
// if !tables.is_empty() {
// let rules = String::from_utf8(Command::new("ip").arg("rule").arg("list").invoke(ErrorKind::Network).await?)?;
// for table in tables {
// for subnet in subnets.iter().filter(|s| s.addr().is_ipv4()) {
// let subnet_string = subnet.trunc().to_string();
// let rule = ["from", &subnet_string, "lookup", &table.to_string()];
// if !rules.contains(&rule.join(" ")) {
// if rules.contains(&rule[..2].join(" ")) {
// Command::new("ip").arg("rule").arg("del").args(&rule[..2]).invoke(ErrorKind::Network).await?;
// }
// Command::new("ip").arg("rule").arg("add").args(rule).invoke(ErrorKind::Network).await?;
// }
// }
// }
// }
let wan_ip = if !subnets.is_empty()
&& !matches!(
device_type,

28
core/src/registry/migrations/m_01_package_s9pk_array.rs Normal file
View File

@@ -0,0 +1,28 @@
use imbl::vector;
use imbl_value::json;
use super::RegistryMigration;
use crate::prelude::*;
pub struct PackageS9pkArray;
impl RegistryMigration for PackageS9pkArray {
fn name(&self) -> &'static str {
"PackageS9pkArray"
}
fn action(&self, db: &mut Value) -> Result<(), Error> {
for (_, info) in db["index"]["package"]["packages"]
.as_object_mut()
.unwrap()
.iter_mut()
{
for (_, info) in info["versions"].as_object_mut().unwrap().iter_mut() {
let hw_req = info["hardwareRequirements"].take();
let mut s9pk = info["s9pk"].take();
s9pk["urls"] = Value::Array(vector![s9pk["url"].take()]);
info["s9pks"] = Value::Array(vector![Value::Array(vector![hw_req, s9pk])]);
}
}
Ok(())
}
}

7
core/src/registry/migrations/mod.rs
View File

@@ -4,14 +4,17 @@ use crate::prelude::*;
use crate::registry::RegistryDatabase;
mod m_00_package_signer_scope;
mod m_01_package_s9pk_array;
pub trait RegistryMigration {
fn name(&self) -> &'static str;
fn action(&self, db: &mut Value) -> Result<(), Error>;
}
pub const MIGRATIONS: &[&dyn RegistryMigration] =
&[&m_00_package_signer_scope::PackageSignerScopeMigration];
pub const MIGRATIONS: &[&dyn RegistryMigration] = &[
&m_00_package_signer_scope::PackageSignerScopeMigration,
&m_01_package_s9pk_array::PackageS9pkArray,
];
#[instrument(skip_all)]
pub fn run_migrations(db: &mut Model<RegistryDatabase>) -> Result<(), Error> {

318
core/src/registry/package/add.rs
View File

@@ -24,13 +24,14 @@ use crate::sign::ed25519::Ed25519;
use crate::sign::{AnySignature, AnyVerifyingKey, SignatureScheme};
use crate::util::VersionString;
use crate::util::io::TrackingIO;
use crate::util::serde::Base64;
#[derive(Debug, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct AddPackageParams {
#[ts(type = "string")]
pub url: Url,
#[ts(type = "string[]")]
pub urls: Vec<Url>,
#[ts(skip)]
#[serde(rename = "__Auth_signer")]
pub uploader: AnyVerifyingKey,
@@ -41,7 +42,7 @@ pub struct AddPackageParams {
pub async fn add_package(
ctx: RegistryContext,
AddPackageParams {
url,
urls,
uploader,
commitment,
signature,
@@ -52,15 +53,31 @@ pub async fn add_package(
.verify_commitment(&uploader, &commitment, SIG_CONTEXT, &signature)?;
let peek = ctx.db.peek().await;
let uploader_guid = peek.as_index().as_signers().get_signer(&uploader)?;
let ([url], rest) = urls.split_at(1) else {
return Err(Error::new(
eyre!("must specify at least 1 url"),
ErrorKind::InvalidRequest,
));
};
let s9pk = S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
Some(&commitment),
)
.await?;
for url in rest {
S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
Some(&commitment),
)
.await?;
}
let manifest = s9pk.as_manifest();
let mut info = PackageVersionInfo::from_s9pk(&s9pk, url).await?;
let mut info = PackageVersionInfo::from_s9pk(&s9pk, urls).await?;
for (_, s9pk) in &mut info.s9pks {
if !s9pk.signatures.contains_key(&uploader) && s9pk.commitment == commitment {
s9pk.signatures.insert(uploader.clone(), signature.clone());
@@ -88,7 +105,7 @@ pub async fn add_package(
.upsert(&manifest.id, || Ok(Default::default()))?;
let v = package.as_versions_mut();
if let Some(prev) = v.as_idx_mut(&manifest.version) {
prev.mutate(|p| p.merge_with(info))?;
prev.mutate(|p| p.merge_with(info, true))?;
} else {
v.insert(&manifest.version, &info)?;
}
@@ -107,7 +124,10 @@ pub async fn add_package(
#[serde(rename_all = "camelCase")]
pub struct CliAddPackageParams {
pub file: PathBuf,
pub url: Url,
#[arg(long)]
pub url: Vec<Url>,
#[arg(long)]
pub no_verify: bool,
}
pub async fn cli_add_package(
@@ -115,7 +135,12 @@ pub async fn cli_add_package(
context: ctx,
parent_method,
method,
params: CliAddPackageParams { file, url },
params:
CliAddPackageParams {
file,
url,
no_verify,
},
..
}: HandlerArgs<CliContext, CliAddPackageParams>,
) -> Result<(), Error> {
@@ -123,7 +148,19 @@ pub async fn cli_add_package(
let progress = FullProgressTracker::new();
let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(1));
let mut verify_phase = progress.add_phase(InternedString::intern("Verifying URL"), Some(100));
let verify = if !no_verify {
url.iter()
.map(|url| {
let phase = progress.add_phase(
InternedString::from_display(&lazy_format!("Verifying {url}")),
Some(100),
);
(url.clone(), phase)
})
.collect()
} else {
Vec::new()
};
let mut index_phase = progress.add_phase(
InternedString::intern("Adding File to Registry Index"),
Some(1),
@@ -137,11 +174,240 @@ pub async fn cli_add_package(
let signature = Ed25519.sign_commitment(ctx.developer_key()?, &commitment, SIG_CONTEXT)?;
sign_phase.complete();
verify_phase.start();
let source = BufferedHttpSource::new(ctx.client.clone(), url.clone(), verify_phase).await?;
let mut src = S9pk::deserialize(&Arc::new(source), Some(&commitment)).await?;
src.serialize(&mut TrackingIO::new(0, &mut tokio::io::sink()), true)
.await?;
for (url, mut phase) in verify {
phase.start();
let source = BufferedHttpSource::new(ctx.client.clone(), url, phase).await?;
let mut src = S9pk::deserialize(&Arc::new(source), Some(&commitment)).await?;
src.serialize(&mut TrackingIO::new(0, &mut tokio::io::sink()), true)
.await?;
}
index_phase.start();
ctx.call_remote::<RegistryContext>(
&parent_method.into_iter().chain(method).join("."),
imbl_value::json!({
"urls": &url,
"signature": AnySignature::Ed25519(signature),
"commitment": commitment,
}),
)
.await?;
index_phase.complete();
progress.complete();
progress_task.await.with_kind(ErrorKind::Unknown)?;
Ok(())
}
#[derive(Debug, Deserialize, Serialize, Parser, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct RemovePackageParams {
pub id: PackageId,
pub version: VersionString,
#[arg(long)]
pub sighash: Option<Base64<[u8; 32]>>,
#[ts(skip)]
#[arg(skip)]
#[serde(rename = "__Auth_signer")]
pub signer: Option<AnyVerifyingKey>,
}
pub async fn remove_package(
ctx: RegistryContext,
RemovePackageParams {
id,
version,
sighash,
signer,
}: RemovePackageParams,
) -> Result<bool, Error> {
let peek = ctx.db.peek().await;
let signer =
signer.ok_or_else(|| Error::new(eyre!("missing signer"), ErrorKind::InvalidRequest))?;
let signer_guid = peek.as_index().as_signers().get_signer(&signer)?;
let rev = ctx
.db
.mutate(|db| {
if db.as_admins().de()?.contains(&signer_guid)
|| db
.as_index()
.as_package()
.as_packages()
.as_idx(&id)
.or_not_found(&id)?
.as_authorized()
.de()?
.get(&signer_guid)
.map_or(false, |v| version.satisfies(v))
{
if let Some(package) = db
.as_index_mut()
.as_package_mut()
.as_packages_mut()
.as_idx_mut(&id)
{
if let Some(sighash) = sighash {
if if let Some(package) = package.as_versions_mut().as_idx_mut(&version) {
package.as_s9pks_mut().mutate(|s| {
s.retain(|(_, asset)| asset.commitment.root_sighash != sighash);
Ok(s.is_empty())
})?
} else {
false
} {
package.as_versions_mut().remove(&version)?;
}
} else {
package.as_versions_mut().remove(&version)?;
}
}
Ok(())
} else {
Err(Error::new(eyre!("UNAUTHORIZED"), ErrorKind::Authorization))
}
})
.await;
rev.result.map(|_| rev.revision.is_some())
}
#[derive(Debug, Deserialize, Serialize, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct AddMirrorParams {
#[ts(type = "string")]
pub url: Url,
#[ts(skip)]
#[serde(rename = "__Auth_signer")]
pub uploader: AnyVerifyingKey,
pub commitment: MerkleArchiveCommitment,
pub signature: AnySignature,
}
pub async fn add_mirror(
ctx: RegistryContext,
AddMirrorParams {
url,
uploader,
commitment,
signature,
}: AddMirrorParams,
) -> Result<(), Error> {
uploader
.scheme()
.verify_commitment(&uploader, &commitment, SIG_CONTEXT, &signature)?;
let peek = ctx.db.peek().await;
let uploader_guid = peek.as_index().as_signers().get_signer(&uploader)?;
let s9pk = S9pk::deserialize(
&Arc::new(HttpSource::new(ctx.client.clone(), url.clone()).await?),
Some(&commitment),
)
.await?;
let manifest = s9pk.as_manifest();
let mut info = PackageVersionInfo::from_s9pk(&s9pk, vec![url]).await?;
for (_, s9pk) in &mut info.s9pks {
if !s9pk.signatures.contains_key(&uploader) && s9pk.commitment == commitment {
s9pk.signatures.insert(uploader.clone(), signature.clone());
}
}
ctx.db
.mutate(|db| {
if db.as_admins().de()?.contains(&uploader_guid)
|| db
.as_index()
.as_package()
.as_packages()
.as_idx(&manifest.id)
.or_not_found(&manifest.id)?
.as_authorized()
.de()?
.get(&uploader_guid)
.map_or(false, |v| manifest.version.satisfies(v))
{
let package = db
.as_index_mut()
.as_package_mut()
.as_packages_mut()
.as_idx_mut(&manifest.id)
.and_then(|p| p.as_versions_mut().as_idx_mut(&manifest.version))
.or_not_found(&lazy_format!("{}@{}", &manifest.id, &manifest.version))?;
package.mutate(|p| p.merge_with(info, false))?;
Ok(())
} else {
Err(Error::new(eyre!("UNAUTHORIZED"), ErrorKind::Authorization))
}
})
.await
.result
}
#[derive(Debug, Deserialize, Serialize, Parser)]
#[command(rename_all = "kebab-case")]
#[serde(rename_all = "camelCase")]
pub struct CliAddMirrorParams {
pub file: PathBuf,
pub url: Url,
pub no_verify: bool,
}
pub async fn cli_add_mirror(
HandlerArgs {
context: ctx,
parent_method,
method,
params:
CliAddMirrorParams {
file,
url,
no_verify,
},
..
}: HandlerArgs<CliContext, CliAddMirrorParams>,
) -> Result<(), Error> {
let s9pk = S9pk::open(&file, None).await?;
let progress = FullProgressTracker::new();
let mut sign_phase = progress.add_phase(InternedString::intern("Signing File"), Some(1));
let verify = if !no_verify {
let url = &url;
vec![(
url.clone(),
progress.add_phase(
InternedString::from_display(&lazy_format!("Verifying {url}")),
Some(100),
),
)]
} else {
Vec::new()
};
let mut index_phase = progress.add_phase(
InternedString::intern("Adding File to Registry Index"),
Some(1),
);
let progress_task =
progress.progress_bar_task(&format!("Adding {} to registry...", file.display()));
sign_phase.start();
let commitment = s9pk.as_archive().commitment().await?;
let signature = Ed25519.sign_commitment(ctx.developer_key()?, &commitment, SIG_CONTEXT)?;
sign_phase.complete();
for (url, mut phase) in verify {
phase.start();
let source = BufferedHttpSource::new(ctx.client.clone(), url, phase).await?;
let mut src = S9pk::deserialize(&Arc::new(source), Some(&commitment)).await?;
src.serialize(&mut TrackingIO::new(0, &mut tokio::io::sink()), true)
.await?;
}
index_phase.start();
ctx.call_remote::<RegistryContext>(
@@ -165,22 +431,26 @@ pub async fn cli_add_package(
#[derive(Debug, Deserialize, Serialize, Parser, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export)]
pub struct RemovePackageParams {
pub struct RemoveMirrorParams {
pub id: PackageId,
pub version: VersionString,
#[arg(long)]
#[ts(type = "string")]
pub url: Url,
#[ts(skip)]
#[arg(skip)]
#[serde(rename = "__Auth_signer")]
pub signer: Option<AnyVerifyingKey>,
}
pub async fn remove_package(
pub async fn remove_mirror(
ctx: RegistryContext,
RemovePackageParams {
RemoveMirrorParams {
id,
version,
url,
signer,
}: RemovePackageParams,
}: RemoveMirrorParams,
) -> Result<(), Error> {
let peek = ctx.db.peek().await;
let signer =
@@ -206,8 +476,20 @@ pub async fn remove_package(
.as_package_mut()
.as_packages_mut()
.as_idx_mut(&id)
.and_then(|p| p.as_versions_mut().as_idx_mut(&version))
{
package.as_versions_mut().remove(&version)?;
package.as_s9pks_mut().mutate(|s| {
s.iter_mut()
.for_each(|(_, asset)| asset.urls.retain(|u| u != &url));
if s.iter().any(|(_, asset)| asset.urls.is_empty()) {
Err(Error::new(
eyre!("cannot remove last mirror from an s9pk"),
ErrorKind::InvalidRequest,
))
} else {
Ok(())
}
})?;
}
Ok(())
} else {

21
core/src/registry/package/index.rs
View File

@@ -145,7 +145,10 @@ pub struct PackageVersionInfo {
pub s9pks: Vec<(HardwareRequirements, RegistryAsset<MerkleArchiveCommitment>)>,
}
impl PackageVersionInfo {
pub async fn from_s9pk<S: FileSource + Clone>(s9pk: &S9pk<S>, url: Url) -> Result<Self, Error> {
pub async fn from_s9pk<S: FileSource + Clone>(
s9pk: &S9pk<S>,
urls: Vec<Url>,
) -> Result<Self, Error> {
Ok(Self {
metadata: PackageMetadata::load(s9pk).await?,
source_version: None, // TODO
@@ -153,7 +156,7 @@ impl PackageVersionInfo {
s9pk.as_manifest().hardware_requirements.clone(),
RegistryAsset {
published_at: Utc::now(),
urls: vec![url],
urls,
commitment: s9pk.as_archive().commitment().await?,
signatures: [(
AnyVerifyingKey::Ed25519(s9pk.as_archive().signer()),
@@ -165,18 +168,22 @@ impl PackageVersionInfo {
)],
})
}
pub fn merge_with(&mut self, other: Self) -> Result<(), Error> {
pub fn merge_with(&mut self, other: Self, replace_urls: bool) -> Result<(), Error> {
for (hw_req, asset) in other.s9pks {
if let Some((_, matching)) = self
.s9pks
.iter_mut()
.find(|(h, s)| s.commitment == asset.commitment && *h == hw_req)
{
for url in asset.urls {
if matching.urls.contains(&url) {
continue;
if replace_urls {
matching.urls = asset.urls;
} else {
for url in asset.urls {
if matching.urls.contains(&url) {
continue;
}
matching.urls.push(url);
}
matching.urls.push(url);
}
} else {
if let Some((h, matching)) = self.s9pks.iter_mut().find(|(h, _)| *h == hw_req) {

34
core/src/registry/package/mod.rs
View File

@@ -32,14 +32,46 @@ pub fn package_api<C: Context>() -> ParentHandler<C> {
.no_display()
.with_about("Add package to registry index"),
)
.subcommand(
"add-mirror",
from_fn_async(add::add_mirror)
.with_metadata("get_signer", Value::Bool(true))
.no_cli(),
)
.subcommand(
"add-mirror",
from_fn_async(add::cli_add_mirror)
.no_display()
.with_about("Add a mirror for an s9pk"),
)
.subcommand(
"remove",
from_fn_async(add::remove_package)
.with_metadata("get_signer", Value::Bool(true))
.no_display()
.with_custom_display_fn(|args, changed| {
if !changed {
tracing::warn!(
"{}@{}{} does not exist, so not removed",
args.params.id,
args.params.version,
args.params
.sighash
.map_or(String::new(), |h| format!("#{h}"))
);
}
Ok(())
})
.with_about("Remove package from registry index")
.with_call_remote::<CliContext>(),
)
.subcommand(
"remove-mirror",
from_fn_async(add::remove_mirror)
.with_metadata("get_signer", Value::Bool(true))
.no_display()
.with_about("Remove a mirror from a package")
.with_call_remote::<CliContext>(),
)
.subcommand(
"signer",
signer::signer_api::<C>().with_about("Add, remove, and list package signers"),