wip(fix): Working on fixing the migration. (#2771)

* wip(fix): Working on fixing the migration.

* get s9pk compat key

* wip: Change to just using the archive to not use the manifest parsing.

* fix: Fix the rebuild

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>

core/startos/src/s9pk/v2/mod.rs

@@ -102,6 +102,19 @@ impl<S: FileSource + Clone> S9pk<S> {
         })
     }
 
+    pub fn new_with_manifest(
+        archive: MerkleArchive<S>,
+        size: Option<u64>,
+        manifest: Manifest,
+    ) -> Self {
+        Self {
+            manifest,
+            manifest_dirty: true,
+            archive,
+            size,
+        }
+    }
+
     pub fn validate_and_filter(&mut self, arch: Option<&str>) -> Result<(), Error> {
         let filter = self.manifest.validate_for(arch, self.archive.contents())?;
         filter.keep_checked(self.archive.contents_mut())
@@ -263,10 +276,10 @@ impl<S: From<TmpSource<PackSource>> + FileSource + Clone> S9pk<S> {
 
 impl<S: ArchiveSource + Clone> S9pk<Section<S>> {
     #[instrument(skip_all)]
-    pub async fn deserialize(
+    pub async fn archive(
         source: &S,
         commitment: Option<&MerkleArchiveCommitment>,
-    ) -> Result<Self, Error> {
+    ) -> Result<MerkleArchive<Section<S>>, Error> {
         use tokio::io::AsyncReadExt;
 
         let mut header = source
@@ -283,9 +296,14 @@ impl<S: ArchiveSource + Clone> S9pk<Section<S>> {
             ErrorKind::ParseS9pk,
             "Invalid Magic or Unexpected Version"
         );
-
-        let mut archive =
-            MerkleArchive::deserialize(source, SIG_CONTEXT, &mut header, commitment).await?;
+        MerkleArchive::deserialize(source, SIG_CONTEXT, &mut header, commitment).await
+    }
+    #[instrument(skip_all)]
+    pub async fn deserialize(
+        source: &S,
+        commitment: Option<&MerkleArchiveCommitment>,
+    ) -> Result<Self, Error> {
+        let mut archive = Self::archive(source, commitment).await?;
 
         archive.sort_by(|a, b| match (priority(a), priority(b)) {
             (Some(a), Some(b)) => a.cmp(&b),

core/startos/src/version/mod.rs

@@ -26,8 +26,9 @@ mod v0_3_6_alpha_4;
 mod v0_3_6_alpha_5;
 mod v0_3_6_alpha_6;
 mod v0_3_6_alpha_7;
+mod v0_3_6_alpha_8;
 
-pub type Current = v0_3_6_alpha_7::Version; // VERSION_BUMP
+pub type Current = v0_3_6_alpha_8::Version; // VERSION_BUMP
 
 impl Current {
     #[instrument(skip(self, db))]
@@ -104,6 +105,7 @@ enum Version {
     V0_3_6_alpha_5(Wrapper<v0_3_6_alpha_5::Version>),
     V0_3_6_alpha_6(Wrapper<v0_3_6_alpha_6::Version>),
     V0_3_6_alpha_7(Wrapper<v0_3_6_alpha_7::Version>),
+    V0_3_6_alpha_8(Wrapper<v0_3_6_alpha_8::Version>),
     Other(exver::Version),
 }
 
@@ -135,6 +137,7 @@ impl Version {
             Self::V0_3_6_alpha_5(v) => DynVersion(Box::new(v.0)),
             Self::V0_3_6_alpha_6(v) => DynVersion(Box::new(v.0)),
             Self::V0_3_6_alpha_7(v) => DynVersion(Box::new(v.0)),
+            Self::V0_3_6_alpha_8(v) => DynVersion(Box::new(v.0)),
             Self::Other(v) => {
                 return Err(Error::new(
                     eyre!("unknown version {v}"),
@@ -158,6 +161,7 @@ impl Version {
             Version::V0_3_6_alpha_5(Wrapper(x)) => x.semver(),
             Version::V0_3_6_alpha_6(Wrapper(x)) => x.semver(),
             Version::V0_3_6_alpha_7(Wrapper(x)) => x.semver(),
+            Version::V0_3_6_alpha_8(Wrapper(x)) => x.semver(),
             Version::Other(x) => x.clone(),
         }
     }

core/startos/src/version/v0_3_6_alpha_8.rs

@@ -0,0 +1,125 @@
+use exver::{PreReleaseSegment, VersionRange};
+use tokio::fs::File;
+
+use super::v0_3_5::V0_3_0_COMPAT;
+use super::{v0_3_6_alpha_7, VersionT};
+use crate::s9pk::manifest::{DeviceFilter, Manifest};
+use crate::s9pk::merkle_archive::MerkleArchive;
+use crate::s9pk::v2::SIG_CONTEXT;
+use crate::s9pk::S9pk;
+use crate::util::io::create_file;
+use crate::{
+    install::PKG_ARCHIVE_DIR, s9pk::merkle_archive::source::multi_cursor_file::MultiCursorFile,
+};
+use crate::{prelude::*, service::LoadDisposition};
+
+lazy_static::lazy_static! {
+    static ref V0_3_6_alpha_8: exver::Version = exver::Version::new(
+        [0, 3, 6],
+        [PreReleaseSegment::String("alpha".into()), 8.into()]
+    );
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct Version;
+
+impl VersionT for Version {
+    type Previous = v0_3_6_alpha_7::Version;
+    type PreUpRes = ();
+
+    async fn pre_up(self) -> Result<Self::PreUpRes, Error> {
+        Ok(())
+    }
+    fn semver(self) -> exver::Version {
+        V0_3_6_alpha_8.clone()
+    }
+    fn compat(self) -> &'static VersionRange {
+        &V0_3_0_COMPAT
+    }
+    fn up(self, _: &mut Value, _: Self::PreUpRes) -> Result<(), Error> {
+        Ok(())
+    }
+    async fn post_up(self, ctx: &crate::context::RpcContext) -> Result<(), Error> {
+        let s9pk_dir = ctx.datadir.join(PKG_ARCHIVE_DIR).join("installed");
+
+        for s9pk_path in s9pk_dir.read_dir()? {
+            let s9pk_path = s9pk_path?.path();
+            let matches_s9pk = s9pk_path.extension().map(|x| x == "s9pk").unwrap_or(false);
+            if !matches_s9pk {
+                continue;
+            }
+
+            let get_archive = async {
+                let multi_cursor = MultiCursorFile::from(File::open(&s9pk_path).await?);
+                Ok::<_, Error>(S9pk::archive(&multi_cursor, None).await?)
+            };
+
+            let archive: MerkleArchive<
+                crate::s9pk::merkle_archive::source::Section<MultiCursorFile>,
+            > = match get_archive.await {
+                Ok(a) => a,
+                Err(e) => {
+                    tracing::error!("Error opening s9pk for install: {e}");
+                    tracing::debug!("{e:?}");
+                    continue;
+                }
+            };
+
+            let previous_manifest: Value = serde_json::from_slice::<serde_json::Value>(
+                &archive
+                    .contents()
+                    .get_path("manifest.json")
+                    .or_not_found("manifest.json")?
+                    .read_file_to_vec()
+                    .await?,
+            )
+            .with_kind(ErrorKind::Deserialization)?
+            .into();
+
+            let mut manifest = previous_manifest.clone();
+
+            if let Some(device) = previous_manifest["hardwareRequirements"]["device"].as_object() {
+                manifest["hardwareRequirements"]["device"] = to_value(
+                    &device
+                        .into_iter()
+                        .map(|(class, product)| {
+                            Ok::<_, Error>(DeviceFilter {
+                                pattern_description: format!(
+                                    "a {class} device matching the expression {}",
+                                    &product
+                                ),
+                                class: class.clone(),
+                                pattern: from_value(product.clone())?,
+                            })
+                        })
+                        .fold(Ok::<_, Error>(Vec::new()), |acc, value| {
+                            let mut acc = acc?;
+                            acc.push(value?);
+                            Ok(acc)
+                        })?,
+                )?;
+            }
+
+            if previous_manifest != manifest {
+                let tmp_path = s9pk_path.with_extension("s9pk.tmp");
+                let mut tmp_file = create_file(&tmp_path).await?;
+                // TODO, wouldn't this break in the later versions of the manifest that would need changes, this doesn't seem to be a good way to handle this
+                let manifest: Manifest = from_value(manifest.clone())?;
+                let id = manifest.id.clone();
+                let mut s9pk: S9pk<_> = S9pk::new_with_manifest(archive, None, manifest);
+                let s9pk_compat_key = ctx.account.read().await.compat_s9pk_key.clone();
+                s9pk.as_archive_mut()
+                    .set_signer(s9pk_compat_key, SIG_CONTEXT);
+                s9pk.serialize(&mut tmp_file, true).await?;
+                tmp_file.sync_all().await?;
+                tokio::fs::rename(&tmp_path, &s9pk_path).await?;
+                ctx.services.load(ctx, &id, LoadDisposition::Retry).await?;
+            }
+        }
+
+        Ok(())
+    }
+    fn down(self, _db: &mut Value) -> Result<(), Error> {
+        Ok(())
+    }
+}