Merge branch 'next/patch' of github.com:Start9Labs/start-os into next/minor

.gitignore

@@ -27,4 +27,5 @@ secrets.db
 /results
 /dpkg-workdir
 /compiled.tar
-/compiled-*.tar
+/compiled-*.tar
+/firmware

Makefile

@@ -8,7 +8,8 @@ ARCH := $(shell if [ "$(PLATFORM)" = "raspberrypi" ]; then echo aarch64; else ec
 IMAGE_TYPE=$(shell if [ "$(PLATFORM)" = raspberrypi ]; then echo img; else echo iso; fi)
 BINS := core/target/$(ARCH)-unknown-linux-gnu/release/startbox core/target/aarch64-unknown-linux-musl/release/container-init core/target/x86_64-unknown-linux-musl/release/container-init
 WEB_UIS := web/dist/raw/ui web/dist/raw/setup-wizard web/dist/raw/diagnostic-ui web/dist/raw/install-wizard
-BUILD_SRC := $(shell git ls-files build) build/lib/depends build/lib/conflicts
+FIRMWARE_ROMS := ./firmware/$(PLATFORM) $(shell jq --raw-output '.[] | select(.platform[] | contains("$(PLATFORM)")) | "./firmware/$(PLATFORM)/" + .id + ".rom.gz"' build/lib/firmware.json)
+BUILD_SRC := $(shell git ls-files build) build/lib/depends build/lib/conflicts $(FIRMWARE_ROMS)
 DEBIAN_SRC := $(shell git ls-files debian/)
 IMAGE_RECIPE_SRC := $(shell git ls-files image-recipe/)
 STARTD_SRC := core/startos/startd.service $(BUILD_SRC)
@@ -72,6 +73,7 @@ clean:
 	rm -rf dpkg-workdir
 	rm -rf image-recipe/deb
 	rm -rf results
+	rm -rf build/lib/firmware
 	rm -f ENVIRONMENT.txt
 	rm -f PLATFORM.txt
 	rm -f GIT_HASH.txt
@@ -134,6 +136,8 @@ install: $(ALL_TARGETS)
 	$(call cp,system-images/compat/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/startos/system-images/compat.tar)
 	$(call cp,system-images/utils/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/startos/system-images/utils.tar)
 	$(call cp,system-images/binfmt/docker-images/$(ARCH).tar,$(DESTDIR)/usr/lib/startos/system-images/binfmt.tar)
+	
+	$(call cp,firmware/$(PLATFORM),$(DESTDIR)/usr/lib/startos/firmware)
 
 update-overlay: $(ALL_TARGETS)
 	@echo "\033[33m!!! THIS WILL ONLY REFLASH YOUR DEVICE IN MEMORY !!!\033[0m"
@@ -165,6 +169,9 @@ upload-ota: results/$(BASENAME).squashfs
 build/lib/depends build/lib/conflicts: build/dpkg-deps/*
 	build/dpkg-deps/generate.sh
 
+$(FIRMWARE_ROMS): build/lib/firmware.json download-firmware.sh $(PLATFORM_FILE)
+	./download-firmware.sh $(PLATFORM)
+
 system-images/compat/docker-images/$(ARCH).tar: $(COMPAT_SRC) core/Cargo.lock
 	cd system-images/compat && make docker-images/$(ARCH).tar && touch docker-images/$(ARCH).tar
 

build/lib/firmware.json

@@ -0,0 +1,13 @@
+[
+  {
+    "id": "pureboot-librem_mini_v2-basic_usb_autoboot_blob_jail-Release-28.3",
+    "platform": ["x86_64"],
+    "system-product-name": "librem_mini_v2",
+    "bios-version": {
+      "semver-prefix": "PureBoot-Release-",
+      "semver-range": "<28.3"
+    },
+    "url": "https://source.puri.sm/firmware/releases/-/raw/master/librem_mini_v2/custom/pureboot-librem_mini_v2-basic_usb_autoboot_blob_jail-Release-28.3.rom.gz",
+    "shasum": "5019bcf53f7493c7aa74f8ef680d18b5fc26ec156c705a841433aaa2fdef8f35"
+  }
+]

core/Cargo.lock

@@ -4261,6 +4261,9 @@ name = "semver"
 version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "semver-parser"
@@ -5025,6 +5028,7 @@ dependencies = [
  "rpc-toolkit",
  "rust-argon2",
  "scopeguard",
+ "semver 1.0.20",
  "serde",
  "serde_json",
  "serde_with",

core/startos/Cargo.toml

@@ -130,6 +130,7 @@ rpassword = "7.2.0"
 rpc-toolkit = "0.2.2"
 rust-argon2 = "2.0.0"
 scopeguard = "1.1" # because avahi-sys fucks your shit up
+semver = { version = "1.0.20", features = ["serde"] }
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_cbor = { package = "ciborium", version = "0.2.1" }
 serde_json = "1.0"

core/startos/src/bins/start_init.rs

@@ -9,7 +9,7 @@ use tracing::instrument;
 
 use crate::context::rpc::RpcContextConfig;
 use crate::context::{DiagnosticContext, InstallContext, SetupContext};
-use crate::disk::fsck::RepairStrategy;
+use crate::disk::fsck::{RepairStrategy, RequiresReboot};
 use crate::disk::main::DEFAULT_PASSWORD;
 use crate::disk::REPAIR_DISK_PATH;
 use crate::firmware::update_firmware;
@@ -30,11 +30,18 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Er
         }
     }));
 
-    if update_firmware().await?.0 {
-        return Ok(Some(Shutdown {
-            export_args: None,
-            restart: true,
-        }));
+    match update_firmware().await {
+        Ok(RequiresReboot(true)) => {
+            return Ok(Some(Shutdown {
+                export_args: None,
+                restart: true,
+            }))
+        }
+        Err(e) => {
+            tracing::warn!("Error performing firmware update: {e}");
+            tracing::debug!("{e:?}");
+        }
+        _ => (),
     }
 
     Command::new("ln")

core/startos/src/disk/fsck/mod.rs

@@ -11,7 +11,7 @@ use crate::Error;
 pub mod btrfs;
 pub mod ext4;
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)]
 #[must_use]
 pub struct RequiresReboot(pub bool);
 impl std::ops::BitOrAssign for RequiresReboot {

core/startos/src/firmware.rs

@@ -1,16 +1,63 @@
+use std::collections::BTreeSet;
 use std::path::Path;
 
 use async_compression::tokio::bufread::GzipDecoder;
+use clap::ArgMatches;
+use rpc_toolkit::command;
+use serde::{Deserialize, Serialize};
 use tokio::fs::File;
-use tokio::io::{AsyncRead, BufReader};
+use tokio::io::BufReader;
 use tokio::process::Command;
 
 use crate::disk::fsck::RequiresReboot;
 use crate::prelude::*;
 use crate::util::Invoke;
+use crate::PLATFORM;
 
+/// Part of the Firmware, look there for more about
+#[derive(Clone, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct VersionMatcher {
+    /// Strip this prefix on the version matcher
+    semver_prefix: Option<String>,
+    /// Match the semver to this range
+    semver_range: Option<semver::VersionReq>,
+    /// Strip this suffix on the version matcher
+    semver_suffix: Option<String>,
+}
+
+/// Inside a file that is firmware.json, we
+/// wanted a structure that could help decide what to do
+/// for each of the firmware versions
+#[derive(Clone, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Firmware {
+    id: String,
+    /// This is the platform(s) the firmware was built for
+    platform: BTreeSet<String>,
+    /// This usally comes from the dmidecode
+    system_product_name: Option<String>,
+    /// The version comes from dmidecode, then we decide if it matches
+    bios_version: Option<VersionMatcher>,
+    /// the hash of the firmware rom.gz
+    shasum: String,
+}
+
+fn display_firmware_update_result(arg: RequiresReboot, _: &ArgMatches) {
+    if arg.0 {
+        println!("Firmware successfully updated! Reboot to apply changes.");
+    } else {
+        println!("No firmware update available.");
+    }
+}
+
+/// We wanted to make sure during every init
+/// that the firmware was the correct and updated for
+/// systems like the Pure System that a new firmware
+/// was released and the updates where pushed through the pure os.
+#[command(rename = "update-firmware", display(display_firmware_update_result))]
 pub async fn update_firmware() -> Result<RequiresReboot, Error> {
-    let product_name = String::from_utf8(
+    let system_product_name = String::from_utf8(
         Command::new("dmidecode")
             .arg("-s")
             .arg("system-product-name")
@@ -19,52 +66,84 @@ pub async fn update_firmware() -> Result<RequiresReboot, Error> {
     )?
     .trim()
     .to_owned();
-    if product_name.is_empty() {
+    let bios_version = String::from_utf8(
+        Command::new("dmidecode")
+            .arg("-s")
+            .arg("bios-version")
+            .invoke(ErrorKind::Firmware)
+            .await?,
+    )?
+    .trim()
+    .to_owned();
+    if system_product_name.is_empty() || bios_version.is_empty() {
         return Ok(RequiresReboot(false));
     }
-    let firmware_dir = Path::new("/usr/lib/startos/firmware").join(&product_name);
-    if tokio::fs::metadata(&firmware_dir).await.is_ok() {
-        let current_firmware = String::from_utf8(
-            Command::new("dmidecode")
-                .arg("-s")
-                .arg("bios-version")
-                .invoke(ErrorKind::Firmware)
-                .await?,
-        )?
-        .trim()
-        .to_owned();
-        if tokio::fs::metadata(firmware_dir.join(format!("{current_firmware}.rom.gz")))
-            .await
-            .is_err()
-            && tokio::fs::metadata(firmware_dir.join(format!("{current_firmware}.rom")))
-                .await
-                .is_err()
-        {
-            let mut firmware_read_dir = tokio::fs::read_dir(&firmware_dir).await?;
-            while let Some(entry) = firmware_read_dir.next_entry().await? {
-                let filename = entry.file_name().to_string_lossy().into_owned();
-                let rdr: Option<Box<dyn AsyncRead + Unpin + Send>> =
-                    if filename.ends_with(".rom.gz") {
-                        Some(Box::new(GzipDecoder::new(BufReader::new(
-                            File::open(entry.path()).await?,
-                        ))))
-                    } else if filename.ends_with(".rom") {
-                        Some(Box::new(File::open(entry.path()).await?))
-                    } else {
-                        None
-                    };
-                if let Some(mut rdr) = rdr {
-                    Command::new("flashrom")
-                        .arg("-p")
-                        .arg("internal")
-                        .arg("-w-")
-                        .input(Some(&mut rdr))
-                        .invoke(ErrorKind::Firmware)
-                        .await?;
-                    return Ok(RequiresReboot(true));
+
+    let firmware_dir = Path::new("/usr/lib/startos/firmware");
+
+    for firmware in serde_json::from_str::<Vec<Firmware>>(
+        &tokio::fs::read_to_string("/usr/lib/startos/firmware.json").await?,
+    )
+    .with_kind(ErrorKind::Deserialization)?
+    {
+        let id = firmware.id;
+        let matches_product_name = firmware
+            .system_product_name
+            .map_or(true, |spn| spn == system_product_name);
+        let matches_bios_version = firmware
+            .bios_version
+            .map_or(Some(true), |bv| {
+                let mut semver_str = bios_version.as_str();
+                if let Some(prefix) = &bv.semver_prefix {
+                    semver_str = semver_str.strip_prefix(prefix)?;
                 }
-            }
+                if let Some(suffix) = &bv.semver_suffix {
+                    semver_str = semver_str.strip_suffix(suffix)?;
+                }
+                let semver = semver_str
+                    .split(".")
+                    .filter_map(|v| v.parse().ok())
+                    .chain(std::iter::repeat(0))
+                    .take(3)
+                    .collect::<Vec<_>>();
+                let semver = semver::Version::new(semver[0], semver[1], semver[2]);
+                Some(
+                    bv.semver_range
+                        .as_ref()
+                        .map_or(true, |r| r.matches(&semver)),
+                )
+            })
+            .unwrap_or(false);
+        if firmware.platform.contains(&*PLATFORM) && matches_product_name && matches_bios_version {
+            let filename = format!("{id}.rom.gz");
+            let firmware_path = firmware_dir.join(&filename);
+            Command::new("sha256sum")
+                .arg("-c")
+                .input(Some(&mut std::io::Cursor::new(format!(
+                    "{} {}",
+                    firmware.shasum,
+                    firmware_path.display()
+                ))))
+                .invoke(ErrorKind::Filesystem)
+                .await?;
+            let mut rdr = if tokio::fs::metadata(&firmware_path).await.is_ok() {
+                GzipDecoder::new(BufReader::new(File::open(&firmware_path).await?))
+            } else {
+                return Err(Error::new(
+                    eyre!("Firmware {id}.rom.gz not found in {firmware_dir:?}"),
+                    ErrorKind::NotFound,
+                ));
+            };
+            Command::new("flashrom")
+                .arg("-p")
+                .arg("internal")
+                .arg("-w-")
+                .input(Some(&mut rdr))
+                .invoke(ErrorKind::Firmware)
+                .await?;
+            return Ok(RequiresReboot(true));
         }
     }
+
     Ok(RequiresReboot(false))
 }

core/startos/src/init.rs

@@ -96,44 +96,64 @@ pub async fn init_postgres(datadir: impl AsRef<Path>) -> Result<(), Error> {
 
     let pg_version_string = pg_version.to_string();
     let pg_version_path = db_dir.join(&pg_version_string);
-    if tokio::fs::metadata(&pg_version_path).await.is_err() {
-        let conf_dir = Path::new("/etc/postgresql").join(pg_version.to_string());
-        let conf_dir_tmp = {
-            let mut tmp = conf_dir.clone();
-            tmp.set_extension("tmp");
-            tmp
-        };
-        if tokio::fs::metadata(&conf_dir).await.is_ok() {
-            Command::new("mv")
-                .arg(&conf_dir)
-                .arg(&conf_dir_tmp)
-                .invoke(ErrorKind::Filesystem)
-                .await?;
-        }
-        let mut old_version = pg_version;
-        while old_version > 13
-        /* oldest pg version included in startos */
+    if exists
+    // maybe migrate
+    {
+        let incomplete_path = db_dir.join(format!("{pg_version}.migration.incomplete"));
+        if tokio::fs::metadata(&incomplete_path).await.is_ok() // previous migration was incomplete
+        && tokio::fs::metadata(&pg_version_path).await.is_ok()
         {
-            old_version -= 1;
-            let old_datadir = db_dir.join(old_version.to_string());
-            if tokio::fs::metadata(&old_datadir).await.is_ok() {
-                Command::new("pg_upgradecluster")
-                    .arg(old_version.to_string())
-                    .arg("main")
-                    .invoke(crate::ErrorKind::Database)
-                    .await?;
-                break;
-            }
+            tokio::fs::remove_dir_all(&pg_version_path).await?;
         }
-        if tokio::fs::metadata(&conf_dir).await.is_ok() {
+        if tokio::fs::metadata(&pg_version_path).await.is_err()
+        // need to migrate
+        {
+            let conf_dir = Path::new("/etc/postgresql").join(pg_version.to_string());
+            let conf_dir_tmp = {
+                let mut tmp = conf_dir.clone();
+                tmp.set_extension("tmp");
+                tmp
+            };
             if tokio::fs::metadata(&conf_dir).await.is_ok() {
-                tokio::fs::remove_dir_all(&conf_dir).await?;
+                Command::new("mv")
+                    .arg(&conf_dir)
+                    .arg(&conf_dir_tmp)
+                    .invoke(ErrorKind::Filesystem)
+                    .await?;
             }
-            Command::new("mv")
-                .arg(&conf_dir_tmp)
-                .arg(&conf_dir)
-                .invoke(ErrorKind::Filesystem)
-                .await?;
+            let mut old_version = pg_version;
+            while old_version > 13
+            /* oldest pg version included in startos */
+            {
+                old_version -= 1;
+                let old_datadir = db_dir.join(old_version.to_string());
+                if tokio::fs::metadata(&old_datadir).await.is_ok() {
+                    tokio::fs::File::create(&incomplete_path)
+                        .await?
+                        .sync_all()
+                        .await?;
+                    Command::new("pg_upgradecluster")
+                        .arg(old_version.to_string())
+                        .arg("main")
+                        .invoke(crate::ErrorKind::Database)
+                        .await?;
+                    break;
+                }
+            }
+            if tokio::fs::metadata(&conf_dir).await.is_ok() {
+                if tokio::fs::metadata(&conf_dir).await.is_ok() {
+                    tokio::fs::remove_dir_all(&conf_dir).await?;
+                }
+                Command::new("mv")
+                    .arg(&conf_dir_tmp)
+                    .arg(&conf_dir)
+                    .invoke(ErrorKind::Filesystem)
+                    .await?;
+            }
+            tokio::fs::remove_file(&incomplete_path).await?;
+        }
+        if tokio::fs::metadata(&incomplete_path).await.is_ok() {
+            unreachable!() // paranoia
         }
     }
 
@@ -306,12 +326,13 @@ pub async fn init(cfg: &RpcContextConfig) -> Result<InitResult, Error> {
             tracing::info!("Created Docker Network");
         }
 
+        let datadir = cfg.datadir();
         tracing::info!("Loading System Docker Images");
-        crate::install::load_images("/usr/lib/startos/system-images").await?;
+        crate::install::rebuild_from("/usr/lib/startos/system-images", &datadir).await?;
         tracing::info!("Loaded System Docker Images");
 
         tracing::info!("Loading Package Docker Images");
-        crate::install::load_images(cfg.datadir().join(PKG_ARCHIVE_DIR)).await?;
+        crate::install::rebuild_from(datadir.join(PKG_ARCHIVE_DIR), &datadir).await?;
         tracing::info!("Loaded Package Docker Images");
     }
 

core/startos/src/install/mod.rs

@@ -891,102 +891,11 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
     }
     tracing::info!("Install {}@{}: Fetched Dependency Info", pkg_id, version);
 
-    let public_dir_path = ctx
-        .datadir
-        .join(PKG_PUBLIC_DIR)
-        .join(pkg_id)
-        .join(version.as_str());
-    tokio::fs::create_dir_all(&public_dir_path).await?;
-
-    tracing::info!("Install {}@{}: Unpacking LICENSE.md", pkg_id, version);
-    progress
-        .track_read_during(ctx.db.clone(), pkg_id, || async {
-            let license_path = public_dir_path.join("LICENSE.md");
-            let mut dst = File::create(&license_path).await?;
-            tokio::io::copy(&mut rdr.license().await?, &mut dst).await?;
-            dst.sync_all().await?;
-            Ok(())
+    let icon = progress
+        .track_read_during(ctx.db.clone(), pkg_id, || {
+            unpack_s9pk(&ctx.datadir, &manifest, rdr)
         })
         .await?;
-    tracing::info!("Install {}@{}: Unpacked LICENSE.md", pkg_id, version);
-
-    tracing::info!("Install {}@{}: Unpacking INSTRUCTIONS.md", pkg_id, version);
-    progress
-        .track_read_during(ctx.db.clone(), pkg_id, || async {
-            let instructions_path = public_dir_path.join("INSTRUCTIONS.md");
-            let mut dst = File::create(&instructions_path).await?;
-            tokio::io::copy(&mut rdr.instructions().await?, &mut dst).await?;
-            dst.sync_all().await?;
-            Ok(())
-        })
-        .await?;
-    tracing::info!("Install {}@{}: Unpacked INSTRUCTIONS.md", pkg_id, version);
-
-    let icon_filename = Path::new("icon").with_extension(manifest.assets.icon_type());
-    let icon_path = public_dir_path.join(&icon_filename);
-    tracing::info!(
-        "Install {}@{}: Unpacking {}",
-        pkg_id,
-        version,
-        icon_path.display()
-    );
-    let icon_buf = progress
-        .track_read_during(ctx.db.clone(), pkg_id, || async {
-            Ok(rdr.icon().await?.to_vec().await?)
-        })
-        .await?;
-    let mut dst = File::create(&icon_path).await?;
-    dst.write_all(&icon_buf).await?;
-    dst.sync_all().await?;
-    let icon = DataUrl::from_vec(
-        mime(manifest.assets.icon_type()).unwrap_or("image/png"),
-        icon_buf,
-    );
-    tracing::info!(
-        "Install {}@{}: Unpacked {}",
-        pkg_id,
-        version,
-        icon_filename.display()
-    );
-
-    tracing::info!("Install {}@{}: Unpacking Docker Images", pkg_id, version);
-    progress
-        .track_read_during(ctx.db.clone(), pkg_id, || async {
-            Command::new(CONTAINER_TOOL)
-                .arg("load")
-                .input(Some(&mut rdr.docker_images().await?))
-                .invoke(ErrorKind::Docker)
-                .await
-        })
-        .await?;
-    tracing::info!("Install {}@{}: Unpacked Docker Images", pkg_id, version,);
-
-    tracing::info!("Install {}@{}: Unpacking Assets", pkg_id, version);
-    progress
-        .track_read_during(ctx.db.clone(), pkg_id, || async {
-            let asset_dir = asset_dir(&ctx.datadir, pkg_id, version);
-            if tokio::fs::metadata(&asset_dir).await.is_err() {
-                tokio::fs::create_dir_all(&asset_dir).await?;
-            }
-            let mut tar = tokio_tar::Archive::new(rdr.assets().await?);
-            tar.unpack(asset_dir).await?;
-
-            let script_dir = script_dir(&ctx.datadir, pkg_id, version);
-            if tokio::fs::metadata(&script_dir).await.is_err() {
-                tokio::fs::create_dir_all(&script_dir).await?;
-            }
-            if let Some(mut hdl) = rdr.scripts().await? {
-                tokio::io::copy(
-                    &mut hdl,
-                    &mut File::create(script_dir.join("embassy.js")).await?,
-                )
-                .await?;
-            }
-
-            Ok(())
-        })
-        .await?;
-    tracing::info!("Install {}@{}: Unpacked Assets", pkg_id, version);
 
     progress.unpack_complete.store(true, Ordering::SeqCst);
 
@@ -1107,6 +1016,8 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
     let mut auto_start = false;
     let mut configured = false;
 
+    let mut to_cleanup = None;
+
     if let PackageDataEntry::Updating(PackageDataEntryUpdating {
         installed: prev, ..
     }) = &prev
@@ -1148,7 +1059,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
             auto_start = prev.status.main.running();
         }
         if &prev.manifest.version != version {
-            cleanup(&ctx, &prev.manifest.id, &prev.manifest.version).await?;
+            to_cleanup = Some((prev.manifest.id.clone(), prev.manifest.version.clone()));
         }
     } else if let PackageDataEntry::Restoring(PackageDataEntryRestoring { .. }) = prev {
         next.installed.marketplace_url = manifest
@@ -1191,6 +1102,10 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
         })
         .await?;
 
+    if let Some((id, version)) = to_cleanup {
+        cleanup(&ctx, &id, &version).await?;
+    }
+
     if configured && manifest.config.is_some() {
         let breakages = BTreeMap::new();
         let overrides = Default::default();
@@ -1237,15 +1152,103 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
 }
 
 #[instrument(skip_all)]
-pub fn load_images<'a, P: AsRef<Path> + 'a + Send + Sync>(
-    datadir: P,
+pub async fn unpack_s9pk<R: AsyncRead + AsyncSeek + Unpin + Send + Sync>(
+    datadir: impl AsRef<Path>,
+    manifest: &Manifest,
+    rdr: &mut S9pkReader<R>,
+) -> Result<DataUrl<'static>, Error> {
+    let datadir = datadir.as_ref();
+    let pkg_id = &manifest.id;
+    let version = &manifest.version;
+
+    let public_dir_path = datadir
+        .join(PKG_PUBLIC_DIR)
+        .join(pkg_id)
+        .join(version.as_str());
+    tokio::fs::create_dir_all(&public_dir_path).await?;
+
+    tracing::info!("Install {}@{}: Unpacking LICENSE.md", pkg_id, version);
+    let license_path = public_dir_path.join("LICENSE.md");
+    let mut dst = File::create(&license_path).await?;
+    tokio::io::copy(&mut rdr.license().await?, &mut dst).await?;
+    dst.sync_all().await?;
+    tracing::info!("Install {}@{}: Unpacked LICENSE.md", pkg_id, version);
+
+    tracing::info!("Install {}@{}: Unpacking INSTRUCTIONS.md", pkg_id, version);
+    let instructions_path = public_dir_path.join("INSTRUCTIONS.md");
+    let mut dst = File::create(&instructions_path).await?;
+    tokio::io::copy(&mut rdr.instructions().await?, &mut dst).await?;
+    dst.sync_all().await?;
+    tracing::info!("Install {}@{}: Unpacked INSTRUCTIONS.md", pkg_id, version);
+
+    let icon_filename = Path::new("icon").with_extension(manifest.assets.icon_type());
+    let icon_path = public_dir_path.join(&icon_filename);
+    tracing::info!(
+        "Install {}@{}: Unpacking {}",
+        pkg_id,
+        version,
+        icon_path.display()
+    );
+    let icon_buf = rdr.icon().await?.to_vec().await?;
+    let mut dst = File::create(&icon_path).await?;
+    dst.write_all(&icon_buf).await?;
+    dst.sync_all().await?;
+    let icon = DataUrl::from_vec(
+        mime(manifest.assets.icon_type()).unwrap_or("image/png"),
+        icon_buf,
+    );
+    tracing::info!(
+        "Install {}@{}: Unpacked {}",
+        pkg_id,
+        version,
+        icon_filename.display()
+    );
+
+    tracing::info!("Install {}@{}: Unpacking Docker Images", pkg_id, version);
+    Command::new(CONTAINER_TOOL)
+        .arg("load")
+        .input(Some(&mut rdr.docker_images().await?))
+        .invoke(ErrorKind::Docker)
+        .await?;
+    tracing::info!("Install {}@{}: Unpacked Docker Images", pkg_id, version,);
+
+    tracing::info!("Install {}@{}: Unpacking Assets", pkg_id, version);
+    let asset_dir = asset_dir(datadir, pkg_id, version);
+    if tokio::fs::metadata(&asset_dir).await.is_ok() {
+        tokio::fs::remove_dir_all(&asset_dir).await?;
+    }
+    tokio::fs::create_dir_all(&asset_dir).await?;
+    let mut tar = tokio_tar::Archive::new(rdr.assets().await?);
+    tar.unpack(asset_dir).await?;
+
+    let script_dir = script_dir(datadir, pkg_id, version);
+    if tokio::fs::metadata(&script_dir).await.is_err() {
+        tokio::fs::create_dir_all(&script_dir).await?;
+    }
+    if let Some(mut hdl) = rdr.scripts().await? {
+        tokio::io::copy(
+            &mut hdl,
+            &mut File::create(script_dir.join("embassy.js")).await?,
+        )
+        .await?;
+    }
+    tracing::info!("Install {}@{}: Unpacked Assets", pkg_id, version);
+
+    Ok(icon)
+}
+
+#[instrument(skip_all)]
+pub fn rebuild_from<'a>(
+    source: impl AsRef<Path> + 'a + Send + Sync,
+    datadir: impl AsRef<Path> + 'a + Send + Sync,
 ) -> BoxFuture<'a, Result<(), Error>> {
     async move {
-        let docker_dir = datadir.as_ref();
-        if tokio::fs::metadata(&docker_dir).await.is_ok() {
-            ReadDirStream::new(tokio::fs::read_dir(&docker_dir).await?)
+        let source_dir = source.as_ref();
+        let datadir = datadir.as_ref();
+        if tokio::fs::metadata(&source_dir).await.is_ok() {
+            ReadDirStream::new(tokio::fs::read_dir(&source_dir).await?)
                 .map(|r| {
-                    r.with_ctx(|_| (crate::ErrorKind::Filesystem, format!("{:?}", &docker_dir)))
+                    r.with_ctx(|_| (crate::ErrorKind::Filesystem, format!("{:?}", &source_dir)))
                 })
                 .try_for_each(|entry| async move {
                     let m = entry.metadata().await?;
@@ -1260,26 +1263,21 @@ pub fn load_images<'a, P: AsRef<Path> + 'a + Send + Sync>(
                                             .arg("load")
                                             .input(Some(&mut File::open(&path).await?))
                                             .invoke(ErrorKind::Docker)
-                                            .await
+                                            .await?;
+                                        Ok::<_, Error>(())
                                     }
                                     Some("s9pk") => {
-                                        Command::new(CONTAINER_TOOL)
-                                            .arg("load")
-                                            .input(Some(
-                                                &mut S9pkReader::open(&path, true)
-                                                    .await?
-                                                    .docker_images()
-                                                    .await?,
-                                            ))
-                                            .invoke(ErrorKind::Docker)
-                                            .await
+                                        let mut s9pk = S9pkReader::open(&path, true).await?;
+                                        unpack_s9pk(datadir, &s9pk.manifest().await?, &mut s9pk)
+                                            .await?;
+                                        Ok(())
                                     }
                                     _ => unreachable!(),
                                 }
                             }
                             .await
                             {
-                                tracing::error!("Error loading docker images from s9pk: {e}");
+                                tracing::error!("Error unpacking {path:?}: {e}");
                                 tracing::debug!("{e:?}");
                             }
                             Ok(())
@@ -1287,7 +1285,7 @@ pub fn load_images<'a, P: AsRef<Path> + 'a + Send + Sync>(
                             Ok(())
                         }
                     } else if m.is_dir() {
-                        load_images(entry.path()).await?;
+                        rebuild_from(entry.path(), datadir).await?;
                         Ok(())
                     } else {
                         Ok(())

core/startos/src/lib.rs

@@ -105,6 +105,7 @@ pub fn main_api() -> Result<(), RpcError> {
     shutdown::restart,
     shutdown::rebuild,
     update::update_system,
+    firmware::update_firmware,
 ))]
 pub fn server() -> Result<(), RpcError> {
     Ok(())

download-firmware.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+set -e
+
+PLATFORM=$1
+
+if [ -z "$PLATFORM" ]; then
+	>&2 echo "usage: $0 <PLATFORM>"
+	exit 1
+fi
+
+rm -rf ./firmware/$PLATFORM
+mkdir -p ./firmware/$PLATFORM
+
+cd ./firmware/$PLATFORM
+
+mapfile -t firmwares <<< "$(jq -c ".[] | select(.platform[] | contains(\"$PLATFORM\"))" ../../build/lib/firmware.json)"
+for firmware in "${firmwares[@]}"; do
+	if [ -n "$firmware" ]; then
+		id=$(echo "$firmware" | jq --raw-output '.id')
+		url=$(echo "$firmware" | jq --raw-output '.url')
+		shasum=$(echo "$firmware" | jq --raw-output '.shasum')
+		curl --fail -L -o "${id}.rom.gz" "$url"
+		echo "$shasum ${id}.rom.gz" | sha256sum -c
+	fi
+done

image-recipe/build.sh

@@ -344,4 +344,4 @@ elif [ "${IMAGE_TYPE}" = img ]; then
 
 	mv $TARGET_NAME $RESULTS_DIR/$IMAGE_BASENAME.img
 
-fi
+fi

system-images/compat/Cargo.lock

@@ -3800,9 +3800,12 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.16"
+version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
+checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "serde"
@@ -4475,6 +4478,7 @@ dependencies = [
  "rpc-toolkit",
  "rust-argon2 2.0.0",
  "scopeguard",
+ "semver",
  "serde",
  "serde_json",
  "serde_with",