Feature/multi platform (#1866)

* wip

* wip

* wip

* wip

* wip

* wip

* remove debian dir

* lazy env and git hash

* remove env and git hash on clean

* don't leave project dir

* use docker for native builds

* start9 rust

* correctly mount registry

* remove systemd config

* switch to /usr/bin

* disable sound for now

* wip

* change disk list

* multi-arch images

* multi-arch system images

* default aarch64

* edition 2021

* dynamic wifi interface name

* use wifi interface from config

* bugfixes

* add beep based sound

* wip

* wip

* wip

* separate out raspberry pi specific files

* fixes

* use new initramfs always

* switch journald conf to sed script

* fixes

* fix permissions

* talking about kernel modules not scripts

* fix

* fix

* switch to MBR

* install to /usr/lib

* fixes

* fixes

* fixes

* fixes

* add media config to cfg path

* fixes

* fixes

* fixes

* raspi image fixes

* fix test

* fix workflow

* sync boot partition

* gahhhhh

backend/src/disk/fsck.rs

@@ -63,7 +63,7 @@ fn backup_existing_undo_file<'a>(path: &'a Path) -> BoxFuture<'a, Result<(), Err
 pub async fn e2fsck_aggressive(
     logicalname: impl AsRef<Path> + std::fmt::Debug,
 ) -> Result<RequiresReboot, Error> {
-    let undo_path = Path::new("/embassy-os")
+    let undo_path = Path::new("/media/embassy/config")
         .join(
             logicalname
                 .as_ref()

backend/src/disk/mod.rs

@@ -1,6 +1,10 @@
+use std::path::{Path, PathBuf};
+
 use clap::ArgMatches;
 use rpc_toolkit::command;
+use serde::Deserialize;
 
+use crate::context::RpcContext;
 use crate::disk::util::DiskInfo;
 use crate::util::display_none;
 use crate::util::serde::{display_serializable, IoFormat};
@@ -12,7 +16,19 @@ pub mod mount;
 pub mod util;
 
 pub const BOOT_RW_PATH: &str = "/media/boot-rw";
-pub const REPAIR_DISK_PATH: &str = "/embassy-os/repair-disk";
+pub const REPAIR_DISK_PATH: &str = "/media/embassy/config/repair-disk";
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct OsPartitionInfo {
+    pub boot: PathBuf,
+    pub root: PathBuf,
+}
+impl OsPartitionInfo {
+    pub fn contains(&self, logicalname: impl AsRef<Path>) -> bool {
+        &*self.boot == logicalname.as_ref() || &*self.root == logicalname.as_ref()
+    }
+}
 
 #[command(subcommands(list, repair))]
 pub fn disk() -> Result<(), Error> {
@@ -75,11 +91,12 @@ fn display_disk_info(info: Vec<DiskInfo>, matches: &ArgMatches) {
 
 #[command(display(display_disk_info))]
 pub async fn list(
+    #[context] ctx: RpcContext,
     #[allow(unused_variables)]
     #[arg]
     format: Option<IoFormat>,
 ) -> Result<Vec<DiskInfo>, Error> {
-    crate::disk::util::list().await
+    crate::disk::util::list(&ctx.os_partitions).await
 }
 
 #[command(display(display_none))]

backend/src/disk/mount/filesystem/bind.rs

@@ -0,0 +1,54 @@
+use std::os::unix::ffi::OsStrExt;
+use std::path::Path;
+
+use async_trait::async_trait;
+use digest::generic_array::GenericArray;
+use digest::{Digest, OutputSizeUser};
+use sha2::Sha256;
+
+use super::{FileSystem, MountType, ReadOnly};
+use crate::disk::mount::util::bind;
+use crate::{Error, ResultExt};
+
+pub struct Bind<SrcDir: AsRef<Path>> {
+    src_dir: SrcDir,
+}
+impl<SrcDir: AsRef<Path>> Bind<SrcDir> {
+    pub fn new(src_dir: SrcDir) -> Self {
+        Self { src_dir }
+    }
+}
+#[async_trait]
+impl<SrcDir: AsRef<Path> + Send + Sync> FileSystem for Bind<SrcDir> {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        mount_type: MountType,
+    ) -> Result<(), Error> {
+        bind(
+            self.src_dir.as_ref(),
+            mountpoint,
+            matches!(mount_type, ReadOnly),
+        )
+        .await
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        let mut sha = Sha256::new();
+        sha.update("Bind");
+        sha.update(
+            tokio::fs::canonicalize(self.src_dir.as_ref())
+                .await
+                .with_ctx(|_| {
+                    (
+                        crate::ErrorKind::Filesystem,
+                        self.src_dir.as_ref().display().to_string(),
+                    )
+                })?
+                .as_os_str()
+                .as_bytes(),
+        );
+        Ok(sha.finalize())
+    }
+}

backend/src/disk/mount/filesystem/httpdirfs.rs

@@ -0,0 +1,52 @@
+use std::path::Path;
+
+use async_trait::async_trait;
+use digest::generic_array::GenericArray;
+use digest::{Digest, OutputSizeUser};
+use reqwest::Url;
+use serde::{Deserialize, Serialize};
+use sha2::Sha256;
+
+use super::{FileSystem, MountType};
+use crate::util::Invoke;
+use crate::Error;
+
+pub async fn mount_httpdirfs(url: &Url, mountpoint: impl AsRef<Path>) -> Result<(), Error> {
+    tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
+    let mut cmd = tokio::process::Command::new("httpdirfs");
+    cmd.arg("--cache")
+        .arg("--single-file-mode")
+        .arg(url.as_str())
+        .arg(mountpoint.as_ref());
+    cmd.invoke(crate::ErrorKind::Filesystem).await?;
+    Ok(())
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct HttpDirFS {
+    url: Url,
+}
+impl HttpDirFS {
+    pub fn new(url: Url) -> Self {
+        HttpDirFS { url }
+    }
+}
+#[async_trait]
+impl FileSystem for HttpDirFS {
+    async fn mount<P: AsRef<Path> + Send + Sync>(
+        &self,
+        mountpoint: P,
+        _mount_type: MountType,
+    ) -> Result<(), Error> {
+        mount_httpdirfs(&self.url, mountpoint).await
+    }
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
+        let mut sha = Sha256::new();
+        sha.update("HttpDirFS");
+        sha.update(self.url.as_str());
+        Ok(sha.finalize())
+    }
+}

backend/src/disk/mount/filesystem/mod.rs

@@ -7,9 +7,11 @@ use sha2::Sha256;
 
 use crate::Error;
 
+pub mod bind;
 pub mod block_dev;
 pub mod cifs;
 pub mod ecryptfs;
+pub mod httpdirfs;
 pub mod label;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]

backend/src/disk/mount/guard.rs

@@ -11,7 +11,7 @@ use super::util::unmount;
 use crate::util::Invoke;
 use crate::Error;
 
-pub const TMP_MOUNTPOINT: &'static str = "/media/embassy-os/tmp";
+pub const TMP_MOUNTPOINT: &'static str = "/media/embassy/tmp";
 
 #[async_trait::async_trait]
 pub trait GenericMountGuard: AsRef<Path> + std::fmt::Debug + Send + Sync + 'static {

backend/src/disk/util.rs

@@ -19,6 +19,7 @@ use tracing::instrument;
 use super::mount::filesystem::block_dev::BlockDev;
 use super::mount::filesystem::ReadOnly;
 use super::mount::guard::TmpMountGuard;
+use crate::disk::OsPartitionInfo;
 use crate::util::io::from_yaml_async_reader;
 use crate::util::serde::IoFormat;
 use crate::util::{Invoke, Version};
@@ -232,7 +233,11 @@ pub async fn recovery_info(
 }
 
 #[instrument]
-pub async fn list() -> Result<Vec<DiskInfo>, Error> {
+pub async fn list(os: &OsPartitionInfo) -> Result<Vec<DiskInfo>, Error> {
+    struct DiskIndex {
+        parts: IndexSet<PathBuf>,
+        internal: bool,
+    }
     let disk_guids = pvscan().await?;
     let disks = tokio_stream::wrappers::ReadDirStream::new(
         tokio::fs::read_dir(DISK_PATH)
@@ -245,127 +250,157 @@ pub async fn list() -> Result<Vec<DiskInfo>, Error> {
             crate::ErrorKind::Filesystem,
         )
     })
-    .try_fold(BTreeMap::new(), |mut disks, dir_entry| async move {
-        if let Some(disk_path) = dir_entry.path().file_name().and_then(|s| s.to_str()) {
-            let (disk_path, part_path) = if let Some(end) = PARTITION_REGEX.find(disk_path) {
-                (
-                    disk_path.strip_suffix(end.as_str()).unwrap_or_default(),
-                    Some(disk_path),
-                )
-            } else {
-                (disk_path, None)
-            };
-            let disk_path = Path::new(DISK_PATH).join(disk_path);
-            let disk = tokio::fs::canonicalize(&disk_path).await.with_ctx(|_| {
-                (
-                    crate::ErrorKind::Filesystem,
-                    disk_path.display().to_string(),
-                )
-            })?;
-            if &*disk == Path::new("/dev/mmcblk0") {
-                return Ok(disks);
-            }
-            if !disks.contains_key(&disk) {
-                disks.insert(disk.clone(), IndexSet::new());
-            }
-            if let Some(part_path) = part_path {
-                let part_path = Path::new(DISK_PATH).join(part_path);
-                let part = tokio::fs::canonicalize(&part_path).await.with_ctx(|_| {
+    .try_fold(
+        BTreeMap::<PathBuf, DiskIndex>::new(),
+        |mut disks, dir_entry| async move {
+            if let Some(disk_path) = dir_entry.path().file_name().and_then(|s| s.to_str()) {
+                let (disk_path, part_path) = if let Some(end) = PARTITION_REGEX.find(disk_path) {
+                    (
+                        disk_path.strip_suffix(end.as_str()).unwrap_or_default(),
+                        Some(disk_path),
+                    )
+                } else {
+                    (disk_path, None)
+                };
+                let disk_path = Path::new(DISK_PATH).join(disk_path);
+                let disk = tokio::fs::canonicalize(&disk_path).await.with_ctx(|_| {
                     (
                         crate::ErrorKind::Filesystem,
-                        part_path.display().to_string(),
+                        disk_path.display().to_string(),
                     )
                 })?;
-                disks.get_mut(&disk).unwrap().insert(part);
+                let part = if let Some(part_path) = part_path {
+                    let part_path = Path::new(DISK_PATH).join(part_path);
+                    let part = tokio::fs::canonicalize(&part_path).await.with_ctx(|_| {
+                        (
+                            crate::ErrorKind::Filesystem,
+                            part_path.display().to_string(),
+                        )
+                    })?;
+                    Some(part)
+                } else {
+                    None
+                };
+                if !disks.contains_key(&disk) {
+                    disks.insert(
+                        disk.clone(),
+                        DiskIndex {
+                            parts: IndexSet::new(),
+                            internal: false,
+                        },
+                    );
+                }
+                if let Some(part) = part {
+                    if os.contains(&part) {
+                        disks.get_mut(&disk).unwrap().internal = true;
+                    } else {
+                        disks.get_mut(&disk).unwrap().parts.insert(part);
+                    }
+                }
             }
-        }
-        Ok(disks)
-    })
+            Ok(disks)
+        },
+    )
     .await?;
 
     let mut res = Vec::with_capacity(disks.len());
-    for (disk, parts) in disks {
-        let mut guid: Option<String> = None;
-        let mut partitions = Vec::with_capacity(parts.len());
-        let vendor = get_vendor(&disk)
-            .await
-            .map_err(|e| tracing::warn!("Could not get vendor of {}: {}", disk.display(), e.source))
-            .unwrap_or_default();
-        let model = get_model(&disk)
-            .await
-            .map_err(|e| tracing::warn!("Could not get model of {}: {}", disk.display(), e.source))
-            .unwrap_or_default();
-        let capacity = get_capacity(&disk)
-            .await
-            .map_err(|e| {
-                tracing::warn!("Could not get capacity of {}: {}", disk.display(), e.source)
-            })
-            .unwrap_or_default();
-        if let Some(g) = disk_guids.get(&disk) {
-            guid = g.clone();
-        } else {
-            for part in parts {
-                let mut embassy_os = None;
-                let label = get_label(&part).await?;
-                let capacity = get_capacity(&part)
-                    .await
-                    .map_err(|e| {
-                        tracing::warn!("Could not get capacity of {}: {}", part.display(), e.source)
-                    })
-                    .unwrap_or_default();
-                let mut used = None;
-
-                match TmpMountGuard::mount(&BlockDev::new(&part), ReadOnly).await {
-                    Err(e) => tracing::warn!("Could not collect usage information: {}", e.source),
-                    Ok(mount_guard) => {
-                        used = get_used(&mount_guard)
-                            .await
-                            .map_err(|e| {
-                                tracing::warn!(
-                                    "Could not get usage of {}: {}",
-                                    part.display(),
-                                    e.source
-                                )
-                            })
-                            .ok();
-                        if let Some(recovery_info) = match recovery_info(&mount_guard).await {
-                            Ok(a) => a,
-                            Err(e) => {
-                                tracing::error!(
-                                    "Error fetching unencrypted backup metadata: {}",
-                                    e
-                                );
-                                None
-                            }
-                        } {
-                            embassy_os = Some(recovery_info)
-                        }
-                        mount_guard.unmount().await?;
-                    }
+    for (disk, index) in disks {
+        if index.internal {
+            for part in index.parts {
+                let mut disk_info = disk_info(disk.clone()).await;
+                disk_info.logicalname = part;
+                if let Some(g) = disk_guids.get(&disk_info.logicalname) {
+                    disk_info.guid = g.clone();
+                } else {
+                    disk_info.partitions = vec![part_info(disk_info.logicalname.clone()).await];
                 }
-
-                partitions.push(PartitionInfo {
-                    logicalname: part,
-                    label,
-                    capacity,
-                    used,
-                    embassy_os,
-                });
+                res.push(disk_info);
             }
+        } else {
+            let mut disk_info = disk_info(disk).await;
+            disk_info.partitions = Vec::with_capacity(index.parts.len());
+            if let Some(g) = disk_guids.get(&disk_info.logicalname) {
+                disk_info.guid = g.clone();
+            } else {
+                for part in index.parts {
+                    disk_info.partitions.push(part_info(part).await);
+                }
+            }
+            res.push(disk_info);
         }
-        res.push(DiskInfo {
-            logicalname: disk,
-            vendor,
-            model,
-            partitions,
-            capacity,
-            guid,
-        })
     }
 
     Ok(res)
 }
 
+async fn disk_info(disk: PathBuf) -> DiskInfo {
+    let vendor = get_vendor(&disk)
+        .await
+        .map_err(|e| tracing::warn!("Could not get vendor of {}: {}", disk.display(), e.source))
+        .unwrap_or_default();
+    let model = get_model(&disk)
+        .await
+        .map_err(|e| tracing::warn!("Could not get model of {}: {}", disk.display(), e.source))
+        .unwrap_or_default();
+    let capacity = get_capacity(&disk)
+        .await
+        .map_err(|e| tracing::warn!("Could not get capacity of {}: {}", disk.display(), e.source))
+        .unwrap_or_default();
+    DiskInfo {
+        logicalname: disk,
+        vendor,
+        model,
+        partitions: Vec::new(),
+        capacity,
+        guid: None,
+    }
+}
+
+async fn part_info(part: PathBuf) -> PartitionInfo {
+    let mut embassy_os = None;
+    let label = get_label(&part)
+        .await
+        .map_err(|e| tracing::warn!("Could not get label of {}: {}", part.display(), e.source))
+        .unwrap_or_default();
+    let capacity = get_capacity(&part)
+        .await
+        .map_err(|e| tracing::warn!("Could not get capacity of {}: {}", part.display(), e.source))
+        .unwrap_or_default();
+    let mut used = None;
+
+    match TmpMountGuard::mount(&BlockDev::new(&part), ReadOnly).await {
+        Err(e) => tracing::warn!("Could not collect usage information: {}", e.source),
+        Ok(mount_guard) => {
+            used = get_used(&mount_guard)
+                .await
+                .map_err(|e| {
+                    tracing::warn!("Could not get usage of {}: {}", part.display(), e.source)
+                })
+                .ok();
+            if let Some(recovery_info) = match recovery_info(&mount_guard).await {
+                Ok(a) => a,
+                Err(e) => {
+                    tracing::error!("Error fetching unencrypted backup metadata: {}", e);
+                    None
+                }
+            } {
+                embassy_os = Some(recovery_info)
+            }
+            if let Err(e) = mount_guard.unmount().await {
+                tracing::error!("Error unmounting partition {}: {}", part.display(), e);
+            }
+        }
+    }
+
+    PartitionInfo {
+        logicalname: part,
+        label,
+        capacity,
+        used,
+        embassy_os,
+    }
+}
+
 fn parse_pvscan_output(pvscan_output: &str) -> BTreeMap<PathBuf, Option<String>> {
     fn parse_line(line: &str) -> IResult<&str, (&str, Option<&str>)> {
         let pv_parse = preceded(