allow embassy-cli not as root (#1501)

* allow embassy-cli not as root
* clean up merge

backend/src/backup/target/mod.rs

@@ -6,7 +6,7 @@ use chrono::{DateTime, Utc};
 use clap::ArgMatches;
 use color_eyre::eyre::eyre;
 use digest::generic_array::GenericArray;
-use digest::Digest;
+use digest::{Digest, OutputSizeUser};
 use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
@@ -119,7 +119,9 @@ impl FileSystem for BackupTargetFS {
             BackupTargetFS::Cifs(a) => a.mount(mountpoint, mount_type).await,
         }
     }
-    async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
         match self {
             BackupTargetFS::Disk(a) => a.source_hash().await,
             BackupTargetFS::Cifs(a) => a.source_hash().await,

backend/src/context/cli.rs

@@ -15,6 +15,7 @@ use rpc_toolkit::Context;
 use serde::Deserialize;
 use tracing::instrument;
 
+use crate::util::config::{load_config_from_paths, local_config_path};
 use crate::ResultExt;
 
 #[derive(Debug, Default, Deserialize)]
@@ -60,16 +61,16 @@ impl CliContext {
     /// BLOCKING
     #[instrument(skip(matches))]
     pub fn init(matches: &ArgMatches) -> Result<Self, crate::Error> {
-        let cfg_path = Path::new(matches.value_of("config").unwrap_or(crate::CONFIG_PATH));
-        let base = if cfg_path.exists() {
-            serde_yaml::from_reader(
-                File::open(cfg_path)
-                    .with_ctx(|_| (crate::ErrorKind::Filesystem, cfg_path.display().to_string()))?,
-            )
-            .with_kind(crate::ErrorKind::Deserialization)?
-        } else {
-            CliContextConfig::default()
-        };
+        let local_config_path = local_config_path();
+        let base: CliContextConfig = load_config_from_paths(
+            matches
+                .values_of("config")
+                .into_iter()
+                .flatten()
+                .map(|p| Path::new(p))
+                .chain(local_config_path.as_deref().into_iter())
+                .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+        )?;
         let mut url = if let Some(host) = matches.value_of("host") {
             host.parse()?
         } else if let Some(host) = base.host {
@@ -88,7 +89,9 @@ impl CliContext {
         };
 
         let cookie_path = base.cookie_path.unwrap_or_else(|| {
-            cfg_path
+            local_config_path
+                .as_deref()
+                .unwrap_or_else(|| Path::new(crate::util::config::CONFIG_PATH))
                 .parent()
                 .unwrap_or(Path::new("/"))
                 .join(".cookies.json")

backend/src/context/diagnostic.rs

@@ -28,7 +28,7 @@ impl DiagnosticContextConfig {
         let cfg_path = path
             .as_ref()
             .map(|p| p.as_ref())
-            .unwrap_or(Path::new(crate::CONFIG_PATH));
+            .unwrap_or(Path::new(crate::util::config::CONFIG_PATH));
         if let Some(f) = File::maybe_open(cfg_path)
             .await
             .with_ctx(|_| (crate::ErrorKind::Filesystem, cfg_path.display().to_string()))?

backend/src/context/rpc.rs

@@ -7,8 +7,8 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use bollard::Docker;
-use patch_db::{json_ptr::JsonPointer, LockReceipt};
-use patch_db::{DbHandle, LockType, PatchDb, Revision};
+use patch_db::json_ptr::JsonPointer;
+use patch_db::{DbHandle, LockReceipt, LockType, PatchDb, Revision};
 use reqwest::Url;
 use rpc_toolkit::url::Host;
 use rpc_toolkit::Context;
@@ -20,9 +20,10 @@ use tokio::process::Command;
 use tokio::sync::{broadcast, oneshot, Mutex, RwLock};
 use tracing::instrument;
 
+use crate::core::rpc_continuations::{RequestGuid, RpcContinuation};
 use crate::db::model::{Database, InstalledPackageDataEntry, PackageDataEntry};
 use crate::hostname::{derive_hostname, derive_id, get_product_key};
-use crate::install::cleanup::{cleanup_failed, uninstall};
+use crate::install::cleanup::{cleanup_failed, uninstall, CleanupFailedReceipts};
 use crate::manager::ManagerMap;
 use crate::middleware::auth::HashSessionToken;
 use crate::net::tor::os_key;
@@ -34,10 +35,6 @@ use crate::shutdown::Shutdown;
 use crate::status::{MainStatus, Status};
 use crate::util::io::from_yaml_async_reader;
 use crate::util::{AsyncFileExt, Invoke};
-use crate::{
-    core::rpc_continuations::{RequestGuid, RpcContinuation},
-    install::cleanup::CleanupFailedReceipts,
-};
 use crate::{Error, ResultExt};
 
 #[derive(Debug, Default, Deserialize)]
@@ -57,7 +54,7 @@ impl RpcContextConfig {
         let cfg_path = path
             .as_ref()
             .map(|p| p.as_ref())
-            .unwrap_or(Path::new(crate::CONFIG_PATH));
+            .unwrap_or(Path::new(crate::util::config::CONFIG_PATH));
         if let Some(f) = File::maybe_open(cfg_path)
             .await
             .with_ctx(|_| (crate::ErrorKind::Filesystem, cfg_path.display().to_string()))?

backend/src/context/sdk.rs

@@ -1,5 +1,3 @@
-use std::fs::File;
-use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
@@ -9,6 +7,7 @@ use rpc_toolkit::Context;
 use serde::Deserialize;
 use tracing::instrument;
 
+use crate::util::config::{load_config_from_paths, local_config_path};
 use crate::{Error, ResultExt};
 
 #[derive(Debug, Default, Deserialize)]
@@ -28,22 +27,24 @@ impl SdkContext {
     /// BLOCKING
     #[instrument(skip(matches))]
     pub fn init(matches: &ArgMatches) -> Result<Self, crate::Error> {
-        let cfg_path = Path::new(matches.value_of("config").unwrap_or(crate::CONFIG_PATH));
-        let base = if cfg_path.exists() {
-            serde_yaml::from_reader(
-                File::open(cfg_path)
-                    .with_ctx(|_| (crate::ErrorKind::Filesystem, cfg_path.display().to_string()))?,
-            )
-            .with_kind(crate::ErrorKind::Deserialization)?
-        } else {
-            SdkContextConfig::default()
-        };
+        let local_config_path = local_config_path();
+        let base: SdkContextConfig = load_config_from_paths(
+            matches
+                .values_of("config")
+                .into_iter()
+                .flatten()
+                .map(|p| Path::new(p))
+                .chain(local_config_path.as_deref().into_iter())
+                .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
+        )?;
         Ok(SdkContext(Arc::new(SdkContextSeed {
             developer_key_path: base.developer_key_path.unwrap_or_else(|| {
-                cfg_path
+                local_config_path
+                    .as_deref()
+                    .unwrap_or_else(|| Path::new(crate::util::config::CONFIG_PATH))
                     .parent()
                     .unwrap_or(Path::new("/"))
-                    .join(".developer_key")
+                    .join("developer.key.pem")
             }),
         })))
     }
@@ -53,9 +54,17 @@ impl SdkContext {
         if !self.developer_key_path.exists() {
             return Err(Error::new(eyre!("Developer Key does not exist! Please run `embassy-sdk init` before running this command."), crate::ErrorKind::Uninitialized));
         }
-        let mut keypair_buf = [0; ed25519_dalek::KEYPAIR_LENGTH];
-        File::open(&self.developer_key_path)?.read_exact(&mut keypair_buf)?;
-        Ok(ed25519_dalek::Keypair::from_bytes(&keypair_buf)?)
+        let pair = <ed25519::KeypairBytes as ed25519::pkcs8::DecodePrivateKey>::from_pkcs8_pem(
+            &std::fs::read_to_string(&self.developer_key_path)?,
+        )
+        .with_kind(crate::ErrorKind::Pem)?;
+        let secret = ed25519_dalek::SecretKey::from_bytes(&pair.secret_key[..])?;
+        let public = if let Some(public) = pair.public_key {
+            ed25519_dalek::PublicKey::from_bytes(&public[..])?
+        } else {
+            (&secret).into()
+        };
+        Ok(ed25519_dalek::Keypair { secret, public })
     }
 }
 impl std::ops::Deref for SdkContext {

backend/src/context/setup.rs

@@ -45,7 +45,7 @@ impl SetupContextConfig {
         let cfg_path = path
             .as_ref()
             .map(|p| p.as_ref())
-            .unwrap_or(Path::new(crate::CONFIG_PATH));
+            .unwrap_or(Path::new(crate::util::config::CONFIG_PATH));
         if let Some(f) = File::maybe_open(cfg_path)
             .await
             .with_ctx(|_| (crate::ErrorKind::Filesystem, cfg_path.display().to_string()))?

backend/src/developer/mod.rs

@@ -2,6 +2,7 @@ use std::fs::File;
 use std::io::Write;
 use std::path::Path;
 
+use ed25519::pkcs8::EncodePrivateKey;
 use ed25519_dalek::Keypair;
 use rpc_toolkit::command;
 use tracing::instrument;
@@ -22,8 +23,17 @@ pub fn init(#[context] ctx: SdkContext) -> Result<(), Error> {
         tracing::info!("Generating new developer key...");
         let keypair = Keypair::generate(&mut rand::thread_rng());
         tracing::info!("Writing key to {}", ctx.developer_key_path.display());
+        let keypair_bytes = ed25519::KeypairBytes {
+            secret_key: keypair.secret.to_bytes(),
+            public_key: Some(keypair.public.to_bytes()),
+        };
         let mut dev_key_file = File::create(&ctx.developer_key_path)?;
-        dev_key_file.write_all(&keypair.to_bytes())?;
+        dev_key_file.write_all(
+            keypair_bytes
+                .to_pkcs8_pem(base64ct::LineEnding::default())
+                .with_kind(crate::ErrorKind::Pem)?
+                .as_bytes(),
+        )?;
         dev_key_file.sync_all()?;
     }
     Ok(())

backend/src/disk/mount/filesystem/block_dev.rs

@@ -3,7 +3,7 @@ use std::path::Path;
 
 use async_trait::async_trait;
 use digest::generic_array::GenericArray;
-use digest::Digest;
+use digest::{Digest, OutputSizeUser};
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
 
@@ -45,7 +45,9 @@ impl<LogicalName: AsRef<Path> + Send + Sync> FileSystem for BlockDev<LogicalName
     ) -> Result<(), Error> {
         mount(self.logicalname.as_ref(), mountpoint, mount_type).await
     }
-    async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
         let mut sha = Sha256::new();
         sha.update("BlockDev");
         sha.update(

backend/src/disk/mount/filesystem/cifs.rs

@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
 
 use async_trait::async_trait;
 use digest::generic_array::GenericArray;
-use digest::Digest;
+use digest::{Digest, OutputSizeUser};
 use serde::{Deserialize, Serialize};
 use sha2::Sha256;
 use tokio::process::Command;
@@ -93,7 +93,9 @@ impl FileSystem for Cifs {
         )
         .await
     }
-    async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
         let mut sha = Sha256::new();
         sha.update("Cifs");
         sha.update(self.hostname.as_bytes());

backend/src/disk/mount/filesystem/ecryptfs.rs

@@ -4,7 +4,7 @@ use std::path::Path;
 use async_trait::async_trait;
 use color_eyre::eyre::eyre;
 use digest::generic_array::GenericArray;
-use digest::Digest;
+use digest::{Digest, OutputSizeUser};
 use sha2::Sha256;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
@@ -63,7 +63,9 @@ impl<EncryptedDir: AsRef<Path> + Send + Sync, Key: AsRef<str> + Send + Sync> Fil
     ) -> Result<(), Error> {
         mount_ecryptfs(self.encrypted_dir.as_ref(), mountpoint, self.key.as_ref()).await
     }
-    async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
         let mut sha = Sha256::new();
         sha.update("EcryptFS");
         sha.update(

backend/src/disk/mount/filesystem/label.rs

@@ -2,7 +2,7 @@ use std::path::Path;
 
 use async_trait::async_trait;
 use digest::generic_array::GenericArray;
-use digest::Digest;
+use digest::{Digest, OutputSizeUser};
 use sha2::Sha256;
 
 use super::{FileSystem, MountType, ReadOnly};
@@ -41,7 +41,9 @@ impl<S: AsRef<str> + Send + Sync> FileSystem for Label<S> {
     ) -> Result<(), Error> {
         mount_label(self.label.as_ref(), mountpoint, mount_type).await
     }
-    async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
         let mut sha = Sha256::new();
         sha.update("Label");
         sha.update(self.label.as_ref().as_bytes());

backend/src/disk/mount/filesystem/mod.rs

@@ -2,7 +2,7 @@ use std::path::Path;
 
 use async_trait::async_trait;
 use digest::generic_array::GenericArray;
-use digest::Digest;
+use digest::{Digest, OutputSizeUser};
 use sha2::Sha256;
 
 use crate::Error;
@@ -27,5 +27,7 @@ pub trait FileSystem {
         mountpoint: P,
         mount_type: MountType,
     ) -> Result<(), Error>;
-    async fn source_hash(&self) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error>;
+    async fn source_hash(
+        &self,
+    ) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error>;
 }

backend/src/error.rs

@@ -31,7 +31,7 @@ pub enum ErrorKind {
     InvalidOnionAddress = 22,
     Pack = 23,
     ValidateS9pk = 24,
-    DiskCorrupted = 25,
+    DiskCorrupted = 25, // Remove
     Tor = 26,
     ConfigGen = 27,
     ParseNumber = 28,
@@ -66,6 +66,7 @@ pub enum ErrorKind {
     ProductKeyMismatch = 57,
     LanPortConflict = 58,
     Javascript = 59,
+    Pem = 60,
 }
 impl ErrorKind {
     pub fn as_str(&self) -> &'static str {
@@ -128,8 +129,9 @@ impl ErrorKind {
             Incoherent => "Incoherent",
             InvalidBackupTargetId => "Invalid Backup Target ID",
             ProductKeyMismatch => "Incompatible Product Keys",
-            LanPortConflict => "Incompatible LAN port configuration",
-            Javascript => "Javascript engine error",
+            LanPortConflict => "Incompatible LAN Port Configuration",
+            Javascript => "Javascript Engine Error",
+            Pem => "PEM Encoding Error",
         }
     }
 }

backend/src/lib.rs

@@ -1,4 +1,3 @@
-pub const CONFIG_PATH: &str = "/etc/embassy/config.yaml";
 #[cfg(not(feature = "beta"))]
 pub const DEFAULT_MARKETPLACE: &str = "https://marketplace.start9.com";
 #[cfg(feature = "beta")]

backend/src/s9pk/builder.rs

@@ -1,5 +1,4 @@
-use digest::Digest;
-use sha2::Sha512;
+use sha2_old::{Digest, Sha512};
 use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, SeekFrom};
 use tracing::instrument;
 use typed_builder::TypedBuilder;

backend/src/s9pk/reader.rs

@@ -6,10 +6,10 @@ use std::str::FromStr;
 use std::task::{Context, Poll};
 
 use color_eyre::eyre::eyre;
-use digest::Output;
+use digest_old::Output;
 use ed25519_dalek::PublicKey;
 use futures::TryStreamExt;
-use sha2::{Digest, Sha512};
+use sha2_old::{Digest, Sha512};
 use tokio::fs::File;
 use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, ReadBuf, Take};
 use tracing::instrument;

backend/src/setup.rs

@@ -7,6 +7,7 @@ use std::time::Duration;
 
 use color_eyre::eyre::eyre;
 use digest::generic_array::GenericArray;
+use digest::OutputSizeUser;
 use futures::future::BoxFuture;
 use futures::{FutureExt, TryFutureExt, TryStreamExt};
 use nix::unistd::{Gid, Uid};
@@ -477,7 +478,7 @@ async fn recover(
 
 async fn shasum(
     path: impl AsRef<Path>,
-) -> Result<GenericArray<u8, <Sha256 as Digest>::OutputSize>, Error> {
+) -> Result<GenericArray<u8, <Sha256 as OutputSizeUser>::OutputSize>, Error> {
     use tokio::io::AsyncReadExt;
 
     let mut rdr = tokio::fs::File::open(path).await?;

backend/src/util/config.rs

@@ -0,0 +1,56 @@
+use std::fs::File;
+use std::path::{Path, PathBuf};
+
+use serde::Deserialize;
+use serde_json::Value;
+
+use crate::util::serde::IoFormat;
+use crate::{Config, Error, ResultExt};
+
+pub const CONFIG_PATH: &str = "/etc/embassy/config.yaml";
+pub const CONFIG_PATH_LOCAL: &str = ".embassy/config.yaml";
+
+pub fn local_config_path() -> Option<PathBuf> {
+    if let Ok(home) = std::env::var("HOME") {
+        Some(Path::new(&home).join(CONFIG_PATH_LOCAL))
+    } else {
+        None
+    }
+}
+
+/// BLOCKING
+pub fn load_config_from_paths<'a, T: for<'de> Deserialize<'de>>(
+    paths: impl IntoIterator<Item = impl AsRef<Path>>,
+) -> Result<T, Error> {
+    let mut config = Default::default();
+    for path in paths {
+        if path.as_ref().exists() {
+            let format: IoFormat = path
+                .as_ref()
+                .extension()
+                .and_then(|s| s.to_str())
+                .map(|f| f.parse())
+                .transpose()?
+                .unwrap_or_default();
+            let new = format.from_reader(File::open(path)?)?;
+            config = merge_configs(config, new);
+        }
+    }
+    serde_json::from_value(Value::Object(config)).with_kind(crate::ErrorKind::Deserialization)
+}
+
+pub fn merge_configs(mut first: Config, second: Config) -> Config {
+    for (k, v) in second.into_iter() {
+        let new = match first.remove(&k) {
+            None => v,
+            Some(old) => match (old, v) {
+                (Value::Object(first), Value::Object(second)) => {
+                    Value::Object(merge_configs(first, second))
+                }
+                (first, _) => first,
+            },
+        };
+        first.insert(k, new);
+    }
+    first
+}

backend/src/util/mod.rs

@@ -10,22 +10,21 @@ use std::task::{Context, Poll};
 use async_trait::async_trait;
 use clap::ArgMatches;
 use color_eyre::eyre::{self, eyre};
-use digest::Digest;
 use fd_lock_rs::FdLock;
 use futures::future::BoxFuture;
 use futures::FutureExt;
+pub use helpers::NonDetachingJoinHandle;
 use lazy_static::lazy_static;
+pub use models::Version;
 use pin_project::pin_project;
+use sha2_old::Digest;
 use tokio::fs::File;
 use tokio::sync::{Mutex, OwnedMutexGuard, RwLock};
 use tracing::instrument;
 
 use crate::shutdown::Shutdown;
 use crate::{Error, ResultExt as _};
-
-
-pub use helpers::NonDetachingJoinHandle;
-pub use models::Version;
+pub mod config;
 pub mod io;
 pub mod logger;
 pub mod serde;
@@ -251,7 +250,6 @@ where
     }
 }
 
-
 pub struct GeneralGuard<F: FnOnce() -> T, T = ()>(Option<F>);
 impl<F: FnOnce() -> T, T> GeneralGuard<F, T> {
     pub fn new(f: F) -> Self {