add utils system image, move compat into system-images

system-images/compat/.gitignore

@@ -0,0 +1,5 @@
+/target
+**/*.rs.bk
+.DS_Store
+.vscode
+compat.tar

system-images/compat/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+authors = ["Aiden McClelland <me@drbonez.dev>"]
+edition = "2018"
+name = "compat"
+version = "0.1.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = { version = "1.0.40", features = ["backtrace"] }
+beau_collector = "0.2.1"
+clap = "2.33.3"
+embassy-os = { path = "../appmgr", default-features = false }
+emver = { version = "0.1.2", features = ["serde"] }
+failure = "0.1.8"
+indexmap = { version = "1.6.2", features = ["serde"] }
+itertools = "0.10.0"
+lazy_static = "1.4"
+linear-map = { version = "1.2", features = ["serde_impl"] }
+log = "0.4.11"
+nix = "0.22.0"
+pest = "2.1"
+pest_derive = "2.1"
+rand = "0.7"
+regex = "1.4.2"
+rust-argon2 = "0.8.3"
+serde = { version = "1.0.118", features = ["derive", "rc"] }
+serde_json = "1.0.67"
+serde_yaml = "0.8.17"

system-images/compat/Dockerfile

@@ -0,0 +1,6 @@
+FROM alpine:latest
+
+RUN apk update && apk add duplicity && apk add curl
+ADD ./target/aarch64-unknown-linux-musl/release/compat /usr/local/bin/compat
+
+ENTRYPOINT ["compat"]

system-images/compat/Makefile

@@ -0,0 +1,11 @@
+ COMPAT_SRC := $(shell find ./src) Cargo.toml Cargo.lock
+
+.DELETE_ON_ERROR:
+
+all: compat.tar
+
+compat.tar: Dockerfile build
+	DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --tag start9/x_system/compat --platform=linux/arm64 -o type=docker,dest=compat.tar .
+
+build: $(COMPAT_SRC)
+	./build.sh

system-images/compat/build.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+shopt -s expand_aliases
+
+if [ "$0" != "./build.sh" ]; then
+	>&2 echo "Must be run from compat directory"
+	exit 1
+fi
+
+alias 'rust-musl-builder'='docker run --rm -it -v "$HOME"/.cargo/registry:/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-musl-cross:aarch64-musl'
+
+cd ../..
+rust-musl-builder sh -c "(cd embassy-os/compat && cargo +beta build --release --target=aarch64-unknown-linux-musl --no-default-features)"
+cd embassy-os/compat

system-images/compat/src/backup.rs

@@ -0,0 +1,74 @@
+use std::path::Path;
+
+pub fn create_backup<P: AsRef<Path>>(
+    mountpoint: P,
+    data_path: P,
+    app_id: &str,
+) -> Result<(), anyhow::Error> {
+    let path = std::fs::canonicalize(mountpoint)?;
+    let volume_path = Path::new(embassy::VOLUMES).join(app_id);
+
+    let exclude = if volume_path.is_dir() {
+        let ignore_path = volume_path.join(".backupignore");
+        if ignore_path.is_file() {
+            std::fs::read(ignore_path)?
+        } else {
+            Vec::new()
+        }
+    } else {
+        return Err(anyhow::anyhow!("Volume For {} Does Not Exist", app_id))
+    };
+
+    let mut data_cmd = std::process::Command::new("duplicity");
+    for exclude in exclude {
+        if exclude.to_string().starts_with('!') {
+            data_cmd.arg(format!(
+                "--include={}",
+                volume_path.join(exclude.to_string().trim_start_matches('!')).display()
+            ));
+        } else {
+            data_cmd.arg(format!("--exclude={}", volume_path.join(exclude.to_string()).display()));
+        }
+    }
+    let data_res = data_cmd
+        .arg(volume_path)
+        .arg(format!("file://{}", data_path.as_ref().display().to_string()))
+        .output();
+    data_res?;
+
+    Ok(())
+}
+
+pub fn restore_backup<P: AsRef<Path>>(
+    path: P,
+    data_path: P,
+    app_id: &str,
+) -> Result<(), anyhow::Error> {
+    let path = std::fs::canonicalize(path)?;
+    if !path.is_dir() {
+        anyhow::anyhow!("Backup Path Must Be Directory");
+    }
+    let metadata_path = path.join("metadata.yaml");
+    let volume_path = Path::new(embassy::VOLUMES).join(app_id);
+
+    let mut data_cmd = std::process::Command::new("duplicity");
+    data_cmd
+        .arg("--force")
+        .arg(format!("file://{:#?}", data_path.as_ref().display().to_string()))
+        .arg(&volume_path);
+
+    let data_output = data_cmd.status()?;
+    if !data_output.success() {
+        return Err(anyhow::anyhow!("duplicity error for {}", app_id))
+    }
+
+    std::fs::copy(
+        metadata_path,
+        Path::new(embassy::VOLUMES)
+            .join(app_id)
+            .join("start9")
+            .join("restore.yaml"),
+    )?;
+
+    Ok(())
+}

system-images/compat/src/config/mod.rs

@@ -0,0 +1,85 @@
+use std::borrow::Cow;
+use std::collections::BTreeMap;
+use std::path::Path;
+
+use beau_collector::BeauCollector;
+use embassy::config::action::SetResult;
+use embassy::config::{Config, spec};
+use linear_map::LinearMap;
+
+pub mod rules;
+
+use anyhow::anyhow;
+pub use rules::{ConfigRuleEntry, ConfigRuleEntryWithSuggestions};
+
+pub fn validate_configuration(
+    name: &str,
+    config: Config,
+    rules_path: &Path,
+    config_path: &Path,
+) -> Result<SetResult, anyhow::Error> {
+    let rules: Vec<ConfigRuleEntry> = serde_yaml::from_reader(std::fs::File::open(rules_path)?)?;
+    let mut cfgs = LinearMap::new();
+    cfgs.insert(name, Cow::Borrowed(&config));
+    let rule_check = rules
+        .into_iter()
+        .map(|r| r.check(&config, &cfgs))
+        .bcollect::<Vec<_>>();
+    match rule_check {
+        Ok(_) => {
+            // create temp config file
+            serde_yaml::to_writer(std::fs::File::create(config_path.with_extension("tmp"))?, &config)?;
+            std::fs::rename(config_path.with_extension("tmp"), config_path)?;
+            // return set result
+            Ok(SetResult {
+                depends_on: BTreeMap::new(),
+                // sending sigterm so service is restarted - in 0.3.x services, this is whatever signal is needed to send to the process to pick up the configuration
+                signal: Some(nix::sys::signal::SIGTERM),
+            })
+        }
+        Err(e) => Err(anyhow!("{}", e))
+    }
+}
+
+pub fn validate_dependency_configuration(
+    name: &str,
+    config: Config,
+    parent_name: &str,
+    parent_config: Config,
+    rules_path: &Path,
+) -> Result<(), anyhow::Error> {
+    let rules: Vec<ConfigRuleEntry> = serde_yaml::from_reader(std::fs::File::open(rules_path)?)?;
+    let mut cfgs = LinearMap::new();
+    cfgs.insert(parent_name, Cow::Borrowed(&parent_config));
+    cfgs.insert(name, Cow::Borrowed(&config));
+    let rule_check = rules
+        .into_iter()
+        .map(|r| r.check(&parent_config, &cfgs))
+        .bcollect::<Vec<_>>();
+    match rule_check {
+        Ok(_) => Ok(()),
+        Err(e) => Err(anyhow!("{}", e))
+    }
+}
+
+pub fn apply_dependency_configuration(
+    name: &str,
+    config: Config,
+    parent_name: &str,
+    mut parent_config: Config,
+    rules_path: &Path,
+) -> Result<Config, anyhow::Error> {
+    let rules: Vec<ConfigRuleEntryWithSuggestions> =
+        serde_yaml::from_reader(std::fs::File::open(rules_path)?)?;
+    let mut cfgs = LinearMap::new();
+    cfgs.insert(parent_name, Cow::Owned(parent_config.clone()));
+    cfgs.insert(name, Cow::Owned(config.clone()));
+    let rule_check = rules
+        .into_iter()
+        .map(|r| r.apply(parent_name, &mut parent_config, &mut cfgs))
+        .bcollect::<Vec<_>>();
+    match rule_check {
+        Ok(_) => Ok(parent_config),
+        Err(e) => Err(anyhow!("{}", e)),
+    }
+}

system-images/compat/src/config/rule_parser.pest

@@ -0,0 +1,76 @@
+num = @{ int ~ ("." ~ ASCII_DIGIT*)? ~ (^"e" ~ int)? }
+    int = @{ ("+" | "-")? ~ ASCII_DIGIT+ }
+
+raw_string = @{ (!("\\" | "\"") ~ ANY)+ }
+predefined = @{ "n" | "r" | "t" | "\\" | "0" | "\"" | "'" }
+escape = @{ "\\" ~ predefined }
+str = @{ "\"" ~ (raw_string | escape)* ~ "\"" }
+
+ident_char = @{ ASCII_ALPHANUMERIC | "-" }
+sub_ident = _{ sub_ident_regular | sub_ident_index | sub_ident_any | sub_ident_all | sub_ident_fn }
+    sub_ident_regular = { sub_ident_regular_base | sub_ident_regular_expr }
+    sub_ident_regular_base = @{ ASCII_ALPHA ~ ident_char* }
+    sub_ident_regular_expr = ${ "[" ~ str_expr ~ "]" }
+    sub_ident_index = { sub_ident_index_base | sub_ident_index_expr }
+    sub_ident_index_base = @{ ASCII_DIGIT+ }
+    sub_ident_index_expr = ${ "[" ~ num_expr ~ "]" }
+    sub_ident_any = @{ "*" }
+    sub_ident_all = @{ "&" }
+    sub_ident_fn = ${ "[" ~ list_access_function ~ "]"}
+        list_access_function = _{ list_access_function_first | list_access_function_last | list_access_function_any | list_access_function_all }
+        list_access_function_first = !{ "first" ~ "(" ~ sub_ident_regular ~ "=>" ~ bool_expr ~ ")" }
+        list_access_function_last = !{ "last" ~ "(" ~ sub_ident_regular ~ "=>" ~ bool_expr ~ ")" }
+        list_access_function_any = !{ "any" ~ "(" ~ sub_ident_regular ~ "=>" ~ bool_expr ~ ")" }
+        list_access_function_all = !{ "all" ~ "(" ~ sub_ident_regular ~ "=>" ~ bool_expr ~ ")" }
+
+app_id = ${ "[" ~ sub_ident_regular ~ "]" }
+ident = _{ (app_id ~ ".")? ~ sub_ident_regular ~ ("." ~ sub_ident)* }
+bool_var = ${ ident ~ "?" }
+num_var = ${ "#" ~ ident }
+str_var = ${ "'" ~ ident }
+any_var = ${ ident }
+
+bool_op = _{ and | or | xor }
+    and = { "AND" }
+    or  = { "OR" }
+    xor = { "XOR" }
+
+num_cmp_op = _{ lt | lte | eq | neq | gt | gte }
+str_cmp_op = _{ lt | lte | eq | neq | gt | gte }
+    lt  = { "<" }
+    lte = { "<=" }
+    eq  = { "=" }
+    neq = { "!=" }
+    gt  = { ">" }
+    gte = { ">=" }
+
+num_op = _{ add | sub | mul | div | pow }
+str_op = _{ add }
+    add = { "+" }
+    sub = { "-" }
+    mul = { "*" }
+    div = { "/" }
+    pow = { "^" }
+
+num_expr = !{ num_term ~ (num_op ~ num_term)* }
+num_term = _{ num | num_var | "(" ~ num_expr ~ ")" }
+
+str_expr = !{ str_term ~ (str_op ~ str_term)* }
+str_term = _{ str | str_var | "(" ~ str_expr ~ ")" }
+
+num_cmp_expr = { num_expr ~ num_cmp_op ~ num_expr }
+str_cmp_expr = { str_expr ~ str_cmp_op ~ str_expr }
+
+bool_expr = !{ bool_term ~ (bool_op ~ bool_term)* }
+inv_bool_expr = { "!(" ~ bool_expr ~ ")" }
+bool_term = _{ bool_var | "(" ~ bool_expr ~ ")" | inv_bool_expr | num_cmp_expr | str_cmp_expr }
+
+val_expr = _{ any_var | str_expr | num_expr | bool_expr }
+
+rule = _{ SOI ~ bool_expr ~ EOI }
+reference = _{ SOI ~ any_var ~ EOI }
+value = _{ SOI ~ val_expr ~ EOI }
+del_action = _{ SOI ~ "FROM" ~ any_var ~ "AS" ~ sub_ident_regular ~ "WHERE" ~ bool_expr ~ EOI }
+obj_key = _{ SOI ~ sub_ident_regular ~ EOI }
+
+WHITESPACE = _{ " " | "\t" }

system-images/compat/src/main.rs

@@ -0,0 +1,317 @@
+use std::{
+    fs::File,
+    io::{stdin, stdout},
+    path::Path,
+};
+
+#[macro_use]
+extern crate failure;
+extern crate pest;
+#[macro_use]
+extern crate pest_derive;
+
+mod backup;
+mod config;
+use anyhow::anyhow;
+use backup::{create_backup, restore_backup};
+use clap::{App, Arg, SubCommand};
+use config::{
+    apply_dependency_configuration, validate_configuration, validate_dependency_configuration,
+};
+use embassy::config::action::ConfigRes;
+pub enum CompatRes {
+    SetResult,
+    ConfigRes,
+}
+
+fn main() {
+    match inner_main() {
+        Ok(a) => a,
+        Err(e) => {
+            eprintln!("{}", e);
+            log::debug!("{:?}", e.backtrace());
+            drop(e);
+            std::process::exit(1)
+        }
+    }
+}
+
+fn inner_main() -> Result<(), anyhow::Error> {
+    let app = App::new("compat")
+        .subcommand(
+            SubCommand::with_name("config")
+                .subcommand(
+                    SubCommand::with_name("get")
+                        .arg(
+                            Arg::with_name("mountpoint")
+                                .help("Path to the data mountpoint")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("spec")
+                                .help("The path to the config spec in the container")
+                                .required(true),
+                        ),
+                )
+                .subcommand(
+                    SubCommand::with_name("set")
+                        .arg(
+                            Arg::with_name("package_id")
+                                .help("The `id` field from the manifest file")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("mountpoint")
+                                .help("Path to the data mountpoint")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("assets")
+                                .help("Path to the rules file")
+                                .required(true),
+                        ),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("dependency")
+                .subcommand(
+                    SubCommand::with_name("check")
+                        .arg(
+                            Arg::with_name("dependent_package_id")
+                                .help("Package identifier of this package (the child/depdendent)")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("dependency_package_id")
+                                .help("Identifier of the dependency")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("mountpoint")
+                                .help(" ountpoint for the dependent's config file")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("assets")
+                                .help("Path to the dependency's config rules file")
+                                .required(true),
+                        ),
+                )
+                .subcommand(
+                    SubCommand::with_name("auto-configure")
+                        .arg(
+                            Arg::with_name("dependent_package_id")
+                                .help("Package identifier of this package (the child/depdendent)")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("dependency_package_id")
+                                .help("Package identifier of the parent/dependency")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("mountpoint")
+                                .help("Mountpoint for the dependent's config file")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("assets")
+                                .help("Path to the dependency's config rules file")
+                                .required(true),
+                        ),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("duplicity")
+                .subcommand(
+                    SubCommand::with_name("create")
+                        .arg(
+                            Arg::with_name("package-id")
+                                .help("The `id` field from the manifest file")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("mountpoint")
+                                .help("The backups mount point")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("datapath")
+                                .help("The path to the data to be backed up in the container")
+                                .required(true),
+                        ),
+                )
+                .subcommand(
+                    SubCommand::with_name("restore")
+                        .arg(
+                            Arg::with_name("package-id")
+                                .help("The `id` field from the manifest file")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("mountpoint")
+                                .help("The backups mount point")
+                                .required(true),
+                        )
+                        .arg(
+                            Arg::with_name("datapath")
+                                .help("The path to the data to be restored to the container")
+                                .required(true),
+                        ),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("properties").arg(
+                Arg::with_name("mountpoint")
+                    .help("The data directory of the service to mount to.")
+                    .required(true),
+            ),
+        );
+    let matches = app.get_matches();
+    match matches.subcommand() {
+        ("config", Some(sub_m)) => match sub_m.subcommand() {
+            ("get", Some(sub_m)) => {
+                let cfg_path =
+                    Path::new(sub_m.value_of("mountpoint").unwrap()).join("start9/config.yaml");
+                let cfg = if cfg_path.exists() {
+                    Some(serde_yaml::from_reader(File::open(cfg_path).unwrap()).unwrap())
+                } else {
+                    None
+                };
+                let spec_path = Path::new(sub_m.value_of("spec").unwrap());
+                let spec = serde_yaml::from_reader(File::open(spec_path).unwrap()).unwrap();
+                serde_yaml::to_writer(stdout(), &ConfigRes { config: cfg, spec })?;
+                Ok(())
+            }
+            ("set", Some(sub_m)) => {
+                let config = serde_yaml::from_reader(stdin())?;
+                let cfg_path = Path::new(sub_m.value_of("mountpoint").unwrap()).join("start9");
+                if !cfg_path.exists() {
+                    std::fs::create_dir_all(&cfg_path).unwrap();
+                };
+                let rules_path = Path::new(sub_m.value_of("assets").unwrap());
+                let name = sub_m.value_of("package_id").unwrap();
+                match validate_configuration(
+                    &name,
+                    config,
+                    rules_path,
+                    &cfg_path.join("config.yaml"),
+                ) {
+                    Ok(a) => {
+                        serde_yaml::to_writer(stdout(), &a)?;
+                        Ok(())
+                    }
+                    Err(e) => Err(e),
+                }
+            }
+            (subcmd, _) => {
+                panic!("Unknown subcommand: {}", subcmd);
+            }
+        },
+        ("dependency", Some(sub_m)) => match sub_m.subcommand() {
+            ("check", Some(sub_m)) => {
+                let parent_config = serde_yaml::from_reader(stdin())?;
+                let config = serde_yaml::from_reader(
+                    File::open(
+                        Path::new(sub_m.value_of("mountpoint").unwrap()).join("start9/config.yaml"),
+                    )
+                    .unwrap(),
+                )?;
+                let rules_path = Path::new(sub_m.value_of("assets").unwrap());
+                let name = sub_m.value_of("dependent_package_id").unwrap();
+                let parent_name = sub_m.value_of("dependency_package_id").unwrap();
+                match validate_dependency_configuration(
+                    name,
+                    config,
+                    parent_name,
+                    parent_config,
+                    rules_path,
+                ) {
+                    Ok(a) => {
+                        serde_yaml::to_writer(stdout(), &a)?;
+                        Ok(())
+                    }
+                    Err(e) => {
+                        // error string is configs rules failure description
+                        Err(e)
+                    }
+                }
+            }
+            ("auto-configure", Some(sub_m)) => {
+                let parent_config = serde_yaml::from_reader(stdin())?;
+                let config = serde_yaml::from_reader(
+                    File::open(
+                        Path::new(sub_m.value_of("mountpoint").unwrap()).join("start9/config.yaml"),
+                    )
+                    .unwrap(),
+                )?;
+                let rules_path = Path::new(sub_m.value_of("assets").unwrap());
+                let name = sub_m.value_of("dependent_package_id").unwrap();
+                let parent_name = sub_m.value_of("dependency_package_id").unwrap();
+                match apply_dependency_configuration(
+                    name,
+                    config,
+                    parent_name,
+                    parent_config,
+                    rules_path,
+                ) {
+                    Ok(a) => {
+                        serde_yaml::to_writer(stdout(), &a)?;
+                        Ok(())
+                    }
+                    Err(e) => Err(e),
+                }
+            }
+            (subcmd, _) => {
+                panic!("Unknown subcommand: {}", subcmd);
+            }
+        },
+        ("duplicity", Some(sub_m)) => match sub_m.subcommand() {
+            ("create", Some(sub_m)) => {
+                let res = create_backup(
+                    sub_m.value_of("mountpoint").unwrap(),
+                    sub_m.value_of("datapath").unwrap(),
+                    sub_m.value_of("package-id").unwrap(),
+                );
+                match res {
+                    Ok(r) => {
+                        serde_yaml::to_writer(stdout(), &r)?;
+                        Ok(())
+                    }
+                    Err(e) => Err(anyhow!("Could not create backup: {}", e)),
+                }
+            }
+            ("restore", Some(sub_m)) => {
+                let res = restore_backup(
+                    sub_m.value_of("package-id").unwrap(),
+                    sub_m.value_of("datapath").unwrap(),
+                    sub_m.value_of("mountpoint").unwrap(),
+                );
+                match res {
+                    Ok(r) => {
+                        serde_yaml::to_writer(stdout(), &r)?;
+                        Ok(())
+                    }
+                    Err(e) => Err(anyhow!("Could not restore backup: {}", e)),
+                }
+            }
+            (subcmd, _) => {
+                panic!("Unknown subcommand: {}", subcmd);
+            }
+        },
+        ("properties", Some(sub_m)) => {
+            let stats_path =
+                Path::new(sub_m.value_of("mountpoint").unwrap()).join("start9/stats.yaml");
+            let stats: serde_json::Value = if stats_path.exists() {
+                serde_yaml::from_reader(File::open(stats_path).unwrap()).unwrap()
+            } else {
+                serde_json::Value::from("{}")
+            };
+            serde_json::to_writer(stdout(), &stats)?;
+            Ok(())
+        }
+        (subcmd, _) => {
+            panic!("Unknown subcommand: {}", subcmd);
+        }
+    }
+}

system-images/utils/Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine:latest
+
+RUN apk update && apk add yq jq

system-images/utils/Makefile

@@ -0,0 +1,6 @@
+.DELETE_ON_ERROR:
+
+all: utils.tar
+
+utils.tar: Dockerfile
+	DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --tag start9/x_system/utils --platform=linux/arm64 -o type=docker,dest=utils.tar .