Update compatibility image (#440)

* backup create compiling

* backup restore compiling

* wip

* wip

* backups compiling

* wip

* wip update set result

* fix config set

* wip remove async

* make synchronous

* add build steps

* remove tokio

* add restore subcommand

* fix set to read stdin

* add package properties key

* proper error handle

* create config path if it doesnt exist

* fix merge conflict

* make config a path and koin config file when needed

* update cargo lock

* fix cargo lock

* update properties sub command

* fix dependency check and auto configure; clean up error handling

* fix errors

* fix properties subcommand order

* fix cargo lock

* update cargo.lock

* update appmgr cargo lock with build-portable

* fix error strings and input dependency config in dependency commands

* rename temp file instead

* read dependent config file

* fix temp config file creation

* create config file regardless if it exists

* add curl

* wip refactor config

* wip refactor config

* update rules to use embassyos config type

Co-authored-by: Chris Guida <chrisguida@gmail.com>
This commit is contained in:
Lucy C
2021-09-23 17:44:49 -06:00
committed by Aiden McClelland
parent 737edfe944
commit 6b62f0a883
13 changed files with 2145 additions and 79 deletions

74
compat/src/backup.rs Normal file
View File

@@ -0,0 +1,74 @@
use std::path::Path;
pub fn create_backup<P: AsRef<Path>>(
mountpoint: P,
data_path: P,
app_id: &str,
) -> Result<(), anyhow::Error> {
let path = std::fs::canonicalize(mountpoint)?;
let volume_path = Path::new(embassy::VOLUMES).join(app_id);
let exclude = if volume_path.is_dir() {
let ignore_path = volume_path.join(".backupignore");
if ignore_path.is_file() {
std::fs::read(ignore_path)?
} else {
Vec::new()
}
} else {
return Err(anyhow::anyhow!("Volume For {} Does Not Exist", app_id))
};
let mut data_cmd = std::process::Command::new("duplicity");
for exclude in exclude {
if exclude.to_string().starts_with('!') {
data_cmd.arg(format!(
"--include={}",
volume_path.join(exclude.to_string().trim_start_matches('!')).display()
));
} else {
data_cmd.arg(format!("--exclude={}", volume_path.join(exclude.to_string()).display()));
}
}
let data_res = data_cmd
.arg(volume_path)
.arg(format!("file://{}", data_path.as_ref().display().to_string()))
.output();
data_res?;
Ok(())
}
pub fn restore_backup<P: AsRef<Path>>(
path: P,
data_path: P,
app_id: &str,
) -> Result<(), anyhow::Error> {
let path = std::fs::canonicalize(path)?;
if !path.is_dir() {
anyhow::anyhow!("Backup Path Must Be Directory");
}
let metadata_path = path.join("metadata.yaml");
let volume_path = Path::new(embassy::VOLUMES).join(app_id);
let mut data_cmd = std::process::Command::new("duplicity");
data_cmd
.arg("--force")
.arg(format!("file://{:#?}", data_path.as_ref().display().to_string()))
.arg(&volume_path);
let data_output = data_cmd.status()?;
if !data_output.success() {
return Err(anyhow::anyhow!("duplicity error for {}", app_id))
}
std::fs::copy(
metadata_path,
Path::new(embassy::VOLUMES)
.join(app_id)
.join("start9")
.join("restore.yaml"),
)?;
Ok(())
}

83
compat/src/config/mod.rs Normal file
View File

@@ -0,0 +1,83 @@
use std::borrow::Cow;
use std::path::Path;
use beau_collector::BeauCollector;
use embassy::config::action::SetResult;
use embassy::config::{Config, spec};
use linear_map::LinearMap;
pub mod rules;
use anyhow::anyhow;
pub use rules::{ConfigRuleEntry, ConfigRuleEntryWithSuggestions};
pub fn validate_configuration(
name: &str,
config: Config,
rules_path: &Path,
config_path: &Path,
) -> Result<SetResult, anyhow::Error> {
let rules: Vec<ConfigRuleEntry> = serde_yaml::from_reader(std::fs::File::open(rules_path)?)?;
let mut cfgs = LinearMap::new();
cfgs.insert(name, Cow::Borrowed(&config));
let rule_check = rules
.into_iter()
.map(|r| r.check(&config, &cfgs))
.bcollect::<Vec<_>>();
match rule_check {
Ok(_) => {
// create temp config file
serde_yaml::to_writer(std::fs::File::create(config_path.with_extension("tmp"))?, &config)?;
std::fs::rename(config_path.with_extension("tmp"), config_path)?;
// return set result
Ok(SetResult {
depends_on: indexmap::IndexMap::new(),
// sending sigterm so service is restarted - in 0.3.x services, this is whatever signal is needed to send to the process to pick up the configuration
signal: Some(nix::sys::signal::SIGTERM),
})
}
Err(e) => Err(anyhow!("{}", e))
}
}
pub fn validate_dependency_configuration(
name: &str,
config: Config,
rules_path: &Path,
dependent_config: Config,
) -> Result<(), anyhow::Error> {
let rules: Vec<ConfigRuleEntry> = serde_yaml::from_reader(std::fs::File::open(rules_path)?)?;
let mut cfgs = LinearMap::new();
cfgs.insert(name, Cow::Borrowed(&config));
cfgs.insert(name, Cow::Borrowed(&dependent_config));
let rule_check = rules
.into_iter()
.map(|r| r.check(&config, &cfgs))
.bcollect::<Vec<_>>();
match rule_check {
Ok(_) => Ok(()),
Err(e) => Err(anyhow!("{}", e))
}
}
pub fn apply_dependency_configuration(
name: &str,
mut config: Config,
rules_path: &Path,
dependent_config: Config,
) -> Result<Config, anyhow::Error> {
let rules: Vec<ConfigRuleEntryWithSuggestions> =
serde_yaml::from_reader(std::fs::File::open(rules_path)?)?;
let mut cfgs = LinearMap::new();
cfgs.insert(name, Cow::Owned(config.clone()));
cfgs.insert(name, Cow::Owned(dependent_config.clone()));
let rule_check = rules
.into_iter()
.map(|r| r.apply(name, &mut config, &mut cfgs))
.bcollect::<Vec<_>>();
match rule_check {
Ok(_) => Ok(config),
Err(e) => Err(anyhow!("{}", e))
}
}

View File

@@ -0,0 +1,76 @@
num = @{ int ~ ("." ~ ASCII_DIGIT*)? ~ (^"e" ~ int)? }
int = @{ ("+" | "-")? ~ ASCII_DIGIT+ }
raw_string = @{ (!("\\" | "\"") ~ ANY)+ }
predefined = @{ "n" | "r" | "t" | "\\" | "0" | "\"" | "'" }
escape = @{ "\\" ~ predefined }
str = @{ "\"" ~ (raw_string | escape)* ~ "\"" }
ident_char = @{ ASCII_ALPHANUMERIC | "-" }
sub_ident = _{ sub_ident_regular | sub_ident_index | sub_ident_any | sub_ident_all | sub_ident_fn }
sub_ident_regular = { sub_ident_regular_base | sub_ident_regular_expr }
sub_ident_regular_base = @{ ASCII_ALPHA ~ ident_char* }
sub_ident_regular_expr = ${ "[" ~ str_expr ~ "]" }
sub_ident_index = { sub_ident_index_base | sub_ident_index_expr }
sub_ident_index_base = @{ ASCII_DIGIT+ }
sub_ident_index_expr = ${ "[" ~ num_expr ~ "]" }
sub_ident_any = @{ "*" }
sub_ident_all = @{ "&" }
sub_ident_fn = ${ "[" ~ list_access_function ~ "]"}
list_access_function = _{ list_access_function_first | list_access_function_last | list_access_function_any | list_access_function_all }
list_access_function_first = !{ "first" ~ "(" ~ sub_ident_regular ~ "=>" ~ bool_expr ~ ")" }
list_access_function_last = !{ "last" ~ "(" ~ sub_ident_regular ~ "=>" ~ bool_expr ~ ")" }
list_access_function_any = !{ "any" ~ "(" ~ sub_ident_regular ~ "=>" ~ bool_expr ~ ")" }
list_access_function_all = !{ "all" ~ "(" ~ sub_ident_regular ~ "=>" ~ bool_expr ~ ")" }
app_id = ${ "[" ~ sub_ident_regular ~ "]" }
ident = _{ (app_id ~ ".")? ~ sub_ident_regular ~ ("." ~ sub_ident)* }
bool_var = ${ ident ~ "?" }
num_var = ${ "#" ~ ident }
str_var = ${ "'" ~ ident }
any_var = ${ ident }
bool_op = _{ and | or | xor }
and = { "AND" }
or = { "OR" }
xor = { "XOR" }
num_cmp_op = _{ lt | lte | eq | neq | gt | gte }
str_cmp_op = _{ lt | lte | eq | neq | gt | gte }
lt = { "<" }
lte = { "<=" }
eq = { "=" }
neq = { "!=" }
gt = { ">" }
gte = { ">=" }
num_op = _{ add | sub | mul | div | pow }
str_op = _{ add }
add = { "+" }
sub = { "-" }
mul = { "*" }
div = { "/" }
pow = { "^" }
num_expr = !{ num_term ~ (num_op ~ num_term)* }
num_term = _{ num | num_var | "(" ~ num_expr ~ ")" }
str_expr = !{ str_term ~ (str_op ~ str_term)* }
str_term = _{ str | str_var | "(" ~ str_expr ~ ")" }
num_cmp_expr = { num_expr ~ num_cmp_op ~ num_expr }
str_cmp_expr = { str_expr ~ str_cmp_op ~ str_expr }
bool_expr = !{ bool_term ~ (bool_op ~ bool_term)* }
inv_bool_expr = { "!(" ~ bool_expr ~ ")" }
bool_term = _{ bool_var | "(" ~ bool_expr ~ ")" | inv_bool_expr | num_cmp_expr | str_cmp_expr }
val_expr = _{ any_var | str_expr | num_expr | bool_expr }
rule = _{ SOI ~ bool_expr ~ EOI }
reference = _{ SOI ~ any_var ~ EOI }
value = _{ SOI ~ val_expr ~ EOI }
del_action = _{ SOI ~ "FROM" ~ any_var ~ "AS" ~ sub_ident_regular ~ "WHERE" ~ bool_expr ~ EOI }
obj_key = _{ SOI ~ sub_ident_regular ~ EOI }
WHITESPACE = _{ " " | "\t" }

1259
compat/src/config/rules.rs Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -1,24 +1,162 @@
use std::{fs::File, io::stdout, path::Path};
use std::{
fs::File,
io::{stdin, stdout},
path::Path,
};
#[macro_use]
extern crate failure;
extern crate pest;
#[macro_use]
extern crate pest_derive;
mod backup;
mod config;
use anyhow::anyhow;
use backup::{create_backup, restore_backup};
use clap::{App, Arg, SubCommand};
use embassy::config::action::{ConfigRes, SetResult};
use config::{
apply_dependency_configuration, validate_configuration, validate_dependency_configuration,
};
use embassy::config::action::ConfigRes;
pub enum CompatRes {
SetResult,
ConfigRes,
}
fn main() {
let app = App::new("compat").subcommand(
SubCommand::with_name("config").subcommand(
SubCommand::with_name("get")
.arg(
Arg::with_name("mountpoint")
.help("The `mount` field from manifest.yaml")
.required(true),
match inner_main() {
Ok(a) => a,
Err(e) => {
eprintln!("{}", e);
log::debug!("{:?}", e.backtrace());
drop(e);
std::process::exit(1)
}
}
}
fn inner_main() -> Result<(), anyhow::Error> {
let app = App::new("compat")
.subcommand(
SubCommand::with_name("config")
.subcommand(
SubCommand::with_name("get")
.arg(
Arg::with_name("mountpoint")
.help("Path to the data mountpoint")
.required(true),
)
.arg(
Arg::with_name("spec")
.help("The path to the config spec in the container")
.required(true),
),
)
.arg(
Arg::with_name("spec")
.help("The path to the config spec in the container")
.required(true),
.subcommand(
SubCommand::with_name("set")
.arg(
Arg::with_name("package_id")
.help("The `id` field from the manifest file")
.required(true),
)
.arg(
Arg::with_name("mountpoint")
.help("Path to the data mountpoint")
.required(true),
)
.arg(
Arg::with_name("assets")
.help("Path to the rules file")
.required(true),
),
),
),
);
)
.subcommand(
SubCommand::with_name("dependency")
.subcommand(
SubCommand::with_name("check")
.arg(
Arg::with_name("dependency_package_id")
.help("Identifier of the dependency")
.required(true),
)
.arg(
Arg::with_name("mountpoint")
.help(" ountpoint for the dependent's config file")
.required(true),
)
.arg(
Arg::with_name("assets")
.help("Path to the dependency's config rules file")
.required(true),
),
)
.subcommand(
SubCommand::with_name("auto-configure")
.arg(
Arg::with_name("dependency_package_id")
.help("Package identifier of the dependency")
.required(true),
)
.arg(
Arg::with_name("mountpoint")
.help("Mountpoint for the dependent's config file")
.required(true),
)
.arg(
Arg::with_name("assets")
.help("Path to the dependency's config rules file")
.required(true),
),
),
)
.subcommand(
SubCommand::with_name("duplicity")
.subcommand(
SubCommand::with_name("create")
.arg(
Arg::with_name("package-id")
.help("The `id` field from the manifest file")
.required(true),
)
.arg(
Arg::with_name("mountpoint")
.help("The backups mount point")
.required(true),
)
.arg(
Arg::with_name("datapath")
.help("The path to the data to be backed up in the container")
.required(true),
),
)
.subcommand(
SubCommand::with_name("restore")
.arg(
Arg::with_name("package-id")
.help("The `id` field from the manifest file")
.required(true),
)
.arg(
Arg::with_name("mountpoint")
.help("The backups mount point")
.required(true),
)
.arg(
Arg::with_name("datapath")
.help("The path to the data to be restored to the container")
.required(true),
),
),
)
.subcommand(
SubCommand::with_name("properties").arg(
Arg::with_name("mountpoint")
.help("The data directory of the service to mount to.")
.required(true),
),
);
let matches = app.get_matches();
match matches.subcommand() {
("config", Some(sub_m)) => match sub_m.subcommand() {
@@ -32,14 +170,134 @@ fn main() {
};
let spec_path = Path::new(sub_m.value_of("spec").unwrap());
let spec = serde_yaml::from_reader(File::open(spec_path).unwrap()).unwrap();
serde_yaml::to_writer(stdout(), &ConfigRes { config: cfg, spec }).unwrap();
serde_yaml::to_writer(stdout(), &ConfigRes { config: cfg, spec })?;
Ok(())
}
("set", Some(sub_m)) => {
let config = serde_yaml::from_reader(stdin())?;
let cfg_path = Path::new(sub_m.value_of("mountpoint").unwrap()).join("start9");
if !cfg_path.exists() {
std::fs::create_dir_all(&cfg_path).unwrap();
};
let rules_path = Path::new(sub_m.value_of("assets").unwrap());
let name = sub_m.value_of("package_id").unwrap();
match validate_configuration(
&name,
config,
rules_path,
&cfg_path.join("config.yaml"),
) {
Ok(a) => {
serde_yaml::to_writer(stdout(), &a)?;
Ok(())
}
Err(e) => Err(e),
}
}
(subcmd, _) => {
panic!("unknown subcommand: {}", subcmd);
panic!("Unknown subcommand: {}", subcmd);
}
},
("dependency", Some(sub_m)) => match sub_m.subcommand() {
("check", Some(sub_m)) => {
let dependency_config = serde_yaml::from_reader(stdin())?;
let dependent_config = serde_yaml::from_reader(
File::open(
Path::new(sub_m.value_of("mountpoint").unwrap()).join("start9/config.yaml"),
)
.unwrap(),
)?;
let rules_path = Path::new(sub_m.value_of("assets").unwrap());
let name = sub_m.value_of("dependency_package_id").unwrap();
match validate_dependency_configuration(
&name,
dependency_config,
rules_path,
dependent_config,
) {
Ok(a) => {
serde_yaml::to_writer(stdout(), &a)?;
Ok(())
}
Err(e) => {
// error string is configs rules failure description
Err(e)
}
}
}
("auto-configure", Some(sub_m)) => {
let dependency_config = serde_yaml::from_reader(stdin())?;
let dependent_config = serde_yaml::from_reader(
File::open(
Path::new(sub_m.value_of("mountpoint").unwrap()).join("start9/config.yaml"),
)
.unwrap(),
)?;
let rules_path = Path::new(sub_m.value_of("assets").unwrap());
let name = sub_m.value_of("dependency_package_id").unwrap();
match apply_dependency_configuration(
name,
dependency_config,
rules_path,
dependent_config,
) {
Ok(a) => {
serde_yaml::to_writer(stdout(), &a)?;
Ok(())
}
Err(e) => Err(e),
}
}
(subcmd, _) => {
panic!("Unknown subcommand: {}", subcmd);
}
},
("duplicity", Some(sub_m)) => match sub_m.subcommand() {
("create", Some(sub_m)) => {
let res = create_backup(
sub_m.value_of("mountpoint").unwrap(),
sub_m.value_of("datapath").unwrap(),
sub_m.value_of("package-id").unwrap(),
);
match res {
Ok(r) => {
serde_yaml::to_writer(stdout(), &r)?;
Ok(())
}
Err(e) => Err(anyhow!("Could not create backup: {}", e)),
}
}
("restore", Some(sub_m)) => {
let res = restore_backup(
sub_m.value_of("package-id").unwrap(),
sub_m.value_of("datapath").unwrap(),
sub_m.value_of("mountpoint").unwrap(),
);
match res {
Ok(r) => {
serde_yaml::to_writer(stdout(), &r)?;
Ok(())
}
Err(e) => Err(anyhow!("Could not restore backup: {}", e)),
}
}
(subcmd, _) => {
panic!("Unknown subcommand: {}", subcmd);
}
},
("properties", Some(sub_m)) => {
let stats_path =
Path::new(sub_m.value_of("mountpoint").unwrap()).join("start9/stats.yaml");
let stats: serde_json::Value = if stats_path.exists() {
serde_yaml::from_reader(File::open(stats_path).unwrap()).unwrap()
} else {
serde_json::Value::from("{}")
};
serde_json::to_writer(stdout(), &stats)?;
Ok(())
}
(subcmd, _) => {
panic!("unknown subcommand: {}", subcmd);
panic!("Unknown subcommand: {}", subcmd);
}
}
}