appmgr 0.3.0 rewrite pt 1

appmgr: split bins

update cargo.toml and .gitignore

context

appmgr: refactor error module

appmgr: context

begin new s9pk format

appmgr: add fields to manifest

appmgr: start action abstraction

appmgr: volume abstraction

appmgr: improved volumes

appmgr: install wip

appmgr: health daemon

appmgr: health checks

appmgr: wip

config get

appmgr: secret store

wip

appmgr: config rewritten

appmgr: delete non-reusable code

appmgr: wip

appmgr: please the borrow-checker

appmgr: technically runs now

appmgr: cli

appmgr: clean up cli

appmgr: rpc-toolkit in action

appmgr: wrap up config

appmgr: account for updates during install

appmgr: fix: #308

appmgr: impl Display for Version

appmgr: cleanup

appmgr: set dependents on install

appmgr: dependency health checks
This commit is contained in:
Aiden McClelland
2021-04-08 11:16:25 -06:00
committed by Aiden McClelland
parent 5741cf084f
commit 8954e3e338
84 changed files with 7510 additions and 9950 deletions

106
appmgr/src/s9pk/builder.rs Normal file
View File

@@ -0,0 +1,106 @@
use std::io::{Read, Seek, SeekFrom, Write};
use typed_builder::TypedBuilder;
use super::header::{FileSection, Header};
use super::manifest::Manifest;
use crate::{Error, ResultExt};
#[derive(TypedBuilder)]
pub struct S9pkPacker<
'a,
W: Write + Seek,
RLicense: Read,
RInstructions: Read,
RIcon: Read,
RDockerImages: Read,
> {
writer: W,
manifest: &'a Manifest,
license: RLicense,
instructions: RInstructions,
icon: RIcon,
docker_images: RDockerImages,
}
impl<
'a,
W: Write + Seek,
RLicense: Read,
RInstructions: Read,
RIcon: Read,
RDockerImages: Read,
> S9pkPacker<'a, W, RLicense, RInstructions, RIcon, RDockerImages>
{
/// BLOCKING
pub fn pack(mut self) -> Result<(), Error> {
let header_pos = self.writer.stream_position()?;
if header_pos != 0 {
log::warn!("Appending to non-empty file.");
}
let mut header = Header::placeholder();
header.serialize(&mut self.writer).with_ctx(|_| {
(
crate::ErrorKind::Serialization,
"Writing Placeholder Header",
)
})?;
let mut position = self.writer.stream_position()?;
// manifest
serde_cbor::to_writer(&mut self.writer, self.manifest).with_ctx(|_| {
(
crate::ErrorKind::Serialization,
"Serializing Manifest (CBOR)",
)
})?;
let new_pos = self.writer.stream_position()?;
header.table_of_contents.manifest = FileSection {
position,
length: new_pos - position,
};
position = new_pos;
// license
std::io::copy(&mut self.license, &mut self.writer)
.with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying License"))?;
let new_pos = self.writer.stream_position()?;
header.table_of_contents.license = FileSection {
position,
length: new_pos - position,
};
position = new_pos;
// instructions
std::io::copy(&mut self.instructions, &mut self.writer)
.with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Instructions"))?;
let new_pos = self.writer.stream_position()?;
header.table_of_contents.instructions = FileSection {
position,
length: new_pos - position,
};
position = new_pos;
// icon
std::io::copy(&mut self.icon, &mut self.writer)
.with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying Icon"))?;
let new_pos = self.writer.stream_position()?;
header.table_of_contents.icon = FileSection {
position,
length: new_pos - position,
};
position = new_pos;
// docker_images
std::io::copy(&mut self.docker_images, &mut self.writer)
.with_ctx(|_| (crate::ErrorKind::Filesystem, "Copying App Image"))?;
let new_pos = self.writer.stream_position()?;
header.table_of_contents.docker_images = FileSection {
position,
length: new_pos - position,
};
position = new_pos;
// header
self.writer.seek(SeekFrom::Start(header_pos))?;
header
.serialize(&mut self.writer)
.with_ctx(|_| (crate::ErrorKind::Serialization, "Writing Header"))?;
self.writer.seek(SeekFrom::Start(position))?;
Ok(())
}
}

169
appmgr/src/s9pk/header.rs Normal file
View File

@@ -0,0 +1,169 @@
use std::collections::HashMap;
use std::io::Write;
use anyhow::anyhow;
use ed25519_dalek::{PublicKey, Signature};
use tokio::io::{AsyncRead, AsyncReadExt};
use crate::Error;
pub const MAGIC: [u8; 2] = [59, 59];
pub const VERSION: u8 = 1;
pub struct Header {
pub pubkey: PublicKey,
pub signature: Signature,
pub table_of_contents: TableOfContents,
}
impl Header {
pub fn placeholder() -> Self {
Header {
pubkey: PublicKey::default(),
signature: Signature::new([0; 64]),
table_of_contents: Default::default(),
}
}
// MUST BE SAME SIZE REGARDLESS OF DATA
pub fn serialize<W: Write>(&self, mut writer: W) -> std::io::Result<()> {
writer.write_all(&MAGIC)?;
writer.write_all(&[VERSION])?;
writer.write_all(self.pubkey.as_bytes())?;
writer.write_all(self.signature.as_ref())?;
self.table_of_contents.serialize(writer)?;
Ok(())
}
pub async fn deserialize<R: AsyncRead + Unpin>(mut reader: R) -> Result<Self, Error> {
let mut magic = [0; 2];
reader.read_exact(&mut magic).await?;
if magic != MAGIC {
return Err(Error::new(
anyhow!("Incorrect Magic"),
crate::ErrorKind::ParseS9pk,
));
}
let mut version = [0];
reader.read_exact(&mut version).await?;
if version[0] != VERSION {
return Err(Error::new(
anyhow!("Unknown Version"),
crate::ErrorKind::ParseS9pk,
));
}
let mut pubkey_bytes = [0; 32];
reader.read_exact(&mut pubkey_bytes).await?;
let pubkey = PublicKey::from_bytes(&pubkey_bytes)
.map_err(|e| Error::new(e, crate::ErrorKind::ParseS9pk))?;
let mut sig_bytes = [0; 64];
reader.read_exact(&mut sig_bytes).await?;
let signature = Signature::new(sig_bytes);
let table_of_contents = TableOfContents::deserialize(reader).await?;
Ok(Header {
pubkey,
signature,
table_of_contents,
})
}
}
#[derive(Debug, Default)]
pub struct TableOfContents {
pub manifest: FileSection,
pub license: FileSection,
pub instructions: FileSection,
pub icon: FileSection,
pub docker_images: FileSection,
}
impl TableOfContents {
pub fn serialize<W: Write>(&self, mut writer: W) -> std::io::Result<()> {
let len: u32 = 16 // size of FileSection
* (
1 + // manifest
1 + // license
1 + // instructions
1 + // icon
1 // docker_images
);
writer.write_all(&u32::to_be_bytes(len))?;
self.manifest.serialize_entry("manifest", &mut writer)?;
self.license.serialize_entry("license", &mut writer)?;
self.instructions
.serialize_entry("instructions", &mut writer)?;
self.icon.serialize_entry("icon", &mut writer)?;
self.docker_images
.serialize_entry("docker_images", &mut writer)?;
Ok(())
}
pub async fn deserialize<R: AsyncRead + Unpin>(mut reader: R) -> std::io::Result<Self> {
let mut toc_len = [0; 4];
reader.read_exact(&mut toc_len).await?;
let toc_len = u32::from_be_bytes(toc_len);
let mut reader = reader.take(toc_len as u64);
let mut table = HashMap::new();
while let Some((label, section)) = FileSection::deserialize_entry(&mut reader).await? {
table.insert(label, section);
}
fn from_table(
table: &HashMap<Vec<u8>, FileSection>,
label: &str,
) -> std::io::Result<FileSection> {
table.get(label.as_bytes()).copied().ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
format!("Missing Required Label: {}", label),
)
})
}
fn as_opt(fs: FileSection) -> Option<FileSection> {
if fs.position | fs.length == 0 {
// 0/0 is not a valid file section
None
} else {
Some(fs)
}
}
Ok(TableOfContents {
manifest: from_table(&table, "manifest")?,
license: from_table(&table, "license")?,
instructions: from_table(&table, "instructions")?,
icon: from_table(&table, "icon")?,
docker_images: from_table(&table, "docker_images")?,
})
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct FileSection {
pub position: u64,
pub length: u64,
}
impl FileSection {
pub fn serialize_entry<W: Write>(self, label: &str, mut writer: W) -> std::io::Result<()> {
writer.write_all(&[label.len() as u8])?;
writer.write_all(label.as_bytes())?;
writer.write_all(&u64::to_be_bytes(self.position))?;
writer.write_all(&u64::to_be_bytes(self.length))?;
Ok(())
}
pub async fn deserialize_entry<R: AsyncRead + Unpin>(
mut reader: R,
) -> std::io::Result<Option<(Vec<u8>, Self)>> {
let mut label_len = [0];
let read = reader.read(&mut label_len).await?;
if read == 0 {
return Ok(None);
}
let label = vec![0; label_len[0] as usize];
let mut pos = [0; 8];
reader.read_exact(&mut pos).await?;
let mut len = [0; 8];
reader.read_exact(&mut len).await?;
Ok(Some((
label,
FileSection {
position: u64::from_be_bytes(pos),
length: u64::from_be_bytes(len),
},
)))
}
}

222
appmgr/src/s9pk/manifest.rs Normal file
View File

@@ -0,0 +1,222 @@
use std::borrow::Borrow;
use std::net::Ipv4Addr;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use chrono::{DateTime, Utc};
use indexmap::IndexMap;
use patch_db::HasModel;
use serde::{Deserialize, Serialize, Serializer};
use url::Url;
use crate::action::{ActionImplementation, Actions};
use crate::backup::BackupActions;
use crate::config::action::ConfigActions;
use crate::db::model::InterfaceInfo;
use crate::dependencies::Dependencies;
use crate::id::{Id, InterfaceId, InvalidId, SYSTEM_ID};
use crate::migration::Migrations;
use crate::net::host::Hosts;
use crate::net::tor::HiddenServiceVersion;
use crate::status::health_check::{HealthCheckResult, HealthChecks};
use crate::util::Version;
use crate::volume::Volumes;
use crate::Error;
pub const SYSTEM_PACKAGE_ID: PackageId<&'static str> = PackageId(SYSTEM_ID);
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct PackageId<S: AsRef<str> = String>(Id<S>);
impl FromStr for PackageId {
type Err = InvalidId;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(PackageId(Id::try_from(s.to_owned())?))
}
}
impl<S: AsRef<str>> AsRef<PackageId<S>> for PackageId<S> {
fn as_ref(&self) -> &PackageId<S> {
self
}
}
impl<S: AsRef<str>> std::fmt::Display for PackageId<S> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", &self.0)
}
}
impl<S: AsRef<str>> AsRef<str> for PackageId<S> {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> Borrow<str> for PackageId<S> {
fn borrow(&self) -> &str {
self.0.as_ref()
}
}
impl<S: AsRef<str>> AsRef<Path> for PackageId<S> {
fn as_ref(&self) -> &Path {
self.0.as_ref().as_ref()
}
}
impl<'de, S> Deserialize<'de> for PackageId<S>
where
S: AsRef<str>,
Id<S>: Deserialize<'de>,
{
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::de::Deserializer<'de>,
{
Ok(PackageId(Deserialize::deserialize(deserializer)?))
}
}
impl<S> Serialize for PackageId<S>
where
S: AsRef<str>,
{
fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
where
Ser: Serializer,
{
Serialize::serialize(&self.0, serializer)
}
}
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
pub struct Manifest {
pub id: PackageId,
pub title: String,
pub version: Version,
pub description: Description,
#[serde(default)]
pub assets: Assets,
#[serde(default)]
pub build: Option<Vec<String>>,
pub release_notes: String,
pub license: String, // type of license
pub wrapper_repo: Url,
pub upstream_repo: Url,
pub support_site: Option<Url>,
pub marketing_site: Option<Url>,
#[serde(default)]
pub alerts: Alerts,
#[model]
pub main: ActionImplementation,
pub health_checks: HealthChecks,
#[model]
pub config: Option<ConfigActions>,
#[model]
pub volumes: Volumes,
// #[serde(default = "current_version")]
pub min_os_version: Version,
// #[serde(default)]
pub interfaces: Interfaces,
// #[serde(default)]
#[model]
pub backup: BackupActions,
#[serde(default)]
#[model]
pub migrations: Migrations,
#[serde(default)]
pub actions: Actions,
// #[serde(default)]
// pub permissions: Permissions,
#[serde(default)]
#[model]
pub dependencies: Dependencies,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct Interfaces(IndexMap<InterfaceId, Interface>); // TODO
impl Interfaces {
pub async fn install(&self, ip: Ipv4Addr) -> Result<InterfaceInfo, Error> {
todo!()
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct Interface {
tor_config: Option<TorConfig>,
lan_config: Option<IndexMap<u16, LanPortConfig>>,
ui: bool,
protocols: Vec<String>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct TorConfig {
#[serde(default)]
hidden_service_version: HiddenServiceVersion,
port_mapping: IndexMap<u16, u16>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct LanPortConfig {
ssl: bool,
mapping: u16,
}
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct Assets {
#[serde(default)]
license: Option<PathBuf>,
#[serde(default)]
icon: Option<PathBuf>,
#[serde(default)]
docker_images: Option<PathBuf>,
#[serde(default)]
instructions: Option<PathBuf>,
}
impl Assets {
pub fn license_path(&self) -> &Path {
self.license
.as_ref()
.map(|a| a.as_path())
.unwrap_or(Path::new("LICENSE.md"))
}
pub fn icon_path(&self) -> &Path {
self.icon
.as_ref()
.map(|a| a.as_path())
.unwrap_or(Path::new("icon.png"))
}
pub fn icon_type(&self) -> &str {
self.icon
.as_ref()
.and_then(|icon| icon.extension())
.and_then(|ext| ext.to_str())
.unwrap_or("png")
}
pub fn docker_images_path(&self) -> &Path {
self.docker_images
.as_ref()
.map(|a| a.as_path())
.unwrap_or(Path::new("images.tar"))
}
pub fn instructions_path(&self) -> &Path {
self.instructions
.as_ref()
.map(|a| a.as_path())
.unwrap_or(Path::new("INSTRUCTIONS.md"))
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Description {
pub short: String,
pub long: String,
}
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct Alerts {
pub install: Option<String>,
pub uninstall: Option<String>,
pub restore: Option<String>,
pub start: Option<String>,
pub stop: Option<String>,
}

6
appmgr/src/s9pk/mod.rs Normal file
View File

@@ -0,0 +1,6 @@
pub mod builder;
pub mod header;
pub mod manifest;
pub mod reader;
pub const SIG_CONTEXT: &'static [u8] = b"s9pk";

144
appmgr/src/s9pk/reader.rs Normal file
View File

@@ -0,0 +1,144 @@
use std::io::SeekFrom;
use std::path::Path;
use std::pin::Pin;
use std::task::{Context, Poll};
use digest::Output;
use sha2::{Digest, Sha512};
use tokio::fs::File;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, ReadBuf, Take};
use super::header::{FileSection, Header, TableOfContents};
use super::manifest::Manifest;
use super::SIG_CONTEXT;
use crate::install::progress::InstallProgressTracker;
use crate::{Error, ResultExt};
#[pin_project::pin_project]
pub struct ReadHandle<'a, R: AsyncRead + AsyncSeek + Unpin = File> {
pos: &'a mut u64,
#[pin]
rdr: Take<&'a mut R>,
}
impl<'a, R: AsyncRead + AsyncSeek + Unpin> ReadHandle<'a, R> {
pub async fn to_vec(mut self) -> std::io::Result<Vec<u8>> {
let mut buf = vec![0; self.rdr.limit() as usize];
self.rdr.read_exact(&mut buf).await?;
Ok(buf)
}
}
impl<'a, R: AsyncRead + AsyncSeek + Unpin> AsyncRead for ReadHandle<'a, R> {
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
let start = buf.filled().len();
let this = self.project();
let pos = this.pos;
AsyncRead::poll_read(this.rdr, cx, buf).map(|res| {
**pos += (buf.filled().len() - start) as u64;
res
})
}
}
pub struct S9pkReader<R: AsyncRead + AsyncSeek + Unpin = File> {
hash: Output<Sha512>,
hash_string: String,
toc: TableOfContents,
pos: u64,
rdr: R,
}
impl S9pkReader {
pub async fn open<P: AsRef<Path>>(path: P) -> Result<Self, Error> {
let p = path.as_ref();
let rdr = File::open(p)
.await
.with_ctx(|_| (crate::error::ErrorKind::Filesystem, p.display().to_string()))?;
Self::from_reader(rdr).await
}
}
impl<R: AsyncRead + AsyncSeek + Unpin> S9pkReader<InstallProgressTracker<R>> {
pub fn validated(&mut self) {
self.rdr.validated()
}
}
impl<R: AsyncRead + AsyncSeek + Unpin> S9pkReader<R> {
pub async fn validate(&mut self) -> Result<(), Error> {
todo!()
}
pub async fn from_reader(mut rdr: R) -> Result<Self, Error> {
let header = Header::deserialize(&mut rdr).await?;
let pos = rdr.stream_position().await?;
let mut hasher = Sha512::new();
let mut buf = [0; 1024];
let mut read;
while {
read = rdr.read(&mut buf).await?;
read != 0
} {
hasher.update(&buf[0..read]);
}
let hash = hasher.clone().finalize();
header
.pubkey
.verify_prehashed(hasher, Some(SIG_CONTEXT), &header.signature)?;
Ok(S9pkReader {
hash_string: base32::encode(
base32::Alphabet::RFC4648 { padding: false },
hash.as_slice(),
),
hash,
toc: header.table_of_contents,
pos,
rdr,
})
}
pub fn hash(&self) -> &Output<Sha512> {
&self.hash
}
pub fn hash_str(&self) -> &str {
self.hash_string.as_str()
}
async fn read_handle<'a>(
&'a mut self,
section: FileSection,
) -> Result<ReadHandle<'a, R>, Error> {
if self.pos != section.position {
self.rdr.seek(SeekFrom::Start(section.position)).await?;
self.pos = section.position;
}
Ok(ReadHandle {
pos: &mut self.pos,
rdr: (&mut self.rdr).take(section.length),
})
}
pub async fn manifest(&mut self) -> Result<Manifest, Error> {
serde_cbor::from_slice(&self.read_handle(self.toc.manifest).await?.to_vec().await?)
.with_ctx(|_| (crate::ErrorKind::ParseS9pk, "Deserializing Manifest (CBOR)"))
}
pub async fn license<'a>(&'a mut self) -> Result<ReadHandle<'a, R>, Error> {
Ok(self.read_handle(self.toc.license).await?)
}
pub async fn instructions<'a>(&'a mut self) -> Result<ReadHandle<'a, R>, Error> {
Ok(self.read_handle(self.toc.instructions).await?)
}
pub async fn icon<'a>(&'a mut self) -> Result<ReadHandle<'a, R>, Error> {
Ok(self.read_handle(self.toc.icon).await?)
}
pub async fn docker_images<'a>(&'a mut self) -> Result<ReadHandle<'a, R>, Error> {
Ok(self.read_handle(self.toc.docker_images).await?)
}
}