Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-30 20:14:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge 036, everything broken

Browse Source
This commit is contained in:
Matt Hill
2024-03-20 13:32:57 -06:00
parent f4fadd366e
commit 5e6a7e134f
429 changed files with 42285 additions and 27221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
core/startos/Cargo.toml
View File

@@ -21,20 +21,26 @@ license = "MIT"
name = "startos"
path = "src/lib.rs"
[[bin]]
name = "containerbox"
path = "src/main.rs"
[[bin]]
name = "start-cli"
path = "src/main.rs"
[[bin]]
name = "startbox"
path = "src/main.rs"
[features]
avahi = ["avahi-sys"]
avahi-alias = ["avahi"]
cli = []
container-runtime = []
daemon = []
default = ["cli", "sdk", "daemon", "js-engine"]
default = ["cli", "daemon"]
dev = []
docker = []
sdk = []
unstable = ["console-subscriber", "tokio/tracing"]
docker = []
[dependencies]
aes = { version = "0.7.5", features = ["ctr"] }
@@ -45,18 +51,16 @@ async-compression = { version = "0.4.4", features = [
] }
async-stream = "0.3.5"
async-trait = "0.1.74"
avahi-sys = { git = "https://github.com/Start9Labs/avahi-sys", version = "0.10.0", branch = "feature/dynamic-linking", features = [
"dynamic",
], optional = true }
axum = { version = "0.7.3", features = ["ws"] }
axum-server = "0.6.0"
base32 = "0.4.0"
base64 = "0.21.4"
base64ct = "1.6.0"
basic-cookies = "0.1.4"
bimap = { version = "0.6.2", features = ["serde"] }
blake3 = "1.5.0"
bytes = "1"
chrono = { version = "0.4.31", features = ["serde"] }
clap = "3.2.25"
clap = "4.4.12"
color-eyre = "0.6.2"
console = "0.15.7"
console-subscriber = { version = "0.2", optional = true }
@@ -73,7 +77,6 @@ ed25519-dalek = { version = "2.0.0", features = [
"digest",
] }
ed25519-dalek-v1 = { package = "ed25519-dalek", version = "1" }
container-init = { path = "../container-init" }
emver = { version = "0.1.7", git = "https://github.com/Start9Labs/emver-rs.git", features = [
"serde",
] }
@@ -83,13 +86,11 @@ gpt = "3.1.0"
helpers = { path = "../helpers" }
hex = "0.4.3"
hmac = "0.12.1"
http = "0.2.9"
hyper = { version = "0.14.27", features = ["full"] }
hyper-ws-listener = "0.3.0"
id-pool = { version = "0.2.2", features = [
"u16",
http = "1.0.0"
id-pool = { version = "0.2.2", default-features = false, features = [
"serde",
], default-features = false }
"u16",
] }
imbl = "2.0.2"
imbl-value = { git = "https://github.com/Start9Labs/imbl-value.git" }
include_dir = "0.7.3"
@@ -99,12 +100,13 @@ integer-encoding = { version = "4.0.0", features = ["tokio_async"] }
ipnet = { version = "2.8.0", features = ["serde"] }
iprange = { version = "0.6.7", features = ["serde"] }
isocountry = "0.3.2"
itertools = "0.11.0"
itertools = "0.12.0"
jaq-core = "0.10.1"
jaq-std = "0.10.0"
josekit = "0.8.4"
js-engine = { path = '../js-engine', optional = true }
jsonpath_lib = { git = "https://github.com/Start9Labs/jsonpath.git" }
lazy_async_pool = "0.3.3"
lazy_format = "2.0"
lazy_static = "1.4.0"
libc = "0.2.149"
log = "0.4.20"
@@ -115,6 +117,7 @@ nix = { version = "0.27.1", features = ["user", "process", "signal", "fs"] }
nom = "7.1.3"
num = "0.4.1"
num_enum = "0.7.0"
once_cell = "1.19.0"
openssh-keys = "0.6.2"
openssl = { version = "0.10.57", features = ["vendored"] }
p256 = { version = "0.13.2", features = ["pem"] }
@@ -129,12 +132,12 @@ proptest = "1.3.1"
proptest-derive = "0.4.0"
rand = { version = "0.8.5", features = ["std"] }
regex = "1.10.2"
reqwest = { version = "0.11.22", features = ["stream", "json", "socks"] }
reqwest = { version = "0.11.23", features = ["stream", "json", "socks"] }
reqwest_cookie_store = "0.6.0"
rpassword = "7.2.0"
rpc-toolkit = "0.2.2"
rpc-toolkit = { git = "https://github.com/Start9Labs/rpc-toolkit.git", branch = "refactor/traits" }
rust-argon2 = "2.0.0"
scopeguard = "1.1" # because avahi-sys fucks your shit up
rustyline-async = "0.4.1"
semver = { version = "1.0.20", features = ["serde"] }
serde = { version = "1.0", features = ["derive", "rc"] }
serde_cbor = { package = "ciborium", version = "0.2.1" }
@@ -143,6 +146,7 @@ serde_toml = { package = "toml", version = "0.8.2" }
serde_with = { version = "3.4.0", features = ["macros", "json"] }
serde_yaml = "0.9.25"
sha2 = "0.10.2"
shell-words = "1"
simple-logging = "2.0.2"
sqlx = { version = "0.7.2", features = [
"chrono",
@@ -155,20 +159,23 @@ stderrlog = "0.5.4"
tar = "0.4.40"
thiserror = "1.0.49"
tokio = { version = "1", features = ["full"] }
tokio-rustls = "0.24.1"
tokio-rustls = "0.25.0"
tokio-socks = "0.5.1"
tokio-stream = { version = "0.1.14", features = ["io-util", "sync", "net"] }
tokio-tar = { git = "https://github.com/dr-bonez/tokio-tar.git" }
tokio-tungstenite = { version = "0.20.1", features = ["native-tls"] }
tokio-tungstenite = { version = "0.21.0", features = ["native-tls"] }
tokio-util = { version = "0.7.9", features = ["io"] }
torut = "0.2.1"
torut = { git = "https://github.com/Start9Labs/torut.git", branch = "update/dependencies", features = [
"serialize",
] }
tracing = "0.1.39"
tracing-error = "0.2.0"
tracing-futures = "0.2.5"
tracing-journald = "0.3.0"
tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
trust-dns-server = "0.23.1"
typed-builder = "0.17.0"
ts-rs = "7.1.1"
typed-builder = "0.18.0"
url = { version = "2.4.1", features = ["serde"] }
urlencoding = "2.1.3"
uuid = { version = "1.4.1", features = ["v4"] }

3
core/startos/Effects.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface SetStoreParams { value: any, path: string, }

3
core/startos/bindings/AddSslOptions.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface AddSslOptions { scheme: string | null, preferredExternalPort: number, addXForwardedHeaders: boolean | null, }

4
core/startos/bindings/AddressInfo.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { BindOptions } from "./BindOptions";
export interface AddressInfo { username: string | null, hostId: string, bindOptions: BindOptions, suffix: string, }

3
core/startos/bindings/Algorithm.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type Algorithm = "ecdsa" | "ed25519";

3
core/startos/bindings/AllowedStatuses.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type AllowedStatuses = "only-running" | "only-stopped" | "any" | "disabled";

3
core/startos/bindings/BindKind.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type BindKind = "static" | "single" | "multi";

5
core/startos/bindings/BindOptions.ts Normal file
View File

@@ -0,0 +1,5 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { AddSslOptions } from "./AddSslOptions";
import type { BindOptionsSecure } from "./BindOptionsSecure";
export interface BindOptions { scheme: string | null, preferredExternalPort: number, addSsl: AddSslOptions | null, secure: BindOptionsSecure | null, }

3
core/startos/bindings/BindOptionsSecure.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface BindOptionsSecure { ssl: boolean, }

6
core/startos/bindings/BindParams.ts Normal file
View File

@@ -0,0 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { AddSslOptions } from "./AddSslOptions";
import type { BindKind } from "./BindKind";
import type { BindOptionsSecure } from "./BindOptionsSecure";
export interface BindParams { kind: BindKind, id: string, internalPort: number, scheme: string, preferredExternalPort: number, addSsl: AddSslOptions | null, secure: BindOptionsSecure | null, }

3
core/startos/bindings/Callback.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type Callback = () => void;

3
core/startos/bindings/ChrootParams.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface ChrootParams { env: string | null, workdir: string | null, user: string | null, path: string, command: string, args: string[], }

3
core/startos/bindings/CreateOverlayedImageParams.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface CreateOverlayedImageParams { imageId: string, }

3
core/startos/bindings/DependencyKind.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type DependencyKind = "exists" | "running";

4
core/startos/bindings/DependencyRequirement.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DependencyKind } from "./DependencyKind";
export interface DependencyRequirement { id: string, kind: DependencyKind, healthChecks: string[], }

3
core/startos/bindings/DestroyOverlayedImageParams.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface DestroyOverlayedImageParams { guid: string, }

3
core/startos/bindings/ExecuteAction.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface ExecuteAction { serviceId: string | null, actionId: string, input: any, }

4
core/startos/bindings/ExportActionParams.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { AllowedStatuses } from "./AllowedStatuses";
export interface ExportActionParams { name: string, description: string, id: string, input: {[key: string]: any}, allowedStatuses: AllowedStatuses, group: string | null, }

5
core/startos/bindings/ExportServiceInterfaceParams.ts Normal file
View File

@@ -0,0 +1,5 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { AddressInfo } from "./AddressInfo";
import type { ServiceInterfaceType } from "./ServiceInterfaceType";
export interface ExportServiceInterfaceParams { id: string, name: string, description: string, hasPrimary: boolean, disabled: boolean, masked: boolean, addressInfo: AddressInfo, type: ServiceInterfaceType, }

3
core/startos/bindings/ExposeForDependentsParams.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface ExposeForDependentsParams { paths: string[], }

3
core/startos/bindings/ExposeUiParams.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type ExposeUiParams = { "type": "object", value: {[key: string]: ExposeUiParams}, } | { "type": "string", path: string, description: string | null, masked: boolean, copyable: boolean | null, qr: boolean | null, };

3
core/startos/bindings/ExposedUI.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type ExposedUI = { "type": "object", value: {[key: string]: ExposedUI}, description: string | null, } | { "type": "string", path: string, description: string | null, masked: boolean, copyable: boolean | null, qr: boolean | null, };

5
core/startos/bindings/GetHostInfoParams.ts Normal file
View File

@@ -0,0 +1,5 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Callback } from "./Callback";
import type { GetHostInfoParamsKind } from "./GetHostInfoParamsKind";
export interface GetHostInfoParams { kind: GetHostInfoParamsKind | null, serviceInterfaceId: string, packageId: string | null, callback: Callback, }

3
core/startos/bindings/GetHostInfoParamsKind.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type GetHostInfoParamsKind = "multi";

4
core/startos/bindings/GetPrimaryUrlParams.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Callback } from "./Callback";
export interface GetPrimaryUrlParams { packageId: string | null, serviceInterfaceId: string, callback: Callback, }

4
core/startos/bindings/GetServiceInterfaceParams.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Callback } from "./Callback";
export interface GetServiceInterfaceParams { packageId: string | null, serviceInterfaceId: string, callback: Callback, }

3
core/startos/bindings/GetServicePortForwardParams.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface GetServicePortForwardParams { packageId: string | null, internalPort: number, }

4
core/startos/bindings/GetSslCertificateParams.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Algorithm } from "./Algorithm";
export interface GetSslCertificateParams { packageId: string | null, hostId: string, algorithm: Algorithm | null, }

4
core/startos/bindings/GetSslKeyParams.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Algorithm } from "./Algorithm";
export interface GetSslKeyParams { packageId: string | null, hostId: string, algorithm: Algorithm | null, }

3
core/startos/bindings/GetStoreParams.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface GetStoreParams { packageId: string | null, path: string, }

4
core/startos/bindings/GetSystemSmtpParams.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Callback } from "./Callback";
export interface GetSystemSmtpParams { callback: Callback, }

3
core/startos/bindings/HealthCheckString.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type HealthCheckString = "passing" | "disabled" | "starting" | "warning" | "failure";

4
core/startos/bindings/ListServiceInterfacesParams.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Callback } from "./Callback";
export interface ListServiceInterfacesParams { packageId: string | null, callback: Callback, }

4
core/startos/bindings/MountParams.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { MountTarget } from "./MountTarget";
export interface MountParams { location: string, target: MountTarget, }

3
core/startos/bindings/MountTarget.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface MountTarget { packageId: string, volumeId: string, subpath: string | null, readonly: boolean, }

3
core/startos/bindings/ParamsMaybePackageId.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface ParamsMaybePackageId { packageId: string | null, }

3
core/startos/bindings/ParamsPackageId.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface ParamsPackageId { packageId: string, }

3
core/startos/bindings/RemoveActionParams.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface RemoveActionParams { id: string, }

3
core/startos/bindings/RemoveAddressParams.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface RemoveAddressParams { id: string, }

3
core/startos/bindings/ReverseProxyBind.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface ReverseProxyBind { ip: string | null, port: number, ssl: boolean, }

3
core/startos/bindings/ReverseProxyDestination.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface ReverseProxyDestination { ip: string | null, port: number, ssl: boolean, }

3
core/startos/bindings/ReverseProxyHttp.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface ReverseProxyHttp { headers: null | {[key: string]: string}, }

6
core/startos/bindings/ReverseProxyParams.ts Normal file
View File

@@ -0,0 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ReverseProxyBind } from "./ReverseProxyBind";
import type { ReverseProxyDestination } from "./ReverseProxyDestination";
import type { ReverseProxyHttp } from "./ReverseProxyHttp";
export interface ReverseProxyParams { bind: ReverseProxyBind, dst: ReverseProxyDestination, http: ReverseProxyHttp, }

3
core/startos/bindings/ServiceInterfaceType.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type ServiceInterfaceType = "ui" | "p2p" | "api";

3
core/startos/bindings/SetConfigured.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface SetConfigured { configured: boolean, }

4
core/startos/bindings/SetDependenciesParams.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DependencyRequirement } from "./DependencyRequirement";
export interface SetDependenciesParams { dependencies: Array<DependencyRequirement>, }

4
core/startos/bindings/SetHealth.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { HealthCheckString } from "./HealthCheckString";
export interface SetHealth { name: string, status: HealthCheckString, message: string | null, }

4
core/startos/bindings/SetMainStatus.ts Normal file
View File

@@ -0,0 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { Status } from "./Status";
export interface SetMainStatus { status: Status, }

3
core/startos/bindings/SetStoreParams.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface SetStoreParams { value: any, path: string, }

3
core/startos/bindings/Status.ts Normal file
View File

@@ -0,0 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type Status = "running" | "stopped";

10
core/startos/deny.toml
View File

@@ -14,9 +14,15 @@ allow = [
"BSD-3-Clause",
"LGPL-3.0",
"OpenSSL",
"Unicode-DFS-2016",
"Zlib",
]
clarify = [
{ name = "webpki", expression = "ISC", license-files = [ { path = "LICENSE", hash = 0x001c7e6c } ] },
{ name = "ring", expression = "OpenSSL", license-files = [ { path = "LICENSE", hash = 0xbd0eed23 } ] },
{ name = "webpki", expression = "ISC", license-files = [
{ path = "LICENSE", hash = 0x001c7e6c },
] },
{ name = "ring", expression = "OpenSSL", license-files = [
{ path = "LICENSE", hash = 0xbd0eed23 },
] },
]

125
core/startos/src/account.rs
View File

@@ -1,15 +1,14 @@
use std::time::SystemTime;
use ed25519_dalek::SecretKey;
use openssl::pkey::{PKey, Private};
use openssl::x509::X509;
use sqlx::PgExecutor;
use torut::onion::TorSecretKeyV3;
use crate::db::model::DatabaseModel;
use crate::hostname::{generate_hostname, generate_id, Hostname};
use crate::net::keys::Key;
use crate::net::ssl::{generate_key, make_root_cert};
use crate::prelude::*;
use crate::util::crypto::ed25519_expand_key;
use crate::util::serde::Pem;
fn hash_password(password: &str) -> Result<String, Error> {
argon2::hash_encoded(
@@ -25,103 +24,83 @@ pub struct AccountInfo {
pub server_id: String,
pub hostname: Hostname,
pub password: String,
pub key: Key,
pub tor_key: TorSecretKeyV3,
pub root_ca_key: PKey<Private>,
pub root_ca_cert: X509,
pub ssh_key: ssh_key::PrivateKey,
}
impl AccountInfo {
pub fn new(password: &str, start_time: SystemTime) -> Result<Self, Error> {
let server_id = generate_id();
let hostname = generate_hostname();
let tor_key = TorSecretKeyV3::generate();
let root_ca_key = generate_key()?;
let root_ca_cert = make_root_cert(&root_ca_key, &hostname, start_time)?;
let ssh_key = ssh_key::PrivateKey::from(ssh_key::private::Ed25519Keypair::random(
&mut rand::thread_rng(),
));
Ok(Self {
server_id,
hostname,
password: hash_password(password)?,
key: Key::new(None),
tor_key,
root_ca_key,
root_ca_cert,
ssh_key,
})
}
pub async fn load(secrets: impl PgExecutor<'_>) -> Result<Self, Error> {
let r = sqlx::query!("SELECT * FROM account WHERE id = 0")
.fetch_one(secrets)
.await?;
let server_id = r.server_id.unwrap_or_else(generate_id);
let hostname = r.hostname.map(Hostname).unwrap_or_else(generate_hostname);
let password = r.password;
let network_key = SecretKey::try_from(r.network_key).map_err(|e| {
Error::new(
eyre!("expected vec of len 32, got len {}", e.len()),
ErrorKind::ParseDbField,
)
})?;
let tor_key = if let Some(k) = &r.tor_key {
<[u8; 64]>::try_from(&k[..]).map_err(|_| {
Error::new(
eyre!("expected vec of len 64, got len {}", k.len()),
ErrorKind::ParseDbField,
)
})?
} else {
ed25519_expand_key(&network_key)
};
let key = Key::from_pair(None, network_key, tor_key);
let root_ca_key = PKey::private_key_from_pem(r.root_ca_key_pem.as_bytes())?;
let root_ca_cert = X509::from_pem(r.root_ca_cert_pem.as_bytes())?;
pub fn load(db: &DatabaseModel) -> Result<Self, Error> {
let server_id = db.as_public().as_server_info().as_id().de()?;
let hostname = Hostname(db.as_public().as_server_info().as_hostname().de()?);
let password = db.as_private().as_password().de()?;
let key_store = db.as_private().as_key_store();
let tor_addr = db.as_public().as_server_info().as_onion_address().de()?;
let tor_key = key_store.as_onion().get_key(&tor_addr)?;
let cert_store = key_store.as_local_certs();
let root_ca_key = cert_store.as_root_key().de()?.0;
let root_ca_cert = cert_store.as_root_cert().de()?.0;
let ssh_key = db.as_private().as_ssh_privkey().de()?.0;
Ok(Self {
server_id,
hostname,
password,
key,
tor_key,
root_ca_key,
root_ca_cert,
ssh_key,
})
}
pub async fn save(&self, secrets: impl PgExecutor<'_>) -> Result<(), Error> {
let server_id = self.server_id.as_str();
let hostname = self.hostname.0.as_str();
let password = self.password.as_str();
let network_key = self.key.as_bytes();
let network_key = network_key.as_slice();
let root_ca_key = String::from_utf8(self.root_ca_key.private_key_to_pem_pkcs8()?)?;
let root_ca_cert = String::from_utf8(self.root_ca_cert.to_pem()?)?;
sqlx::query!(
r#"
INSERT INTO account (
id,
server_id,
hostname,
password,
network_key,
root_ca_key_pem,
root_ca_cert_pem
) VALUES (
0, $1, $2, $3, $4, $5, $6
) ON CONFLICT (id) DO UPDATE SET
server_id = EXCLUDED.server_id,
hostname = EXCLUDED.hostname,
password = EXCLUDED.password,
network_key = EXCLUDED.network_key,
root_ca_key_pem = EXCLUDED.root_ca_key_pem,
root_ca_cert_pem = EXCLUDED.root_ca_cert_pem
"#,
server_id,
hostname,
password,
network_key,
root_ca_key,
root_ca_cert,
)
.execute(secrets)
.await?;
pub fn save(&self, db: &mut DatabaseModel) -> Result<(), Error> {
let server_info = db.as_public_mut().as_server_info_mut();
server_info.as_id_mut().ser(&self.server_id)?;
server_info.as_hostname_mut().ser(&self.hostname.0)?;
server_info
.as_lan_address_mut()
.ser(&self.hostname.lan_address().parse()?)?;
server_info
.as_pubkey_mut()
.ser(&self.ssh_key.public_key().to_openssh()?)?;
let onion_address = self.tor_key.public().get_onion_address();
server_info.as_onion_address_mut().ser(&onion_address)?;
server_info
.as_tor_address_mut()
.ser(&format!("https://{onion_address}").parse()?)?;
db.as_private_mut().as_password_mut().ser(&self.password)?;
db.as_private_mut()
.as_ssh_privkey_mut()
.ser(Pem::new_ref(&self.ssh_key))?;
let key_store = db.as_private_mut().as_key_store_mut();
key_store.as_onion_mut().insert_key(&self.tor_key)?;
let cert_store = key_store.as_local_certs_mut();
cert_store
.as_root_key_mut()
.ser(Pem::new_ref(&self.root_ca_key))?;
cert_store
.as_root_cert_mut()
.ser(Pem::new_ref(&self.root_ca_cert))?;
Ok(())
}

156
core/startos/src/action.rs
View File

@@ -1,26 +1,14 @@
use std::collections::{BTreeMap, BTreeSet};
use clap::ArgMatches;
use color_eyre::eyre::eyre;
use indexmap::IndexSet;
use clap::Parser;
pub use models::ActionId;
use models::ImageId;
use models::PackageId;
use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
use tracing::instrument;
use crate::config::{Config, ConfigSpec};
use crate::config::Config;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::procedure::docker::DockerContainers;
use crate::procedure::{PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
use crate::util::Version;
use crate::volume::Volumes;
use crate::{Error, ResultExt};
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct Actions(pub BTreeMap<ActionId, Action>);
use crate::util::serde::{display_serializable, StdinDeserializable, WithIoFormat};
#[derive(Debug, Serialize, Deserialize)]
#[serde(tag = "version")]
@@ -44,72 +32,11 @@ pub enum DockerStatus {
Stopped,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct Action {
pub name: String,
pub description: String,
#[serde(default)]
pub warning: Option<String>,
pub implementation: PackageProcedure,
pub allowed_statuses: IndexSet<DockerStatus>,
#[serde(default)]
pub input_spec: ConfigSpec,
}
impl Action {
#[instrument(skip_all)]
pub fn validate(
&self,
_container: &Option<DockerContainers>,
eos_version: &Version,
volumes: &Volumes,
image_ids: &BTreeSet<ImageId>,
) -> Result<(), Error> {
self.implementation
.validate(eos_version, volumes, image_ids, true)
.with_ctx(|_| {
(
crate::ErrorKind::ValidateS9pk,
format!("Action {}", self.name),
)
})
pub fn display_action_result(params: WithIoFormat<ActionParams>, result: ActionResult) {
if let Some(format) = params.format {
return display_serializable(format, result);
}
#[instrument(skip_all)]
pub async fn execute(
&self,
ctx: &RpcContext,
pkg_id: &PackageId,
pkg_version: &Version,
action_id: &ActionId,
volumes: &Volumes,
input: Option<Config>,
) -> Result<ActionResult, Error> {
if let Some(ref input) = input {
self.input_spec
.matches(&input)
.with_kind(crate::ErrorKind::ConfigSpecViolation)?;
}
self.implementation
.execute(
ctx,
pkg_id,
pkg_version,
ProcedureName::Action(action_id.clone()),
volumes,
input,
None,
)
.await?
.map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::Action))
}
}
fn display_action_result(action_result: ActionResult, matches: &ArgMatches) {
if matches.is_present("format") {
return display_serializable(action_result, matches);
}
match action_result {
match result {
ActionResult::V0(ar) => {
println!(
"{}: {}",
@@ -120,44 +47,39 @@ fn display_action_result(action_result: ActionResult, matches: &ArgMatches) {
}
}
#[command(about = "Executes an action", display(display_action_result))]
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct ActionParams {
#[arg(id = "id")]
#[serde(rename = "id")]
pub package_id: PackageId,
#[arg(id = "action-id")]
#[serde(rename = "action-id")]
pub action_id: ActionId,
#[command(flatten)]
pub input: StdinDeserializable<Option<Config>>,
}
// impl C
// #[command(about = "Executes an action", display(display_action_result))]
#[instrument(skip_all)]
pub async fn action(
#[context] ctx: RpcContext,
#[arg(rename = "id")] pkg_id: PackageId,
#[arg(rename = "action-id")] action_id: ActionId,
#[arg(stdin, parse(parse_stdin_deserializable))] input: Option<Config>,
#[allow(unused_variables)]
#[arg(long = "format")]
format: Option<IoFormat>,
ctx: RpcContext,
ActionParams {
package_id,
action_id,
input: StdinDeserializable(input),
}: ActionParams,
) -> Result<ActionResult, Error> {
let manifest = ctx
.db
.peek()
ctx.services
.get(&package_id)
.await
.as_ref()
.or_not_found(lazy_format!("Manager for {}", package_id))?
.action(
action_id,
input.map(|c| to_value(&c)).transpose()?.unwrap_or_default(),
)
.await
.as_package_data()
.as_idx(&pkg_id)
.or_not_found(&pkg_id)?
.as_installed()
.or_not_found(&pkg_id)?
.as_manifest()
.de()?;
if let Some(action) = manifest.actions.0.get(&action_id) {
action
.execute(
&ctx,
&manifest.id,
&manifest.version,
&action_id,
&manifest.volumes,
input,
)
.await
} else {
Err(Error::new(
eyre!("Action not found in manifest"),
crate::ErrorKind::NotFound,
))
}
}

376
core/startos/src/auth.rs
View File

@@ -1,25 +1,43 @@
use std::collections::BTreeMap;
use std::marker::PhantomData;
use chrono::{DateTime, Utc};
use clap::ArgMatches;
use clap::Parser;
use color_eyre::eyre::eyre;
use imbl_value::{json, InternedString};
use josekit::jwk::Jwk;
use rpc_toolkit::command;
use rpc_toolkit::command_helpers::prelude::{RequestParts, ResponseParts};
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::{command, from_fn_async, AnyContext, CallRemote, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sqlx::{Executor, Postgres};
use tracing::instrument;
use crate::context::{CliContext, RpcContext};
use crate::middleware::auth::{AsLogoutSessionId, HasLoggedOutSessions, HashSessionToken};
use crate::middleware::encrypt::EncryptedWire;
use crate::db::model::DatabaseModel;
use crate::middleware::auth::{
AsLogoutSessionId, HasLoggedOutSessions, HashSessionToken, LoginRes,
};
use crate::prelude::*;
use crate::util::display_none;
use crate::util::serde::{display_serializable, IoFormat};
use crate::util::crypto::EncryptedWire;
use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat};
use crate::{ensure_code, Error, ResultExt};
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
pub struct Sessions(pub BTreeMap<InternedString, Session>);
impl Sessions {
pub fn new() -> Self {
Self(BTreeMap::new())
}
}
impl Map for Sessions {
type Key = InternedString;
type Value = Session;
fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
Ok(key)
}
fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
Ok(key.clone())
}
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(untagged)]
pub enum PasswordType {
@@ -61,20 +79,39 @@ impl std::str::FromStr for PasswordType {
})
}
}
#[command(subcommands(login, logout, session, reset_password, get_pubkey))]
pub fn auth() -> Result<(), Error> {
Ok(())
}
pub fn cli_metadata() -> Value {
serde_json::json!({
"platforms": ["cli"],
})
}
pub fn parse_metadata(_: &str, _: &ArgMatches) -> Result<Value, Error> {
Ok(cli_metadata())
pub fn auth() -> ParentHandler {
ParentHandler::new()
.subcommand(
"login",
from_fn_async(login_impl)
.with_metadata("login", Value::Bool(true))
.no_cli(),
)
.subcommand("login", from_fn_async(cli_login).no_display())
.subcommand(
"logout",
from_fn_async(logout)
.with_metadata("get-session", Value::Bool(true))
.with_remote_cli::<CliContext>()
// TODO @dr-bonez
.no_display(),
)
.subcommand("session", session())
.subcommand(
"reset-password",
from_fn_async(reset_password_impl).no_cli(),
)
.subcommand(
"reset-password",
from_fn_async(cli_reset_password).no_display(),
)
.subcommand(
"get-pubkey",
from_fn_async(get_pubkey)
.with_metadata("authenticated", Value::Bool(false))
.no_display()
.with_remote_cli::<CliContext>(),
)
}
#[test]
@@ -89,12 +126,17 @@ fn gen_pwd() {
.unwrap()
)
}
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct CliLoginParams {
password: Option<PasswordType>,
}
#[instrument(skip_all)]
async fn cli_login(
ctx: CliContext,
password: Option<PasswordType>,
metadata: Value,
CliLoginParams { password }: CliLoginParams,
) -> Result<(), RpcError> {
let password = if let Some(password) = password {
password.decrypt(&ctx)?
@@ -102,14 +144,16 @@ async fn cli_login(
rpassword::prompt_password("Password: ")?
};
rpc_toolkit::command_helpers::call_remote(
ctx,
ctx.call_remote(
"auth.login",
serde_json::json!({ "password": password, "metadata": metadata }),
PhantomData::<()>,
json!({
"password": password,
"metadata": {
"platforms": ["cli"],
},
}),
)
.await?
.result?;
.await?;
Ok(())
}
@@ -128,99 +172,110 @@ pub fn check_password(hash: &str, password: &str) -> Result<(), Error> {
Ok(())
}
pub async fn check_password_against_db<Ex>(secrets: &mut Ex, password: &str) -> Result<(), Error>
where
for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
{
let pw_hash = sqlx::query!("SELECT password FROM account")
.fetch_one(secrets)
.await?
.password;
pub fn check_password_against_db(db: &DatabaseModel, password: &str) -> Result<(), Error> {
let pw_hash = db.as_private().as_password().de()?;
check_password(&pw_hash, password)?;
Ok(())
}
#[command(
custom_cli(cli_login(async, context(CliContext))),
display(display_none),
metadata(authenticated = false)
)]
#[instrument(skip_all)]
pub async fn login(
#[context] ctx: RpcContext,
#[request] req: &RequestParts,
#[response] res: &mut ResponseParts,
#[arg] password: Option<PasswordType>,
#[arg(
parse(parse_metadata),
default = "cli_metadata",
help = "RPC Only: This value cannot be overidden from the cli"
)]
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct LoginParams {
password: Option<PasswordType>,
#[serde(default)]
user_agent: Option<String>,
#[serde(default)]
metadata: Value,
) -> Result<(), Error> {
let password = password.unwrap_or_default().decrypt(&ctx)?;
let mut handle = ctx.secret_store.acquire().await?;
check_password_against_db(handle.as_mut(), &password).await?;
}
let hash_token = HashSessionToken::new();
let user_agent = req.headers.get("user-agent").and_then(|h| h.to_str().ok());
let metadata = serde_json::to_string(&metadata).with_kind(crate::ErrorKind::Database)?;
let hash_token_hashed = hash_token.hashed();
sqlx::query!(
"INSERT INTO session (id, user_agent, metadata) VALUES ($1, $2, $3)",
hash_token_hashed,
#[instrument(skip_all)]
pub async fn login_impl(
ctx: RpcContext,
LoginParams {
password,
user_agent,
metadata,
)
.execute(handle.as_mut())
.await?;
res.headers.insert(
"set-cookie",
hash_token.header_value()?, // Should be impossible, but don't want to panic
);
}: LoginParams,
) -> Result<LoginRes, Error> {
let password = password.unwrap_or_default().decrypt(&ctx)?;
Ok(())
ctx.db
.mutate(|db| {
check_password_against_db(db, &password)?;
let hash_token = HashSessionToken::new();
db.as_private_mut().as_sessions_mut().insert(
hash_token.hashed(),
&Session {
logged_in: Utc::now(),
last_active: Utc::now(),
user_agent,
metadata,
},
)?;
Ok(hash_token.to_login_res())
})
.await
}
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct LogoutParams {
session: InternedString,
}
#[command(display(display_none), metadata(authenticated = false))]
#[instrument(skip_all)]
pub async fn logout(
#[context] ctx: RpcContext,
#[request] req: &RequestParts,
ctx: RpcContext,
LogoutParams { session }: LogoutParams,
) -> Result<Option<HasLoggedOutSessions>, Error> {
let auth = match HashSessionToken::from_request_parts(req) {
Err(_) => return Ok(None),
Ok(a) => a,
};
Ok(Some(HasLoggedOutSessions::new(vec![auth], &ctx).await?))
Ok(Some(
HasLoggedOutSessions::new(vec![HashSessionToken::from_token(session)], &ctx).await?,
))
}
#[derive(Deserialize, Serialize)]
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct Session {
logged_in: DateTime<Utc>,
last_active: DateTime<Utc>,
user_agent: Option<String>,
metadata: Value,
pub logged_in: DateTime<Utc>,
pub last_active: DateTime<Utc>,
pub user_agent: Option<String>,
pub metadata: Value,
}
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct SessionList {
current: String,
sessions: BTreeMap<String, Session>,
current: InternedString,
sessions: Sessions,
}
#[command(subcommands(list, kill))]
pub async fn session() -> Result<(), Error> {
Ok(())
pub fn session() -> ParentHandler {
ParentHandler::new()
.subcommand(
"list",
from_fn_async(list)
.with_metadata("get-session", Value::Bool(true))
.with_display_serializable()
.with_custom_display_fn::<AnyContext, _>(|handle, result| {
Ok(display_sessions(handle.params, result))
})
.with_remote_cli::<CliContext>(),
)
.subcommand(
"kill",
from_fn_async(kill)
.no_display()
.with_remote_cli::<CliContext>(),
)
}
fn display_sessions(arg: SessionList, matches: &ArgMatches) {
fn display_sessions(params: WithIoFormat<ListParams>, arg: SessionList) {
use prettytable::*;
if matches.is_present("format") {
return display_serializable(arg, matches);
if let Some(format) = params.format {
return display_serializable(format, arg);
}
let mut table = Table::new();
@@ -231,7 +286,7 @@ fn display_sessions(arg: SessionList, matches: &ArgMatches) {
"USER AGENT",
"METADATA",
]);
for (id, session) in arg.sessions {
for (id, session) in arg.sessions.0 {
let mut row = row![
&id,
&format!("{}", session.logged_in),
@@ -249,67 +304,71 @@ fn display_sessions(arg: SessionList, matches: &ArgMatches) {
table.print_tty(false).unwrap();
}
#[command(display(display_sessions))]
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct ListParams {
#[arg(skip)]
session: InternedString,
}
// #[command(display(display_sessions))]
#[instrument(skip_all)]
pub async fn list(
#[context] ctx: RpcContext,
#[request] req: &RequestParts,
#[allow(unused_variables)]
#[arg(long = "format")]
format: Option<IoFormat>,
ctx: RpcContext,
ListParams { session, .. }: ListParams,
) -> Result<SessionList, Error> {
Ok(SessionList {
current: HashSessionToken::from_request_parts(req)?.as_hash(),
sessions: sqlx::query!(
"SELECT * FROM session WHERE logged_out IS NULL OR logged_out > CURRENT_TIMESTAMP"
)
.fetch_all(ctx.secret_store.acquire().await?.as_mut())
.await?
.into_iter()
.map(|row| {
Ok((
row.id,
Session {
logged_in: DateTime::from_utc(row.logged_in, Utc),
last_active: DateTime::from_utc(row.last_active, Utc),
user_agent: row.user_agent,
metadata: serde_json::from_str(&row.metadata)
.with_kind(crate::ErrorKind::Database)?,
},
))
})
.collect::<Result<_, Error>>()?,
current: HashSessionToken::from_token(session).hashed().clone(),
sessions: ctx.db.peek().await.into_private().into_sessions().de()?,
})
}
fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<Vec<String>, RpcError> {
Ok(arg.split(",").map(|s| s.trim().to_owned()).collect())
#[derive(Debug, Clone, Serialize, Deserialize)]
struct KillSessionId(InternedString);
impl KillSessionId {
fn new(id: String) -> Self {
Self(InternedString::from(id))
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct KillSessionId(String);
impl AsLogoutSessionId for KillSessionId {
fn as_logout_session_id(self) -> String {
fn as_logout_session_id(self) -> InternedString {
self.0
}
}
#[command(display(display_none))]
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct KillParams {
ids: Vec<String>,
}
#[instrument(skip_all)]
pub async fn kill(
#[context] ctx: RpcContext,
#[arg(parse(parse_comma_separated))] ids: Vec<String>,
) -> Result<(), Error> {
HasLoggedOutSessions::new(ids.into_iter().map(KillSessionId), &ctx).await?;
pub async fn kill(ctx: RpcContext, KillParams { ids }: KillParams) -> Result<(), Error> {
HasLoggedOutSessions::new(ids.into_iter().map(KillSessionId::new), &ctx).await?;
Ok(())
}
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct ResetPasswordParams {
#[arg(name = "old-password")]
old_password: Option<PasswordType>,
#[arg(name = "new-password")]
new_password: Option<PasswordType>,
}
#[instrument(skip_all)]
async fn cli_reset_password(
ctx: CliContext,
old_password: Option<PasswordType>,
new_password: Option<PasswordType>,
ResetPasswordParams {
old_password,
new_password,
}: ResetPasswordParams,
) -> Result<(), RpcError> {
let old_password = if let Some(old_password) = old_password {
old_password.decrypt(&ctx)?
@@ -331,28 +390,22 @@ async fn cli_reset_password(
new_password
};
rpc_toolkit::command_helpers::call_remote(
ctx,
ctx.call_remote(
"auth.reset-password",
serde_json::json!({ "old-password": old_password, "new-password": new_password }),
PhantomData::<()>,
imbl_value::json!({ "old-password": old_password, "new-password": new_password }),
)
.await?
.result?;
.await?;
Ok(())
}
#[command(
rename = "reset-password",
custom_cli(cli_reset_password(async, context(CliContext))),
display(display_none)
)]
#[instrument(skip_all)]
pub async fn reset_password(
#[context] ctx: RpcContext,
#[arg(rename = "old-password")] old_password: Option<PasswordType>,
#[arg(rename = "new-password")] new_password: Option<PasswordType>,
pub async fn reset_password_impl(
ctx: RpcContext,
ResetPasswordParams {
old_password,
new_password,
}: ResetPasswordParams,
) -> Result<(), Error> {
let old_password = old_password.unwrap_or_default().decrypt(&ctx)?;
let new_password = new_password.unwrap_or_default().decrypt(&ctx)?;
@@ -367,24 +420,23 @@ pub async fn reset_password(
));
}
account.set_password(&new_password)?;
account.save(&ctx.secret_store).await?;
let account_password = &account.password;
let account = account.clone();
ctx.db
.mutate(|d| {
d.as_server_info_mut()
d.as_public_mut()
.as_server_info_mut()
.as_password_hash_mut()
.ser(account_password)
.ser(account_password)?;
account.save(d)?;
Ok(())
})
.await
}
#[command(
rename = "get-pubkey",
display(display_none),
metadata(authenticated = false)
)]
#[instrument(skip_all)]
pub async fn get_pubkey(#[context] ctx: RpcContext) -> Result<Jwk, RpcError> {
pub async fn get_pubkey(ctx: RpcContext) -> Result<Jwk, RpcError> {
let secret = ctx.as_ref().clone();
let pub_key = secret.to_public_key()?;
Ok(pub_key)

448
core/startos/src/backup/backup_bulk.rs
View File

@@ -1,17 +1,15 @@
use std::collections::BTreeMap;
use std::panic::UnwindSafe;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use chrono::Utc;
use clap::ArgMatches;
use clap::Parser;
use color_eyre::eyre::eyre;
use helpers::AtomicFile;
use imbl::OrdSet;
use models::Version;
use rpc_toolkit::command;
use models::PackageId;
use serde::{Deserialize, Serialize};
use tokio::io::AsyncWriteExt;
use tokio::sync::Mutex;
use tracing::instrument;
use super::target::BackupTargetId;
@@ -20,259 +18,264 @@ use crate::auth::check_password_against_db;
use crate::backup::os::OsBackup;
use crate::backup::{BackupReport, ServerBackupReport};
use crate::context::RpcContext;
use crate::db::model::BackupProgress;
use crate::db::package::get_packages;
use crate::db::model::public::BackupProgress;
use crate::db::model::DatabaseModel;
use crate::disk::mount::backup::BackupMountGuard;
use crate::disk::mount::filesystem::ReadWrite;
use crate::disk::mount::guard::TmpMountGuard;
use crate::manager::BackupReturn;
use crate::notifications::NotificationLevel;
use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard};
use crate::notifications::{notify, NotificationLevel};
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::util::display_none;
use crate::util::io::dir_copy;
use crate::util::serde::IoFormat;
use crate::version::VersionT;
fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<OrdSet<PackageId>, Error> {
arg.split(',')
.map(|s| s.trim().parse::<PackageId>().map_err(Error::from))
.collect()
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct BackupParams {
target_id: BackupTargetId,
#[arg(long = "old-password")]
old_password: Option<crate::auth::PasswordType>,
#[arg(long = "package-ids")]
package_ids: Option<Vec<PackageId>>,
password: crate::auth::PasswordType,
}
struct BackupStatusGuard(Option<PatchDb>);
impl BackupStatusGuard {
fn new(db: PatchDb) -> Self {
Self(Some(db))
}
async fn handle_result(
mut self,
result: Result<BTreeMap<PackageId, PackageBackupReport>, Error>,
) -> Result<(), Error> {
if let Some(db) = self.0.as_ref() {
db.mutate(|v| {
v.as_public_mut()
.as_server_info_mut()
.as_status_info_mut()
.as_backup_progress_mut()
.ser(&None)
})
.await?;
}
if let Some(db) = self.0.take() {
match result {
Ok(report) if report.iter().all(|(_, rep)| rep.error.is_none()) => {
db.mutate(|db| {
notify(
db,
None,
NotificationLevel::Success,
"Backup Complete".to_owned(),
"Your backup has completed".to_owned(),
BackupReport {
server: ServerBackupReport {
attempted: true,
error: None,
},
packages: report,
},
)
})
.await
}
Ok(report) => {
db.mutate(|db| {
notify(
db,
None,
NotificationLevel::Warning,
"Backup Complete".to_owned(),
"Your backup has completed, but some package(s) failed to backup"
.to_owned(),
BackupReport {
server: ServerBackupReport {
attempted: true,
error: None,
},
packages: report,
},
)
})
.await
}
Err(e) => {
tracing::error!("Backup Failed: {}", e);
tracing::debug!("{:?}", e);
let err_string = e.to_string();
db.mutate(|db| {
notify(
db,
None,
NotificationLevel::Error,
"Backup Failed".to_owned(),
"Your backup failed to complete.".to_owned(),
BackupReport {
server: ServerBackupReport {
attempted: true,
error: Some(err_string),
},
packages: BTreeMap::new(),
},
)
})
.await
}
}?;
}
Ok(())
}
}
impl Drop for BackupStatusGuard {
fn drop(&mut self) {
if let Some(db) = self.0.take() {
tokio::spawn(async move {
db.mutate(|v| {
v.as_public_mut()
.as_server_info_mut()
.as_status_info_mut()
.as_backup_progress_mut()
.ser(&None)
})
.await
.unwrap()
});
}
}
}
#[command(rename = "create", display(display_none))]
#[instrument(skip(ctx, old_password, password))]
pub async fn backup_all(
#[context] ctx: RpcContext,
#[arg(rename = "target-id")] target_id: BackupTargetId,
#[arg(rename = "old-password", long = "old-password")] old_password: Option<
crate::auth::PasswordType,
>,
#[arg(
rename = "package-ids",
long = "package-ids",
parse(parse_comma_separated)
)]
package_ids: Option<OrdSet<PackageId>>,
#[arg] password: crate::auth::PasswordType,
ctx: RpcContext,
BackupParams {
target_id,
old_password,
package_ids,
password,
}: BackupParams,
) -> Result<(), Error> {
let db = ctx.db.peek().await;
let old_password_decrypted = old_password
.as_ref()
.unwrap_or(&password)
.clone()
.decrypt(&ctx)?;
let password = password.decrypt(&ctx)?;
check_password_against_db(ctx.secret_store.acquire().await?.as_mut(), &password).await?;
let fs = target_id
.load(ctx.secret_store.acquire().await?.as_mut())
.await?;
let ((fs, package_ids), status_guard) = (
ctx.db
.mutate(|db| {
check_password_against_db(db, &password)?;
let fs = target_id.load(db)?;
let package_ids = if let Some(ids) = package_ids {
ids.into_iter().collect()
} else {
db.as_public()
.as_package_data()
.as_entries()?
.into_iter()
.filter(|(_, m)| m.as_state_info().expect_installed().is_ok())
.map(|(id, _)| id)
.collect()
};
assure_backing_up(db, &package_ids)?;
Ok((fs, package_ids))
})
.await?,
BackupStatusGuard::new(ctx.db.clone()),
);
let mut backup_guard = BackupMountGuard::mount(
TmpMountGuard::mount(&fs, ReadWrite).await?,
&old_password_decrypted,
)
.await?;
let package_ids = if let Some(ids) = package_ids {
ids.into_iter()
.flat_map(|package_id| {
let version = db
.as_package_data()
.as_idx(&package_id)?
.as_manifest()
.as_version()
.de()
.ok()?;
Some((package_id, version))
})
.collect()
} else {
get_packages(db.clone())?.into_iter().collect()
};
if old_password.is_some() {
backup_guard.change_password(&password)?;
}
assure_backing_up(&ctx.db, &package_ids).await?;
tokio::task::spawn(async move {
let backup_res = perform_backup(&ctx, backup_guard, &package_ids).await;
match backup_res {
Ok(report) if report.iter().all(|(_, rep)| rep.error.is_none()) => ctx
.notification_manager
.notify(
ctx.db.clone(),
None,
NotificationLevel::Success,
"Backup Complete".to_owned(),
"Your backup has completed".to_owned(),
BackupReport {
server: ServerBackupReport {
attempted: true,
error: None,
},
packages: report
.into_iter()
.map(|((package_id, _), value)| (package_id, value))
.collect(),
},
None,
)
.await
.expect("failed to send notification"),
Ok(report) => ctx
.notification_manager
.notify(
ctx.db.clone(),
None,
NotificationLevel::Warning,
"Backup Complete".to_owned(),
"Your backup has completed, but some package(s) failed to backup".to_owned(),
BackupReport {
server: ServerBackupReport {
attempted: true,
error: None,
},
packages: report
.into_iter()
.map(|((package_id, _), value)| (package_id, value))
.collect(),
},
None,
)
.await
.expect("failed to send notification"),
Err(e) => {
tracing::error!("Backup Failed: {}", e);
tracing::debug!("{:?}", e);
ctx.notification_manager
.notify(
ctx.db.clone(),
None,
NotificationLevel::Error,
"Backup Failed".to_owned(),
"Your backup failed to complete.".to_owned(),
BackupReport {
server: ServerBackupReport {
attempted: true,
error: Some(e.to_string()),
},
packages: BTreeMap::new(),
},
None,
)
.await
.expect("failed to send notification");
}
}
ctx.db
.mutate(|v| {
v.as_server_info_mut()
.as_status_info_mut()
.as_backup_progress_mut()
.ser(&None)
})
.await?;
Ok::<(), Error>(())
status_guard
.handle_result(perform_backup(&ctx, backup_guard, &package_ids).await)
.await
.unwrap();
});
Ok(())
}
#[instrument(skip(db, packages))]
async fn assure_backing_up(
db: &PatchDb,
packages: impl IntoIterator<Item = &(PackageId, Version)> + UnwindSafe + Send,
fn assure_backing_up<'a>(
db: &mut DatabaseModel,
packages: impl IntoIterator<Item = &'a PackageId>,
) -> Result<(), Error> {
db.mutate(|v| {
let backing_up = v
.as_server_info_mut()
.as_status_info_mut()
.as_backup_progress_mut();
if backing_up
.clone()
.de()?
.iter()
.flat_map(|x| x.values())
.fold(false, |acc, x| {
if !x.complete {
return true;
}
acc
})
{
return Err(Error::new(
eyre!("Server is already backing up!"),
ErrorKind::InvalidRequest,
));
}
backing_up.ser(&Some(
packages
.into_iter()
.map(|(x, _)| (x.clone(), BackupProgress { complete: false }))
.collect(),
))?;
Ok(())
})
.await
let backing_up = db
.as_public_mut()
.as_server_info_mut()
.as_status_info_mut()
.as_backup_progress_mut();
if backing_up
.clone()
.de()?
.iter()
.flat_map(|x| x.values())
.fold(false, |acc, x| {
if !x.complete {
return true;
}
acc
})
{
return Err(Error::new(
eyre!("Server is already backing up!"),
ErrorKind::InvalidRequest,
));
}
backing_up.ser(&Some(
packages
.into_iter()
.map(|x| (x.clone(), BackupProgress { complete: false }))
.collect(),
))?;
Ok(())
}
#[instrument(skip(ctx, backup_guard))]
async fn perform_backup(
ctx: &RpcContext,
backup_guard: BackupMountGuard<TmpMountGuard>,
package_ids: &OrdSet<(PackageId, Version)>,
) -> Result<BTreeMap<(PackageId, Version), PackageBackupReport>, Error> {
package_ids: &OrdSet<PackageId>,
) -> Result<BTreeMap<PackageId, PackageBackupReport>, Error> {
let mut backup_report = BTreeMap::new();
let backup_guard = Arc::new(Mutex::new(backup_guard));
let backup_guard = Arc::new(backup_guard);
for package_id in package_ids {
let (response, _report) = match ctx
.managers
.get(package_id)
.await
.ok_or_else(|| Error::new(eyre!("Manager not found"), ErrorKind::InvalidRequest))?
.backup(backup_guard.clone())
.await
{
BackupReturn::Ran { report, res } => (res, report),
BackupReturn::AlreadyRunning(report) => {
backup_report.insert(package_id.clone(), report);
continue;
}
BackupReturn::Error(error) => {
tracing::warn!("Backup thread error");
tracing::debug!("{error:?}");
backup_report.insert(
package_id.clone(),
PackageBackupReport {
error: Some("Backup thread error".to_owned()),
},
);
continue;
}
};
backup_report.insert(
package_id.clone(),
PackageBackupReport {
error: response.as_ref().err().map(|e| e.to_string()),
},
);
if let Ok(pkg_meta) = response {
backup_guard
.lock()
.await
.metadata
.package_backups
.insert(package_id.0.clone(), pkg_meta);
for id in package_ids {
if let Some(service) = &*ctx.services.get(id).await {
backup_report.insert(
id.clone(),
PackageBackupReport {
error: service
.backup(backup_guard.package_backup(id))
.await
.err()
.map(|e| e.to_string()),
},
);
}
}
let ui = ctx.db.peek().await.into_ui().de()?;
let mut backup_guard = Arc::try_unwrap(backup_guard).map_err(|_| {
Error::new(
eyre!("leaked reference to BackupMountGuard"),
ErrorKind::Incoherent,
)
})?;
let mut os_backup_file = AtomicFile::new(
backup_guard.lock().await.as_ref().join("os-backup.cbor"),
None::<PathBuf>,
)
.await
.with_kind(ErrorKind::Filesystem)?;
let ui = ctx.db.peek().await.into_public().into_ui().de()?;
let mut os_backup_file =
AtomicFile::new(backup_guard.path().join("os-backup.cbor"), None::<PathBuf>)
.await
.with_kind(ErrorKind::Filesystem)?;
os_backup_file
.write_all(&IoFormat::Cbor.to_vec(&OsBackup {
account: ctx.account.read().await.clone(),
@@ -284,11 +287,11 @@ async fn perform_backup(
.await
.with_kind(ErrorKind::Filesystem)?;
let luks_folder_old = backup_guard.lock().await.as_ref().join("luks.old");
let luks_folder_old = backup_guard.path().join("luks.old");
if tokio::fs::metadata(&luks_folder_old).await.is_ok() {
tokio::fs::remove_dir_all(&luks_folder_old).await?;
}
let luks_folder_bak = backup_guard.lock().await.as_ref().join("luks");
let luks_folder_bak = backup_guard.path().join("luks");
if tokio::fs::metadata(&luks_folder_bak).await.is_ok() {
tokio::fs::rename(&luks_folder_bak, &luks_folder_old).await?;
}
@@ -298,14 +301,6 @@ async fn perform_backup(
}
let timestamp = Some(Utc::now());
let mut backup_guard = Arc::try_unwrap(backup_guard)
.map_err(|_err| {
Error::new(
eyre!("Backup guard could not ensure that the others where dropped"),
ErrorKind::Unknown,
)
})?
.into_inner();
backup_guard.unencrypted_metadata.version = crate::version::Current::new().semver().into();
backup_guard.unencrypted_metadata.full = true;
@@ -315,7 +310,12 @@ async fn perform_backup(
backup_guard.save_and_unmount().await?;
ctx.db
.mutate(|v| v.as_server_info_mut().as_last_backup_mut().ser(&timestamp))
.mutate(|v| {
v.as_public_mut()
.as_server_info_mut()
.as_last_backup_mut()
.ser(&timestamp)
})
.await?;
Ok(backup_report)

206
core/startos/src/backup/mod.rs
View File

@@ -1,33 +1,15 @@
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::collections::BTreeMap;
use chrono::{DateTime, Utc};
use color_eyre::eyre::eyre;
use helpers::AtomicFile;
use models::{ImageId, OptionExt};
use models::{HostId, PackageId};
use reqwest::Url;
use rpc_toolkit::command;
use rpc_toolkit::{from_fn_async, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
use tracing::instrument;
use self::target::PackageBackupInfo;
use crate::context::RpcContext;
use crate::install::PKG_ARCHIVE_DIR;
use crate::manager::manager_seed::ManagerSeed;
use crate::net::interface::InterfaceId;
use crate::net::keys::Key;
use crate::context::CliContext;
#[allow(unused_imports)]
use crate::prelude::*;
use crate::procedure::docker::DockerContainers;
use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
use crate::util::serde::{Base32, Base64, IoFormat};
use crate::util::Version;
use crate::version::{Current, VersionT};
use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR};
use crate::{Error, ErrorKind, ResultExt};
use crate::util::serde::{Base32, Base64};
pub mod backup_bulk;
pub mod os;
@@ -51,176 +33,24 @@ pub struct PackageBackupReport {
pub error: Option<String>,
}
#[command(subcommands(backup_bulk::backup_all, target::target))]
pub fn backup() -> Result<(), Error> {
Ok(())
}
#[command(rename = "backup", subcommands(restore::restore_packages_rpc))]
pub fn package_backup() -> Result<(), Error> {
Ok(())
// #[command(subcommands(backup_bulk::backup_all, target::target))]
pub fn backup() -> ParentHandler {
ParentHandler::new()
.subcommand(
"create",
from_fn_async(backup_bulk::backup_all)
.no_display()
.with_remote_cli::<CliContext>(),
)
.subcommand("target", target::target())
}
#[derive(Deserialize, Serialize)]
struct BackupMetadata {
pub timestamp: DateTime<Utc>,
#[serde(default)]
pub network_keys: BTreeMap<InterfaceId, Base64<[u8; 32]>>,
pub network_keys: BTreeMap<HostId, Base64<[u8; 32]>>,
#[serde(default)]
pub tor_keys: BTreeMap<InterfaceId, Base32<[u8; 64]>>, // DEPRECATED
pub tor_keys: BTreeMap<HostId, Base32<[u8; 64]>>, // DEPRECATED
pub marketplace_url: Option<Url>,
}
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
#[model = "Model<Self>"]
pub struct BackupActions {
pub create: PackageProcedure,
pub restore: PackageProcedure,
}
impl BackupActions {
pub fn validate(
&self,
_container: &Option<DockerContainers>,
eos_version: &Version,
volumes: &Volumes,
image_ids: &BTreeSet<ImageId>,
) -> Result<(), Error> {
self.create
.validate(eos_version, volumes, image_ids, false)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Create"))?;
self.restore
.validate(eos_version, volumes, image_ids, false)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Restore"))?;
Ok(())
}
#[instrument(skip_all)]
pub async fn create(&self, seed: Arc<ManagerSeed>) -> Result<PackageBackupInfo, Error> {
let manifest = &seed.manifest;
let mut volumes = seed.manifest.volumes.to_readonly();
let ctx = &seed.ctx;
let pkg_id = &manifest.id;
let pkg_version = &manifest.version;
volumes.insert(VolumeId::Backup, Volume::Backup { readonly: false });
let backup_dir = backup_dir(&manifest.id);
if tokio::fs::metadata(&backup_dir).await.is_err() {
tokio::fs::create_dir_all(&backup_dir).await?
}
self.create
.execute::<(), NoOutput>(
ctx,
pkg_id,
pkg_version,
ProcedureName::CreateBackup,
&volumes,
None,
None,
)
.await?
.map_err(|e| eyre!("{}", e.1))
.with_kind(crate::ErrorKind::Backup)?;
let (network_keys, tor_keys): (Vec<_>, Vec<_>) =
Key::for_package(&ctx.secret_store, pkg_id)
.await?
.into_iter()
.filter_map(|k| {
let interface = k.interface().map(|(_, i)| i)?;
Some((
(interface.clone(), Base64(k.as_bytes())),
(interface, Base32(k.tor_key().as_bytes())),
))
})
.unzip();
let marketplace_url = ctx
.db
.peek()
.await
.as_package_data()
.as_idx(&pkg_id)
.or_not_found(pkg_id)?
.expect_as_installed()?
.as_installed()
.as_marketplace_url()
.de()?;
let tmp_path = Path::new(BACKUP_DIR)
.join(pkg_id)
.join(format!("{}.s9pk", pkg_id));
let s9pk_path = ctx
.datadir
.join(PKG_ARCHIVE_DIR)
.join(pkg_id)
.join(pkg_version.as_str())
.join(format!("{}.s9pk", pkg_id));
let mut infile = File::open(&s9pk_path).await?;
let mut outfile = AtomicFile::new(&tmp_path, None::<PathBuf>)
.await
.with_kind(ErrorKind::Filesystem)?;
tokio::io::copy(&mut infile, &mut *outfile)
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
format!("cp {} -> {}", s9pk_path.display(), tmp_path.display()),
)
})?;
outfile.save().await.with_kind(ErrorKind::Filesystem)?;
let timestamp = Utc::now();
let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
let mut outfile = AtomicFile::new(&metadata_path, None::<PathBuf>)
.await
.with_kind(ErrorKind::Filesystem)?;
let network_keys = network_keys.into_iter().collect();
let tor_keys = tor_keys.into_iter().collect();
outfile
.write_all(&IoFormat::Cbor.to_vec(&BackupMetadata {
timestamp,
network_keys,
tor_keys,
marketplace_url,
})?)
.await?;
outfile.save().await.with_kind(ErrorKind::Filesystem)?;
Ok(PackageBackupInfo {
os_version: Current::new().semver().into(),
title: manifest.title.clone(),
version: pkg_version.clone(),
timestamp,
})
}
#[instrument(skip_all)]
pub async fn restore(
&self,
ctx: &RpcContext,
pkg_id: &PackageId,
pkg_version: &Version,
volumes: &Volumes,
) -> Result<Option<Url>, Error> {
let mut volumes = volumes.clone();
volumes.insert(VolumeId::Backup, Volume::Backup { readonly: true });
self.restore
.execute::<(), NoOutput>(
ctx,
pkg_id,
pkg_version,
ProcedureName::RestoreBackup,
&volumes,
None,
None,
)
.await?
.map_err(|e| eyre!("{}", e.1))
.with_kind(crate::ErrorKind::Restore)?;
let metadata_path = Path::new(BACKUP_DIR).join(pkg_id).join("metadata.cbor");
let metadata: BackupMetadata = IoFormat::Cbor.from_slice(
&tokio::fs::read(&metadata_path).await.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
metadata_path.display().to_string(),
)
})?,
)?;
Ok(metadata.marketplace_url)
}
}

131
core/startos/src/backup/os.rs
View File

@@ -1,13 +1,15 @@
use openssl::pkey::PKey;
use openssl::pkey::{PKey, Private};
use openssl::x509::X509;
use patch_db::Value;
use serde::{Deserialize, Serialize};
use ssh_key::private::Ed25519Keypair;
use torut::onion::TorSecretKeyV3;
use crate::account::AccountInfo;
use crate::hostname::{generate_hostname, generate_id, Hostname};
use crate::net::keys::Key;
use crate::prelude::*;
use crate::util::serde::Base64;
use crate::util::crypto::ed25519_expand_key;
use crate::util::serde::{Base32, Base64, Pem};
pub struct OsBackup {
pub account: AccountInfo,
@@ -19,19 +21,23 @@ impl<'de> Deserialize<'de> for OsBackup {
D: serde::Deserializer<'de>,
{
let tagged = OsBackupSerDe::deserialize(deserializer)?;
match tagged.version {
Ok(match tagged.version {
0 => patch_db::value::from_value::<OsBackupV0>(tagged.rest)
.map_err(serde::de::Error::custom)?
.project()
.map_err(serde::de::Error::custom),
.map_err(serde::de::Error::custom)?,
1 => patch_db::value::from_value::<OsBackupV1>(tagged.rest)
.map_err(serde::de::Error::custom)?
.project()
.map_err(serde::de::Error::custom),
v => Err(serde::de::Error::custom(&format!(
"Unknown backup version {v}"
))),
}
.project(),
2 => patch_db::value::from_value::<OsBackupV2>(tagged.rest)
.map_err(serde::de::Error::custom)?
.project(),
v => {
return Err(serde::de::Error::custom(&format!(
"Unknown backup version {v}"
)))
}
})
}
}
impl Serialize for OsBackup {
@@ -40,11 +46,9 @@ impl Serialize for OsBackup {
S: serde::Serializer,
{
OsBackupSerDe {
version: 1,
rest: patch_db::value::to_value(
&OsBackupV1::unproject(self).map_err(serde::ser::Error::custom)?,
)
.map_err(serde::ser::Error::custom)?,
version: 2,
rest: patch_db::value::to_value(&OsBackupV2::unproject(self))
.map_err(serde::ser::Error::custom)?,
}
.serialize(serializer)
}
@@ -62,10 +66,10 @@ struct OsBackupSerDe {
#[derive(Deserialize)]
#[serde(rename = "kebab-case")]
struct OsBackupV0 {
// tor_key: Base32<[u8; 64]>,
root_ca_key: String, // PEM Encoded OpenSSL Key
root_ca_cert: String, // PEM Encoded OpenSSL X509 Certificate
ui: Value, // JSON Value
tor_key: Base32<[u8; 64]>, // Base32 Encoded Ed25519 Expanded Secret Key
root_ca_key: Pem<PKey<Private>>, // PEM Encoded OpenSSL Key
root_ca_cert: Pem<X509>, // PEM Encoded OpenSSL X509 Certificate
ui: Value, // JSON Value
}
impl OsBackupV0 {
fn project(self) -> Result<OsBackup, Error> {
@@ -74,9 +78,13 @@ impl OsBackupV0 {
server_id: generate_id(),
hostname: generate_hostname(),
password: Default::default(),
key: Key::new(None),
root_ca_key: PKey::private_key_from_pem(self.root_ca_key.as_bytes())?,
root_ca_cert: X509::from_pem(self.root_ca_cert.as_bytes())?,
root_ca_key: self.root_ca_key.0,
root_ca_cert: self.root_ca_cert.0,
ssh_key: ssh_key::PrivateKey::random(
&mut rand::thread_rng(),
ssh_key::Algorithm::Ed25519,
)?,
tor_key: TorSecretKeyV3::from(self.tor_key.0),
},
ui: self.ui,
})
@@ -87,36 +95,67 @@ impl OsBackupV0 {
#[derive(Deserialize, Serialize)]
#[serde(rename = "kebab-case")]
struct OsBackupV1 {
server_id: String, // uuidv4
hostname: String, // embassy-<adjective>-<noun>
net_key: Base64<[u8; 32]>, // Ed25519 Secret Key
root_ca_key: String, // PEM Encoded OpenSSL Key
root_ca_cert: String, // PEM Encoded OpenSSL X509 Certificate
ui: Value, // JSON Value
// TODO add more
server_id: String, // uuidv4
hostname: String, // embassy-<adjective>-<noun>
net_key: Base64<[u8; 32]>, // Ed25519 Secret Key
root_ca_key: Pem<PKey<Private>>, // PEM Encoded OpenSSL Key
root_ca_cert: Pem<X509>, // PEM Encoded OpenSSL X509 Certificate
ui: Value, // JSON Value
}
impl OsBackupV1 {
fn project(self) -> Result<OsBackup, Error> {
Ok(OsBackup {
fn project(self) -> OsBackup {
OsBackup {
account: AccountInfo {
server_id: self.server_id,
hostname: Hostname(self.hostname),
password: Default::default(),
key: Key::from_bytes(None, self.net_key.0),
root_ca_key: PKey::private_key_from_pem(self.root_ca_key.as_bytes())?,
root_ca_cert: X509::from_pem(self.root_ca_cert.as_bytes())?,
root_ca_key: self.root_ca_key.0,
root_ca_cert: self.root_ca_cert.0,
ssh_key: ssh_key::PrivateKey::from(Ed25519Keypair::from_seed(&self.net_key.0)),
tor_key: TorSecretKeyV3::from(ed25519_expand_key(&self.net_key.0)),
},
ui: self.ui,
})
}
fn unproject(backup: &OsBackup) -> Result<Self, Error> {
Ok(Self {
server_id: backup.account.server_id.clone(),
hostname: backup.account.hostname.0.clone(),
net_key: Base64(backup.account.key.as_bytes()),
root_ca_key: String::from_utf8(backup.account.root_ca_key.private_key_to_pem_pkcs8()?)?,
root_ca_cert: String::from_utf8(backup.account.root_ca_cert.to_pem()?)?,
ui: backup.ui.clone(),
})
}
}
}
/// V2
#[derive(Deserialize, Serialize)]
#[serde(rename = "kebab-case")]
struct OsBackupV2 {
server_id: String, // uuidv4
hostname: String, // <adjective>-<noun>
root_ca_key: Pem<PKey<Private>>, // PEM Encoded OpenSSL Key
root_ca_cert: Pem<X509>, // PEM Encoded OpenSSL X509 Certificate
ssh_key: Pem<ssh_key::PrivateKey>, // PEM Encoded OpenSSH Key
tor_key: TorSecretKeyV3, // Base64 Encoded Ed25519 Expanded Secret Key
ui: Value, // JSON Value
}
impl OsBackupV2 {
fn project(self) -> OsBackup {
OsBackup {
account: AccountInfo {
server_id: self.server_id,
hostname: Hostname(self.hostname),
password: Default::default(),
root_ca_key: self.root_ca_key.0,
root_ca_cert: self.root_ca_cert.0,
ssh_key: self.ssh_key.0,
tor_key: self.tor_key,
},
ui: self.ui,
}
}
fn unproject(backup: &OsBackup) -> Self {
Self {
server_id: backup.account.server_id.clone(),
hostname: backup.account.hostname.0.clone(),
root_ca_key: Pem(backup.account.root_ca_key.clone()),
root_ca_cert: Pem(backup.account.root_ca_cert.clone()),
ssh_key: Pem(backup.account.ssh_key.clone()),
tor_key: backup.account.tor_key.clone(),
ui: backup.ui.clone(),
}
}
}

440
core/startos/src/backup/restore.rs
View File

@@ -1,170 +1,72 @@
use std::collections::BTreeMap;
use std::path::Path;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;
use clap::ArgMatches;
use futures::future::BoxFuture;
use futures::{stream, FutureExt, StreamExt};
use clap::Parser;
use futures::{stream, StreamExt};
use models::PackageId;
use openssl::x509::X509;
use rpc_toolkit::command;
use sqlx::Connection;
use tokio::fs::File;
use patch_db::json_ptr::ROOT;
use serde::{Deserialize, Serialize};
use torut::onion::OnionAddressV3;
use tracing::instrument;
use super::target::BackupTargetId;
use crate::backup::os::OsBackup;
use crate::backup::BackupMetadata;
use crate::context::rpc::RpcContextConfig;
use crate::context::{RpcContext, SetupContext};
use crate::db::model::{PackageDataEntry, PackageDataEntryRestoring, StaticFiles};
use crate::disk::mount::backup::{BackupMountGuard, PackageBackupMountGuard};
use crate::db::model::Database;
use crate::disk::mount::backup::BackupMountGuard;
use crate::disk::mount::filesystem::ReadWrite;
use crate::disk::mount::guard::TmpMountGuard;
use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard};
use crate::hostname::Hostname;
use crate::init::init;
use crate::install::progress::InstallProgress;
use crate::install::{download_install_s9pk, PKG_PUBLIC_DIR};
use crate::notifications::NotificationLevel;
use crate::prelude::*;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::s9pk::reader::S9pkReader;
use crate::setup::SetupStatus;
use crate::util::display_none;
use crate::util::io::dir_size;
use crate::s9pk::S9pk;
use crate::service::service_map::DownloadInstallFuture;
use crate::util::serde::IoFormat;
use crate::volume::{backup_dir, BACKUP_DIR, PKG_VOLUME_DIR};
fn parse_comma_separated(arg: &str, _: &ArgMatches) -> Result<Vec<PackageId>, Error> {
arg.split(',')
.map(|s| s.trim().parse().map_err(Error::from))
.collect()
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct RestorePackageParams {
pub ids: Vec<PackageId>,
pub target_id: BackupTargetId,
pub password: String,
}
#[command(rename = "restore", display(display_none))]
// TODO dr Why doesn't anything use this
// #[command(rename = "restore", display(display_none))]
#[instrument(skip(ctx, password))]
pub async fn restore_packages_rpc(
#[context] ctx: RpcContext,
#[arg(parse(parse_comma_separated))] ids: Vec<PackageId>,
#[arg(rename = "target-id")] target_id: BackupTargetId,
#[arg] password: String,
ctx: RpcContext,
RestorePackageParams {
ids,
target_id,
password,
}: RestorePackageParams,
) -> Result<(), Error> {
let fs = target_id
.load(ctx.secret_store.acquire().await?.as_mut())
.await?;
let fs = target_id.load(&ctx.db.peek().await)?;
let backup_guard =
BackupMountGuard::mount(TmpMountGuard::mount(&fs, ReadWrite).await?, &password).await?;
let (backup_guard, tasks, _) = restore_packages(&ctx, backup_guard, ids).await?;
let tasks = restore_packages(&ctx, backup_guard, ids).await?;
tokio::spawn(async move {
stream::iter(tasks.into_iter().map(|x| (x, ctx.clone())))
.for_each_concurrent(5, |(res, ctx)| async move {
match res.await {
(Ok(_), _) => (),
(Err(err), package_id) => {
if let Err(err) = ctx
.notification_manager
.notify(
ctx.db.clone(),
Some(package_id.clone()),
NotificationLevel::Error,
"Restoration Failure".to_string(),
format!("Error restoring package {}: {}", package_id, err),
(),
None,
)
.await
{
tracing::error!("Failed to notify: {}", err);
tracing::debug!("{:?}", err);
};
tracing::error!("Error restoring package {}: {}", package_id, err);
stream::iter(tasks)
.for_each_concurrent(5, |(id, res)| async move {
match async { res.await?.await }.await {
Ok(_) => (),
Err(err) => {
tracing::error!("Error restoring package {}: {}", id, err);
tracing::debug!("{:?}", err);
}
}
})
.await;
if let Err(e) = backup_guard.unmount().await {
tracing::error!("Error unmounting backup drive: {}", e);
tracing::debug!("{:?}", e);
}
});
Ok(())
}
async fn approximate_progress(
rpc_ctx: &RpcContext,
progress: &mut ProgressInfo,
) -> Result<(), Error> {
for (id, size) in &mut progress.target_volume_size {
let dir = rpc_ctx.datadir.join(PKG_VOLUME_DIR).join(id).join("data");
if tokio::fs::metadata(&dir).await.is_err() {
*size = 0;
} else {
*size = dir_size(&dir, None).await?;
}
}
Ok(())
}
async fn approximate_progress_loop(
ctx: &SetupContext,
rpc_ctx: &RpcContext,
mut starting_info: ProgressInfo,
) {
loop {
if let Err(e) = approximate_progress(rpc_ctx, &mut starting_info).await {
tracing::error!("Failed to approximate restore progress: {}", e);
tracing::debug!("{:?}", e);
} else {
*ctx.setup_status.write().await = Some(Ok(starting_info.flatten()));
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
}
#[derive(Debug, Default)]
struct ProgressInfo {
package_installs: BTreeMap<PackageId, Arc<InstallProgress>>,
src_volume_size: BTreeMap<PackageId, u64>,
target_volume_size: BTreeMap<PackageId, u64>,
}
impl ProgressInfo {
fn flatten(&self) -> SetupStatus {
let mut total_bytes = 0;
let mut bytes_transferred = 0;
for progress in self.package_installs.values() {
total_bytes += ((progress.size.unwrap_or(0) as f64) * 2.2) as u64;
bytes_transferred += progress.downloaded.load(Ordering::SeqCst);
bytes_transferred += ((progress.validated.load(Ordering::SeqCst) as f64) * 0.2) as u64;
bytes_transferred += progress.unpacked.load(Ordering::SeqCst);
}
for size in self.src_volume_size.values() {
total_bytes += *size;
}
for size in self.target_volume_size.values() {
bytes_transferred += *size;
}
if bytes_transferred > total_bytes {
bytes_transferred = total_bytes;
}
SetupStatus {
total_bytes: Some(total_bytes),
bytes_transferred,
complete: false,
}
}
}
#[instrument(skip(ctx))]
pub async fn recover_full_embassy(
ctx: SetupContext,
@@ -179,7 +81,7 @@ pub async fn recover_full_embassy(
)
.await?;
let os_backup_path = backup_guard.as_ref().join("os-backup.cbor");
let os_backup_path = backup_guard.path().join("os-backup.cbor");
let mut os_backup: OsBackup = IoFormat::Cbor.from_slice(
&tokio::fs::read(&os_backup_path)
.await
@@ -193,17 +95,13 @@ pub async fn recover_full_embassy(
)
.with_kind(ErrorKind::PasswordHashGeneration)?;
let secret_store = ctx.secret_store().await?;
let db = ctx.db().await?;
db.put(&ROOT, &Database::init(&os_backup.account)?).await?;
drop(db);
os_backup.account.save(&secret_store).await?;
init(&ctx.config).await?;
secret_store.close().await;
let cfg = RpcContextConfig::load(ctx.config_path.clone()).await?;
init(&cfg).await?;
let rpc_ctx = RpcContext::init(ctx.config_path.clone(), disk_guid.clone()).await?;
let rpc_ctx = RpcContext::init(&ctx.config, disk_guid.clone()).await?;
let ids: Vec<_> = backup_guard
.metadata
@@ -211,43 +109,25 @@ pub async fn recover_full_embassy(
.keys()
.cloned()
.collect();
let (backup_guard, tasks, progress_info) =
restore_packages(&rpc_ctx, backup_guard, ids).await?;
let task_consumer_rpc_ctx = rpc_ctx.clone();
tokio::select! {
_ = async move {
stream::iter(tasks.into_iter().map(|x| (x, task_consumer_rpc_ctx.clone())))
.for_each_concurrent(5, |(res, ctx)| async move {
match res.await {
(Ok(_), _) => (),
(Err(err), package_id) => {
if let Err(err) = ctx.notification_manager.notify(
ctx.db.clone(),
Some(package_id.clone()),
NotificationLevel::Error,
"Restoration Failure".to_string(), format!("Error restoring package {}: {}", package_id,err), (), None).await{
tracing::error!("Failed to notify: {}", err);
tracing::debug!("{:?}", err);
};
tracing::error!("Error restoring package {}: {}", package_id, err);
tracing::debug!("{:?}", err);
},
}
}).await;
let tasks = restore_packages(&rpc_ctx, backup_guard, ids).await?;
stream::iter(tasks)
.for_each_concurrent(5, |(id, res)| async move {
match async { res.await?.await }.await {
Ok(_) => (),
Err(err) => {
tracing::error!("Error restoring package {}: {}", id, err);
tracing::debug!("{:?}", err);
}
}
})
.await;
} => {
},
_ = approximate_progress_loop(&ctx, &rpc_ctx, progress_info) => unreachable!(concat!(module_path!(), "::approximate_progress_loop should not terminate")),
}
backup_guard.unmount().await?;
rpc_ctx.shutdown().await?;
Ok((
disk_guid,
os_backup.account.hostname,
os_backup.account.key.tor_address(),
os_backup.account.tor_key.public().get_onion_address(),
os_backup.account.root_ca_cert,
))
}
@@ -257,205 +137,25 @@ async fn restore_packages(
ctx: &RpcContext,
backup_guard: BackupMountGuard<TmpMountGuard>,
ids: Vec<PackageId>,
) -> Result<
(
BackupMountGuard<TmpMountGuard>,
Vec<BoxFuture<'static, (Result<(), Error>, PackageId)>>,
ProgressInfo,
),
Error,
> {
let guards = assure_restoring(ctx, ids, &backup_guard).await?;
let mut progress_info = ProgressInfo::default();
let mut tasks = Vec::with_capacity(guards.len());
for (manifest, guard) in guards {
let id = manifest.id.clone();
let (progress, task) = restore_package(ctx.clone(), manifest, guard).await?;
progress_info
.package_installs
.insert(id.clone(), progress.clone());
progress_info
.src_volume_size
.insert(id.clone(), dir_size(backup_dir(&id), None).await?);
progress_info.target_volume_size.insert(id.clone(), 0);
let package_id = id.clone();
tasks.push(
async move {
if let Err(e) = task.await {
tracing::error!("Error restoring package {}: {}", id, e);
tracing::debug!("{:?}", e);
Err(e)
} else {
Ok(())
}
}
.map(|x| (x, package_id))
.boxed(),
);
}
Ok((backup_guard, tasks, progress_info))
}
#[instrument(skip(ctx, backup_guard))]
async fn assure_restoring(
ctx: &RpcContext,
ids: Vec<PackageId>,
backup_guard: &BackupMountGuard<TmpMountGuard>,
) -> Result<Vec<(Manifest, PackageBackupMountGuard)>, Error> {
let mut guards = Vec::with_capacity(ids.len());
let mut insert_packages = BTreeMap::new();
) -> Result<BTreeMap<PackageId, DownloadInstallFuture>, Error> {
let backup_guard = Arc::new(backup_guard);
let mut tasks = BTreeMap::new();
for id in ids {
let peek = ctx.db.peek().await;
let model = peek.as_package_data().as_idx(&id);
if !model.is_none() {
return Err(Error::new(
eyre!("Can't restore over existing package: {}", id),
crate::ErrorKind::InvalidRequest,
));
}
let guard = backup_guard.mount_package_backup(&id).await?;
let s9pk_path = Path::new(BACKUP_DIR).join(&id).join(format!("{}.s9pk", id));
let mut rdr = S9pkReader::open(&s9pk_path, false).await?;
let manifest = rdr.manifest().await?;
let version = manifest.version.clone();
let progress = Arc::new(InstallProgress::new(Some(
tokio::fs::metadata(&s9pk_path).await?.len(),
)));
let public_dir_path = ctx
.datadir
.join(PKG_PUBLIC_DIR)
.join(&id)
.join(version.as_str());
tokio::fs::create_dir_all(&public_dir_path).await?;
let license_path = public_dir_path.join("LICENSE.md");
let mut dst = File::create(&license_path).await?;
tokio::io::copy(&mut rdr.license().await?, &mut dst).await?;
dst.sync_all().await?;
let instructions_path = public_dir_path.join("INSTRUCTIONS.md");
let mut dst = File::create(&instructions_path).await?;
tokio::io::copy(&mut rdr.instructions().await?, &mut dst).await?;
dst.sync_all().await?;
let icon_path = Path::new("icon").with_extension(&manifest.assets.icon_type());
let icon_path = public_dir_path.join(&icon_path);
let mut dst = File::create(&icon_path).await?;
tokio::io::copy(&mut rdr.icon().await?, &mut dst).await?;
dst.sync_all().await?;
insert_packages.insert(
id.clone(),
PackageDataEntry::Restoring(PackageDataEntryRestoring {
install_progress: progress.clone(),
static_files: StaticFiles::local(&id, &version, manifest.assets.icon_type()),
manifest: manifest.clone(),
}),
);
guards.push((manifest, guard));
}
ctx.db
.mutate(|db| {
for (id, package) in insert_packages {
db.as_package_data_mut().insert(&id, &package)?;
}
Ok(())
})
.await?;
Ok(guards)
}
#[instrument(skip(ctx, guard))]
async fn restore_package<'a>(
ctx: RpcContext,
manifest: Manifest,
guard: PackageBackupMountGuard,
) -> Result<(Arc<InstallProgress>, BoxFuture<'static, Result<(), Error>>), Error> {
let id = manifest.id.clone();
let s9pk_path = Path::new(BACKUP_DIR)
.join(&manifest.id)
.join(format!("{}.s9pk", id));
let metadata_path = Path::new(BACKUP_DIR).join(&id).join("metadata.cbor");
let metadata: BackupMetadata = IoFormat::Cbor.from_slice(
&tokio::fs::read(&metadata_path)
.await
.with_ctx(|_| (ErrorKind::Filesystem, metadata_path.display().to_string()))?,
)?;
let mut secrets = ctx.secret_store.acquire().await?;
let mut secrets_tx = secrets.begin().await?;
for (iface, key) in metadata.network_keys {
let k = key.0.as_slice();
sqlx::query!(
"INSERT INTO network_keys (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
id.to_string(),
iface.to_string(),
k,
)
.execute(secrets_tx.as_mut()).await?;
}
// DEPRECATED
for (iface, key) in metadata.tor_keys {
let k = key.0.as_slice();
sqlx::query!(
"INSERT INTO tor (package, interface, key) VALUES ($1, $2, $3) ON CONFLICT (package, interface) DO NOTHING",
id.to_string(),
iface.to_string(),
k,
)
.execute(secrets_tx.as_mut()).await?;
}
secrets_tx.commit().await?;
drop(secrets);
let len = tokio::fs::metadata(&s9pk_path)
.await
.with_ctx(|_| (ErrorKind::Filesystem, s9pk_path.display().to_string()))?
.len();
let file = File::open(&s9pk_path)
.await
.with_ctx(|_| (ErrorKind::Filesystem, s9pk_path.display().to_string()))?;
let progress = InstallProgress::new(Some(len));
let marketplace_url = metadata.marketplace_url;
let progress = Arc::new(progress);
ctx.db
.mutate(|db| {
db.as_package_data_mut().insert(
&id,
&PackageDataEntry::Restoring(PackageDataEntryRestoring {
install_progress: progress.clone(),
static_files: StaticFiles::local(
&id,
&manifest.version,
manifest.assets.icon_type(),
),
manifest: manifest.clone(),
}),
let backup_dir = backup_guard.clone().package_backup(&id);
let task = ctx
.services
.install(
ctx.clone(),
S9pk::open(
backup_dir.path().join(&id).with_extension("s9pk"),
Some(&id),
)
.await?,
Some(backup_dir),
)
})
.await?;
Ok((
progress.clone(),
async move {
download_install_s9pk(ctx, manifest, marketplace_url, progress, file, None).await?;
.await?;
tasks.insert(id, task);
}
guard.unmount().await?;
Ok(())
}
.boxed(),
))
Ok(tasks)
}

274
core/startos/src/backup/target/cifs.rs
View File

@@ -1,21 +1,40 @@
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use clap::Parser;
use color_eyre::eyre::eyre;
use futures::TryStreamExt;
use rpc_toolkit::command;
use imbl_value::InternedString;
use rpc_toolkit::{command, from_fn_async, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use sqlx::{Executor, Postgres};
use super::{BackupTarget, BackupTargetId};
use crate::context::RpcContext;
use crate::context::{CliContext, RpcContext};
use crate::db::model::DatabaseModel;
use crate::disk::mount::filesystem::cifs::Cifs;
use crate::disk::mount::filesystem::ReadOnly;
use crate::disk::mount::guard::TmpMountGuard;
use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard};
use crate::disk::util::{recovery_info, EmbassyOsRecoveryInfo};
use crate::prelude::*;
use crate::util::display_none;
use crate::util::serde::KeyVal;
#[derive(Debug, Default, Deserialize, Serialize)]
pub struct CifsTargets(pub BTreeMap<u32, Cifs>);
impl CifsTargets {
pub fn new() -> Self {
Self(BTreeMap::new())
}
}
impl Map for CifsTargets {
type Key = u32;
type Value = Cifs;
fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
Self::key_string(key)
}
fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
Ok(InternedString::from_display(key))
}
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct CifsBackupTarget {
@@ -26,38 +45,70 @@ pub struct CifsBackupTarget {
embassy_os: Option<EmbassyOsRecoveryInfo>,
}
#[command(subcommands(add, update, remove))]
pub fn cifs() -> Result<(), Error> {
Ok(())
pub fn cifs() -> ParentHandler {
ParentHandler::new()
.subcommand(
"add",
from_fn_async(add)
.no_display()
.with_remote_cli::<CliContext>(),
)
.subcommand(
"update",
from_fn_async(update)
.no_display()
.with_remote_cli::<CliContext>(),
)
.subcommand(
"remove",
from_fn_async(remove)
.no_display()
.with_remote_cli::<CliContext>(),
)
}
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct AddParams {
pub hostname: String,
pub path: PathBuf,
pub username: String,
pub password: Option<String>,
}
#[command(display(display_none))]
pub async fn add(
#[context] ctx: RpcContext,
#[arg] hostname: String,
#[arg] path: PathBuf,
#[arg] username: String,
#[arg] password: Option<String>,
) -> Result<KeyVal<BackupTargetId, BackupTarget>, Error> {
let cifs = Cifs {
ctx: RpcContext,
AddParams {
hostname,
path,
username,
password,
}: AddParams,
) -> Result<KeyVal<BackupTargetId, BackupTarget>, Error> {
let cifs = Cifs {
hostname,
path: Path::new("/").join(path),
username,
password,
};
let guard = TmpMountGuard::mount(&cifs, ReadOnly).await?;
let embassy_os = recovery_info(&guard).await?;
let embassy_os = recovery_info(guard.path()).await?;
guard.unmount().await?;
let path_string = Path::new("/").join(&cifs.path).display().to_string();
let id: i32 = sqlx::query!(
"INSERT INTO cifs_shares (hostname, path, username, password) VALUES ($1, $2, $3, $4) RETURNING id",
cifs.hostname,
path_string,
cifs.username,
cifs.password,
)
.fetch_one(&ctx.secret_store)
.await?.id;
let id = ctx
.db
.mutate(|db| {
let id = db
.as_private()
.as_cifs()
.keys()?
.into_iter()
.max()
.map_or(0, |a| a + 1);
db.as_private_mut().as_cifs_mut().insert(&id, &cifs)?;
Ok(id)
})
.await?;
Ok(KeyVal {
key: BackupTargetId::Cifs { id },
value: BackupTarget::Cifs(CifsBackupTarget {
@@ -70,14 +121,26 @@ pub async fn add(
})
}
#[command(display(display_none))]
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct UpdateParams {
pub id: BackupTargetId,
pub hostname: String,
pub path: PathBuf,
pub username: String,
pub password: Option<String>,
}
pub async fn update(
#[context] ctx: RpcContext,
#[arg] id: BackupTargetId,
#[arg] hostname: String,
#[arg] path: PathBuf,
#[arg] username: String,
#[arg] password: Option<String>,
ctx: RpcContext,
UpdateParams {
id,
hostname,
path,
username,
password,
}: UpdateParams,
) -> Result<KeyVal<BackupTargetId, BackupTarget>, Error> {
let id = if let BackupTargetId::Cifs { id } = id {
id
@@ -89,32 +152,27 @@ pub async fn update(
};
let cifs = Cifs {
hostname,
path,
path: Path::new("/").join(path),
username,
password,
};
let guard = TmpMountGuard::mount(&cifs, ReadOnly).await?;
let embassy_os = recovery_info(&guard).await?;
let embassy_os = recovery_info(guard.path()).await?;
guard.unmount().await?;
let path_string = Path::new("/").join(&cifs.path).display().to_string();
if sqlx::query!(
"UPDATE cifs_shares SET hostname = $1, path = $2, username = $3, password = $4 WHERE id = $5",
cifs.hostname,
path_string,
cifs.username,
cifs.password,
id,
)
.execute(&ctx.secret_store)
.await?
.rows_affected()
== 0
{
return Err(Error::new(
eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
ErrorKind::NotFound,
));
};
ctx.db
.mutate(|db| {
db.as_private_mut()
.as_cifs_mut()
.as_idx_mut(&id)
.ok_or_else(|| {
Error::new(
eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
ErrorKind::NotFound,
)
})?
.ser(&cifs)
})
.await?;
Ok(KeyVal {
key: BackupTargetId::Cifs { id },
value: BackupTarget::Cifs(CifsBackupTarget {
@@ -127,8 +185,14 @@ pub async fn update(
})
}
#[command(display(display_none))]
pub async fn remove(#[context] ctx: RpcContext, #[arg] id: BackupTargetId) -> Result<(), Error> {
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct RemoveParams {
pub id: BackupTargetId,
}
pub async fn remove(ctx: RpcContext, RemoveParams { id }: RemoveParams) -> Result<(), Error> {
let id = if let BackupTargetId::Cifs { id } = id {
id
} else {
@@ -137,74 +201,46 @@ pub async fn remove(#[context] ctx: RpcContext, #[arg] id: BackupTargetId) -> Re
ErrorKind::NotFound,
));
};
if sqlx::query!("DELETE FROM cifs_shares WHERE id = $1", id)
.execute(&ctx.secret_store)
.await?
.rows_affected()
== 0
{
return Err(Error::new(
eyre!("Backup Target ID {} Not Found", BackupTargetId::Cifs { id }),
ErrorKind::NotFound,
));
};
ctx.db
.mutate(|db| db.as_private_mut().as_cifs_mut().remove(&id))
.await?;
Ok(())
}
pub async fn load<Ex>(secrets: &mut Ex, id: i32) -> Result<Cifs, Error>
where
for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
{
let record = sqlx::query!(
"SELECT hostname, path, username, password FROM cifs_shares WHERE id = $1",
id
)
.fetch_one(secrets)
.await?;
Ok(Cifs {
hostname: record.hostname,
path: PathBuf::from(record.path),
username: record.username,
password: record.password,
})
pub fn load(db: &DatabaseModel, id: u32) -> Result<Cifs, Error> {
db.as_private()
.as_cifs()
.as_idx(&id)
.ok_or_else(|| {
Error::new(
eyre!("Backup Target ID {} Not Found", id),
ErrorKind::NotFound,
)
})?
.de()
}
pub async fn list<Ex>(secrets: &mut Ex) -> Result<Vec<(i32, CifsBackupTarget)>, Error>
where
for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
{
let mut records =
sqlx::query!("SELECT id, hostname, path, username, password FROM cifs_shares")
.fetch_many(secrets);
pub async fn list(db: &DatabaseModel) -> Result<Vec<(u32, CifsBackupTarget)>, Error> {
let mut cifs = Vec::new();
while let Some(query_result) = records.try_next().await? {
if let Some(record) = query_result.right() {
let mount_info = Cifs {
hostname: record.hostname,
path: PathBuf::from(record.path),
username: record.username,
password: record.password,
};
let embassy_os = async {
let guard = TmpMountGuard::mount(&mount_info, ReadOnly).await?;
let embassy_os = recovery_info(&guard).await?;
guard.unmount().await?;
Ok::<_, Error>(embassy_os)
}
.await;
cifs.push((
record.id,
CifsBackupTarget {
hostname: mount_info.hostname,
path: mount_info.path,
username: mount_info.username,
mountable: embassy_os.is_ok(),
embassy_os: embassy_os.ok().and_then(|a| a),
},
));
for (id, model) in db.as_private().as_cifs().as_entries()? {
let mount_info = model.de()?;
let embassy_os = async {
let guard = TmpMountGuard::mount(&mount_info, ReadOnly).await?;
let embassy_os = recovery_info(guard.path()).await?;
guard.unmount().await?;
Ok::<_, Error>(embassy_os)
}
.await;
cifs.push((
id,
CifsBackupTarget {
hostname: mount_info.hostname,
path: mount_info.path,
username: mount_info.username,
mountable: embassy_os.is_ok(),
embassy_os: embassy_os.ok().and_then(|a| a),
},
));
}
Ok(cifs)

148
core/startos/src/backup/target/mod.rs
View File

@@ -1,31 +1,34 @@
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use clap::ArgMatches;
use clap::builder::ValueParserFactory;
use clap::Parser;
use color_eyre::eyre::eyre;
use digest::generic_array::GenericArray;
use digest::OutputSizeUser;
use rpc_toolkit::command;
use models::PackageId;
use rpc_toolkit::{command, from_fn_async, AnyContext, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use sha2::Sha256;
use sqlx::{Executor, Postgres};
use tokio::sync::Mutex;
use tracing::instrument;
use self::cifs::CifsBackupTarget;
use crate::context::RpcContext;
use crate::context::{CliContext, RpcContext};
use crate::db::model::DatabaseModel;
use crate::disk::mount::backup::BackupMountGuard;
use crate::disk::mount::filesystem::block_dev::BlockDev;
use crate::disk::mount::filesystem::cifs::Cifs;
use crate::disk::mount::filesystem::{FileSystem, MountType, ReadWrite};
use crate::disk::mount::guard::TmpMountGuard;
use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard};
use crate::disk::util::PartitionInfo;
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::util::serde::{deserialize_from_str, display_serializable, serialize_display};
use crate::util::{display_none, Version};
use crate::util::clap::FromStrParser;
use crate::util::serde::{
deserialize_from_str, display_serializable, serialize_display, HandlerExtSerde, WithIoFormat,
};
use crate::util::Version;
pub mod cifs;
@@ -46,18 +49,15 @@ pub enum BackupTarget {
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
pub enum BackupTargetId {
Disk { logicalname: PathBuf },
Cifs { id: i32 },
Cifs { id: u32 },
}
impl BackupTargetId {
pub async fn load<Ex>(self, secrets: &mut Ex) -> Result<BackupTargetFS, Error>
where
for<'a> &'a mut Ex: Executor<'a, Database = Postgres>,
{
pub fn load(self, db: &DatabaseModel) -> Result<BackupTargetFS, Error> {
Ok(match self {
BackupTargetId::Disk { logicalname } => {
BackupTargetFS::Disk(BlockDev::new(logicalname))
}
BackupTargetId::Cifs { id } => BackupTargetFS::Cifs(cifs::load(secrets, id).await?),
BackupTargetId::Cifs { id } => BackupTargetFS::Cifs(cifs::load(db, id)?),
})
}
}
@@ -84,6 +84,12 @@ impl std::str::FromStr for BackupTargetId {
}
}
}
impl ValueParserFactory for BackupTargetId {
type Parser = FromStrParser<Self>;
fn value_parser() -> Self::Parser {
FromStrParser::new()
}
}
impl<'de> Deserialize<'de> for BackupTargetId {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
@@ -108,9 +114,8 @@ pub enum BackupTargetFS {
Disk(BlockDev<PathBuf>),
Cifs(Cifs),
}
#[async_trait]
impl FileSystem for BackupTargetFS {
async fn mount<P: AsRef<Path> + Send + Sync>(
async fn mount<P: AsRef<Path> + Send>(
&self,
mountpoint: P,
mount_type: MountType,
@@ -130,19 +135,33 @@ impl FileSystem for BackupTargetFS {
}
}
#[command(subcommands(cifs::cifs, list, info, mount, umount))]
pub fn target() -> Result<(), Error> {
Ok(())
// #[command(subcommands(cifs::cifs, list, info, mount, umount))]
pub fn target() -> ParentHandler {
ParentHandler::new()
.subcommand("cifs", cifs::cifs())
.subcommand(
"list",
from_fn_async(list)
.with_display_serializable()
.with_remote_cli::<CliContext>(),
)
.subcommand(
"info",
from_fn_async(info)
.with_display_serializable()
.with_custom_display_fn::<AnyContext, _>(|params, info| {
Ok(display_backup_info(params.params, info))
})
.with_remote_cli::<CliContext>(),
)
}
#[command(display(display_serializable))]
pub async fn list(
#[context] ctx: RpcContext,
) -> Result<BTreeMap<BackupTargetId, BackupTarget>, Error> {
let mut sql_handle = ctx.secret_store.acquire().await?;
// #[command(display(display_serializable))]
pub async fn list(ctx: RpcContext) -> Result<BTreeMap<BackupTargetId, BackupTarget>, Error> {
let peek = ctx.db.peek().await;
let (disks_res, cifs) = tokio::try_join!(
crate::disk::util::list(&ctx.os_partitions),
cifs::list(sql_handle.as_mut()),
cifs::list(&peek),
)?;
Ok(disks_res
.into_iter()
@@ -187,11 +206,11 @@ pub struct PackageBackupInfo {
pub timestamp: DateTime<Utc>,
}
fn display_backup_info(info: BackupInfo, matches: &ArgMatches) {
fn display_backup_info(params: WithIoFormat<InfoParams>, info: BackupInfo) {
use prettytable::*;
if matches.is_present("format") {
return display_serializable(info, matches);
if let Some(format) = params.format {
return display_serializable(format, info);
}
let mut table = Table::new();
@@ -223,21 +242,24 @@ fn display_backup_info(info: BackupInfo, matches: &ArgMatches) {
table.print_tty(false).unwrap();
}
#[command(display(display_backup_info))]
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct InfoParams {
target_id: BackupTargetId,
password: String,
}
#[instrument(skip(ctx, password))]
pub async fn info(
#[context] ctx: RpcContext,
#[arg(rename = "target-id")] target_id: BackupTargetId,
#[arg] password: String,
ctx: RpcContext,
InfoParams {
target_id,
password,
}: InfoParams,
) -> Result<BackupInfo, Error> {
let guard = BackupMountGuard::mount(
TmpMountGuard::mount(
&target_id
.load(ctx.secret_store.acquire().await?.as_mut())
.await?,
ReadWrite,
)
.await?,
TmpMountGuard::mount(&target_id.load(&ctx.db.peek().await)?, ReadWrite).await?,
&password,
)
.await?;
@@ -254,45 +276,51 @@ lazy_static::lazy_static! {
Mutex::new(BTreeMap::new());
}
#[command]
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct MountParams {
target_id: BackupTargetId,
password: String,
}
#[instrument(skip_all)]
pub async fn mount(
#[context] ctx: RpcContext,
#[arg(rename = "target-id")] target_id: BackupTargetId,
#[arg] password: String,
ctx: RpcContext,
MountParams {
target_id,
password,
}: MountParams,
) -> Result<String, Error> {
let mut mounts = USER_MOUNTS.lock().await;
if let Some(existing) = mounts.get(&target_id) {
return Ok(existing.as_ref().display().to_string());
return Ok(existing.path().display().to_string());
}
let guard = BackupMountGuard::mount(
TmpMountGuard::mount(
&target_id
.clone()
.load(ctx.secret_store.acquire().await?.as_mut())
.await?,
ReadWrite,
)
.await?,
TmpMountGuard::mount(&target_id.clone().load(&ctx.db.peek().await)?, ReadWrite).await?,
&password,
)
.await?;
let res = guard.as_ref().display().to_string();
let res = guard.path().display().to_string();
mounts.insert(target_id, guard);
Ok(res)
}
#[command(display(display_none))]
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct UmountParams {
target_id: Option<BackupTargetId>,
}
#[instrument(skip_all)]
pub async fn umount(
#[context] _ctx: RpcContext,
#[arg(rename = "target-id")] target_id: Option<BackupTargetId>,
) -> Result<(), Error> {
let mut mounts = USER_MOUNTS.lock().await;
pub async fn umount(_: RpcContext, UmountParams { target_id }: UmountParams) -> Result<(), Error> {
let mut mounts = USER_MOUNTS.lock().await; // TODO: move to context
if let Some(target_id) = target_id {
if let Some(existing) = mounts.remove(&target_id) {
existing.unmount().await?;

163
core/startos/src/bins/avahi_alias.rs
View File

@@ -1,163 +0,0 @@
use avahi_sys::{
self, avahi_client_errno, avahi_entry_group_add_service, avahi_entry_group_commit,
avahi_strerror, AvahiClient,
};
fn log_str_error(action: &str, e: i32) {
unsafe {
let e_str = avahi_strerror(e);
eprintln!(
"Could not {}: {:?}",
action,
std::ffi::CStr::from_ptr(e_str)
);
}
}
pub fn main() {
let aliases: Vec<_> = std::env::args().skip(1).collect();
unsafe {
let simple_poll = avahi_sys::avahi_simple_poll_new();
let poll = avahi_sys::avahi_simple_poll_get(simple_poll);
let mut box_err = Box::pin(0 as i32);
let err_c: *mut i32 = box_err.as_mut().get_mut();
let avahi_client = avahi_sys::avahi_client_new(
poll,
avahi_sys::AvahiClientFlags::AVAHI_CLIENT_NO_FAIL,
Some(client_callback),
std::ptr::null_mut(),
err_c,
);
if avahi_client == std::ptr::null_mut::<AvahiClient>() {
log_str_error("create Avahi client", *box_err);
panic!("Failed to create Avahi Client");
}
let group = avahi_sys::avahi_entry_group_new(
avahi_client,
Some(entry_group_callback),
std::ptr::null_mut(),
);
if group == std::ptr::null_mut() {
log_str_error("create Avahi entry group", avahi_client_errno(avahi_client));
panic!("Failed to create Avahi Entry Group");
}
let mut hostname_buf = vec![0];
let hostname_raw = avahi_sys::avahi_client_get_host_name_fqdn(avahi_client);
hostname_buf.extend_from_slice(std::ffi::CStr::from_ptr(hostname_raw).to_bytes_with_nul());
let buflen = hostname_buf.len();
debug_assert!(hostname_buf.ends_with(b".local\0"));
debug_assert!(!hostname_buf[..(buflen - 7)].contains(&b'.'));
// assume fixed length prefix on hostname due to local address
hostname_buf[0] = (buflen - 8) as u8; // set the prefix length to len - 8 (leading byte, .local, nul) for the main address
hostname_buf[buflen - 7] = 5; // set the prefix length to 5 for "local"
let mut res;
let http_tcp_cstr =
std::ffi::CString::new("_http._tcp").expect("Could not cast _http._tcp to c string");
res = avahi_entry_group_add_service(
group,
avahi_sys::AVAHI_IF_UNSPEC,
avahi_sys::AVAHI_PROTO_UNSPEC,
avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_USE_MULTICAST,
hostname_raw,
http_tcp_cstr.as_ptr(),
std::ptr::null(),
std::ptr::null(),
443,
// below is a secret final argument that the type signature of this function does not tell you that it
// needs. This is because the C lib function takes a variable number of final arguments indicating the
// desired TXT records to add to this service entry. The way it decides when to stop taking arguments
// from the stack and dereferencing them is when it finds a null pointer...because fuck you, that's why.
// The consequence of this is that forgetting this last argument will cause segfaults or other undefined
// behavior. Welcome back to the stone age motherfucker.
std::ptr::null::<libc::c_char>(),
);
if res < avahi_sys::AVAHI_OK {
log_str_error("add service to Avahi entry group", res);
panic!("Failed to load Avahi services");
}
eprintln!("Published {:?}", std::ffi::CStr::from_ptr(hostname_raw));
for alias in aliases {
let lan_address = alias + ".local";
let lan_address_ptr = std::ffi::CString::new(lan_address)
.expect("Could not cast lan address to c string");
res = avahi_sys::avahi_entry_group_add_record(
group,
avahi_sys::AVAHI_IF_UNSPEC,
avahi_sys::AVAHI_PROTO_UNSPEC,
avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_USE_MULTICAST
| avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_ALLOW_MULTIPLE,
lan_address_ptr.as_ptr(),
avahi_sys::AVAHI_DNS_CLASS_IN as u16,
avahi_sys::AVAHI_DNS_TYPE_CNAME as u16,
avahi_sys::AVAHI_DEFAULT_TTL,
hostname_buf.as_ptr().cast(),
hostname_buf.len(),
);
if res < avahi_sys::AVAHI_OK {
log_str_error("add CNAME record to Avahi entry group", res);
panic!("Failed to load Avahi services");
}
eprintln!("Published {:?}", lan_address_ptr);
}
let commit_err = avahi_entry_group_commit(group);
if commit_err < avahi_sys::AVAHI_OK {
log_str_error("reset Avahi entry group", commit_err);
panic!("Failed to load Avahi services: reset");
}
}
std::thread::park()
}
unsafe extern "C" fn entry_group_callback(
_group: *mut avahi_sys::AvahiEntryGroup,
state: avahi_sys::AvahiEntryGroupState,
_userdata: *mut core::ffi::c_void,
) {
match state {
avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_FAILURE => {
eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_FAILURE");
}
avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_COLLISION => {
eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_COLLISION");
}
avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_UNCOMMITED => {
eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_UNCOMMITED");
}
avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_ESTABLISHED => {
eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_ESTABLISHED");
}
avahi_sys::AvahiEntryGroupState_AVAHI_ENTRY_GROUP_REGISTERING => {
eprintln!("AvahiCallback: EntryGroupState = AVAHI_ENTRY_GROUP_REGISTERING");
}
other => {
eprintln!("AvahiCallback: EntryGroupState = {}", other);
}
}
}
unsafe extern "C" fn client_callback(
_group: *mut avahi_sys::AvahiClient,
state: avahi_sys::AvahiClientState,
_userdata: *mut core::ffi::c_void,
) {
match state {
avahi_sys::AvahiClientState_AVAHI_CLIENT_FAILURE => {
eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_FAILURE");
}
avahi_sys::AvahiClientState_AVAHI_CLIENT_S_RUNNING => {
eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_RUNNING");
}
avahi_sys::AvahiClientState_AVAHI_CLIENT_CONNECTING => {
eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_CONNECTING");
}
avahi_sys::AvahiClientState_AVAHI_CLIENT_S_COLLISION => {
eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_COLLISION");
}
avahi_sys::AvahiClientState_AVAHI_CLIENT_S_REGISTERING => {
eprintln!("AvahiCallback: ClientState = AVAHI_CLIENT_S_REGISTERING");
}
other => {
eprintln!("AvahiCallback: ClientState = {}", other);
}
}
}

38
core/startos/src/bins/container_cli.rs Normal file
View File

@@ -0,0 +1,38 @@
use std::ffi::OsString;
use rpc_toolkit::CliApp;
use serde_json::Value;
use crate::service::cli::{ContainerCliContext, ContainerClientConfig};
use crate::util::logger::EmbassyLogger;
use crate::version::{Current, VersionT};
lazy_static::lazy_static! {
static ref VERSION_STRING: String = Current::new().semver().to_string();
}
pub fn main(args: impl IntoIterator<Item = OsString>) {
EmbassyLogger::init();
if let Err(e) = CliApp::new(
|cfg: ContainerClientConfig| Ok(ContainerCliContext::init(cfg)),
crate::service::service_effect_handler::service_effect_handler(),
)
.run(args)
{
match e.data {
Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
Some(Value::Object(o)) => {
if let Some(Value::String(s)) = o.get("details") {
eprintln!("{}: {}", e.message, s);
if let Some(Value::String(s)) = o.get("debug") {
tracing::debug!("{}", s)
}
}
}
Some(a) => eprintln!("{}: {}", e.message, a),
None => eprintln!("{}", e.message),
}
std::process::exit(e.code);
}
}

59
core/startos/src/bins/mod.rs
View File

@@ -1,49 +1,54 @@
use std::collections::VecDeque;
use std::ffi::OsString;
use std::path::Path;
#[cfg(feature = "avahi-alias")]
pub mod avahi_alias;
#[cfg(feature = "container-runtime")]
pub mod container_cli;
pub mod deprecated;
#[cfg(feature = "cli")]
pub mod start_cli;
#[cfg(feature = "js-engine")]
pub mod start_deno;
#[cfg(feature = "daemon")]
pub mod start_init;
#[cfg(feature = "sdk")]
pub mod start_sdk;
#[cfg(feature = "daemon")]
pub mod startd;
fn select_executable(name: &str) -> Option<fn()> {
fn select_executable(name: &str) -> Option<fn(VecDeque<OsString>)> {
match name {
#[cfg(feature = "avahi-alias")]
"avahi-alias" => Some(avahi_alias::main),
#[cfg(feature = "js-engine")]
"start-deno" => Some(start_deno::main),
#[cfg(feature = "cli")]
"start-cli" => Some(start_cli::main),
#[cfg(feature = "sdk")]
"start-sdk" => Some(start_sdk::main),
#[cfg(feature = "container-runtime")]
"start-cli" => Some(container_cli::main),
#[cfg(feature = "daemon")]
"startd" => Some(startd::main),
"embassy-cli" => Some(|| deprecated::renamed("embassy-cli", "start-cli")),
"embassy-sdk" => Some(|| deprecated::renamed("embassy-sdk", "start-sdk")),
"embassyd" => Some(|| deprecated::renamed("embassyd", "startd")),
"embassy-init" => Some(|| deprecated::removed("embassy-init")),
"embassy-cli" => Some(|_| deprecated::renamed("embassy-cli", "start-cli")),
"embassy-sdk" => Some(|_| deprecated::renamed("embassy-sdk", "start-sdk")),
"embassyd" => Some(|_| deprecated::renamed("embassyd", "startd")),
"embassy-init" => Some(|_| deprecated::removed("embassy-init")),
_ => None,
}
}
pub fn startbox() {
let args = std::env::args().take(2).collect::<Vec<_>>();
let executable = args
.get(0)
.and_then(|s| Path::new(&*s).file_name())
.and_then(|s| s.to_str());
if let Some(x) = executable.and_then(|s| select_executable(&s)) {
x()
} else {
eprintln!("unknown executable: {}", executable.unwrap_or("N/A"));
std::process::exit(1);
let mut args = std::env::args_os().collect::<VecDeque<_>>();
for _ in 0..2 {
if let Some(s) = args.pop_front() {
if let Some(x) = Path::new(&*s)
.file_name()
.and_then(|s| s.to_str())
.and_then(|s| select_executable(&s))
{
args.push_front(s);
return x(args);
}
}
}
let args = std::env::args().collect::<VecDeque<_>>();
eprintln!(
"unknown executable: {}",
args.get(1)
.or_else(|| args.get(0))
.map(|s| s.as_str())
.unwrap_or("N/A")
);
std::process::exit(1);
}

61
core/startos/src/bins/start_cli.rs
View File

@@ -1,62 +1,39 @@
use clap::Arg;
use rpc_toolkit::run_cli;
use rpc_toolkit::yajrc::RpcError;
use std::ffi::OsString;
use rpc_toolkit::CliApp;
use serde_json::Value;
use crate::context::config::ClientConfig;
use crate::context::CliContext;
use crate::util::logger::EmbassyLogger;
use crate::version::{Current, VersionT};
use crate::Error;
lazy_static::lazy_static! {
static ref VERSION_STRING: String = Current::new().semver().to_string();
}
fn inner_main() -> Result<(), Error> {
run_cli!({
command: crate::main_api,
app: app => app
.name("StartOS CLI")
.version(&**VERSION_STRING)
.arg(
clap::Arg::with_name("config")
.short('c')
.long("config")
.takes_value(true),
)
.arg(Arg::with_name("host").long("host").short('h').takes_value(true))
.arg(Arg::with_name("proxy").long("proxy").short('p').takes_value(true)),
context: matches => {
EmbassyLogger::init();
CliContext::init(matches)?
},
exit: |e: RpcError| {
match e.data {
Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
Some(Value::Object(o)) => if let Some(Value::String(s)) = o.get("details") {
pub fn main(args: impl IntoIterator<Item = OsString>) {
EmbassyLogger::init();
if let Err(e) = CliApp::new(
|cfg: ClientConfig| Ok(CliContext::init(cfg.load()?)?),
crate::main_api(),
)
.run(args)
{
match e.data {
Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
Some(Value::Object(o)) => {
if let Some(Value::String(s)) = o.get("details") {
eprintln!("{}: {}", e.message, s);
if let Some(Value::String(s)) = o.get("debug") {
tracing::debug!("{}", s)
}
}
Some(a) => eprintln!("{}: {}", e.message, a),
None => eprintln!("{}", e.message),
}
std::process::exit(e.code);
Some(a) => eprintln!("{}: {}", e.message, a),
None => eprintln!("{}", e.message),
}
});
Ok(())
}
pub fn main() {
match inner_main() {
Ok(_) => (),
Err(e) => {
eprintln!("{}", e.source);
tracing::debug!("{:?}", e.source);
drop(e.source);
std::process::exit(e.kind as i32)
}
std::process::exit(e.code);
}
}

140
core/startos/src/bins/start_deno.rs
View File

@@ -1,140 +0,0 @@
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::{command, run_cli, Context};
use serde_json::Value;
use crate::procedure::js_scripts::ExecuteArgs;
use crate::s9pk::manifest::PackageId;
use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
use crate::version::{Current, VersionT};
use crate::Error;
lazy_static::lazy_static! {
static ref VERSION_STRING: String = Current::new().semver().to_string();
}
struct DenoContext;
impl Context for DenoContext {}
#[command(subcommands(execute, sandbox))]
fn deno_api() -> Result<(), Error> {
Ok(())
}
#[command(cli_only, display(display_serializable))]
async fn execute(
#[arg(stdin, parse(parse_stdin_deserializable))] arg: ExecuteArgs,
#[allow(unused_variables)]
#[arg(long = "format")]
format: Option<IoFormat>,
) -> Result<Result<Value, (i32, String)>, Error> {
let ExecuteArgs {
procedure,
directory,
pkg_id,
pkg_version,
name,
volumes,
input,
} = arg;
PackageLogger::init(&pkg_id);
procedure
.execute_impl(&directory, &pkg_id, &pkg_version, name, &volumes, input)
.await
}
#[command(cli_only, display(display_serializable))]
async fn sandbox(
#[arg(stdin, parse(parse_stdin_deserializable))] arg: ExecuteArgs,
#[allow(unused_variables)]
#[arg(long = "format")]
format: Option<IoFormat>,
) -> Result<Result<Value, (i32, String)>, Error> {
let ExecuteArgs {
procedure,
directory,
pkg_id,
pkg_version,
name,
volumes,
input,
} = arg;
PackageLogger::init(&pkg_id);
procedure
.sandboxed_impl(&directory, &pkg_id, &pkg_version, &volumes, input, name)
.await
}
use tracing::Subscriber;
use tracing_subscriber::util::SubscriberInitExt;
#[derive(Clone)]
struct PackageLogger {}
impl PackageLogger {
fn base_subscriber(id: &PackageId) -> impl Subscriber {
use tracing_error::ErrorLayer;
use tracing_subscriber::prelude::*;
use tracing_subscriber::{fmt, EnvFilter};
let filter_layer = EnvFilter::default().add_directive(
format!("{}=warn", std::module_path!().split("::").next().unwrap())
.parse()
.unwrap(),
);
let fmt_layer = fmt::layer().with_writer(std::io::stderr).with_target(true);
let journald_layer = tracing_journald::layer()
.unwrap()
.with_syslog_identifier(format!("{id}.embassy"));
let sub = tracing_subscriber::registry()
.with(filter_layer)
.with(fmt_layer)
.with(journald_layer)
.with(ErrorLayer::default());
sub
}
pub fn init(id: &PackageId) -> Self {
Self::base_subscriber(id).init();
color_eyre::install().unwrap_or_else(|_| tracing::warn!("tracing too many times"));
Self {}
}
}
fn inner_main() -> Result<(), Error> {
run_cli!({
command: deno_api,
app: app => app
.name("StartOS Deno Executor")
.version(&**VERSION_STRING),
context: _m => DenoContext,
exit: |e: RpcError| {
match e.data {
Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
Some(Value::Object(o)) => if let Some(Value::String(s)) = o.get("details") {
eprintln!("{}: {}", e.message, s);
if let Some(Value::String(s)) = o.get("debug") {
tracing::debug!("{}", s)
}
}
Some(a) => eprintln!("{}: {}", e.message, a),
None => eprintln!("{}", e.message),
}
std::process::exit(e.code);
}
});
Ok(())
}
pub fn main() {
match inner_main() {
Ok(_) => (),
Err(e) => {
eprintln!("{}", e.source);
tracing::debug!("{:?}", e.source);
drop(e.source);
std::process::exit(e.kind as i32)
}
}
}

58
core/startos/src/bins/start_init.rs
View File

@@ -1,5 +1,5 @@
use std::net::{Ipv6Addr, SocketAddr};
use std::path::{Path, PathBuf};
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
@@ -7,7 +7,7 @@ use helpers::NonDetachingJoinHandle;
use tokio::process::Command;
use tracing::instrument;
use crate::context::rpc::RpcContextConfig;
use crate::context::config::ServerConfig;
use crate::context::{DiagnosticContext, InstallContext, SetupContext};
use crate::disk::fsck::{RepairStrategy, RequiresReboot};
use crate::disk::main::DEFAULT_PASSWORD;
@@ -21,7 +21,7 @@ use crate::util::Invoke;
use crate::{Error, ErrorKind, ResultExt, PLATFORM};
#[instrument(skip_all)]
async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
async fn setup_or_init(config: &ServerConfig) -> Result<Option<Shutdown>, Error> {
let song = NonDetachingJoinHandle::from(tokio::spawn(async {
loop {
BEP.play().await.unwrap();
@@ -82,13 +82,12 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Er
.invoke(crate::ErrorKind::OpenSsh)
.await?;
let ctx = InstallContext::init(cfg_path).await?;
let ctx = InstallContext::init().await?;
let server = WebServer::install(
SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
ctx.clone(),
)
.await?;
)?;
drop(song);
tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
@@ -109,26 +108,24 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Er
.await
.is_err()
{
let ctx = SetupContext::init(cfg_path).await?;
let ctx = SetupContext::init(config)?;
let server = WebServer::setup(
SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
ctx.clone(),
)
.await?;
)?;
drop(song);
tokio::time::sleep(Duration::from_secs(1)).await; // let the record state that I hate this
CHIME.play().await?;
ctx.shutdown
.subscribe()
.recv()
.await
.expect("context dropped");
let mut shutdown = ctx.shutdown.subscribe();
shutdown.recv().await.expect("context dropped");
server.shutdown().await;
drop(shutdown);
tokio::task::yield_now().await;
if let Err(e) = Command::new("killall")
.arg("firefox-esr")
@@ -139,13 +136,12 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Er
tracing::debug!("{:?}", e);
}
} else {
let cfg = RpcContextConfig::load(cfg_path).await?;
let guid_string = tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
.await?;
let guid = guid_string.trim();
let requires_reboot = crate::disk::main::import(
guid,
cfg.datadir(),
config.datadir(),
if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() {
RepairStrategy::Aggressive
} else {
@@ -164,13 +160,13 @@ async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Er
.with_ctx(|_| (crate::ErrorKind::Filesystem, REPAIR_DISK_PATH))?;
}
if requires_reboot.0 {
crate::disk::main::export(guid, cfg.datadir()).await?;
crate::disk::main::export(guid, config.datadir()).await?;
Command::new("reboot")
.invoke(crate::ErrorKind::Unknown)
.await?;
}
tracing::info!("Loaded Disk");
crate::init::init(&cfg).await?;
crate::init::init(config).await?;
drop(song);
}
@@ -196,7 +192,7 @@ async fn run_script_if_exists<P: AsRef<Path>>(path: P) {
}
#[instrument(skip_all)]
async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
async fn inner_main(config: &ServerConfig) -> Result<Option<Shutdown>, Error> {
if &*PLATFORM == "raspberrypi" && tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() {
tokio::fs::remove_file(STANDBY_MODE_PATH).await?;
Command::new("sync").invoke(ErrorKind::Filesystem).await?;
@@ -208,7 +204,7 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
run_script_if_exists("/media/embassy/config/preinit.sh").await;
let res = match setup_or_init(cfg_path.clone()).await {
let res = match setup_or_init(config).await {
Err(e) => {
async move {
tracing::error!("{}", e.source);
@@ -216,7 +212,7 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
crate::sound::BEETHOVEN.play().await?;
let ctx = DiagnosticContext::init(
cfg_path,
config,
if tokio::fs::metadata("/media/embassy/config/disk.guid")
.await
.is_ok()
@@ -231,14 +227,12 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
None
},
e,
)
.await?;
)?;
let server = WebServer::diagnostic(
SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
ctx.clone(),
)
.await?;
)?;
let shutdown = ctx.shutdown.subscribe().recv().await.unwrap();
@@ -256,23 +250,13 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
res
}
pub fn main() {
let matches = clap::App::new("start-init")
.arg(
clap::Arg::with_name("config")
.short('c')
.long("config")
.takes_value(true),
)
.get_matches();
let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
pub fn main(config: &ServerConfig) {
let res = {
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.expect("failed to initialize runtime");
rt.block_on(inner_main(cfg_path))
rt.block_on(inner_main(config))
};
match res {

61
core/startos/src/bins/start_sdk.rs
View File

@@ -1,61 +0,0 @@
use rpc_toolkit::run_cli;
use rpc_toolkit::yajrc::RpcError;
use serde_json::Value;
use crate::context::SdkContext;
use crate::util::logger::EmbassyLogger;
use crate::version::{Current, VersionT};
use crate::Error;
lazy_static::lazy_static! {
static ref VERSION_STRING: String = Current::new().semver().to_string();
}
fn inner_main() -> Result<(), Error> {
run_cli!({
command: crate::portable_api,
app: app => app
.name("StartOS SDK")
.version(&**VERSION_STRING)
.arg(
clap::Arg::with_name("config")
.short('c')
.long("config")
.takes_value(true),
),
context: matches => {
if let Err(_) = std::env::var("RUST_LOG") {
std::env::set_var("RUST_LOG", "embassy=warn,js_engine=warn");
}
EmbassyLogger::init();
SdkContext::init(matches)?
},
exit: |e: RpcError| {
match e.data {
Some(Value::String(s)) => eprintln!("{}: {}", e.message, s),
Some(Value::Object(o)) => if let Some(Value::String(s)) = o.get("details") {
eprintln!("{}: {}", e.message, s);
if let Some(Value::String(s)) = o.get("debug") {
tracing::debug!("{}", s)
}
}
Some(a) => eprintln!("{}: {}", e.message, a),
None => eprintln!("{}", e.message),
}
std::process::exit(e.code);
}
});
Ok(())
}
pub fn main() {
match inner_main() {
Ok(_) => (),
Err(e) => {
eprintln!("{}", e.source);
tracing::debug!("{:?}", e.source);
drop(e.source);
std::process::exit(e.kind as i32)
}
}
}

39
core/startos/src/bins/startd.rs
View File

@@ -1,12 +1,15 @@
use std::ffi::OsString;
use std::net::{Ipv6Addr, SocketAddr};
use std::path::{Path, PathBuf};
use std::path::Path;
use std::sync::Arc;
use clap::Parser;
use color_eyre::eyre::eyre;
use futures::{FutureExt, TryFutureExt};
use tokio::signal::unix::signal;
use tracing::instrument;
use crate::context::config::ServerConfig;
use crate::context::{DiagnosticContext, RpcContext};
use crate::net::web_server::WebServer;
use crate::shutdown::Shutdown;
@@ -15,10 +18,10 @@ use crate::util::logger::EmbassyLogger;
use crate::{Error, ErrorKind, ResultExt};
#[instrument(skip_all)]
async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
async fn inner_main(config: &ServerConfig) -> Result<Option<Shutdown>, Error> {
let (rpc_ctx, server, shutdown) = async {
let rpc_ctx = RpcContext::init(
cfg_path,
config,
Arc::new(
tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
.await?
@@ -31,8 +34,7 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
let server = WebServer::main(
SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
rpc_ctx.clone(),
)
.await?;
)?;
let mut shutdown_recv = rpc_ctx.shutdown.subscribe();
@@ -102,32 +104,23 @@ async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error
Ok(shutdown)
}
pub fn main() {
pub fn main(args: impl IntoIterator<Item = OsString>) {
EmbassyLogger::init();
let config = ServerConfig::parse_from(args).load().unwrap();
if !Path::new("/run/embassy/initialized").exists() {
super::start_init::main();
super::start_init::main(&config);
std::fs::write("/run/embassy/initialized", "").unwrap();
}
let matches = clap::App::new("startd")
.arg(
clap::Arg::with_name("config")
.short('c')
.long("config")
.takes_value(true),
)
.get_matches();
let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
let res = {
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.expect("failed to initialize runtime");
rt.block_on(async {
match inner_main(cfg_path.clone()).await {
match inner_main(&config).await {
Ok(a) => Ok(a),
Err(e) => {
async {
@@ -135,7 +128,7 @@ pub fn main() {
tracing::debug!("{:?}", e.source);
crate::sound::BEETHOVEN.play().await?;
let ctx = DiagnosticContext::init(
cfg_path,
&config,
if tokio::fs::metadata("/media/embassy/config/disk.guid")
.await
.is_ok()
@@ -150,14 +143,12 @@ pub fn main() {
None
},
e,
)
.await?;
)?;
let server = WebServer::diagnostic(
SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), 80),
ctx.clone(),
)
.await?;
)?;
let mut shutdown = ctx.shutdown.subscribe();

98
core/startos/src/config/action.rs
View File

@@ -1,22 +1,12 @@
use std::collections::{BTreeMap, BTreeSet};
use color_eyre::eyre::eyre;
use models::ImageId;
use patch_db::HasModel;
use models::PackageId;
use serde::{Deserialize, Serialize};
use tracing::instrument;
use super::{Config, ConfigSpec};
use crate::context::RpcContext;
use crate::dependencies::Dependencies;
#[allow(unused_imports)]
use crate::prelude::*;
use crate::procedure::docker::DockerContainers;
use crate::procedure::{PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
use crate::status::health_check::HealthCheckId;
use crate::util::Version;
use crate::volume::Volumes;
use crate::{Error, ResultExt};
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
@@ -25,90 +15,6 @@ pub struct ConfigRes {
pub spec: ConfigSpec,
}
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
#[model = "Model<Self>"]
pub struct ConfigActions {
pub get: PackageProcedure,
pub set: PackageProcedure,
}
impl ConfigActions {
#[instrument(skip_all)]
pub fn validate(
&self,
_container: &Option<DockerContainers>,
eos_version: &Version,
volumes: &Volumes,
image_ids: &BTreeSet<ImageId>,
) -> Result<(), Error> {
self.get
.validate(eos_version, volumes, image_ids, true)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Get"))?;
self.set
.validate(eos_version, volumes, image_ids, true)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Set"))?;
Ok(())
}
#[instrument(skip_all)]
pub async fn get(
&self,
ctx: &RpcContext,
pkg_id: &PackageId,
pkg_version: &Version,
volumes: &Volumes,
) -> Result<ConfigRes, Error> {
self.get
.execute(
ctx,
pkg_id,
pkg_version,
ProcedureName::GetConfig,
volumes,
None::<()>,
None,
)
.await
.and_then(|res| {
res.map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::ConfigGen))
})
}
#[instrument(skip_all)]
pub async fn set(
&self,
ctx: &RpcContext,
pkg_id: &PackageId,
pkg_version: &Version,
dependencies: &Dependencies,
volumes: &Volumes,
input: &Config,
) -> Result<SetResult, Error> {
let res: SetResult = self
.set
.execute(
ctx,
pkg_id,
pkg_version,
ProcedureName::SetConfig,
volumes,
Some(input),
None,
)
.await
.and_then(|res| {
res.map_err(|e| {
Error::new(eyre!("{}", e.1), crate::ErrorKind::ConfigRulesViolation)
})
})?;
Ok(SetResult {
depends_on: res
.depends_on
.into_iter()
.filter(|(pkg, _)| dependencies.0.contains_key(pkg))
.collect(),
})
}
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct SetResult {

213
core/startos/src/config/mod.rs
View File

@@ -1,34 +1,31 @@
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use clap::Parser;
use color_eyre::eyre::eyre;
use indexmap::IndexSet;
use indexmap::{IndexMap, IndexSet};
use itertools::Itertools;
use models::{ErrorKind, OptionExt};
use models::{ErrorKind, OptionExt, PackageId};
use patch_db::value::InternedString;
use patch_db::Value;
use regex::Regex;
use rpc_toolkit::command;
use rpc_toolkit::{from_fn_async, Empty, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use tracing::instrument;
use crate::context::RpcContext;
use crate::context::{CliContext, RpcContext};
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::util::display_none;
use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
use crate::Error;
use crate::util::serde::{HandlerExtSerde, StdinDeserializable};
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ConfigSpec(pub IndexMap<InternedString, Value>);
pub mod action;
pub mod spec;
pub mod util;
pub use spec::{ConfigSpec, Defaultable};
use util::NumRange;
use self::action::ConfigRes;
use self::spec::ValueSpecPointer;
pub type Config = patch_db::value::InOMap<InternedString, Value>;
pub trait TypeOf {
@@ -55,8 +52,6 @@ pub enum ConfigurationError {
NoMatch(#[from] NoMatchWithPath),
#[error("System Error: {0}")]
SystemError(Error),
#[error("Permission Denied: {0}")]
PermissionDenied(ValueSpecPointer),
}
impl From<ConfigurationError> for Error {
fn from(err: ConfigurationError) -> Self {
@@ -124,164 +119,102 @@ pub enum MatchError {
PropertyMatchesUnionTag(InternedString, String),
#[error("Name of Property {0:?} Conflicts With Map Tag Name")]
PropertyNameMatchesMapTag(String),
#[error("Pointer Is Invalid: {0}")]
InvalidPointer(spec::ValueSpecPointer),
#[error("Object Key Is Invalid: {0}")]
InvalidKey(String),
#[error("Value In List Is Not Unique")]
ListUniquenessViolation,
}
#[command(rename = "config-spec", cli_only, blocking, display(display_none))]
pub fn verify_spec(#[arg] path: PathBuf) -> Result<(), Error> {
let mut file = std::fs::File::open(&path)?;
let format = match path.extension().and_then(|s| s.to_str()) {
Some("yaml") | Some("yml") => IoFormat::Yaml,
Some("json") => IoFormat::Json,
Some("toml") => IoFormat::Toml,
Some("cbor") => IoFormat::Cbor,
_ => {
return Err(Error::new(
eyre!("Unknown file format. Expected one of yaml, json, toml, cbor."),
crate::ErrorKind::Deserialization,
));
}
};
let _: ConfigSpec = format.from_reader(&mut file)?;
Ok(())
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct ConfigParams {
pub id: PackageId,
}
#[command(subcommands(get, set))]
pub fn config(#[arg] id: PackageId) -> Result<PackageId, Error> {
Ok(id)
// #[command(subcommands(get, set))]
pub fn config() -> ParentHandler<ConfigParams> {
ParentHandler::new()
.subcommand(
"get",
from_fn_async(get)
.with_inherited(|ConfigParams { id }, _| id)
.with_display_serializable()
.with_remote_cli::<CliContext>(),
)
.subcommand("set", set().with_inherited(|ConfigParams { id }, _| id))
}
#[command(display(display_serializable))]
#[instrument(skip_all)]
pub async fn get(
#[context] ctx: RpcContext,
#[parent_data] id: PackageId,
#[allow(unused_variables)]
#[arg(long = "format")]
format: Option<IoFormat>,
) -> Result<ConfigRes, Error> {
let db = ctx.db.peek().await;
let manifest = db
.as_package_data()
.as_idx(&id)
.or_not_found(&id)?
.as_installed()
.or_not_found(&id)?
.as_manifest();
let action = manifest
.as_config()
.de()?
.ok_or_else(|| Error::new(eyre!("{} has no config", id), crate::ErrorKind::NotFound))?;
let volumes = manifest.as_volumes().de()?;
let version = manifest.as_version().de()?;
action.get(&ctx, &id, &version, &volumes).await
pub async fn get(ctx: RpcContext, _: Empty, id: PackageId) -> Result<ConfigRes, Error> {
ctx.services
.get(&id)
.await
.as_ref()
.or_not_found(lazy_format!("Manager for {id}"))?
.get_config()
.await
}
#[command(
subcommands(self(set_impl(async, context(RpcContext))), set_dry),
display(display_none),
metadata(sync_db = true)
)]
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
pub struct SetParams {
#[arg(long = "timeout")]
pub timeout: Option<crate::util::serde::Duration>,
#[command(flatten)]
pub config: StdinDeserializable<Option<Config>>,
}
// TODO Dr Why isn't this used?
// #[command(
// subcommands(self(set_impl(async, context(RpcContext))), set_dry),
// display(display_none),
// metadata(sync_db = true)
// )]
#[instrument(skip_all)]
pub fn set(
#[parent_data] id: PackageId,
#[allow(unused_variables)]
#[arg(long = "format")]
format: Option<IoFormat>,
#[arg(long = "timeout")] timeout: Option<crate::util::serde::Duration>,
#[arg(stdin, parse(parse_stdin_deserializable))] config: Option<Config>,
) -> Result<(PackageId, Option<Config>, Option<Duration>), Error> {
Ok((id, config, timeout.map(|d| *d)))
}
#[command(rename = "dry", display(display_serializable))]
#[instrument(skip_all)]
pub async fn set_dry(
#[context] ctx: RpcContext,
#[parent_data] (id, config, timeout): (PackageId, Option<Config>, Option<Duration>),
) -> Result<BTreeMap<PackageId, String>, Error> {
let breakages = BTreeMap::new();
let overrides = Default::default();
let configure_context = ConfigureContext {
breakages,
timeout,
config,
dry_run: true,
overrides,
};
let breakages = configure(&ctx, &id, configure_context).await?;
Ok(breakages)
pub fn set() -> ParentHandler<SetParams, PackageId> {
ParentHandler::new().root_handler(
from_fn_async(set_impl)
.with_metadata("sync_db", Value::Bool(true))
.with_inherited(|set_params, id| (id, set_params))
.no_display()
.with_remote_cli::<CliContext>(),
)
}
#[derive(Default)]
pub struct ConfigureContext {
pub breakages: BTreeMap<PackageId, String>,
pub timeout: Option<Duration>,
pub config: Option<Config>,
pub overrides: BTreeMap<PackageId, Config>,
pub dry_run: bool,
}
#[instrument(skip_all)]
pub async fn set_impl(
ctx: RpcContext,
(id, config, timeout): (PackageId, Option<Config>, Option<Duration>),
_: Empty,
(
id,
SetParams {
timeout,
config: StdinDeserializable(config),
},
): (PackageId, SetParams),
) -> Result<(), Error> {
let breakages = BTreeMap::new();
let overrides = Default::default();
let configure_context = ConfigureContext {
breakages,
timeout,
timeout: timeout.map(|t| *t),
config,
dry_run: false,
overrides,
};
configure(&ctx, &id, configure_context).await?;
Ok(())
}
#[instrument(skip_all)]
pub async fn configure(
ctx: &RpcContext,
id: &PackageId,
configure_context: ConfigureContext,
) -> Result<BTreeMap<PackageId, String>, Error> {
let db = ctx.db.peek().await;
let package = db
.as_package_data()
.as_idx(id)
.or_not_found(&id)?
.as_installed()
.or_not_found(&id)?;
let version = package.as_manifest().as_version().de()?;
ctx.managers
.get(&(id.clone(), version.clone()))
ctx.services
.get(&id)
.await
.as_ref()
.ok_or_else(|| {
Error::new(
eyre!("There is no manager running for {id:?} and {version:?}"),
eyre!("There is no manager running for {id}"),
ErrorKind::Unknown,
)
})?
.configure(configure_context)
.await
.await?;
Ok(())
}
macro_rules! not_found {
($x:expr) => {
crate::Error::new(
color_eyre::eyre::eyre!("Could not find {} at {}:{}", $x, module_path!(), line!()),
crate::ErrorKind::Incoherent,
)
};
}
pub(crate) use not_found;

2013
core/startos/src/config/spec.rs
View File

File diff suppressed because it is too large Load Diff

196
core/startos/src/context/cli.rs
View File

@@ -1,43 +1,37 @@
use std::fs::File;
use std::io::BufReader;
use std::net::Ipv4Addr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use clap::ArgMatches;
use color_eyre::eyre::eyre;
use cookie_store::{CookieStore, RawCookie};
use josekit::jwk::Jwk;
use once_cell::sync::OnceCell;
use reqwest::Proxy;
use reqwest_cookie_store::CookieStoreMutex;
use rpc_toolkit::reqwest::{Client, Url};
use rpc_toolkit::url::Host;
use rpc_toolkit::Context;
use serde::Deserialize;
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::{call_remote_http, CallRemote, Context};
use tokio::net::TcpStream;
use tokio::runtime::Runtime;
use tokio_tungstenite::{MaybeTlsStream, WebSocketStream};
use tracing::instrument;
use super::setup::CURRENT_SECRET;
use crate::context::config::{local_config_path, ClientConfig};
use crate::core::rpc_continuations::RequestGuid;
use crate::middleware::auth::LOCAL_AUTH_COOKIE_PATH;
use crate::util::config::{load_config_from_paths, local_config_path};
use crate::ResultExt;
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct CliContextConfig {
pub host: Option<Url>,
#[serde(deserialize_with = "crate::util::serde::deserialize_from_str_opt")]
#[serde(default)]
pub proxy: Option<Url>,
pub cookie_path: Option<PathBuf>,
}
use crate::prelude::*;
#[derive(Debug)]
pub struct CliContextSeed {
pub runtime: OnceCell<Runtime>,
pub base_url: Url,
pub rpc_url: Url,
pub client: Client,
pub cookie_store: Arc<CookieStoreMutex>,
pub cookie_path: PathBuf,
pub developer_key_path: PathBuf,
pub developer_key: OnceCell<ed25519_dalek::SigningKey>,
}
impl Drop for CliContextSeed {
fn drop(&mut self) {
@@ -60,42 +54,22 @@ impl Drop for CliContextSeed {
}
}
const DEFAULT_HOST: Host<&'static str> = Host::Ipv4(Ipv4Addr::new(127, 0, 0, 1));
const DEFAULT_PORT: u16 = 5959;
#[derive(Debug, Clone)]
pub struct CliContext(Arc<CliContextSeed>);
impl CliContext {
/// BLOCKING
#[instrument(skip_all)]
pub fn init(matches: &ArgMatches) -> Result<Self, crate::Error> {
let local_config_path = local_config_path();
let base: CliContextConfig = load_config_from_paths(
matches
.values_of("config")
.into_iter()
.flatten()
.map(|p| Path::new(p))
.chain(local_config_path.as_deref().into_iter())
.chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
)?;
let mut url = if let Some(host) = matches.value_of("host") {
host.parse()?
} else if let Some(host) = base.host {
pub fn init(config: ClientConfig) -> Result<Self, Error> {
let mut url = if let Some(host) = config.host {
host
} else {
"http://localhost".parse()?
};
let proxy = if let Some(proxy) = matches.value_of("proxy") {
Some(proxy.parse()?)
} else {
base.proxy
};
let cookie_path = base.cookie_path.unwrap_or_else(|| {
local_config_path
let cookie_path = config.cookie_path.unwrap_or_else(|| {
local_config_path()
.as_deref()
.unwrap_or_else(|| Path::new(crate::util::config::CONFIG_PATH))
.unwrap_or_else(|| Path::new(super::config::CONFIG_PATH))
.parent()
.unwrap_or(Path::new("/"))
.join(".cookies.json")
@@ -120,6 +94,7 @@ impl CliContext {
}));
Ok(CliContext(Arc::new(CliContextSeed {
runtime: OnceCell::new(),
base_url: url.clone(),
rpc_url: {
url.path_segments_mut()
@@ -131,7 +106,7 @@ impl CliContext {
},
client: {
let mut builder = Client::builder().cookie_provider(cookie_store.clone());
if let Some(proxy) = proxy {
if let Some(proxy) = config.proxy {
builder =
builder.proxy(Proxy::all(proxy).with_kind(crate::ErrorKind::ParseUrl)?)
}
@@ -139,8 +114,90 @@ impl CliContext {
},
cookie_store,
cookie_path,
developer_key_path: config.developer_key_path.unwrap_or_else(|| {
local_config_path()
.as_deref()
.unwrap_or_else(|| Path::new(super::config::CONFIG_PATH))
.parent()
.unwrap_or(Path::new("/"))
.join("developer.key.pem")
}),
developer_key: OnceCell::new(),
})))
}
/// BLOCKING
#[instrument(skip_all)]
pub fn developer_key(&self) -> Result<&ed25519_dalek::SigningKey, Error> {
self.developer_key.get_or_try_init(|| {
if !self.developer_key_path.exists() {
return Err(Error::new(eyre!("Developer Key does not exist! Please run `start-cli init` before running this command."), crate::ErrorKind::Uninitialized));
}
let pair = <ed25519::KeypairBytes as ed25519::pkcs8::DecodePrivateKey>::from_pkcs8_pem(
&std::fs::read_to_string(&self.developer_key_path)?,
)
.with_kind(crate::ErrorKind::Pem)?;
let secret = ed25519_dalek::SecretKey::try_from(&pair.secret_key[..]).map_err(|_| {
Error::new(
eyre!("pkcs8 key is of incorrect length"),
ErrorKind::OpenSsl,
)
})?;
Ok(secret.into())
})
}
pub async fn ws_continuation(
&self,
guid: RequestGuid,
) -> Result<WebSocketStream<MaybeTlsStream<TcpStream>>, Error> {
let mut url = self.base_url.clone();
let ws_scheme = match url.scheme() {
"https" => "wss",
"http" => "ws",
_ => {
return Err(Error::new(
eyre!("Cannot parse scheme from base URL"),
crate::ErrorKind::ParseUrl,
)
.into())
}
};
url.set_scheme(ws_scheme)
.map_err(|_| Error::new(eyre!("Cannot set URL scheme"), crate::ErrorKind::ParseUrl))?;
url.path_segments_mut()
.map_err(|_| eyre!("Url cannot be base"))
.with_kind(crate::ErrorKind::ParseUrl)?
.push("ws")
.push("rpc")
.push(guid.as_ref());
let (stream, _) =
// base_url is "http://127.0.0.1/", with a trailing slash, so we don't put a leading slash in this path:
tokio_tungstenite::connect_async(url).await.with_kind(ErrorKind::Network)?;
Ok(stream)
}
pub async fn rest_continuation(
&self,
guid: RequestGuid,
body: reqwest::Body,
headers: reqwest::header::HeaderMap,
) -> Result<reqwest::Response, Error> {
let mut url = self.base_url.clone();
url.path_segments_mut()
.map_err(|_| eyre!("Url cannot be base"))
.with_kind(crate::ErrorKind::ParseUrl)?
.push("rest")
.push("rpc")
.push(guid.as_ref());
self.client
.post(url)
.headers(headers)
.body(body)
.send()
.await
.with_kind(ErrorKind::Network)
}
}
impl AsRef<Jwk> for CliContext {
fn as_ref(&self) -> &Jwk {
@@ -154,32 +211,33 @@ impl std::ops::Deref for CliContext {
}
}
impl Context for CliContext {
fn protocol(&self) -> &str {
self.0.base_url.scheme()
}
fn host(&self) -> Host<&str> {
self.0.base_url.host().unwrap_or(DEFAULT_HOST)
}
fn port(&self) -> u16 {
self.0.base_url.port().unwrap_or(DEFAULT_PORT)
}
fn path(&self) -> &str {
self.0.rpc_url.path()
}
fn url(&self) -> Url {
self.0.rpc_url.clone()
}
fn client(&self) -> &Client {
&self.0.client
fn runtime(&self) -> tokio::runtime::Handle {
self.runtime
.get_or_init(|| {
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap()
})
.handle()
.clone()
}
}
/// When we had an empty proxy the system wasn't working like it used to, which allowed empty proxy
#[async_trait::async_trait]
impl CallRemote for CliContext {
async fn call_remote(&self, method: &str, params: Value) -> Result<Value, RpcError> {
call_remote_http(&self.client, self.rpc_url.clone(), method, params).await
}
}
#[test]
fn test_cli_proxy_empty() {
serde_yaml::from_str::<CliContextConfig>(
"
bind_rpc:
",
)
.unwrap();
fn test() {
let ctx = CliContext::init(ClientConfig::default()).unwrap();
ctx.runtime().block_on(async {
reqwest::Client::new()
.get("http://example.com")
.send()
.await
.unwrap();
});
}

169
core/startos/src/context/config.rs Normal file
View File

@@ -0,0 +1,169 @@
use std::fs::File;
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use clap::Parser;
use reqwest::Url;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use sqlx::postgres::PgConnectOptions;
use sqlx::PgPool;
use crate::disk::OsPartitionInfo;
use crate::init::init_postgres;
use crate::prelude::*;
use crate::util::serde::IoFormat;
pub const DEVICE_CONFIG_PATH: &str = "/media/embassy/config/config.yaml"; // "/media/startos/config/config.yaml";
pub const CONFIG_PATH: &str = "/etc/startos/config.yaml";
pub const CONFIG_PATH_LOCAL: &str = ".startos/config.yaml";
pub fn local_config_path() -> Option<PathBuf> {
if let Ok(home) = std::env::var("HOME") {
Some(Path::new(&home).join(CONFIG_PATH_LOCAL))
} else {
None
}
}
pub trait ContextConfig: DeserializeOwned + Default {
fn next(&mut self) -> Option<PathBuf>;
fn merge_with(&mut self, other: Self);
fn from_path(path: impl AsRef<Path>) -> Result<Self, Error> {
let format: IoFormat = path
.as_ref()
.extension()
.and_then(|s| s.to_str())
.map(|f| f.parse())
.transpose()?
.unwrap_or_default();
format.from_reader(File::open(path)?)
}
fn load_path_rec(&mut self, path: Option<impl AsRef<Path>>) -> Result<(), Error> {
if let Some(path) = path.filter(|p| p.as_ref().exists()) {
let mut other = Self::from_path(path)?;
let path = other.next();
self.merge_with(other);
self.load_path_rec(path)?;
}
Ok(())
}
}
#[derive(Debug, Default, Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct ClientConfig {
#[arg(short = 'c', long = "config")]
pub config: Option<PathBuf>,
#[arg(short = 'h', long = "host")]
pub host: Option<Url>,
#[arg(short = 'p', long = "proxy")]
pub proxy: Option<Url>,
#[arg(long = "cookie-path")]
pub cookie_path: Option<PathBuf>,
#[arg(long = "developer-key-path")]
pub developer_key_path: Option<PathBuf>,
}
impl ContextConfig for ClientConfig {
fn next(&mut self) -> Option<PathBuf> {
self.config.take()
}
fn merge_with(&mut self, other: Self) {
self.host = self.host.take().or(other.host);
self.proxy = self.proxy.take().or(other.proxy);
self.cookie_path = self.cookie_path.take().or(other.cookie_path);
}
}
impl ClientConfig {
pub fn load(mut self) -> Result<Self, Error> {
let path = self.next();
self.load_path_rec(path)?;
self.load_path_rec(local_config_path())?;
self.load_path_rec(Some(CONFIG_PATH))?;
Ok(self)
}
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct ServerConfig {
#[arg(short = 'c', long = "config")]
pub config: Option<PathBuf>,
#[arg(long = "wifi-interface")]
pub wifi_interface: Option<String>,
#[arg(long = "ethernet-interface")]
pub ethernet_interface: Option<String>,
#[arg(skip)]
pub os_partitions: Option<OsPartitionInfo>,
#[arg(long = "bind-rpc")]
pub bind_rpc: Option<SocketAddr>,
#[arg(long = "tor-control")]
pub tor_control: Option<SocketAddr>,
#[arg(long = "tor-socks")]
pub tor_socks: Option<SocketAddr>,
#[arg(long = "dns-bind")]
pub dns_bind: Option<Vec<SocketAddr>>,
#[arg(long = "revision-cache-size")]
pub revision_cache_size: Option<usize>,
#[arg(short = 'd', long = "datadir")]
pub datadir: Option<PathBuf>,
#[arg(long = "disable-encryption")]
pub disable_encryption: Option<bool>,
}
impl ContextConfig for ServerConfig {
fn next(&mut self) -> Option<PathBuf> {
self.config.take()
}
fn merge_with(&mut self, other: Self) {
self.wifi_interface = self.wifi_interface.take().or(other.wifi_interface);
self.ethernet_interface = self.ethernet_interface.take().or(other.ethernet_interface);
self.os_partitions = self.os_partitions.take().or(other.os_partitions);
self.bind_rpc = self.bind_rpc.take().or(other.bind_rpc);
self.tor_control = self.tor_control.take().or(other.tor_control);
self.tor_socks = self.tor_socks.take().or(other.tor_socks);
self.dns_bind = self.dns_bind.take().or(other.dns_bind);
self.revision_cache_size = self
.revision_cache_size
.take()
.or(other.revision_cache_size);
self.datadir = self.datadir.take().or(other.datadir);
self.disable_encryption = self.disable_encryption.take().or(other.disable_encryption);
}
}
impl ServerConfig {
pub fn load(mut self) -> Result<Self, Error> {
let path = self.next();
self.load_path_rec(path)?;
self.load_path_rec(Some(DEVICE_CONFIG_PATH))?;
self.load_path_rec(Some(CONFIG_PATH))?;
Ok(self)
}
pub fn datadir(&self) -> &Path {
self.datadir
.as_deref()
.unwrap_or_else(|| Path::new("/embassy-data"))
}
pub async fn db(&self) -> Result<PatchDb, Error> {
let db_path = self.datadir().join("main").join("embassy.db");
let db = PatchDb::open(&db_path)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
Ok(db)
}
#[instrument(skip_all)]
pub async fn secret_store(&self) -> Result<PgPool, Error> {
init_postgres(self.datadir()).await?;
let secret_store =
PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root"))
.await?;
sqlx::migrate!()
.run(&secret_store)
.await
.with_kind(crate::ErrorKind::Database)?;
Ok(secret_store)
}
}

43
core/startos/src/context/diagnostic.rs
View File

@@ -1,47 +1,16 @@
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::sync::Arc;
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::Context;
use serde::Deserialize;
use tokio::sync::broadcast::Sender;
use tracing::instrument;
use crate::context::config::ServerConfig;
use crate::shutdown::Shutdown;
use crate::util::config::load_config_from_paths;
use crate::Error;
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct DiagnosticContextConfig {
pub datadir: Option<PathBuf>,
}
impl DiagnosticContextConfig {
#[instrument(skip_all)]
pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
tokio::task::spawn_blocking(move || {
load_config_from_paths(
path.as_ref()
.into_iter()
.map(|p| p.as_ref())
.chain(std::iter::once(Path::new(
crate::util::config::DEVICE_CONFIG_PATH,
)))
.chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
)
})
.await
.unwrap()
}
pub fn datadir(&self) -> &Path {
self.datadir
.as_deref()
.unwrap_or_else(|| Path::new("/embassy-data"))
}
}
pub struct DiagnosticContextSeed {
pub datadir: PathBuf,
pub shutdown: Sender<Option<Shutdown>>,
@@ -53,20 +22,18 @@ pub struct DiagnosticContextSeed {
pub struct DiagnosticContext(Arc<DiagnosticContextSeed>);
impl DiagnosticContext {
#[instrument(skip_all)]
pub async fn init<P: AsRef<Path> + Send + 'static>(
path: Option<P>,
pub fn init(
config: &ServerConfig,
disk_guid: Option<Arc<String>>,
error: Error,
) -> Result<Self, Error> {
tracing::error!("Error: {}: Starting diagnostic UI", error);
tracing::debug!("{:?}", error);
let cfg = DiagnosticContextConfig::load(path).await?;
let (shutdown, _) = tokio::sync::broadcast::channel(1);
Ok(Self(Arc::new(DiagnosticContextSeed {
datadir: cfg.datadir().to_owned(),
datadir: config.datadir().to_owned(),
shutdown,
disk_guid,
error: Arc::new(error.into()),

25
core/startos/src/context/install.rs
View File

@@ -1,35 +1,13 @@
use std::ops::Deref;
use std::path::Path;
use std::sync::Arc;
use rpc_toolkit::Context;
use serde::Deserialize;
use tokio::sync::broadcast::Sender;
use tracing::instrument;
use crate::net::utils::find_eth_iface;
use crate::util::config::load_config_from_paths;
use crate::Error;
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct InstallContextConfig {}
impl InstallContextConfig {
#[instrument(skip_all)]
pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
tokio::task::spawn_blocking(move || {
load_config_from_paths(
path.as_ref()
.into_iter()
.map(|p| p.as_ref())
.chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
)
})
.await
.unwrap()
}
}
pub struct InstallContextSeed {
pub ethernet_interface: String,
pub shutdown: Sender<()>,
@@ -39,8 +17,7 @@ pub struct InstallContextSeed {
pub struct InstallContext(Arc<InstallContextSeed>);
impl InstallContext {
#[instrument(skip_all)]
pub async fn init<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
let _cfg = InstallContextConfig::load(path.as_ref().map(|p| p.as_ref().to_owned())).await?;
pub async fn init() -> Result<Self, Error> {
let (shutdown, _) = tokio::sync::broadcast::channel(1);
Ok(Self(Arc::new(InstallContextSeed {
ethernet_interface: find_eth_iface().await?,

34
core/startos/src/context/mod.rs
View File

@@ -1,44 +1,12 @@
pub mod cli;
pub mod config;
pub mod diagnostic;
pub mod install;
pub mod rpc;
pub mod sdk;
pub mod setup;
pub use cli::CliContext;
pub use diagnostic::DiagnosticContext;
pub use install::InstallContext;
pub use rpc::RpcContext;
pub use sdk::SdkContext;
pub use setup::SetupContext;
impl From<CliContext> for () {
fn from(_: CliContext) -> Self {
()
}
}
impl From<DiagnosticContext> for () {
fn from(_: DiagnosticContext) -> Self {
()
}
}
impl From<RpcContext> for () {
fn from(_: RpcContext) -> Self {
()
}
}
impl From<SdkContext> for () {
fn from(_: SdkContext) -> Self {
()
}
}
impl From<SetupContext> for () {
fn from(_: SetupContext) -> Self {
()
}
}
impl From<InstallContext> for () {
fn from(_: InstallContext) -> Self {
()
}
}

338
core/startos/src/context/rpc.rs
View File

@@ -1,107 +1,39 @@
use std::collections::BTreeMap;
use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
use helpers::to_tmp_path;
use imbl_value::InternedString;
use josekit::jwk::Jwk;
use patch_db::json_ptr::JsonPointer;
use patch_db::PatchDb;
use reqwest::{Client, Proxy, Url};
use reqwest::{Client, Proxy};
use rpc_toolkit::Context;
use serde::Deserialize;
use sqlx::postgres::PgConnectOptions;
use sqlx::PgPool;
use tokio::sync::{broadcast, oneshot, Mutex, RwLock};
use tokio::time::Instant;
use tracing::instrument;
use super::setup::CURRENT_SECRET;
use crate::account::AccountInfo;
use crate::core::rpc_continuations::{RequestGuid, RestHandler, RpcContinuation};
use crate::db::model::{CurrentDependents, Database, PackageDataEntryMatchModelRef};
use crate::context::config::ServerConfig;
use crate::core::rpc_continuations::{RequestGuid, RestHandler, RpcContinuation, WebSocketHandler};
use crate::db::model::package::CurrentDependents;
use crate::db::prelude::PatchDbExt;
use crate::dependencies::compute_dependency_config_errs;
use crate::disk::OsPartitionInfo;
use crate::init::{check_time_is_synchronized, init_postgres};
use crate::install::cleanup::{cleanup_failed, uninstall};
use crate::manager::ManagerMap;
use crate::init::check_time_is_synchronized;
use crate::lxc::{LxcContainer, LxcManager};
use crate::middleware::auth::HashSessionToken;
use crate::net::net_controller::NetController;
use crate::net::ssl::{root_ca_start_time, SslManager};
use crate::net::utils::find_eth_iface;
use crate::net::wifi::WpaCli;
use crate::notifications::NotificationManager;
use crate::prelude::*;
use crate::service::ServiceMap;
use crate::shutdown::Shutdown;
use crate::status::MainStatus;
use crate::system::get_mem_info;
use crate::util::config::load_config_from_paths;
use crate::util::lshw::{lshw, LshwDevice};
use crate::{Error, ErrorKind, ResultExt};
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct RpcContextConfig {
pub wifi_interface: Option<String>,
pub ethernet_interface: String,
pub os_partitions: OsPartitionInfo,
pub migration_batch_rows: Option<usize>,
pub migration_prefetch_rows: Option<usize>,
pub bind_rpc: Option<SocketAddr>,
pub tor_control: Option<SocketAddr>,
pub tor_socks: Option<SocketAddr>,
pub dns_bind: Option<Vec<SocketAddr>>,
pub revision_cache_size: Option<usize>,
pub datadir: Option<PathBuf>,
pub log_server: Option<Url>,
}
impl RpcContextConfig {
pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
tokio::task::spawn_blocking(move || {
load_config_from_paths(
path.as_ref()
.into_iter()
.map(|p| p.as_ref())
.chain(std::iter::once(Path::new(
crate::util::config::DEVICE_CONFIG_PATH,
)))
.chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
)
})
.await
.unwrap()
}
pub fn datadir(&self) -> &Path {
self.datadir
.as_deref()
.unwrap_or_else(|| Path::new("/embassy-data"))
}
pub async fn db(&self, account: &AccountInfo) -> Result<PatchDb, Error> {
let db_path = self.datadir().join("main").join("embassy.db");
let db = PatchDb::open(&db_path)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
if !db.exists(&<JsonPointer>::default()).await {
db.put(&<JsonPointer>::default(), &Database::init(account))
.await?;
}
Ok(db)
}
#[instrument(skip_all)]
pub async fn secret_store(&self) -> Result<PgPool, Error> {
init_postgres(self.datadir()).await?;
let secret_store =
PgPool::connect_with(PgConnectOptions::new().database("secrets").username("root"))
.await?;
sqlx::migrate!()
.run(&secret_store)
.await
.with_kind(crate::ErrorKind::Database)?;
Ok(secret_store)
}
}
pub struct RpcContextSeed {
is_closed: AtomicBool,
@@ -111,14 +43,13 @@ pub struct RpcContextSeed {
pub datadir: PathBuf,
pub disk_guid: Arc<String>,
pub db: PatchDb,
pub secret_store: PgPool,
pub account: RwLock<AccountInfo>,
pub net_controller: Arc<NetController>,
pub managers: ManagerMap,
pub services: ServiceMap,
pub metrics_cache: RwLock<Option<crate::system::Metrics>>,
pub shutdown: broadcast::Sender<Option<Shutdown>>,
pub tor_socks: SocketAddr,
pub notification_manager: NotificationManager,
pub lxc_manager: Arc<LxcManager>,
pub open_authed_websockets: Mutex<BTreeMap<HashSessionToken, Vec<oneshot::Sender<()>>>>,
pub rpc_stream_continuations: Mutex<BTreeMap<RequestGuid, RpcContinuation>>,
pub wifi_manager: Option<Arc<RwLock<WpaCli>>>,
@@ -126,6 +57,11 @@ pub struct RpcContextSeed {
pub client: Client,
pub hardware: Hardware,
pub start_time: Instant,
pub dev: Dev,
}
pub struct Dev {
pub lxc: Mutex<BTreeMap<InternedString, LxcContainer>>,
}
pub struct Hardware {
@@ -137,77 +73,95 @@ pub struct Hardware {
pub struct RpcContext(Arc<RpcContextSeed>);
impl RpcContext {
#[instrument(skip_all)]
pub async fn init<P: AsRef<Path> + Send + Sync + 'static>(
cfg_path: Option<P>,
disk_guid: Arc<String>,
) -> Result<Self, Error> {
let base = RpcContextConfig::load(cfg_path).await?;
pub async fn init(config: &ServerConfig, disk_guid: Arc<String>) -> Result<Self, Error> {
tracing::info!("Loaded Config");
let tor_proxy = base.tor_socks.unwrap_or(SocketAddr::V4(SocketAddrV4::new(
let tor_proxy = config.tor_socks.unwrap_or(SocketAddr::V4(SocketAddrV4::new(
Ipv4Addr::new(127, 0, 0, 1),
9050,
)));
let (shutdown, _) = tokio::sync::broadcast::channel(1);
let secret_store = base.secret_store().await?;
tracing::info!("Opened Pg DB");
let account = AccountInfo::load(&secret_store).await?;
let db = base.db(&account).await?;
let db = config.db().await?;
let peek = db.peek().await;
let account = AccountInfo::load(&peek)?;
tracing::info!("Opened PatchDB");
let net_controller = Arc::new(
NetController::init(
base.tor_control
db.clone(),
config
.tor_control
.unwrap_or(SocketAddr::from(([127, 0, 0, 1], 9051))),
tor_proxy,
base.dns_bind
config
.dns_bind
.as_deref()
.unwrap_or(&[SocketAddr::from(([127, 0, 0, 1], 53))]),
SslManager::new(&account, root_ca_start_time().await?)?,
&account.hostname,
&account.key,
account.tor_key.clone(),
)
.await?,
);
tracing::info!("Initialized Net Controller");
let managers = ManagerMap::default();
let services = ServiceMap::default();
let metrics_cache = RwLock::<Option<crate::system::Metrics>>::new(None);
let notification_manager = NotificationManager::new(secret_store.clone());
tracing::info!("Initialized Notification Manager");
let tor_proxy_url = format!("socks5h://{tor_proxy}");
let devices = lshw().await?;
let ram = get_mem_info().await?.total.0 as u64 * 1024 * 1024;
if !db.peek().await.as_server_info().as_ntp_synced().de()? {
if !db
.peek()
.await
.as_public()
.as_server_info()
.as_ntp_synced()
.de()?
{
let db = db.clone();
tokio::spawn(async move {
while !check_time_is_synchronized().await.unwrap() {
tokio::time::sleep(Duration::from_secs(30)).await;
}
db.mutate(|v| v.as_server_info_mut().as_ntp_synced_mut().ser(&true))
.await
.unwrap()
db.mutate(|v| {
v.as_public_mut()
.as_server_info_mut()
.as_ntp_synced_mut()
.ser(&true)
})
.await
.unwrap()
});
}
let seed = Arc::new(RpcContextSeed {
is_closed: AtomicBool::new(false),
datadir: base.datadir().to_path_buf(),
os_partitions: base.os_partitions,
wifi_interface: base.wifi_interface.clone(),
ethernet_interface: base.ethernet_interface,
datadir: config.datadir().to_path_buf(),
os_partitions: config.os_partitions.clone().ok_or_else(|| {
Error::new(
eyre!("OS Partition Information Missing"),
ErrorKind::Filesystem,
)
})?,
wifi_interface: config.wifi_interface.clone(),
ethernet_interface: if let Some(eth) = config.ethernet_interface.clone() {
eth
} else {
find_eth_iface().await?
},
disk_guid,
db,
secret_store,
account: RwLock::new(account),
net_controller,
managers,
services,
metrics_cache,
shutdown,
tor_socks: tor_proxy,
notification_manager,
lxc_manager: Arc::new(LxcManager::new()),
open_authed_websockets: Mutex::new(BTreeMap::new()),
rpc_stream_continuations: Mutex::new(BTreeMap::new()),
wifi_manager: base
wifi_manager: config
.wifi_interface
.clone()
.map(|i| Arc::new(RwLock::new(WpaCli::init(i)))),
current_secret: Arc::new(
Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).map_err(|e| {
@@ -231,6 +185,9 @@ impl RpcContext {
.with_kind(crate::ErrorKind::ParseUrl)?,
hardware: Hardware { devices, ram },
start_time: Instant::now(),
dev: Dev {
lxc: Mutex::new(BTreeMap::new()),
},
});
let res = Self(seed.clone());
@@ -241,8 +198,7 @@ impl RpcContext {
#[instrument(skip_all)]
pub async fn shutdown(self) -> Result<(), Error> {
self.managers.empty().await?;
self.secret_store.close().await;
self.services.shutdown_all().await?;
self.is_closed.store(true, Ordering::SeqCst);
tracing::info!("RPC Context is shutdown");
// TODO: shutdown http servers
@@ -254,18 +210,16 @@ impl RpcContext {
self.db
.mutate(|f| {
let mut current_dependents = f
.as_public_mut()
.as_package_data()
.keys()?
.into_iter()
.map(|k| (k.clone(), BTreeMap::new()))
.collect::<BTreeMap<_, _>>();
for (package_id, package) in f.as_package_data_mut().as_entries_mut()? {
for (k, v) in package
.as_installed_mut()
.into_iter()
.flat_map(|i| i.clone().into_current_dependencies().into_entries())
.flatten()
{
for (package_id, package) in
f.as_public_mut().as_package_data_mut().as_entries_mut()?
{
for (k, v) in package.clone().into_current_dependencies().into_entries()? {
let mut entry: BTreeMap<_, _> =
current_dependents.remove(&k).unwrap_or_default();
entry.insert(package_id.clone(), v.de()?);
@@ -274,17 +228,10 @@ impl RpcContext {
}
for (package_id, current_dependents) in current_dependents {
if let Some(deps) = f
.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package_id)
.and_then(|pde| pde.expect_as_installed_mut().ok())
.map(|i| i.as_installed_mut().as_current_dependents_mut())
{
deps.ser(&CurrentDependents(current_dependents))?;
} else if let Some(deps) = f
.as_package_data_mut()
.as_idx_mut(&package_id)
.and_then(|pde| pde.expect_as_removing_mut().ok())
.map(|i| i.as_removing_mut().as_current_dependents_mut())
.map(|i| i.as_current_dependents_mut())
{
deps.ser(&CurrentDependents(current_dependents))?;
}
@@ -293,97 +240,33 @@ impl RpcContext {
})
.await?;
let peek = self.db.peek().await;
for (package_id, package) in peek.as_package_data().as_entries()?.into_iter() {
let action = match package.as_match() {
PackageDataEntryMatchModelRef::Installing(_)
| PackageDataEntryMatchModelRef::Restoring(_)
| PackageDataEntryMatchModelRef::Updating(_) => {
cleanup_failed(self, &package_id).await
}
PackageDataEntryMatchModelRef::Removing(_) => {
uninstall(
self,
self.secret_store.acquire().await?.as_mut(),
&package_id,
)
.await
}
PackageDataEntryMatchModelRef::Installed(m) => {
let version = m.as_manifest().as_version().clone().de()?;
let volumes = m.as_manifest().as_volumes().de()?;
for (volume_id, volume_info) in &*volumes {
let tmp_path = to_tmp_path(volume_info.path_for(
&self.datadir,
&package_id,
&version,
volume_id,
))
.with_kind(ErrorKind::Filesystem)?;
if tokio::fs::metadata(&tmp_path).await.is_ok() {
tokio::fs::remove_dir_all(&tmp_path).await?;
}
}
Ok(())
}
_ => continue,
};
if let Err(e) = action {
tracing::error!("Failed to clean up package {}: {}", package_id, e);
tracing::debug!("{:?}", e);
}
}
let peek = self
.db
.mutate(|v| {
for (_, pde) in v.as_package_data_mut().as_entries_mut()? {
let status = pde
.expect_as_installed_mut()?
.as_installed_mut()
.as_status_mut()
.as_main_mut();
let running = status.clone().de()?.running();
status.ser(&if running {
MainStatus::Starting
} else {
MainStatus::Stopped
})?;
}
Ok(v.clone())
})
.await?;
self.managers.init(self.clone(), peek.clone()).await?;
self.services.init(&self).await?;
tracing::info!("Initialized Package Managers");
let mut all_dependency_config_errs = BTreeMap::new();
for (package_id, package) in peek.as_package_data().as_entries()?.into_iter() {
let peek = self.db.peek().await;
for (package_id, package) in peek.as_public().as_package_data().as_entries()?.into_iter() {
let package = package.clone();
if let Some(current_dependencies) = package
.as_installed()
.and_then(|x| x.as_current_dependencies().de().ok())
{
let manifest = package.as_manifest().de()?;
all_dependency_config_errs.insert(
package_id.clone(),
compute_dependency_config_errs(
self,
&peek,
&manifest,
&current_dependencies,
&Default::default(),
)
.await?,
);
}
let current_dependencies = package.as_current_dependencies().de()?;
all_dependency_config_errs.insert(
package_id.clone(),
compute_dependency_config_errs(
self,
&peek,
&package_id,
&current_dependencies,
&Default::default(),
)
.await?,
);
}
self.db
.mutate(|v| {
for (package_id, errs) in all_dependency_config_errs {
if let Some(config_errors) = v
.as_public_mut()
.as_package_data_mut()
.as_idx_mut(&package_id)
.and_then(|pde| pde.as_installed_mut())
.map(|i| i.as_status_mut().as_dependency_config_errors_mut())
{
config_errors.ser(&errs)?;
@@ -419,33 +302,30 @@ impl RpcContext {
.insert(guid, handler);
}
pub async fn get_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
pub async fn get_ws_continuation_handler(
&self,
guid: &RequestGuid,
) -> Option<WebSocketHandler> {
let mut continuations = self.rpc_stream_continuations.lock().await;
if let Some(cont) = continuations.remove(guid) {
cont.into_handler().await
} else {
None
}
}
pub async fn get_ws_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
let continuations = self.rpc_stream_continuations.lock().await;
if matches!(continuations.get(guid), Some(RpcContinuation::WebSocket(_))) {
drop(continuations);
self.get_continuation_handler(guid).await
} else {
None
if !matches!(continuations.get(guid), Some(RpcContinuation::WebSocket(_))) {
return None;
}
let Some(RpcContinuation::WebSocket(x)) = continuations.remove(guid) else {
return None;
};
x.get().await
}
pub async fn get_rest_continuation_handler(&self, guid: &RequestGuid) -> Option<RestHandler> {
let continuations = self.rpc_stream_continuations.lock().await;
if matches!(continuations.get(guid), Some(RpcContinuation::Rest(_))) {
drop(continuations);
self.get_continuation_handler(guid).await
} else {
None
let mut continuations: tokio::sync::MutexGuard<'_, BTreeMap<RequestGuid, RpcContinuation>> =
self.rpc_stream_continuations.lock().await;
if !matches!(continuations.get(guid), Some(RpcContinuation::Rest(_))) {
return None;
}
let Some(RpcContinuation::Rest(x)) = continuations.remove(guid) else {
return None;
};
x.get().await
}
}
impl AsRef<Jwk> for RpcContext {

28
core/startos/src/context/sdk.rs
View File

@@ -8,13 +8,6 @@ use serde::Deserialize;
use tracing::instrument;
use crate::prelude::*;
use crate::util::config::{load_config_from_paths, local_config_path};
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct SdkContextConfig {
pub developer_key_path: Option<PathBuf>,
}
#[derive(Debug)]
pub struct SdkContextSeed {
@@ -26,7 +19,7 @@ pub struct SdkContext(Arc<SdkContextSeed>);
impl SdkContext {
/// BLOCKING
#[instrument(skip_all)]
pub fn init(matches: &ArgMatches) -> Result<Self, crate::Error> {
pub fn init(config: ) -> Result<Self, crate::Error> {
let local_config_path = local_config_path();
let base: SdkContextConfig = load_config_from_paths(
matches
@@ -48,24 +41,7 @@ impl SdkContext {
}),
})))
}
/// BLOCKING
#[instrument(skip_all)]
pub fn developer_key(&self) -> Result<ed25519_dalek::SigningKey, Error> {
if !self.developer_key_path.exists() {
return Err(Error::new(eyre!("Developer Key does not exist! Please run `start-sdk init` before running this command."), crate::ErrorKind::Uninitialized));
}
let pair = <ed25519::KeypairBytes as ed25519::pkcs8::DecodePrivateKey>::from_pkcs8_pem(
&std::fs::read_to_string(&self.developer_key_path)?,
)
.with_kind(crate::ErrorKind::Pem)?;
let secret = ed25519_dalek::SecretKey::try_from(&pair.secret_key[..]).map_err(|_| {
Error::new(
eyre!("pkcs8 key is of incorrect length"),
ErrorKind::OpenSsl,
)
})?;
Ok(secret.into())
}
}
impl std::ops::Deref for SdkContext {
type Target = SdkContextSeed;

71
core/startos/src/context/setup.rs
View File

@@ -1,9 +1,8 @@
use std::ops::Deref;
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::sync::Arc;
use josekit::jwk::Jwk;
use patch_db::json_ptr::JsonPointer;
use patch_db::PatchDb;
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::Context;
@@ -14,13 +13,11 @@ use tokio::sync::broadcast::Sender;
use tokio::sync::RwLock;
use tracing::instrument;
use crate::account::AccountInfo;
use crate::db::model::Database;
use crate::context::config::ServerConfig;
use crate::disk::OsPartitionInfo;
use crate::init::init_postgres;
use crate::prelude::*;
use crate::setup::SetupStatus;
use crate::util::config::load_config_from_paths;
use crate::{Error, ResultExt};
lazy_static::lazy_static! {
pub static ref CURRENT_SECRET: Jwk = Jwk::generate_ec_key(josekit::jwk::alg::ec::EcCurve::P256).unwrap_or_else(|e| {
@@ -38,45 +35,9 @@ pub struct SetupResult {
pub root_ca: String,
}
#[derive(Debug, Default, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct SetupContextConfig {
pub os_partitions: OsPartitionInfo,
pub migration_batch_rows: Option<usize>,
pub migration_prefetch_rows: Option<usize>,
pub datadir: Option<PathBuf>,
#[serde(default)]
pub disable_encryption: bool,
}
impl SetupContextConfig {
#[instrument(skip_all)]
pub async fn load<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
tokio::task::spawn_blocking(move || {
load_config_from_paths(
path.as_ref()
.into_iter()
.map(|p| p.as_ref())
.chain(std::iter::once(Path::new(
crate::util::config::DEVICE_CONFIG_PATH,
)))
.chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))),
)
})
.await
.unwrap()
}
pub fn datadir(&self) -> &Path {
self.datadir
.as_deref()
.unwrap_or_else(|| Path::new("/embassy-data"))
}
}
pub struct SetupContextSeed {
pub config: ServerConfig,
pub os_partitions: OsPartitionInfo,
pub config_path: Option<PathBuf>,
pub migration_batch_rows: usize,
pub migration_prefetch_rows: usize,
pub disable_encryption: bool,
pub shutdown: Sender<()>,
pub datadir: PathBuf,
@@ -96,16 +57,18 @@ impl AsRef<Jwk> for SetupContextSeed {
pub struct SetupContext(Arc<SetupContextSeed>);
impl SetupContext {
#[instrument(skip_all)]
pub async fn init<P: AsRef<Path> + Send + 'static>(path: Option<P>) -> Result<Self, Error> {
let cfg = SetupContextConfig::load(path.as_ref().map(|p| p.as_ref().to_owned())).await?;
pub fn init(config: &ServerConfig) -> Result<Self, Error> {
let (shutdown, _) = tokio::sync::broadcast::channel(1);
let datadir = cfg.datadir().to_owned();
let datadir = config.datadir().to_owned();
Ok(Self(Arc::new(SetupContextSeed {
os_partitions: cfg.os_partitions,
config_path: path.as_ref().map(|p| p.as_ref().to_owned()),
migration_batch_rows: cfg.migration_batch_rows.unwrap_or(25000),
migration_prefetch_rows: cfg.migration_prefetch_rows.unwrap_or(100_000),
disable_encryption: cfg.disable_encryption,
config: config.clone(),
os_partitions: config.os_partitions.clone().ok_or_else(|| {
Error::new(
eyre!("missing required configuration: `os-partitions`"),
ErrorKind::NotFound,
)
})?,
disable_encryption: config.disable_encryption.unwrap_or(false),
shutdown,
datadir,
selected_v2_drive: RwLock::new(None),
@@ -115,15 +78,11 @@ impl SetupContext {
})))
}
#[instrument(skip_all)]
pub async fn db(&self, account: &AccountInfo) -> Result<PatchDb, Error> {
pub async fn db(&self) -> Result<PatchDb, Error> {
let db_path = self.datadir.join("main").join("embassy.db");
let db = PatchDb::open(&db_path)
.await
.with_ctx(|_| (crate::ErrorKind::Filesystem, db_path.display().to_string()))?;
if !db.exists(&<JsonPointer>::default()).await {
db.put(&<JsonPointer>::default(), &Database::init(account))
.await?;
}
Ok(db)
}
#[instrument(skip_all)]

109
core/startos/src/control.rs
View File

@@ -1,92 +1,55 @@
use clap::Parser;
use color_eyre::eyre::eyre;
use models::PackageId;
use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
use tracing::instrument;
use crate::context::RpcContext;
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
use crate::status::MainStatus;
use crate::util::display_none;
use crate::Error;
#[command(display(display_none), metadata(sync_db = true))]
#[instrument(skip_all)]
pub async fn start(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<(), Error> {
let peek = ctx.db.peek().await;
let version = peek
.as_package_data()
.as_idx(&id)
.or_not_found(&id)?
.as_installed()
.or_not_found(&id)?
.as_manifest()
.as_version()
.de()?;
ctx.managers
.get(&(id, version))
.await
.ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
.start()
.await;
Ok(())
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct ControlParams {
pub id: PackageId,
}
#[command(display(display_none), metadata(sync_db = true))]
pub async fn stop(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<MainStatus, Error> {
let peek = ctx.db.peek().await;
let version = peek
.as_package_data()
.as_idx(&id)
.or_not_found(&id)?
.as_installed()
.or_not_found(&id)?
.as_manifest()
.as_version()
.de()?;
let last_statuts = ctx
.db
.mutate(|v| {
v.as_package_data_mut()
.as_idx_mut(&id)
.and_then(|x| x.as_installed_mut())
.ok_or_else(|| Error::new(eyre!("{} is not installed", id), ErrorKind::NotFound))?
.as_status_mut()
.as_main_mut()
.replace(&MainStatus::Stopping)
})
#[instrument(skip_all)]
pub async fn start(ctx: RpcContext, ControlParams { id }: ControlParams) -> Result<(), Error> {
ctx.services
.get(&id)
.await
.as_ref()
.or_not_found(lazy_format!("Manager for {id}"))?
.start()
.await?;
ctx.managers
.get(&(id, version))
.await
.ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
.stop()
.await;
Ok(last_statuts)
Ok(())
}
#[command(display(display_none), metadata(sync_db = true))]
pub async fn restart(#[context] ctx: RpcContext, #[arg] id: PackageId) -> Result<(), Error> {
let peek = ctx.db.peek().await;
let version = peek
.as_package_data()
.as_idx(&id)
.or_not_found(&id)?
.expect_as_installed()?
.as_manifest()
.as_version()
.de()?;
ctx.managers
.get(&(id, version))
pub async fn stop(ctx: RpcContext, ControlParams { id }: ControlParams) -> Result<(), Error> {
// TODO: why did this return last_status before?
ctx.services
.get(&id)
.await
.as_ref()
.ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
.restart()
.await;
.stop()
.await?;
Ok(())
}
pub async fn restart(ctx: RpcContext, ControlParams { id }: ControlParams) -> Result<(), Error> {
ctx.services
.get(&id)
.await
.as_ref()
.ok_or_else(|| Error::new(eyre!("Manager not found"), crate::ErrorKind::InvalidRequest))?
.restart()
.await?;
Ok(())
}

83
core/startos/src/core/rpc_continuations.rs
View File

@@ -1,27 +1,21 @@
use std::sync::Arc;
use std::time::Duration;
use axum::extract::ws::WebSocket;
use axum::extract::Request;
use axum::response::Response;
use futures::future::BoxFuture;
use futures::FutureExt;
use helpers::TimedResource;
use hyper::upgrade::Upgraded;
use hyper::{Body, Error as HyperError, Request, Response};
use rand::RngCore;
use tokio::task::JoinError;
use tokio_tungstenite::WebSocketStream;
use imbl_value::InternedString;
use crate::{Error, ResultExt};
#[allow(unused_imports)]
use crate::prelude::*;
use crate::util::new_guid;
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)]
pub struct RequestGuid<T: AsRef<str> = String>(Arc<T>);
pub struct RequestGuid(InternedString);
impl RequestGuid {
pub fn new() -> Self {
let mut buf = [0; 40];
rand::thread_rng().fill_bytes(&mut buf);
RequestGuid(Arc::new(base32::encode(
base32::Alphabet::RFC4648 { padding: false },
&buf,
)))
Self(new_guid())
}
pub fn from(r: &str) -> Option<RequestGuid> {
@@ -33,9 +27,15 @@ impl RequestGuid {
return None;
}
}
Some(RequestGuid(Arc::new(r.to_owned())))
Some(RequestGuid(InternedString::intern(r)))
}
}
impl AsRef<str> for RequestGuid {
fn as_ref(&self) -> &str {
self.0.as_ref()
}
}
#[test]
fn parse_guid() {
println!(
@@ -44,22 +44,16 @@ fn parse_guid() {
)
}
impl<T: AsRef<str>> std::fmt::Display for RequestGuid<T> {
impl std::fmt::Display for RequestGuid {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
(&*self.0).as_ref().fmt(f)
self.0.fmt(f)
}
}
pub type RestHandler = Box<
dyn FnOnce(Request<Body>) -> BoxFuture<'static, Result<Response<Body>, crate::Error>> + Send,
>;
pub type RestHandler =
Box<dyn FnOnce(Request) -> BoxFuture<'static, Result<Response, crate::Error>> + Send>;
pub type WebSocketHandler = Box<
dyn FnOnce(
BoxFuture<'static, Result<Result<WebSocketStream<Upgraded>, HyperError>, JoinError>>,
) -> BoxFuture<'static, Result<(), Error>>
+ Send,
>;
pub type WebSocketHandler = Box<dyn FnOnce(WebSocket) -> BoxFuture<'static, ()> + Send>;
pub enum RpcContinuation {
Rest(TimedResource<RestHandler>),
@@ -78,39 +72,4 @@ impl RpcContinuation {
RpcContinuation::WebSocket(a) => a.is_timed_out(),
}
}
pub async fn into_handler(self) -> Option<RestHandler> {
match self {
RpcContinuation::Rest(handler) => handler.get().await,
RpcContinuation::WebSocket(handler) => {
if let Some(handler) = handler.get().await {
Some(Box::new(
|req: Request<Body>| -> BoxFuture<'static, Result<Response<Body>, Error>> {
async move {
let (parts, body) = req.into_parts();
let req = Request::from_parts(parts, body);
let (res, ws_fut) = hyper_ws_listener::create_ws(req)
.with_kind(crate::ErrorKind::Network)?;
if let Some(ws_fut) = ws_fut {
tokio::task::spawn(async move {
match handler(ws_fut.boxed()).await {
Ok(()) => (),
Err(e) => {
tracing::error!("WebSocket Closed: {}", e);
tracing::debug!("{:?}", e);
}
}
});
}
Ok(res)
}
.boxed()
},
))
} else {
None
}
}
}
}
}

287
core/startos/src/db/mod.rs
View File

@@ -1,61 +1,56 @@
pub mod model;
pub mod package;
pub mod prelude;
use std::future::Future;
use std::path::PathBuf;
use std::sync::Arc;
use futures::{FutureExt, SinkExt, StreamExt};
use patch_db::json_ptr::JsonPointer;
use axum::extract::ws::{self, WebSocket};
use axum::extract::WebSocketUpgrade;
use axum::response::Response;
use clap::Parser;
use futures::{FutureExt, StreamExt};
use http::header::COOKIE;
use http::HeaderMap;
use patch_db::json_ptr::{JsonPointer, ROOT};
use patch_db::{Dump, Revision};
use rpc_toolkit::command;
use rpc_toolkit::hyper::upgrade::Upgraded;
use rpc_toolkit::hyper::{Body, Error as HyperError, Request, Response};
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::{command, from_fn_async, CallRemote, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio::sync::oneshot;
use tokio::task::JoinError;
use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
use tokio_tungstenite::tungstenite::protocol::CloseFrame;
use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::WebSocketStream;
use tracing::instrument;
use crate::context::{CliContext, RpcContext};
use crate::middleware::auth::{HasValidSession, HashSessionToken};
use crate::prelude::*;
use crate::util::display_none;
use crate::util::serde::{display_serializable, IoFormat};
use crate::util::serde::{apply_expr, HandlerExtSerde};
lazy_static::lazy_static! {
static ref PUBLIC: JsonPointer = "/public".parse().unwrap();
}
#[instrument(skip_all)]
async fn ws_handler<
WSFut: Future<Output = Result<Result<WebSocketStream<Upgraded>, HyperError>, JoinError>>,
>(
async fn ws_handler(
ctx: RpcContext,
session: Option<(HasValidSession, HashSessionToken)>,
ws_fut: WSFut,
mut stream: WebSocket,
) -> Result<(), Error> {
let (dump, sub) = ctx.db.dump_and_sub().await;
let mut stream = ws_fut
.await
.with_kind(ErrorKind::Network)?
.with_kind(ErrorKind::Unknown)?;
let (dump, sub) = ctx.db.dump_and_sub(PUBLIC.clone()).await;
if let Some((session, token)) = session {
let kill = subscribe_to_session_kill(&ctx, token).await;
send_dump(session, &mut stream, dump).await?;
send_dump(session.clone(), &mut stream, dump).await?;
deal_with_messages(session, kill, sub, stream).await?;
} else {
stream
.close(Some(CloseFrame {
code: CloseCode::Error,
.send(ws::Message::Close(Some(ws::CloseFrame {
code: ws::close_code::ERROR,
reason: "UNAUTHORIZED".into(),
}))
})))
.await
.with_kind(ErrorKind::Network)?;
drop(stream);
}
Ok(())
@@ -80,7 +75,7 @@ async fn deal_with_messages(
_has_valid_authentication: HasValidSession,
mut kill: oneshot::Receiver<()>,
mut sub: patch_db::Subscriber,
mut stream: WebSocketStream<Upgraded>,
mut stream: WebSocket,
) -> Result<(), Error> {
let mut timer = tokio::time::interval(tokio::time::Duration::from_secs(5));
@@ -89,18 +84,18 @@ async fn deal_with_messages(
_ = (&mut kill).fuse() => {
tracing::info!("Closing WebSocket: Reason: Session Terminated");
stream
.close(Some(CloseFrame {
code: CloseCode::Error,
reason: "UNAUTHORIZED".into(),
}))
.await
.with_kind(ErrorKind::Network)?;
.send(ws::Message::Close(Some(ws::CloseFrame {
code: ws::close_code::ERROR,
reason: "UNAUTHORIZED".into(),
}))).await
.with_kind(ErrorKind::Network)?;
drop(stream);
return Ok(())
}
new_rev = sub.recv().fuse() => {
let rev = new_rev.expect("UNREACHABLE: patch-db is dropped");
stream
.send(Message::Text(serde_json::to_string(&rev).with_kind(ErrorKind::Serialization)?))
.send(ws::Message::Text(serde_json::to_string(&rev).with_kind(ErrorKind::Serialization)?))
.await
.with_kind(ErrorKind::Network)?;
}
@@ -117,7 +112,7 @@ async fn deal_with_messages(
// This is trying to give a health checks to the home to keep the ui alive.
_ = timer.tick().fuse() => {
stream
.send(Message::Ping(vec![]))
.send(ws::Message::Ping(vec![]))
.await
.with_kind(crate::ErrorKind::Network)?;
}
@@ -127,11 +122,11 @@ async fn deal_with_messages(
async fn send_dump(
_has_valid_authentication: HasValidSession,
stream: &mut WebSocketStream<Upgraded>,
stream: &mut WebSocket,
dump: Dump,
) -> Result<(), Error> {
stream
.send(Message::Text(
.send(ws::Message::Text(
serde_json::to_string(&dump).with_kind(ErrorKind::Serialization)?,
))
.await
@@ -139,11 +134,14 @@ async fn send_dump(
Ok(())
}
pub async fn subscribe(ctx: RpcContext, req: Request<Body>) -> Result<Response<Body>, Error> {
let (parts, body) = req.into_parts();
pub async fn subscribe(
ctx: RpcContext,
headers: HeaderMap,
ws: WebSocketUpgrade,
) -> Result<Response, Error> {
let session = match async {
let token = HashSessionToken::from_request_parts(&parts)?;
let session = HasValidSession::from_request_parts(&parts, &ctx).await?;
let token = HashSessionToken::from_header(headers.get(COOKIE))?;
let session = HasValidSession::from_header(headers.get(COOKIE), &ctx).await?;
Ok::<_, Error>((session, token))
}
.await
@@ -157,26 +155,24 @@ pub async fn subscribe(ctx: RpcContext, req: Request<Body>) -> Result<Response<B
None
}
};
let req = Request::from_parts(parts, body);
let (res, ws_fut) = hyper_ws_listener::create_ws(req).with_kind(ErrorKind::Network)?;
if let Some(ws_fut) = ws_fut {
tokio::task::spawn(async move {
match ws_handler(ctx, session, ws_fut).await {
Ok(()) => (),
Err(e) => {
tracing::error!("WebSocket Closed: {}", e);
tracing::debug!("{:?}", e);
}
Ok(ws.on_upgrade(|ws| async move {
match ws_handler(ctx, session, ws).await {
Ok(()) => (),
Err(e) => {
tracing::error!("WebSocket Closed: {}", e);
tracing::debug!("{:?}", e);
}
});
}
Ok(res)
}
}))
}
#[command(subcommands(dump, put, apply))]
pub fn db() -> Result<(), RpcError> {
Ok(())
pub fn db() -> ParentHandler {
ParentHandler::new()
.subcommand("dump", from_fn_async(cli_dump).with_display_serializable())
.subcommand("dump", from_fn_async(dump).no_cli())
.subcommand("put", put())
.subcommand("apply", from_fn_async(cli_apply).no_display())
.subcommand("apply", from_fn_async(apply).no_cli())
}
#[derive(Deserialize, Serialize)]
@@ -186,97 +182,64 @@ pub enum RevisionsRes {
Dump(Dump),
}
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct CliDumpParams {
#[arg(long = "include-private", short = 'p')]
#[serde(default)]
include_private: bool,
path: Option<PathBuf>,
}
#[instrument(skip_all)]
async fn cli_dump(
ctx: CliContext,
_format: Option<IoFormat>,
path: Option<PathBuf>,
CliDumpParams {
path,
include_private,
}: CliDumpParams,
) -> Result<Dump, RpcError> {
let dump = if let Some(path) = path {
PatchDb::open(path).await?.dump().await
PatchDb::open(path).await?.dump(&ROOT).await
} else {
rpc_toolkit::command_helpers::call_remote(
ctx,
"db.dump",
serde_json::json!({}),
std::marker::PhantomData::<Dump>,
)
.await?
.result?
from_value::<Dump>(
ctx.call_remote(
"db.dump",
imbl_value::json!({ "include-private":include_private }),
)
.await?,
)?
};
Ok(dump)
}
#[command(
custom_cli(cli_dump(async, context(CliContext))),
display(display_serializable)
)]
pub async fn dump(
#[context] ctx: RpcContext,
#[allow(unused_variables)]
#[arg(long = "format")]
format: Option<IoFormat>,
#[allow(unused_variables)]
#[arg]
path: Option<PathBuf>,
) -> Result<Dump, Error> {
Ok(ctx.db.dump().await)
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct DumpParams {
#[arg(long = "include-private", short = 'p')]
#[serde(default)]
include_private: bool,
}
fn apply_expr(input: jaq_core::Val, expr: &str) -> Result<jaq_core::Val, Error> {
let (expr, errs) = jaq_core::parse::parse(expr, jaq_core::parse::main());
let Some(expr) = expr else {
return Err(Error::new(
eyre!("Failed to parse expression: {:?}", errs),
crate::ErrorKind::InvalidRequest,
));
};
let mut errs = Vec::new();
let mut defs = jaq_core::Definitions::core();
for def in jaq_std::std() {
defs.insert(def, &mut errs);
}
let filter = defs.finish(expr, Vec::new(), &mut errs);
if !errs.is_empty() {
return Err(Error::new(
eyre!("Failed to compile expression: {:?}", errs),
crate::ErrorKind::InvalidRequest,
));
};
let inputs = jaq_core::RcIter::new(std::iter::empty());
let mut res_iter = filter.run(jaq_core::Ctx::new([], &inputs), input);
let Some(res) = res_iter
.next()
.transpose()
.map_err(|e| eyre!("{e}"))
.with_kind(crate::ErrorKind::Deserialization)?
else {
return Err(Error::new(
eyre!("expr returned no results"),
crate::ErrorKind::InvalidRequest,
));
};
if res_iter.next().is_some() {
return Err(Error::new(
eyre!("expr returned too many results"),
crate::ErrorKind::InvalidRequest,
));
}
Ok(res)
pub async fn dump(
ctx: RpcContext,
DumpParams { include_private }: DumpParams,
) -> Result<Dump, Error> {
Ok(if include_private {
ctx.db.dump(&ROOT).await
} else {
ctx.db.dump(&PUBLIC).await
})
}
#[instrument(skip_all)]
async fn cli_apply(ctx: CliContext, expr: String, path: Option<PathBuf>) -> Result<(), RpcError> {
async fn cli_apply(
ctx: CliContext,
ApplyParams { expr, path }: ApplyParams,
) -> Result<(), RpcError> {
if let Some(path) = path {
PatchDb::open(path)
.await?
@@ -301,30 +264,22 @@ async fn cli_apply(ctx: CliContext, expr: String, path: Option<PathBuf>) -> Resu
})
.await?;
} else {
rpc_toolkit::command_helpers::call_remote(
ctx,
"db.apply",
serde_json::json!({ "expr": expr }),
std::marker::PhantomData::<()>,
)
.await?
.result?;
ctx.call_remote("db.apply", imbl_value::json!({ "expr": expr }))
.await?;
}
Ok(())
}
#[command(
custom_cli(cli_apply(async, context(CliContext))),
display(display_none)
)]
pub async fn apply(
#[context] ctx: RpcContext,
#[arg] expr: String,
#[allow(unused_variables)]
#[arg]
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct ApplyParams {
expr: String,
path: Option<PathBuf>,
) -> Result<(), Error> {
}
pub async fn apply(ctx: RpcContext, ApplyParams { expr, .. }: ApplyParams) -> Result<(), Error> {
ctx.db
.mutate(|db| {
let res = apply_expr(
@@ -346,21 +301,25 @@ pub async fn apply(
.await
}
#[command(subcommands(ui))]
pub fn put() -> Result<(), RpcError> {
Ok(())
pub fn put() -> ParentHandler {
ParentHandler::new().subcommand(
"ui",
from_fn_async(ui)
.with_display_serializable()
.with_remote_cli::<CliContext>(),
)
}
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct UiParams {
pointer: JsonPointer,
value: Value,
}
#[command(display(display_serializable))]
// #[command(display(display_serializable))]
#[instrument(skip_all)]
pub async fn ui(
#[context] ctx: RpcContext,
#[arg] pointer: JsonPointer,
#[arg] value: Value,
#[allow(unused_variables)]
#[arg(long = "format")]
format: Option<IoFormat>,
) -> Result<(), Error> {
pub async fn ui(ctx: RpcContext, UiParams { pointer, value, .. }: UiParams) -> Result<(), Error> {
let ptr = "/ui"
.parse::<JsonPointer>()
.with_kind(ErrorKind::Database)?

533
core/startos/src/db/model.rs
View File

@@ -1,533 +0,0 @@
use std::collections::{BTreeMap, BTreeSet};
use std::net::{Ipv4Addr, Ipv6Addr};
use std::sync::Arc;
use chrono::{DateTime, Utc};
use emver::VersionRange;
use imbl_value::InternedString;
use ipnet::{Ipv4Net, Ipv6Net};
use isocountry::CountryCode;
use itertools::Itertools;
use models::{DataUrl, HealthCheckId, InterfaceId};
use openssl::hash::MessageDigest;
use patch_db::{HasModel, Value};
use reqwest::Url;
use serde::{Deserialize, Serialize};
use ssh_key::public::Ed25519PublicKey;
use crate::account::AccountInfo;
use crate::config::spec::PackagePointerSpec;
use crate::install::progress::InstallProgress;
use crate::net::forward::LanPortForwards;
use crate::net::utils::{get_iface_ipv4_addr, get_iface_ipv6_addr};
use crate::prelude::*;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::status::Status;
use crate::util::cpupower::{Governor};
use crate::util::Version;
use crate::version::{Current, VersionT};
use crate::{ARCH, PLATFORM};
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
// #[macro_debug]
pub struct Database {
pub server_info: ServerInfo,
pub package_data: AllPackageData,
pub lan_port_forwards: LanPortForwards,
pub ui: Value,
}
impl Database {
pub fn init(account: &AccountInfo) -> Self {
let lan_address = account.hostname.lan_address().parse().unwrap();
Database {
server_info: ServerInfo {
arch: get_arch(),
platform: get_platform(),
id: account.server_id.clone(),
version: Current::new().semver().into(),
hostname: account.hostname.no_dot_host_name(),
last_backup: None,
last_wifi_region: None,
eos_version_compat: Current::new().compat().clone(),
lan_address,
tor_address: format!("https://{}", account.key.tor_address())
.parse()
.unwrap(),
ip_info: BTreeMap::new(),
status_info: ServerStatus {
backup_progress: None,
updated: false,
update_progress: None,
shutting_down: false,
restarting: false,
},
wifi: WifiInfo {
ssids: Vec::new(),
connected: None,
selected: None,
},
unread_notification_count: 0,
connection_addresses: ConnectionAddresses {
tor: Vec::new(),
clearnet: Vec::new(),
},
password_hash: account.password.clone(),
pubkey: ssh_key::PublicKey::from(Ed25519PublicKey::from(&account.key.ssh_key()))
.to_openssh()
.unwrap(),
ca_fingerprint: account
.root_ca_cert
.digest(MessageDigest::sha256())
.unwrap()
.iter()
.map(|x| format!("{x:X}"))
.join(":"),
ntp_synced: false,
zram: true,
governor: None,
},
package_data: AllPackageData::default(),
lan_port_forwards: LanPortForwards::new(),
ui: serde_json::from_str(include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../web/patchdb-ui-seed.json"
)))
.unwrap(),
}
}
}
pub type DatabaseModel = Model<Database>;
fn get_arch() -> InternedString {
(*ARCH).into()
}
fn get_platform() -> InternedString {
(&*PLATFORM).into()
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct ServerInfo {
#[serde(default = "get_arch")]
pub arch: InternedString,
#[serde(default = "get_platform")]
pub platform: InternedString,
pub id: String,
pub hostname: String,
pub version: Version,
pub last_backup: Option<DateTime<Utc>>,
/// Used in the wifi to determine the region to set the system to
pub last_wifi_region: Option<CountryCode>,
pub eos_version_compat: VersionRange,
pub lan_address: Url,
pub tor_address: Url,
pub ip_info: BTreeMap<String, IpInfo>,
#[serde(default)]
pub status_info: ServerStatus,
pub wifi: WifiInfo,
pub unread_notification_count: u64,
pub connection_addresses: ConnectionAddresses,
pub password_hash: String,
pub pubkey: String,
pub ca_fingerprint: String,
#[serde(default)]
pub ntp_synced: bool,
#[serde(default)]
pub zram: bool,
pub governor: Option<Governor>,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct IpInfo {
pub ipv4_range: Option<Ipv4Net>,
pub ipv4: Option<Ipv4Addr>,
pub ipv6_range: Option<Ipv6Net>,
pub ipv6: Option<Ipv6Addr>,
}
impl IpInfo {
pub async fn for_interface(iface: &str) -> Result<Self, Error> {
let (ipv4, ipv4_range) = get_iface_ipv4_addr(iface).await?.unzip();
let (ipv6, ipv6_range) = get_iface_ipv6_addr(iface).await?.unzip();
Ok(Self {
ipv4_range,
ipv4,
ipv6_range,
ipv6,
})
}
}
#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
#[model = "Model<Self>"]
pub struct BackupProgress {
pub complete: bool,
}
#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct ServerStatus {
pub backup_progress: Option<BTreeMap<PackageId, BackupProgress>>,
pub updated: bool,
pub update_progress: Option<UpdateProgress>,
#[serde(default)]
pub shutting_down: bool,
#[serde(default)]
pub restarting: bool,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct UpdateProgress {
pub size: Option<u64>,
pub downloaded: u64,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct WifiInfo {
pub ssids: Vec<String>,
pub selected: Option<String>,
pub connected: Option<String>,
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct ServerSpecs {
pub cpu: String,
pub disk: String,
pub memory: String,
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct ConnectionAddresses {
pub tor: Vec<String>,
pub clearnet: Vec<String>,
}
#[derive(Debug, Default, Deserialize, Serialize)]
pub struct AllPackageData(pub BTreeMap<PackageId, PackageDataEntry>);
impl Map for AllPackageData {
type Key = PackageId;
type Value = PackageDataEntry;
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct StaticFiles {
license: String,
instructions: String,
icon: String,
}
impl StaticFiles {
pub fn local(id: &PackageId, version: &Version, icon_type: &str) -> Self {
StaticFiles {
license: format!("/public/package-data/{}/{}/LICENSE.md", id, version),
instructions: format!("/public/package-data/{}/{}/INSTRUCTIONS.md", id, version),
icon: format!("/public/package-data/{}/{}/icon.{}", id, version, icon_type),
}
}
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct PackageDataEntryInstalling {
pub static_files: StaticFiles,
pub manifest: Manifest,
pub install_progress: Arc<InstallProgress>,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct PackageDataEntryUpdating {
pub static_files: StaticFiles,
pub manifest: Manifest,
pub installed: InstalledPackageInfo,
pub install_progress: Arc<InstallProgress>,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct PackageDataEntryRestoring {
pub static_files: StaticFiles,
pub manifest: Manifest,
pub install_progress: Arc<InstallProgress>,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct PackageDataEntryRemoving {
pub static_files: StaticFiles,
pub manifest: Manifest,
pub removing: InstalledPackageInfo,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct PackageDataEntryInstalled {
pub static_files: StaticFiles,
pub manifest: Manifest,
pub installed: InstalledPackageInfo,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(tag = "state")]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
// #[macro_debug]
pub enum PackageDataEntry {
Installing(PackageDataEntryInstalling),
Updating(PackageDataEntryUpdating),
Restoring(PackageDataEntryRestoring),
Removing(PackageDataEntryRemoving),
Installed(PackageDataEntryInstalled),
}
impl Model<PackageDataEntry> {
pub fn expect_into_installed(self) -> Result<Model<PackageDataEntryInstalled>, Error> {
if let PackageDataEntryMatchModel::Installed(a) = self.into_match() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in installed state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn expect_as_installed(&self) -> Result<&Model<PackageDataEntryInstalled>, Error> {
if let PackageDataEntryMatchModelRef::Installed(a) = self.as_match() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in installed state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn expect_as_installed_mut(
&mut self,
) -> Result<&mut Model<PackageDataEntryInstalled>, Error> {
if let PackageDataEntryMatchModelMut::Installed(a) = self.as_match_mut() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in installed state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn expect_into_removing(self) -> Result<Model<PackageDataEntryRemoving>, Error> {
if let PackageDataEntryMatchModel::Removing(a) = self.into_match() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in removing state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn expect_as_removing(&self) -> Result<&Model<PackageDataEntryRemoving>, Error> {
if let PackageDataEntryMatchModelRef::Removing(a) = self.as_match() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in removing state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn expect_as_removing_mut(
&mut self,
) -> Result<&mut Model<PackageDataEntryRemoving>, Error> {
if let PackageDataEntryMatchModelMut::Removing(a) = self.as_match_mut() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in removing state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn expect_as_installing_mut(
&mut self,
) -> Result<&mut Model<PackageDataEntryInstalling>, Error> {
if let PackageDataEntryMatchModelMut::Installing(a) = self.as_match_mut() {
Ok(a)
} else {
Err(Error::new(
eyre!("package is not in installing state"),
ErrorKind::InvalidRequest,
))
}
}
pub fn into_manifest(self) -> Model<Manifest> {
match self.into_match() {
PackageDataEntryMatchModel::Installing(a) => a.into_manifest(),
PackageDataEntryMatchModel::Updating(a) => a.into_installed().into_manifest(),
PackageDataEntryMatchModel::Restoring(a) => a.into_manifest(),
PackageDataEntryMatchModel::Removing(a) => a.into_manifest(),
PackageDataEntryMatchModel::Installed(a) => a.into_manifest(),
PackageDataEntryMatchModel::Error(_) => Model::from(Value::Null),
}
}
pub fn as_manifest(&self) -> &Model<Manifest> {
match self.as_match() {
PackageDataEntryMatchModelRef::Installing(a) => a.as_manifest(),
PackageDataEntryMatchModelRef::Updating(a) => a.as_installed().as_manifest(),
PackageDataEntryMatchModelRef::Restoring(a) => a.as_manifest(),
PackageDataEntryMatchModelRef::Removing(a) => a.as_manifest(),
PackageDataEntryMatchModelRef::Installed(a) => a.as_manifest(),
PackageDataEntryMatchModelRef::Error(_) => (&Value::Null).into(),
}
}
pub fn into_installed(self) -> Option<Model<InstalledPackageInfo>> {
match self.into_match() {
PackageDataEntryMatchModel::Installing(_) => None,
PackageDataEntryMatchModel::Updating(a) => Some(a.into_installed()),
PackageDataEntryMatchModel::Restoring(_) => None,
PackageDataEntryMatchModel::Removing(_) => None,
PackageDataEntryMatchModel::Installed(a) => Some(a.into_installed()),
PackageDataEntryMatchModel::Error(_) => None,
}
}
pub fn as_installed(&self) -> Option<&Model<InstalledPackageInfo>> {
match self.as_match() {
PackageDataEntryMatchModelRef::Installing(_) => None,
PackageDataEntryMatchModelRef::Updating(a) => Some(a.as_installed()),
PackageDataEntryMatchModelRef::Restoring(_) => None,
PackageDataEntryMatchModelRef::Removing(_) => None,
PackageDataEntryMatchModelRef::Installed(a) => Some(a.as_installed()),
PackageDataEntryMatchModelRef::Error(_) => None,
}
}
pub fn as_installed_mut(&mut self) -> Option<&mut Model<InstalledPackageInfo>> {
match self.as_match_mut() {
PackageDataEntryMatchModelMut::Installing(_) => None,
PackageDataEntryMatchModelMut::Updating(a) => Some(a.as_installed_mut()),
PackageDataEntryMatchModelMut::Restoring(_) => None,
PackageDataEntryMatchModelMut::Removing(_) => None,
PackageDataEntryMatchModelMut::Installed(a) => Some(a.as_installed_mut()),
PackageDataEntryMatchModelMut::Error(_) => None,
}
}
pub fn as_install_progress(&self) -> Option<&Model<Arc<InstallProgress>>> {
match self.as_match() {
PackageDataEntryMatchModelRef::Installing(a) => Some(a.as_install_progress()),
PackageDataEntryMatchModelRef::Updating(a) => Some(a.as_install_progress()),
PackageDataEntryMatchModelRef::Restoring(a) => Some(a.as_install_progress()),
PackageDataEntryMatchModelRef::Removing(_) => None,
PackageDataEntryMatchModelRef::Installed(_) => None,
PackageDataEntryMatchModelRef::Error(_) => None,
}
}
pub fn as_install_progress_mut(&mut self) -> Option<&mut Model<Arc<InstallProgress>>> {
match self.as_match_mut() {
PackageDataEntryMatchModelMut::Installing(a) => Some(a.as_install_progress_mut()),
PackageDataEntryMatchModelMut::Updating(a) => Some(a.as_install_progress_mut()),
PackageDataEntryMatchModelMut::Restoring(a) => Some(a.as_install_progress_mut()),
PackageDataEntryMatchModelMut::Removing(_) => None,
PackageDataEntryMatchModelMut::Installed(_) => None,
PackageDataEntryMatchModelMut::Error(_) => None,
}
}
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct InstalledPackageInfo {
pub status: Status,
pub marketplace_url: Option<Url>,
#[serde(default)]
#[serde(with = "crate::util::serde::ed25519_pubkey")]
pub developer_key: ed25519_dalek::VerifyingKey,
pub manifest: Manifest,
pub last_backup: Option<DateTime<Utc>>,
pub dependency_info: BTreeMap<PackageId, StaticDependencyInfo>,
pub current_dependents: CurrentDependents,
pub current_dependencies: CurrentDependencies,
pub interface_addresses: InterfaceAddressMap,
}
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
pub struct CurrentDependents(pub BTreeMap<PackageId, CurrentDependencyInfo>);
impl CurrentDependents {
pub fn map(
mut self,
transform: impl Fn(
BTreeMap<PackageId, CurrentDependencyInfo>,
) -> BTreeMap<PackageId, CurrentDependencyInfo>,
) -> Self {
self.0 = transform(self.0);
self
}
}
impl Map for CurrentDependents {
type Key = PackageId;
type Value = CurrentDependencyInfo;
}
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
pub struct CurrentDependencies(pub BTreeMap<PackageId, CurrentDependencyInfo>);
impl CurrentDependencies {
pub fn map(
mut self,
transform: impl Fn(
BTreeMap<PackageId, CurrentDependencyInfo>,
) -> BTreeMap<PackageId, CurrentDependencyInfo>,
) -> Self {
self.0 = transform(self.0);
self
}
}
impl Map for CurrentDependencies {
type Key = PackageId;
type Value = CurrentDependencyInfo;
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct StaticDependencyInfo {
pub title: String,
pub icon: DataUrl<'static>,
}
#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct CurrentDependencyInfo {
#[serde(default)]
pub pointers: BTreeSet<PackagePointerSpec>,
pub health_checks: BTreeSet<HealthCheckId>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct InterfaceAddressMap(pub BTreeMap<InterfaceId, InterfaceAddresses>);
impl Map for InterfaceAddressMap {
type Key = InterfaceId;
type Value = InterfaceAddresses;
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct InterfaceAddresses {
pub tor_address: Option<String>,
pub lan_address: Option<String>,
}

48
core/startos/src/db/model/mod.rs Normal file
View File

@@ -0,0 +1,48 @@
use std::collections::BTreeMap;
use patch_db::HasModel;
use serde::{Deserialize, Serialize};
use crate::account::AccountInfo;
use crate::auth::Sessions;
use crate::backup::target::cifs::CifsTargets;
use crate::db::model::private::Private;
use crate::db::model::public::Public;
use crate::net::forward::AvailablePorts;
use crate::net::keys::KeyStore;
use crate::notifications::Notifications;
use crate::prelude::*;
use crate::ssh::SshKeys;
use crate::util::serde::Pem;
pub mod package;
pub mod private;
pub mod public;
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct Database {
pub public: Public,
pub private: Private,
}
impl Database {
pub fn init(account: &AccountInfo) -> Result<Self, Error> {
Ok(Self {
public: Public::init(account)?,
private: Private {
key_store: KeyStore::new(account)?,
password: account.password.clone(),
ssh_privkey: Pem(account.ssh_key.clone()),
ssh_pubkeys: SshKeys::new(),
available_ports: AvailablePorts::new(),
sessions: Sessions::new(),
notifications: Notifications::new(),
cifs: CifsTargets::new(),
package_stores: BTreeMap::new(),
}, // TODO
})
}
}
pub type DatabaseModel = Model<Database>;

446
core/startos/src/db/model/package.rs Normal file
View File

@@ -0,0 +1,446 @@
use std::collections::{BTreeMap, BTreeSet};
use chrono::{DateTime, Utc};
use imbl_value::InternedString;
use models::{DataUrl, HealthCheckId, HostId, PackageId};
use patch_db::json_ptr::JsonPointer;
use patch_db::HasModel;
use reqwest::Url;
use serde::{Deserialize, Serialize};
use ts_rs::TS;
use crate::net::host::HostInfo;
use crate::prelude::*;
use crate::progress::FullProgress;
use crate::s9pk::manifest::Manifest;
use crate::status::Status;
#[derive(Debug, Default, Deserialize, Serialize)]
pub struct AllPackageData(pub BTreeMap<PackageId, PackageDataEntry>);
impl Map for AllPackageData {
type Key = PackageId;
type Value = PackageDataEntry;
fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
Ok(key)
}
fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
Ok(key.clone().into())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum ManifestPreference {
Old,
New,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[serde(tag = "state")]
#[model = "Model<Self>"]
pub enum PackageState {
Installing(InstallingState),
Restoring(InstallingState),
Updating(UpdatingState),
Installed(InstalledState),
Removing(InstalledState),
}
impl PackageState {
pub fn expect_installed(&self) -> Result<&InstalledState, Error> {
match self {
Self::Installed(a) => Ok(a),
a => Err(Error::new(
eyre!(
"Package {} is not in installed state",
self.as_manifest(ManifestPreference::Old).id
),
ErrorKind::InvalidRequest,
)),
}
}
pub fn into_installing_info(self) -> Option<InstallingInfo> {
match self {
Self::Installing(InstallingState { installing_info })
| Self::Restoring(InstallingState { installing_info }) => Some(installing_info),
Self::Updating(UpdatingState {
installing_info, ..
}) => Some(installing_info),
Self::Installed(_) | Self::Removing(_) => None,
}
}
pub fn as_installing_info(&self) -> Option<&InstallingInfo> {
match self {
Self::Installing(InstallingState { installing_info })
| Self::Restoring(InstallingState { installing_info }) => Some(installing_info),
Self::Updating(UpdatingState {
installing_info, ..
}) => Some(installing_info),
Self::Installed(_) | Self::Removing(_) => None,
}
}
pub fn as_installing_info_mut(&mut self) -> Option<&mut InstallingInfo> {
match self {
Self::Installing(InstallingState { installing_info })
| Self::Restoring(InstallingState { installing_info }) => Some(installing_info),
Self::Updating(UpdatingState {
installing_info, ..
}) => Some(installing_info),
Self::Installed(_) | Self::Removing(_) => None,
}
}
pub fn into_manifest(self, preference: ManifestPreference) -> Manifest {
match self {
Self::Installing(InstallingState {
installing_info: InstallingInfo { new_manifest, .. },
})
| Self::Restoring(InstallingState {
installing_info: InstallingInfo { new_manifest, .. },
}) => new_manifest,
Self::Updating(UpdatingState { manifest, .. })
if preference == ManifestPreference::Old =>
{
manifest
}
Self::Updating(UpdatingState {
installing_info: InstallingInfo { new_manifest, .. },
..
}) => new_manifest,
Self::Installed(InstalledState { manifest })
| Self::Removing(InstalledState { manifest }) => manifest,
}
}
pub fn as_manifest(&self, preference: ManifestPreference) -> &Manifest {
match self {
Self::Installing(InstallingState {
installing_info: InstallingInfo { new_manifest, .. },
})
| Self::Restoring(InstallingState {
installing_info: InstallingInfo { new_manifest, .. },
}) => new_manifest,
Self::Updating(UpdatingState { manifest, .. })
if preference == ManifestPreference::Old =>
{
manifest
}
Self::Updating(UpdatingState {
installing_info: InstallingInfo { new_manifest, .. },
..
}) => new_manifest,
Self::Installed(InstalledState { manifest })
| Self::Removing(InstalledState { manifest }) => manifest,
}
}
pub fn as_manifest_mut(&mut self, preference: ManifestPreference) -> &mut Manifest {
match self {
Self::Installing(InstallingState {
installing_info: InstallingInfo { new_manifest, .. },
})
| Self::Restoring(InstallingState {
installing_info: InstallingInfo { new_manifest, .. },
}) => new_manifest,
Self::Updating(UpdatingState { manifest, .. })
if preference == ManifestPreference::Old =>
{
manifest
}
Self::Updating(UpdatingState {
installing_info: InstallingInfo { new_manifest, .. },
..
}) => new_manifest,
Self::Installed(InstalledState { manifest })
| Self::Removing(InstalledState { manifest }) => manifest,
}
}
}
impl Model<PackageState> {
pub fn expect_installed(&self) -> Result<&Model<InstalledState>, Error> {
match self.as_match() {
PackageStateMatchModelRef::Installed(a) => Ok(a),
a => Err(Error::new(
eyre!(
"Package {} is not in installed state",
self.as_manifest(ManifestPreference::Old).as_id().de()?
),
ErrorKind::InvalidRequest,
)),
}
}
pub fn into_installing_info(self) -> Option<Model<InstallingInfo>> {
match self.into_match() {
PackageStateMatchModel::Installing(s) | PackageStateMatchModel::Restoring(s) => {
Some(s.into_installing_info())
}
PackageStateMatchModel::Updating(s) => Some(s.into_installing_info()),
PackageStateMatchModel::Installed(_) | PackageStateMatchModel::Removing(_) => None,
PackageStateMatchModel::Error(_) => None,
}
}
pub fn as_installing_info(&self) -> Option<&Model<InstallingInfo>> {
match self.as_match() {
PackageStateMatchModelRef::Installing(s) | PackageStateMatchModelRef::Restoring(s) => {
Some(s.as_installing_info())
}
PackageStateMatchModelRef::Updating(s) => Some(s.as_installing_info()),
PackageStateMatchModelRef::Installed(_) | PackageStateMatchModelRef::Removing(_) => {
None
}
PackageStateMatchModelRef::Error(_) => None,
}
}
pub fn as_installing_info_mut(&mut self) -> Option<&mut Model<InstallingInfo>> {
match self.as_match_mut() {
PackageStateMatchModelMut::Installing(s) | PackageStateMatchModelMut::Restoring(s) => {
Some(s.as_installing_info_mut())
}
PackageStateMatchModelMut::Updating(s) => Some(s.as_installing_info_mut()),
PackageStateMatchModelMut::Installed(_) | PackageStateMatchModelMut::Removing(_) => {
None
}
PackageStateMatchModelMut::Error(_) => None,
}
}
pub fn into_manifest(self, preference: ManifestPreference) -> Model<Manifest> {
match self.into_match() {
PackageStateMatchModel::Installing(s) | PackageStateMatchModel::Restoring(s) => {
s.into_installing_info().into_new_manifest()
}
PackageStateMatchModel::Updating(s) if preference == ManifestPreference::Old => {
s.into_manifest()
}
PackageStateMatchModel::Updating(s) => s.into_installing_info().into_new_manifest(),
PackageStateMatchModel::Installed(s) | PackageStateMatchModel::Removing(s) => {
s.into_manifest()
}
PackageStateMatchModel::Error(_) => Value::Null.into(),
}
}
pub fn as_manifest(&self, preference: ManifestPreference) -> &Model<Manifest> {
match self.as_match() {
PackageStateMatchModelRef::Installing(s) | PackageStateMatchModelRef::Restoring(s) => {
s.as_installing_info().as_new_manifest()
}
PackageStateMatchModelRef::Updating(s) if preference == ManifestPreference::Old => {
s.as_manifest()
}
PackageStateMatchModelRef::Updating(s) => s.as_installing_info().as_new_manifest(),
PackageStateMatchModelRef::Installed(s) | PackageStateMatchModelRef::Removing(s) => {
s.as_manifest()
}
PackageStateMatchModelRef::Error(_) => (&Value::Null).into(),
}
}
pub fn as_manifest_mut(
&mut self,
preference: ManifestPreference,
) -> Result<&mut Model<Manifest>, Error> {
Ok(match self.as_match_mut() {
PackageStateMatchModelMut::Installing(s) | PackageStateMatchModelMut::Restoring(s) => {
s.as_installing_info_mut().as_new_manifest_mut()
}
PackageStateMatchModelMut::Updating(s) if preference == ManifestPreference::Old => {
s.as_manifest_mut()
}
PackageStateMatchModelMut::Updating(s) => {
s.as_installing_info_mut().as_new_manifest_mut()
}
PackageStateMatchModelMut::Installed(s) | PackageStateMatchModelMut::Removing(s) => {
s.as_manifest_mut()
}
PackageStateMatchModelMut::Error(s) => {
return Err(Error::new(
eyre!("could not determine package state to get manifest"),
ErrorKind::Database,
))
}
})
}
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct InstallingState {
pub installing_info: InstallingInfo,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct UpdatingState {
pub manifest: Manifest,
pub installing_info: InstallingInfo,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct InstalledState {
pub manifest: Manifest,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct InstallingInfo {
pub new_manifest: Manifest,
pub progress: FullProgress,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct PackageDataEntry {
pub state_info: PackageState,
pub status: Status,
pub marketplace_url: Option<Url>,
#[serde(default)]
#[serde(with = "crate::util::serde::ed25519_pubkey")]
pub developer_key: ed25519_dalek::VerifyingKey,
pub icon: DataUrl<'static>,
pub last_backup: Option<DateTime<Utc>>,
pub dependency_info: BTreeMap<PackageId, StaticDependencyInfo>,
pub current_dependents: CurrentDependents,
pub current_dependencies: CurrentDependencies,
pub interface_addresses: InterfaceAddressMap,
pub hosts: HostInfo,
pub store_exposed_ui: StoreExposedUI,
pub store_exposed_dependents: Vec<JsonPointer>,
}
impl AsRef<PackageDataEntry> for PackageDataEntry {
fn as_ref(&self) -> &PackageDataEntry {
self
}
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[model = "Model<Self>"]
pub struct ExposedDependent {
path: String,
title: String,
description: Option<String>,
masked: Option<bool>,
copyable: Option<bool>,
qr: Option<bool>,
}
#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct StoreExposedUI(pub BTreeMap<InternedString, ExposedUI>);
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, TS)]
#[serde(rename_all = "camelCase")]
#[serde(tag = "type")]
#[ts(export)]
pub enum ExposedUI {
Object {
#[ts(type = "{[key: string]: ExposedUI}")]
value: BTreeMap<String, ExposedUI>,
#[serde(default)]
#[ts(type = "string | null")]
description: String,
},
String {
#[ts(type = "string")]
path: JsonPointer,
description: Option<String>,
masked: bool,
copyable: Option<bool>,
qr: Option<bool>,
},
}
impl Default for ExposedUI {
fn default() -> Self {
ExposedUI::Object {
value: BTreeMap::new(),
description: "".to_string(),
}
}
}
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
pub struct CurrentDependents(pub BTreeMap<PackageId, CurrentDependencyInfo>);
impl CurrentDependents {
pub fn map(
mut self,
transform: impl Fn(
BTreeMap<PackageId, CurrentDependencyInfo>,
) -> BTreeMap<PackageId, CurrentDependencyInfo>,
) -> Self {
self.0 = transform(self.0);
self
}
}
impl Map for CurrentDependents {
type Key = PackageId;
type Value = CurrentDependencyInfo;
fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
Ok(key)
}
fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
Ok(key.clone().into())
}
}
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
pub struct CurrentDependencies(pub BTreeMap<PackageId, CurrentDependencyInfo>);
impl CurrentDependencies {
pub fn map(
mut self,
transform: impl Fn(
BTreeMap<PackageId, CurrentDependencyInfo>,
) -> BTreeMap<PackageId, CurrentDependencyInfo>,
) -> Self {
self.0 = transform(self.0);
self
}
}
impl Map for CurrentDependencies {
type Key = PackageId;
type Value = CurrentDependencyInfo;
fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
Ok(key)
}
fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
Ok(key.clone().into())
}
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct StaticDependencyInfo {
pub title: String,
pub icon: DataUrl<'static>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
#[serde(tag = "kind")]
pub enum CurrentDependencyInfo {
Exists,
#[serde(rename_all = "kebab-case")]
Running {
#[serde(default)]
health_checks: BTreeSet<HealthCheckId>,
},
}
#[derive(Debug, Default, Deserialize, Serialize)]
pub struct InterfaceAddressMap(pub BTreeMap<HostId, InterfaceAddresses>);
impl Map for InterfaceAddressMap {
type Key = HostId;
type Value = InterfaceAddresses;
fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
Ok(key)
}
fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
Ok(key.clone().into())
}
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct InterfaceAddresses {
pub tor_address: Option<String>,
pub lan_address: Option<String>,
}

30
core/startos/src/db/model/private.rs Normal file
View File

@@ -0,0 +1,30 @@
use std::collections::BTreeMap;
use models::PackageId;
use patch_db::{HasModel, Value};
use serde::{Deserialize, Serialize};
use crate::auth::Sessions;
use crate::backup::target::cifs::CifsTargets;
use crate::net::forward::AvailablePorts;
use crate::net::keys::KeyStore;
use crate::notifications::Notifications;
use crate::prelude::*;
use crate::ssh::SshKeys;
use crate::util::serde::Pem;
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct Private {
pub key_store: KeyStore,
pub password: String, // argon2 hash
pub ssh_privkey: Pem<ssh_key::PrivateKey>,
pub ssh_pubkeys: SshKeys,
pub available_ports: AvailablePorts,
pub sessions: Sessions,
pub notifications: Notifications,
pub cifs: CifsTargets,
#[serde(default)]
pub package_stores: BTreeMap<PackageId, Value>,
}

210
core/startos/src/db/model/public.rs Normal file
View File

@@ -0,0 +1,210 @@
use std::collections::BTreeMap;
use std::net::{Ipv4Addr, Ipv6Addr};
use chrono::{DateTime, Utc};
use emver::VersionRange;
use imbl_value::InternedString;
use ipnet::{Ipv4Net, Ipv6Net};
use isocountry::CountryCode;
use itertools::Itertools;
use models::PackageId;
use openssl::hash::MessageDigest;
use patch_db::{HasModel, Value};
use reqwest::Url;
use serde::{Deserialize, Serialize};
use torut::onion::OnionAddressV3;
use crate::account::AccountInfo;
use crate::db::model::package::AllPackageData;
use crate::net::utils::{get_iface_ipv4_addr, get_iface_ipv6_addr};
use crate::prelude::*;
use crate::util::cpupower::Governor;
use crate::util::Version;
use crate::version::{Current, VersionT};
use crate::{ARCH, PLATFORM};
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
// #[macro_debug]
pub struct Public {
pub server_info: ServerInfo,
pub package_data: AllPackageData,
pub ui: Value,
}
impl Public {
pub fn init(account: &AccountInfo) -> Result<Self, Error> {
let lan_address = account.hostname.lan_address().parse().unwrap();
Ok(Self {
server_info: ServerInfo {
arch: get_arch(),
platform: get_platform(),
id: account.server_id.clone(),
version: Current::new().semver().into(),
hostname: account.hostname.no_dot_host_name(),
last_backup: None,
last_wifi_region: None,
eos_version_compat: Current::new().compat().clone(),
lan_address,
onion_address: account.tor_key.public().get_onion_address(),
tor_address: format!("https://{}", account.tor_key.public().get_onion_address())
.parse()
.unwrap(),
ip_info: BTreeMap::new(),
status_info: ServerStatus {
backup_progress: None,
updated: false,
update_progress: None,
shutting_down: false,
restarting: false,
},
wifi: WifiInfo {
ssids: Vec::new(),
connected: None,
selected: None,
},
unread_notification_count: 0,
connection_addresses: ConnectionAddresses {
tor: Vec::new(),
clearnet: Vec::new(),
},
password_hash: account.password.clone(),
pubkey: ssh_key::PublicKey::from(&account.ssh_key)
.to_openssh()
.unwrap(),
ca_fingerprint: account
.root_ca_cert
.digest(MessageDigest::sha256())
.unwrap()
.iter()
.map(|x| format!("{x:X}"))
.join(":"),
ntp_synced: false,
zram: true,
governor: None,
},
package_data: AllPackageData::default(),
ui: serde_json::from_str(include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../web/patchdb-ui-seed.json"
)))
.with_kind(ErrorKind::Deserialization)?,
})
}
}
fn get_arch() -> InternedString {
(*ARCH).into()
}
fn get_platform() -> InternedString {
(&*PLATFORM).into()
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct ServerInfo {
#[serde(default = "get_arch")]
pub arch: InternedString,
#[serde(default = "get_platform")]
pub platform: InternedString,
pub id: String,
pub hostname: String,
pub version: Version,
pub last_backup: Option<DateTime<Utc>>,
/// Used in the wifi to determine the region to set the system to
pub last_wifi_region: Option<CountryCode>,
pub eos_version_compat: VersionRange,
pub lan_address: Url,
pub onion_address: OnionAddressV3,
/// for backwards compatibility
pub tor_address: Url,
pub ip_info: BTreeMap<String, IpInfo>,
#[serde(default)]
pub status_info: ServerStatus,
pub wifi: WifiInfo,
pub unread_notification_count: u64,
pub connection_addresses: ConnectionAddresses,
pub password_hash: String,
pub pubkey: String,
pub ca_fingerprint: String,
#[serde(default)]
pub ntp_synced: bool,
#[serde(default)]
pub zram: bool,
pub governor: Option<Governor>,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct IpInfo {
pub ipv4_range: Option<Ipv4Net>,
pub ipv4: Option<Ipv4Addr>,
pub ipv6_range: Option<Ipv6Net>,
pub ipv6: Option<Ipv6Addr>,
}
impl IpInfo {
pub async fn for_interface(iface: &str) -> Result<Self, Error> {
let (ipv4, ipv4_range) = get_iface_ipv4_addr(iface).await?.unzip();
let (ipv6, ipv6_range) = get_iface_ipv6_addr(iface).await?.unzip();
Ok(Self {
ipv4_range,
ipv4,
ipv6_range,
ipv6,
})
}
}
#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
#[model = "Model<Self>"]
pub struct BackupProgress {
pub complete: bool,
}
#[derive(Debug, Default, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct ServerStatus {
pub backup_progress: Option<BTreeMap<PackageId, BackupProgress>>,
pub updated: bool,
pub update_progress: Option<UpdateProgress>,
#[serde(default)]
pub shutting_down: bool,
#[serde(default)]
pub restarting: bool,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct UpdateProgress {
pub size: Option<u64>,
pub downloaded: u64,
}
#[derive(Debug, Deserialize, Serialize, HasModel)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct WifiInfo {
pub ssids: Vec<String>,
pub selected: Option<String>,
pub connected: Option<String>,
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct ServerSpecs {
pub cpu: String,
pub disk: String,
pub memory: String,
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct ConnectionAddresses {
pub tor: Vec<String>,
pub clearnet: Vec<String>,
}

22
core/startos/src/db/package.rs
View File

@@ -1,22 +0,0 @@
use models::Version;
use crate::prelude::*;
use crate::s9pk::manifest::PackageId;
pub fn get_packages(db: Peeked) -> Result<Vec<(PackageId, Version)>, Error> {
Ok(db
.as_package_data()
.keys()?
.into_iter()
.flat_map(|package_id| {
let version = db
.as_package_data()
.as_idx(&package_id)?
.as_manifest()
.as_version()
.de()
.ok()?;
Some((package_id, version))
})
.collect())
}

262
core/startos/src/db/prelude.rs
View File

@@ -1,11 +1,16 @@
use std::collections::BTreeMap;
use std::future::Future;
use std::marker::PhantomData;
use std::panic::UnwindSafe;
use std::str::FromStr;
use chrono::{DateTime, Utc};
pub use imbl_value::Value;
use patch_db::json_ptr::ROOT;
use patch_db::value::InternedString;
pub use patch_db::{HasModel, PatchDb, Value};
pub use patch_db::{HasModel, PatchDb};
use serde::de::DeserializeOwned;
use serde::Serialize;
use serde::{Deserialize, Serialize};
use crate::db::model::DatabaseModel;
use crate::prelude::*;
@@ -26,22 +31,20 @@ where
patch_db::value::from_value(value).with_kind(ErrorKind::Deserialization)
}
#[async_trait::async_trait]
pub trait PatchDbExt {
async fn peek(&self) -> DatabaseModel;
async fn mutate<U: UnwindSafe + Send>(
fn peek(&self) -> impl Future<Output = DatabaseModel> + Send;
fn mutate<U: UnwindSafe + Send>(
&self,
f: impl FnOnce(&mut DatabaseModel) -> Result<U, Error> + UnwindSafe + Send,
) -> Result<U, Error>;
async fn map_mutate(
) -> impl Future<Output = Result<U, Error>> + Send;
fn map_mutate(
&self,
f: impl FnOnce(DatabaseModel) -> Result<DatabaseModel, Error> + UnwindSafe + Send,
) -> Result<DatabaseModel, Error>;
) -> impl Future<Output = Result<DatabaseModel, Error>> + Send;
}
#[async_trait::async_trait]
impl PatchDbExt for PatchDb {
async fn peek(&self) -> DatabaseModel {
DatabaseModel::from(self.dump().await.value)
DatabaseModel::from(self.dump(&ROOT).await.value)
}
async fn mutate<U: UnwindSafe + Send>(
&self,
@@ -90,12 +93,43 @@ impl<T: Serialize> Model<T> {
}
}
impl<T> Serialize for Model<T> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
self.value.serialize(serializer)
}
}
impl<'de, T: Serialize + Deserialize<'de>> Deserialize<'de> for Model<T> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::de::Error;
Self::new(&T::deserialize(deserializer)?).map_err(D::Error::custom)
}
}
impl<T: Serialize + DeserializeOwned> Model<T> {
pub fn replace(&mut self, value: &T) -> Result<T, Error> {
let orig = self.de()?;
self.ser(value)?;
Ok(orig)
}
pub fn mutate<U>(&mut self, f: impl FnOnce(&mut T) -> Result<U, Error>) -> Result<U, Error> {
let mut orig = self.de()?;
let res = f(&mut orig)?;
self.ser(&orig)?;
Ok(res)
}
pub fn map_mutate(&mut self, f: impl FnOnce(T) -> Result<T, Error>) -> Result<T, Error> {
let mut orig = self.de()?;
let res = f(orig)?;
self.ser(&res)?;
Ok(res)
}
}
impl<T> Clone for Model<T> {
fn clone(&self) -> Self {
@@ -179,20 +213,38 @@ impl<T> Model<Option<T>> {
pub trait Map: DeserializeOwned + Serialize {
type Key;
type Value;
fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error>;
fn key_string(key: &Self::Key) -> Result<InternedString, Error> {
Ok(InternedString::intern(Self::key_str(key)?.as_ref()))
}
}
impl<A, B> Map for BTreeMap<A, B>
where
A: serde::Serialize + serde::de::DeserializeOwned + Ord + AsRef<str>,
B: serde::Serialize + serde::de::DeserializeOwned,
{
type Key = A;
type Value = B;
fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
Ok(key.as_ref())
}
}
impl<A, B> Map for BTreeMap<JsonKey<A>, B>
where
A: serde::Serialize + serde::de::DeserializeOwned + Ord,
B: serde::Serialize + serde::de::DeserializeOwned,
{
type Key = A;
type Value = B;
fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
serde_json::to_string(key).with_kind(ErrorKind::Serialization)
}
}
impl<T: Map> Model<T>
where
T::Key: AsRef<str>,
T::Value: Serialize,
{
pub fn insert(&mut self, key: &T::Key, value: &T::Value) -> Result<(), Error> {
@@ -200,7 +252,7 @@ where
let v = patch_db::value::to_value(value)?;
match &mut self.value {
Value::Object(o) => {
o.insert(InternedString::intern(key.as_ref()), v);
o.insert(T::key_string(key)?, v);
Ok(())
}
v => Err(patch_db::value::Error {
@@ -210,13 +262,40 @@ where
.into()),
}
}
pub fn upsert<F, D>(&mut self, key: &T::Key, value: F) -> Result<&mut Model<T::Value>, Error>
where
F: FnOnce() -> D,
D: AsRef<T::Value>,
{
use serde::ser::Error;
match &mut self.value {
Value::Object(o) => {
use patch_db::ModelExt;
let s = T::key_str(key)?;
let exists = o.contains_key(s.as_ref());
let res = self.transmute_mut(|v| {
use patch_db::value::index::Index;
s.as_ref().index_or_insert(v)
});
if !exists {
res.ser(value().as_ref())?;
}
Ok(res)
}
v => Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
kind: patch_db::value::ErrorKind::Serialization,
}
.into()),
}
}
pub fn insert_model(&mut self, key: &T::Key, value: Model<T::Value>) -> Result<(), Error> {
use patch_db::ModelExt;
use serde::ser::Error;
let v = value.into_value();
match &mut self.value {
Value::Object(o) => {
o.insert(InternedString::intern(key.as_ref()), v);
o.insert(T::key_string(key)?, v);
Ok(())
}
v => Err(patch_db::value::Error {
@@ -230,25 +309,16 @@ where
impl<T: Map> Model<T>
where
T::Key: DeserializeOwned + Ord + Clone,
T::Key: FromStr + Ord + Clone,
Error: From<<T::Key as FromStr>::Err>,
{
pub fn keys(&self) -> Result<Vec<T::Key>, Error> {
use serde::de::Error;
use serde::Deserialize;
match &self.value {
Value::Object(o) => o
.keys()
.cloned()
.map(|k| {
T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(k))
.map_err(|e| {
patch_db::value::Error {
kind: patch_db::value::ErrorKind::Deserialization,
source: e,
}
.into()
})
})
.map(|k| Ok(T::Key::from_str(&*k)?))
.collect(),
v => Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
@@ -261,19 +331,10 @@ where
pub fn into_entries(self) -> Result<Vec<(T::Key, Model<T::Value>)>, Error> {
use patch_db::ModelExt;
use serde::de::Error;
use serde::Deserialize;
match self.value {
Value::Object(o) => o
.into_iter()
.map(|(k, v)| {
Ok((
T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(
k,
))
.with_kind(ErrorKind::Deserialization)?,
Model::from_value(v),
))
})
.map(|(k, v)| Ok((T::Key::from_str(&*k)?, Model::from_value(v))))
.collect(),
v => Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
@@ -285,19 +346,10 @@ where
pub fn as_entries(&self) -> Result<Vec<(T::Key, &Model<T::Value>)>, Error> {
use patch_db::ModelExt;
use serde::de::Error;
use serde::Deserialize;
match &self.value {
Value::Object(o) => o
.iter()
.map(|(k, v)| {
Ok((
T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(
k.clone(),
))
.with_kind(ErrorKind::Deserialization)?,
Model::value_as(v),
))
})
.map(|(k, v)| Ok((T::Key::from_str(&**k)?, Model::value_as(v))))
.collect(),
v => Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
@@ -309,19 +361,10 @@ where
pub fn as_entries_mut(&mut self) -> Result<Vec<(T::Key, &mut Model<T::Value>)>, Error> {
use patch_db::ModelExt;
use serde::de::Error;
use serde::Deserialize;
match &mut self.value {
Value::Object(o) => o
.iter_mut()
.map(|(k, v)| {
Ok((
T::Key::deserialize(patch_db::value::de::InternedStringDeserializer::from(
k.clone(),
))
.with_kind(ErrorKind::Deserialization)?,
Model::value_as_mut(v),
))
})
.map(|(k, v)| Ok((T::Key::from_str(&**k)?, Model::value_as_mut(v))))
.collect(),
v => Err(patch_db::value::Error {
source: patch_db::value::ErrorSource::custom(format!("expected object found {v}")),
@@ -331,36 +374,36 @@ where
}
}
}
impl<T: Map> Model<T>
where
T::Key: AsRef<str>,
{
impl<T: Map> Model<T> {
pub fn into_idx(self, key: &T::Key) -> Option<Model<T::Value>> {
use patch_db::ModelExt;
let s = T::key_str(key).ok()?;
match &self.value {
Value::Object(o) if o.contains_key(key.as_ref()) => Some(self.transmute(|v| {
Value::Object(o) if o.contains_key(s.as_ref()) => Some(self.transmute(|v| {
use patch_db::value::index::Index;
key.as_ref().index_into_owned(v).unwrap()
s.as_ref().index_into_owned(v).unwrap()
})),
_ => None,
}
}
pub fn as_idx<'a>(&'a self, key: &T::Key) -> Option<&'a Model<T::Value>> {
use patch_db::ModelExt;
let s = T::key_str(key).ok()?;
match &self.value {
Value::Object(o) if o.contains_key(key.as_ref()) => Some(self.transmute_ref(|v| {
Value::Object(o) if o.contains_key(s.as_ref()) => Some(self.transmute_ref(|v| {
use patch_db::value::index::Index;
key.as_ref().index_into(v).unwrap()
s.as_ref().index_into(v).unwrap()
})),
_ => None,
}
}
pub fn as_idx_mut<'a>(&'a mut self, key: &T::Key) -> Option<&'a mut Model<T::Value>> {
use patch_db::ModelExt;
let s = T::key_str(key).ok()?;
match &mut self.value {
Value::Object(o) if o.contains_key(key.as_ref()) => Some(self.transmute_mut(|v| {
Value::Object(o) if o.contains_key(s.as_ref()) => Some(self.transmute_mut(|v| {
use patch_db::value::index::Index;
key.as_ref().index_or_insert(v)
s.as_ref().index_or_insert(v)
})),
_ => None,
}
@@ -369,7 +412,7 @@ where
use serde::ser::Error;
match &mut self.value {
Value::Object(o) => {
let v = o.remove(key.as_ref());
let v = o.remove(T::key_str(key)?.as_ref());
Ok(v.map(patch_db::ModelExt::from_value))
}
v => Err(patch_db::value::Error {
@@ -380,3 +423,90 @@ where
}
}
}
#[repr(transparent)]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct JsonKey<T>(pub T);
impl<T> From<T> for JsonKey<T> {
fn from(value: T) -> Self {
Self::new(value)
}
}
impl<T> JsonKey<T> {
pub fn new(value: T) -> Self {
Self(value)
}
pub fn unwrap(self) -> T {
self.0
}
pub fn new_ref(value: &T) -> &Self {
unsafe { std::mem::transmute(value) }
}
pub fn new_mut(value: &mut T) -> &mut Self {
unsafe { std::mem::transmute(value) }
}
}
impl<T> std::ops::Deref for JsonKey<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl<T> std::ops::DerefMut for JsonKey<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl<T: Serialize> Serialize for JsonKey<T> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
use serde::ser::Error;
serde_json::to_string(&self.0)
.map_err(S::Error::custom)?
.serialize(serializer)
}
}
// { "foo": "bar" } -> "{ \"foo\": \"bar\" }"
impl<'de, T: Serialize + DeserializeOwned> Deserialize<'de> for JsonKey<T> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::de::Error;
let string = String::deserialize(deserializer)?;
Ok(Self(
serde_json::from_str(&string).map_err(D::Error::custom)?,
))
}
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct WithTimeData<T> {
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub value: T,
}
impl<T> WithTimeData<T> {
pub fn new(value: T) -> Self {
let now = Utc::now();
Self {
created_at: now,
updated_at: now,
value,
}
}
}
impl<T> std::ops::Deref for WithTimeData<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.value
}
}
impl<T> std::ops::DerefMut for WithTimeData<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
self.updated_at = Utc::now();
&mut self.value
}
}

388
core/startos/src/dependencies.rs
View File

@@ -1,31 +1,24 @@
use std::collections::BTreeMap;
use std::time::Duration;
use color_eyre::eyre::eyre;
use clap::Parser;
use emver::VersionRange;
use models::OptionExt;
use rand::SeedableRng;
use rpc_toolkit::command;
use models::{OptionExt, PackageId};
use rpc_toolkit::{command, from_fn_async, Empty, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use tracing::instrument;
use crate::config::action::ConfigRes;
use crate::config::spec::PackagePointerSpec;
use crate::config::{not_found, Config, ConfigSpec, ConfigureContext};
use crate::config::{Config, ConfigSpec, ConfigureContext};
use crate::context::RpcContext;
use crate::db::model::{CurrentDependencies, Database};
use crate::db::model::package::CurrentDependencies;
use crate::db::model::Database;
use crate::prelude::*;
use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::s9pk::manifest::Manifest;
use crate::status::DependencyConfigErrors;
use crate::util::serde::display_serializable;
use crate::util::{display_none, Version};
use crate::volume::Volumes;
use crate::Error;
#[command(subcommands(configure))]
pub fn dependency() -> Result<(), Error> {
Ok(())
pub fn dependency() -> ParentHandler {
ParentHandler::new().subcommand("configure", configure())
}
#[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)]
@@ -34,6 +27,12 @@ pub struct Dependencies(pub BTreeMap<PackageId, DepInfo>);
impl Map for Dependencies {
type Key = PackageId;
type Value = DepInfo;
fn key_str(key: &Self::Key) -> Result<impl AsRef<str>, Error> {
Ok(key)
}
fn key_string(key: &Self::Key) -> Result<imbl_value::InternedString, Error> {
Ok(key.clone().into())
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
@@ -58,94 +57,56 @@ pub struct DepInfo {
pub requirement: DependencyRequirement,
pub description: Option<String>,
#[serde(default)]
pub config: Option<DependencyConfig>,
pub config: Option<Value>, // TODO: remove
}
#[derive(Clone, Debug, Deserialize, Serialize, HasModel)]
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[model = "Model<Self>"]
pub struct DependencyConfig {
check: PackageProcedure,
auto_configure: PackageProcedure,
#[command(rename_all = "kebab-case")]
pub struct ConfigureParams {
#[arg(name = "dependent-id")]
dependent_id: PackageId,
#[arg(name = "dependency-id")]
dependency_id: PackageId,
}
impl DependencyConfig {
pub async fn check(
&self,
ctx: &RpcContext,
dependent_id: &PackageId,
dependent_version: &Version,
dependent_volumes: &Volumes,
dependency_id: &PackageId,
dependency_config: &Config,
) -> Result<Result<NoOutput, String>, Error> {
Ok(self
.check
.sandboxed(
ctx,
dependent_id,
dependent_version,
dependent_volumes,
Some(dependency_config),
None,
ProcedureName::Check(dependency_id.clone()),
)
.await?
.map_err(|(_, e)| e))
}
pub async fn auto_configure(
&self,
ctx: &RpcContext,
dependent_id: &PackageId,
dependent_version: &Version,
dependent_volumes: &Volumes,
old: &Config,
) -> Result<Config, Error> {
self.auto_configure
.sandboxed(
ctx,
dependent_id,
dependent_version,
dependent_volumes,
Some(old),
None,
ProcedureName::AutoConfig(dependent_id.clone()),
)
.await?
.map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::AutoConfigure))
}
}
#[command(
subcommands(self(configure_impl(async)), configure_dry),
display(display_none)
)]
pub async fn configure(
#[arg(rename = "dependent-id")] dependent_id: PackageId,
#[arg(rename = "dependency-id")] dependency_id: PackageId,
) -> Result<(PackageId, PackageId), Error> {
Ok((dependent_id, dependency_id))
pub fn configure() -> ParentHandler<ConfigureParams> {
ParentHandler::new().root_handler(
from_fn_async(configure_impl)
.with_inherited(|params, _| params)
.no_cli(),
)
}
pub async fn configure_impl(
ctx: RpcContext,
(pkg_id, dep_id): (PackageId, PackageId),
_: Empty,
ConfigureParams {
dependent_id,
dependency_id,
}: ConfigureParams,
) -> Result<(), Error> {
let breakages = BTreeMap::new();
let overrides = Default::default();
let ConfigDryRes {
old_config: _,
new_config,
spec: _,
} = configure_logic(ctx.clone(), (pkg_id, dep_id.clone())).await?;
} = configure_logic(ctx.clone(), (dependent_id, dependency_id.clone())).await?;
let configure_context = ConfigureContext {
breakages,
timeout: Some(Duration::from_secs(3).into()),
config: Some(new_config),
dry_run: false,
overrides,
};
crate::config::configure(&ctx, &dep_id, configure_context).await?;
ctx.services
.get(&dependency_id)
.await
.as_ref()
.ok_or_else(|| {
Error::new(
eyre!("There is no manager running for {dependency_id}"),
ErrorKind::Unknown,
)
})?
.configure(configure_context)
.await?;
Ok(())
}
@@ -157,206 +118,97 @@ pub struct ConfigDryRes {
pub spec: ConfigSpec,
}
#[command(rename = "dry", display(display_serializable))]
#[instrument(skip_all)]
pub async fn configure_dry(
#[context] ctx: RpcContext,
#[parent_data] (pkg_id, dependency_id): (PackageId, PackageId),
) -> Result<ConfigDryRes, Error> {
configure_logic(ctx, (pkg_id, dependency_id)).await
}
pub async fn configure_logic(
ctx: RpcContext,
(pkg_id, dependency_id): (PackageId, PackageId),
(dependent_id, dependency_id): (PackageId, PackageId),
) -> Result<ConfigDryRes, Error> {
let db = ctx.db.peek().await;
let pkg = db
.as_package_data()
.as_idx(&pkg_id)
.or_not_found(&pkg_id)?
.as_installed()
.or_not_found(&pkg_id)?;
let pkg_version = pkg.as_manifest().as_version().de()?;
let pkg_volumes = pkg.as_manifest().as_volumes().de()?;
let dependency = db
.as_package_data()
.as_idx(&dependency_id)
.or_not_found(&dependency_id)?
.as_installed()
.or_not_found(&dependency_id)?;
let dependency_config_action = dependency
.as_manifest()
.as_config()
.de()?
.ok_or_else(|| not_found!("Manifest Config"))?;
let dependency_version = dependency.as_manifest().as_version().de()?;
let dependency_volumes = dependency.as_manifest().as_volumes().de()?;
let dependency = pkg
.as_manifest()
.as_dependencies()
.as_idx(&dependency_id)
.or_not_found(&dependency_id)?;
// let db = ctx.db.peek().await;
// let pkg = db
// .as_package_data()
// .as_idx(&pkg_id)
// .or_not_found(&pkg_id)?
// .as_installed()
// .or_not_found(&pkg_id)?;
// let pkg_version = pkg.as_manifest().as_version().de()?;
// let pkg_volumes = pkg.as_manifest().as_volumes().de()?;
// let dependency = db
// .as_package_data()
// .as_idx(&dependency_id)
// .or_not_found(&dependency_id)?
// .as_installed()
// .or_not_found(&dependency_id)?;
// let dependency_config_action = dependency
// .as_manifest()
// .as_config()
// .de()?
// .ok_or_else(|| not_found!("Manifest Config"))?;
// let dependency_version = dependency.as_manifest().as_version().de()?;
// let dependency_volumes = dependency.as_manifest().as_volumes().de()?;
// let dependency = pkg
// .as_manifest()
// .as_dependencies()
// .as_idx(&dependency_id)
// .or_not_found(&dependency_id)?;
let ConfigRes {
config: maybe_config,
spec,
} = dependency_config_action
.get(
&ctx,
&dependency_id,
&dependency_version,
&dependency_volumes,
)
.await?;
// let ConfigRes {
// config: maybe_config,
// spec,
// } = dependency_config_action
// .get(
// &ctx,
// &dependency_id,
// &dependency_version,
// &dependency_volumes,
// )
// .await?;
let old_config = if let Some(config) = maybe_config {
config
} else {
spec.gen(
&mut rand::rngs::StdRng::from_entropy(),
&Some(Duration::new(10, 0)),
)?
};
// let old_config = if let Some(config) = maybe_config {
// config
// } else {
// spec.gen(
// &mut rand::rngs::StdRng::from_entropy(),
// &Some(Duration::new(10, 0)),
// )?
// };
let new_config = dependency
.as_config()
.de()?
.ok_or_else(|| not_found!("Config"))?
.auto_configure
.sandboxed(
&ctx,
&pkg_id,
&pkg_version,
&pkg_volumes,
Some(&old_config),
None,
ProcedureName::AutoConfig(dependency_id.clone()),
)
.await?
.map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::AutoConfigure))?;
// let new_config = dependency
// .as_config()
// .de()?
// .ok_or_else(|| not_found!("Config"))?
// .auto_configure
// .sandboxed(
// &ctx,
// &pkg_id,
// &pkg_version,
// &pkg_volumes,
// Some(&old_config),
// None,
// ProcedureName::AutoConfig(dependency_id.clone()),
// )
// .await?
// .map_err(|e| Error::new(eyre!("{}", e.1), crate::ErrorKind::AutoConfigure))?;
Ok(ConfigDryRes {
old_config,
new_config,
spec,
})
}
#[instrument(skip_all)]
pub fn add_dependent_to_current_dependents_lists(
db: &mut Model<Database>,
dependent_id: &PackageId,
current_dependencies: &CurrentDependencies,
) -> Result<(), Error> {
for (dependency, dep_info) in &current_dependencies.0 {
if let Some(dependency_dependents) = db
.as_package_data_mut()
.as_idx_mut(dependency)
.and_then(|pde| pde.as_installed_mut())
.map(|i| i.as_current_dependents_mut())
{
dependency_dependents.insert(dependent_id, dep_info)?;
}
}
Ok(())
}
pub fn set_dependents_with_live_pointers_to_needs_config(
db: &mut Peeked,
id: &PackageId,
) -> Result<Vec<(PackageId, Version)>, Error> {
let mut res = Vec::new();
for (dep, info) in db
.as_package_data()
.as_idx(id)
.or_not_found(id)?
.as_installed()
.or_not_found(id)?
.as_current_dependents()
.de()?
.0
{
if info.pointers.iter().any(|ptr| match ptr {
// dependency id matches the package being uninstalled
PackagePointerSpec::TorAddress(ptr) => &ptr.package_id == id && &dep != id,
PackagePointerSpec::LanAddress(ptr) => &ptr.package_id == id && &dep != id,
// we never need to retarget these
PackagePointerSpec::TorKey(_) => false,
PackagePointerSpec::Config(_) => false,
}) {
let installed = db
.as_package_data_mut()
.as_idx_mut(&dep)
.or_not_found(&dep)?
.as_installed_mut()
.or_not_found(&dep)?;
let version = installed.as_manifest().as_version().de()?;
let configured = installed.as_status_mut().as_configured_mut();
if configured.de()? {
configured.ser(&false)?;
res.push((dep, version));
}
}
}
Ok(res)
// Ok(ConfigDryRes {
// old_config,
// new_config,
// spec,
// })
todo!()
}
#[instrument(skip_all)]
pub async fn compute_dependency_config_errs(
ctx: &RpcContext,
db: &Peeked,
manifest: &Manifest,
id: &PackageId,
current_dependencies: &CurrentDependencies,
dependency_config: &BTreeMap<PackageId, Config>,
) -> Result<DependencyConfigErrors, Error> {
let mut dependency_config_errs = BTreeMap::new();
for (dependency, _dep_info) in current_dependencies
.0
.iter()
.filter(|(dep_id, _)| dep_id != &&manifest.id)
{
for (dependency, _dep_info) in current_dependencies.0.iter() {
// check if config passes dependency check
if let Some(cfg) = &manifest
.dependencies
.0
.get(dependency)
.or_not_found(dependency)?
.config
{
if let Err(error) = cfg
.check(
ctx,
&manifest.id,
&manifest.version,
&manifest.volumes,
dependency,
&if let Some(config) = dependency_config.get(dependency) {
config.clone()
} else if let Some(manifest) = db
.as_package_data()
.as_idx(dependency)
.and_then(|pde| pde.as_installed())
.map(|i| i.as_manifest().de())
.transpose()?
{
if let Some(config) = &manifest.config {
config
.get(ctx, &manifest.id, &manifest.version, &manifest.volumes)
.await?
.config
.unwrap_or_default()
} else {
Config::default()
}
} else {
Config::default()
},
)
.await?
{
dependency_config_errs.insert(dependency.clone(), error);
}
if let Some(error) = todo!() {
dependency_config_errs.insert(dependency.clone(), error);
}
}
Ok(DependencyConfigErrors(dependency_config_errs))

12
core/startos/src/developer/mod.rs
View File

@@ -5,16 +5,13 @@ use std::path::Path;
use ed25519::pkcs8::EncodePrivateKey;
use ed25519::PublicKeyBytes;
use ed25519_dalek::{SigningKey, VerifyingKey};
use rpc_toolkit::command;
use tracing::instrument;
use crate::context::SdkContext;
use crate::util::display_none;
use crate::context::CliContext;
use crate::{Error, ResultExt};
#[command(cli_only, blocking, display(display_none))]
#[instrument(skip_all)]
pub fn init(#[context] ctx: SdkContext) -> Result<(), Error> {
pub fn init(ctx: CliContext) -> Result<(), Error> {
if !ctx.developer_key_path.exists() {
let parent = ctx.developer_key_path.parent().unwrap_or(Path::new("/"));
if !parent.exists() {
@@ -48,8 +45,3 @@ pub fn init(#[context] ctx: SdkContext) -> Result<(), Error> {
}
Ok(())
}
#[command(subcommands(crate::s9pk::verify, crate::config::verify_spec))]
pub fn verify() -> Result<(), Error> {
Ok(())
}

77
core/startos/src/diagnostic.rs
View File

@@ -1,44 +1,70 @@
use std::path::Path;
use std::sync::Arc;
use rpc_toolkit::command;
use clap::Parser;
use rpc_toolkit::yajrc::RpcError;
use rpc_toolkit::{command, from_fn, from_fn_async, AnyContext, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use crate::context::DiagnosticContext;
use crate::disk::repair;
use crate::context::{CliContext, DiagnosticContext};
use crate::init::SYSTEM_REBUILD_PATH;
use crate::logs::{fetch_logs, LogResponse, LogSource};
use crate::shutdown::Shutdown;
use crate::util::display_none;
use crate::Error;
#[command(subcommands(error, logs, exit, restart, forget_disk, disk, rebuild))]
pub fn diagnostic() -> Result<(), Error> {
Ok(())
pub fn diagnostic() -> ParentHandler {
ParentHandler::new()
.subcommand("error", from_fn(error).with_remote_cli::<CliContext>())
.subcommand("logs", from_fn_async(logs).no_cli())
.subcommand(
"exit",
from_fn(exit).no_display().with_remote_cli::<CliContext>(),
)
.subcommand(
"restart",
from_fn(restart)
.no_display()
.with_remote_cli::<CliContext>(),
)
.subcommand("disk", disk())
.subcommand(
"rebuild",
from_fn_async(rebuild)
.no_display()
.with_remote_cli::<CliContext>(),
)
}
#[command]
pub fn error(#[context] ctx: DiagnosticContext) -> Result<Arc<RpcError>, Error> {
// #[command]
pub fn error(ctx: DiagnosticContext) -> Result<Arc<RpcError>, Error> {
Ok(ctx.error.clone())
}
#[command(rpc_only)]
#[derive(Deserialize, Serialize, Parser)]
#[serde(rename_all = "kebab-case")]
#[command(rename_all = "kebab-case")]
pub struct LogsParams {
limit: Option<usize>,
cursor: Option<String>,
before: bool,
}
pub async fn logs(
#[arg] limit: Option<usize>,
#[arg] cursor: Option<String>,
#[arg] before: bool,
_: AnyContext,
LogsParams {
limit,
cursor,
before,
}: LogsParams,
) -> Result<LogResponse, Error> {
Ok(fetch_logs(LogSource::System, limit, cursor, before).await?)
}
#[command(display(display_none))]
pub fn exit(#[context] ctx: DiagnosticContext) -> Result<(), Error> {
pub fn exit(ctx: DiagnosticContext) -> Result<(), Error> {
ctx.shutdown.send(None).expect("receiver dropped");
Ok(())
}
#[command(display(display_none))]
pub fn restart(#[context] ctx: DiagnosticContext) -> Result<(), Error> {
pub fn restart(ctx: DiagnosticContext) -> Result<(), Error> {
ctx.shutdown
.send(Some(Shutdown {
export_args: ctx
@@ -50,20 +76,21 @@ pub fn restart(#[context] ctx: DiagnosticContext) -> Result<(), Error> {
.expect("receiver dropped");
Ok(())
}
#[command(display(display_none))]
pub async fn rebuild(#[context] ctx: DiagnosticContext) -> Result<(), Error> {
pub async fn rebuild(ctx: DiagnosticContext) -> Result<(), Error> {
tokio::fs::write(SYSTEM_REBUILD_PATH, b"").await?;
restart(ctx)
}
#[command(subcommands(forget_disk, repair))]
pub fn disk() -> Result<(), Error> {
Ok(())
pub fn disk() -> ParentHandler {
ParentHandler::new().subcommand(
"forget",
from_fn_async(forget_disk)
.no_display()
.with_remote_cli::<CliContext>(),
)
}
#[command(rename = "forget", display(display_none))]
pub async fn forget_disk() -> Result<(), Error> {
pub async fn forget_disk(_: AnyContext) -> Result<(), Error> {
let disk_guid = Path::new("/media/embassy/config/disk.guid");
if tokio::fs::metadata(disk_guid).await.is_ok() {
tokio::fs::remove_file(disk_guid).await?;

12
core/startos/src/disk/main.rs
View File

@@ -7,8 +7,8 @@ use tracing::instrument;
use super::fsck::{RepairStrategy, RequiresReboot};
use super::util::pvscan;
use crate::disk::mount::filesystem::block_dev::mount;
use crate::disk::mount::filesystem::ReadWrite;
use crate::disk::mount::filesystem::block_dev::BlockDev;
use crate::disk::mount::filesystem::{FileSystem, ReadWrite};
use crate::disk::mount::util::unmount;
use crate::util::Invoke;
use crate::{Error, ErrorKind, ResultExt};
@@ -142,7 +142,9 @@ pub async fn create_fs<P: AsRef<Path>>(
.arg(&blockdev_path)
.invoke(crate::ErrorKind::DiskManagement)
.await?;
mount(&blockdev_path, datadir.as_ref().join(name), ReadWrite).await?;
BlockDev::new(&blockdev_path)
.mount(datadir.as_ref().join(name), ReadWrite)
.await?;
Ok(())
}
@@ -318,7 +320,9 @@ pub async fn mount_fs<P: AsRef<Path>>(
tokio::fs::rename(&tmp_luks_bak, &luks_bak).await?;
}
mount(&blockdev_path, datadir.as_ref().join(name), ReadWrite).await?;
BlockDev::new(&blockdev_path)
.mount(datadir.as_ref().join(name), ReadWrite)
.await?;
Ok(reboot)
}

48
core/startos/src/disk/mod.rs
View File

@@ -1,13 +1,11 @@
use std::path::{Path, PathBuf};
use clap::ArgMatches;
use rpc_toolkit::command;
use rpc_toolkit::{from_fn_async, AnyContext, Empty, HandlerExt, ParentHandler};
use serde::{Deserialize, Serialize};
use crate::context::RpcContext;
use crate::context::{CliContext, RpcContext};
use crate::disk::util::DiskInfo;
use crate::util::display_none;
use crate::util::serde::{display_serializable, IoFormat};
use crate::util::serde::{display_serializable, HandlerExtSerde, WithIoFormat};
use crate::Error;
pub mod fsck;
@@ -42,16 +40,30 @@ impl OsPartitionInfo {
}
}
#[command(subcommands(list, repair))]
pub fn disk() -> Result<(), Error> {
Ok(())
pub fn disk() -> ParentHandler {
ParentHandler::new()
.subcommand(
"list",
from_fn_async(list)
.with_display_serializable()
.with_custom_display_fn::<AnyContext, _>(|handle, result| {
Ok(display_disk_info(handle.params, result))
})
.with_remote_cli::<CliContext>(),
)
.subcommand(
"repair",
from_fn_async(repair)
.no_display()
.with_remote_cli::<CliContext>(),
)
}
fn display_disk_info(info: Vec<DiskInfo>, matches: &ArgMatches) {
fn display_disk_info(params: WithIoFormat<Empty>, args: Vec<DiskInfo>) {
use prettytable::*;
if matches.is_present("format") {
return display_serializable(info, matches);
if let Some(format) = params.format {
return display_serializable(format, args);
}
let mut table = Table::new();
@@ -60,9 +72,9 @@ fn display_disk_info(info: Vec<DiskInfo>, matches: &ArgMatches) {
"LABEL",
"CAPACITY",
"USED",
"EMBASSY OS VERSION"
"STARTOS VERSION"
]);
for disk in info {
for disk in args {
let row = row![
disk.logicalname.display(),
"N/A",
@@ -101,17 +113,11 @@ fn display_disk_info(info: Vec<DiskInfo>, matches: &ArgMatches) {
table.print_tty(false).unwrap();
}
#[command(display(display_disk_info))]
pub async fn list(
#[context] ctx: RpcContext,
#[allow(unused_variables)]
#[arg]
format: Option<IoFormat>,
) -> Result<Vec<DiskInfo>, Error> {
// #[command(display(display_disk_info))]
pub async fn list(ctx: RpcContext, _: Empty) -> Result<Vec<DiskInfo>, Error> {
crate::disk::util::list(&ctx.os_partitions).await
}
#[command(display(display_none))]
pub async fn repair() -> Result<(), Error> {
tokio::fs::write(REPAIR_DISK_PATH, b"").await?;
Ok(())

103
core/startos/src/disk/mount/backup.rs
View File

@@ -1,24 +1,24 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
use color_eyre::eyre::eyre;
use helpers::AtomicFile;
use models::PackageId;
use tokio::io::AsyncWriteExt;
use tracing::instrument;
use super::filesystem::ecryptfs::EcryptFS;
use super::guard::{GenericMountGuard, TmpMountGuard};
use super::util::{bind, unmount};
use crate::auth::check_password;
use crate::backup::target::BackupInfo;
use crate::disk::mount::filesystem::ReadWrite;
use crate::disk::mount::guard::SubPath;
use crate::disk::util::EmbassyOsRecoveryInfo;
use crate::middleware::encrypt::{decrypt_slice, encrypt_slice};
use crate::s9pk::manifest::PackageId;
use crate::util::crypto::{decrypt_slice, encrypt_slice};
use crate::util::serde::IoFormat;
use crate::util::FileLock;
use crate::volume::BACKUP_DIR;
use crate::{Error, ErrorKind, ResultExt};
#[derive(Clone, Debug)]
pub struct BackupMountGuard<G: GenericMountGuard> {
backup_disk_mount_guard: Option<G>,
encrypted_guard: Option<TmpMountGuard>,
@@ -29,7 +29,7 @@ pub struct BackupMountGuard<G: GenericMountGuard> {
impl<G: GenericMountGuard> BackupMountGuard<G> {
fn backup_disk_path(&self) -> &Path {
if let Some(guard) = &self.backup_disk_mount_guard {
guard.as_ref()
guard.path()
} else {
unreachable!()
}
@@ -37,7 +37,7 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
#[instrument(skip_all)]
pub async fn mount(backup_disk_mount_guard: G, password: &str) -> Result<Self, Error> {
let backup_disk_path = backup_disk_mount_guard.as_ref();
let backup_disk_path = backup_disk_mount_guard.path();
let unencrypted_metadata_path =
backup_disk_path.join("EmbassyBackups/unencrypted-metadata.cbor");
let mut unencrypted_metadata: EmbassyOsRecoveryInfo =
@@ -108,7 +108,7 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
let encrypted_guard =
TmpMountGuard::mount(&EcryptFS::new(&crypt_path, &enc_key), ReadWrite).await?;
let metadata_path = encrypted_guard.as_ref().join("metadata.cbor");
let metadata_path = encrypted_guard.path().join("metadata.cbor");
let metadata: BackupInfo = if tokio::fs::metadata(&metadata_path).await.is_ok() {
IoFormat::Cbor.from_slice(&tokio::fs::read(&metadata_path).await.with_ctx(|_| {
(
@@ -146,22 +146,13 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
}
#[instrument(skip_all)]
pub async fn mount_package_backup(
&self,
id: &PackageId,
) -> Result<PackageBackupMountGuard, Error> {
let lock = FileLock::new(Path::new(BACKUP_DIR).join(format!("{}.lock", id)), false).await?;
let mountpoint = Path::new(BACKUP_DIR).join(id);
bind(self.as_ref().join(id), &mountpoint, false).await?;
Ok(PackageBackupMountGuard {
mountpoint: Some(mountpoint),
lock: Some(lock),
})
pub fn package_backup(self: &Arc<Self>, id: &PackageId) -> SubPath<Arc<Self>> {
SubPath::new(self.clone(), id)
}
#[instrument(skip_all)]
pub async fn save(&self) -> Result<(), Error> {
let metadata_path = self.as_ref().join("metadata.cbor");
let metadata_path = self.path().join("metadata.cbor");
let backup_disk_path = self.backup_disk_path();
let mut file = AtomicFile::new(&metadata_path, None::<PathBuf>)
.await
@@ -181,7 +172,22 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
}
#[instrument(skip_all)]
pub async fn unmount(mut self) -> Result<(), Error> {
pub async fn save_and_unmount(self) -> Result<(), Error> {
self.save().await?;
self.unmount().await?;
Ok(())
}
}
#[async_trait::async_trait]
impl<G: GenericMountGuard> GenericMountGuard for BackupMountGuard<G> {
fn path(&self) -> &Path {
if let Some(guard) = &self.encrypted_guard {
guard.path()
} else {
unreachable!()
}
}
async fn unmount(mut self) -> Result<(), Error> {
if let Some(guard) = self.encrypted_guard.take() {
guard.unmount().await?;
}
@@ -190,22 +196,6 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
}
Ok(())
}
#[instrument(skip_all)]
pub async fn save_and_unmount(self) -> Result<(), Error> {
self.save().await?;
self.unmount().await?;
Ok(())
}
}
impl<G: GenericMountGuard> AsRef<Path> for BackupMountGuard<G> {
fn as_ref(&self) -> &Path {
if let Some(guard) = &self.encrypted_guard {
guard.as_ref()
} else {
unreachable!()
}
}
}
impl<G: GenericMountGuard> Drop for BackupMountGuard<G> {
fn drop(&mut self) {
@@ -221,42 +211,3 @@ impl<G: GenericMountGuard> Drop for BackupMountGuard<G> {
});
}
}
pub struct PackageBackupMountGuard {
mountpoint: Option<PathBuf>,
lock: Option<FileLock>,
}
impl PackageBackupMountGuard {
pub async fn unmount(mut self) -> Result<(), Error> {
if let Some(mountpoint) = self.mountpoint.take() {
unmount(&mountpoint).await?;
}
if let Some(lock) = self.lock.take() {
lock.unlock().await?;
}
Ok(())
}
}
impl AsRef<Path> for PackageBackupMountGuard {
fn as_ref(&self) -> &Path {
if let Some(mountpoint) = &self.mountpoint {
mountpoint
} else {
unreachable!()
}
}
}
impl Drop for PackageBackupMountGuard {
fn drop(&mut self) {
let mountpoint = self.mountpoint.take();
let lock = self.lock.take();
tokio::spawn(async move {
if let Some(mountpoint) = mountpoint {
unmount(&mountpoint).await.unwrap();
}
if let Some(lock) = lock {
lock.unlock().await.unwrap();
}
});
}
}

27
core/startos/src/disk/mount/filesystem/bind.rs
View File

@@ -1,14 +1,12 @@
use std::os::unix::ffi::OsStrExt;
use std::path::Path;
use async_trait::async_trait;
use digest::generic_array::GenericArray;
use digest::{Digest, OutputSizeUser};
use sha2::Sha256;
use super::{FileSystem, MountType, ReadOnly};
use crate::disk::mount::util::bind;
use crate::{Error, ResultExt};
use super::FileSystem;
use crate::prelude::*;
pub struct Bind<SrcDir: AsRef<Path>> {
src_dir: SrcDir,
@@ -18,19 +16,16 @@ impl<SrcDir: AsRef<Path>> Bind<SrcDir> {
Self { src_dir }
}
}
#[async_trait]
impl<SrcDir: AsRef<Path> + Send + Sync> FileSystem for Bind<SrcDir> {
async fn mount<P: AsRef<Path> + Send + Sync>(
&self,
mountpoint: P,
mount_type: MountType,
) -> Result<(), Error> {
bind(
self.src_dir.as_ref(),
mountpoint,
matches!(mount_type, ReadOnly),
)
.await
async fn source(&self) -> Result<Option<impl AsRef<Path>>, Error> {
Ok(Some(&self.src_dir))
}
fn extra_args(&self) -> impl IntoIterator<Item = impl AsRef<std::ffi::OsStr>> {
["--bind"]
}
async fn pre_mount(&self) -> Result<(), Error> {
tokio::fs::create_dir_all(self.src_dir.as_ref()).await?;
Ok(())
}
async fn source_hash(
&self,

30
core/startos/src/disk/mount/filesystem/block_dev.rs
View File

@@ -1,30 +1,13 @@
use std::os::unix::ffi::OsStrExt;
use std::path::Path;
use async_trait::async_trait;
use digest::generic_array::GenericArray;
use digest::{Digest, OutputSizeUser};
use serde::{Deserialize, Serialize};
use sha2::Sha256;
use super::{FileSystem, MountType, ReadOnly};
use crate::util::Invoke;
use crate::{Error, ResultExt};
pub async fn mount(
logicalname: impl AsRef<Path>,
mountpoint: impl AsRef<Path>,
mount_type: MountType,
) -> Result<(), Error> {
tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
let mut cmd = tokio::process::Command::new("mount");
cmd.arg(logicalname.as_ref()).arg(mountpoint.as_ref());
if mount_type == ReadOnly {
cmd.arg("-o").arg("ro");
}
cmd.invoke(crate::ErrorKind::Filesystem).await?;
Ok(())
}
use super::FileSystem;
use crate::prelude::*;
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "kebab-case")]
@@ -36,14 +19,9 @@ impl<LogicalName: AsRef<Path>> BlockDev<LogicalName> {
BlockDev { logicalname }
}
}
#[async_trait]
impl<LogicalName: AsRef<Path> + Send + Sync> FileSystem for BlockDev<LogicalName> {
async fn mount<P: AsRef<Path> + Send + Sync>(
&self,
mountpoint: P,
mount_type: MountType,
) -> Result<(), Error> {
mount(self.logicalname.as_ref(), mountpoint, mount_type).await
async fn source(&self) -> Result<Option<impl AsRef<Path>>, Error> {
Ok(Some(&self.logicalname))
}
async fn source_hash(
&self,

6
core/startos/src/disk/mount/filesystem/cifs.rs
View File

@@ -2,7 +2,6 @@ use std::net::IpAddr;
use std::os::unix::ffi::OsStrExt;
use std::path::{Path, PathBuf};
use async_trait::async_trait;
use digest::generic_array::GenericArray;
use digest::{Digest, OutputSizeUser};
use serde::{Deserialize, Serialize};
@@ -11,7 +10,7 @@ use tokio::process::Command;
use tracing::instrument;
use super::{FileSystem, MountType, ReadOnly};
use crate::disk::mount::guard::TmpMountGuard;
use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard};
use crate::util::Invoke;
use crate::Error;
@@ -78,9 +77,8 @@ impl Cifs {
Ok(())
}
}
#[async_trait]
impl FileSystem for Cifs {
async fn mount<P: AsRef<std::path::Path> + Send + Sync>(
async fn mount<P: AsRef<std::path::Path> + Send>(
&self,
mountpoint: P,
mount_type: MountType,

65
core/startos/src/disk/mount/filesystem/ecryptfs.rs
View File

@@ -1,33 +1,17 @@
use std::fmt::Display;
use std::os::unix::ffi::OsStrExt;
use std::path::Path;
use async_trait::async_trait;
use digest::generic_array::GenericArray;
use digest::{Digest, OutputSizeUser};
use lazy_format::lazy_format;
use sha2::Sha256;
use tokio::process::Command;
use super::{FileSystem, MountType};
use super::FileSystem;
use crate::disk::mount::filesystem::default_mount_command;
use crate::prelude::*;
use crate::util::Invoke;
use crate::{Error, ResultExt};
pub async fn mount_ecryptfs<P0: AsRef<Path>, P1: AsRef<Path>>(
src: P0,
dst: P1,
key: &str,
) -> Result<(), Error> {
tokio::fs::create_dir_all(dst.as_ref()).await?;
tokio::process::Command::new("mount")
.arg("-t")
.arg("ecryptfs")
.arg(src.as_ref())
.arg(dst.as_ref())
.arg("-o")
// for more information `man ecryptfs`
.arg(format!("key=passphrase:passphrase_passwd={},ecryptfs_cipher=aes,ecryptfs_key_bytes=32,ecryptfs_passthrough=n,ecryptfs_enable_filename_crypto=y,no_sig_cache", key))
.input(Some(&mut std::io::Cursor::new(b"\n")))
.invoke(crate::ErrorKind::Filesystem).await?;
Ok(())
}
pub struct EcryptFS<EncryptedDir: AsRef<Path>, Key: AsRef<str>> {
encrypted_dir: EncryptedDir,
@@ -38,16 +22,45 @@ impl<EncryptedDir: AsRef<Path>, Key: AsRef<str>> EcryptFS<EncryptedDir, Key> {
EcryptFS { encrypted_dir, key }
}
}
#[async_trait]
impl<EncryptedDir: AsRef<Path> + Send + Sync, Key: AsRef<str> + Send + Sync> FileSystem
for EcryptFS<EncryptedDir, Key>
{
async fn mount<P: AsRef<Path> + Send + Sync>(
fn mount_type(&self) -> Option<impl AsRef<str>> {
Some("ecryptfs")
}
async fn source(&self) -> Result<Option<impl AsRef<Path>>, Error> {
Ok(Some(&self.encrypted_dir))
}
fn mount_options(&self) -> impl IntoIterator<Item = impl Display> {
[
Box::new(lazy_format!(
"key=passphrase:passphrase_passwd={}",
self.key.as_ref()
)) as Box<dyn Display>,
Box::new("ecryptfs_cipher=aes"),
Box::new("ecryptfs_key_bytes=32"),
Box::new("ecryptfs_passthrough=n"),
Box::new("ecryptfs_enable_filename_crypto=y"),
Box::new("no_sig_cache"),
]
}
async fn mount<P: AsRef<Path> + Send>(
&self,
mountpoint: P,
_mount_type: MountType, // ignored - inherited from parent fs
mount_type: super::MountType,
) -> Result<(), Error> {
mount_ecryptfs(self.encrypted_dir.as_ref(), mountpoint, self.key.as_ref()).await
self.pre_mount().await?;
tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
Command::new("mount")
.args(
default_mount_command(self, mountpoint, mount_type)
.await?
.get_args(),
)
.input(Some(&mut std::io::Cursor::new(b"\n")))
.invoke(crate::ErrorKind::Filesystem)
.await?;
Ok(())
}
async fn source_hash(
&self,

28
core/startos/src/disk/mount/filesystem/efivarfs.rs
View File

@@ -1,33 +1,19 @@
use std::path::Path;
use async_trait::async_trait;
use digest::generic_array::GenericArray;
use digest::{Digest, OutputSizeUser};
use sha2::Sha256;
use super::{FileSystem, MountType, ReadOnly};
use crate::util::Invoke;
use crate::Error;
use super::FileSystem;
use crate::prelude::*;
pub struct EfiVarFs;
#[async_trait]
impl FileSystem for EfiVarFs {
async fn mount<P: AsRef<Path> + Send + Sync>(
&self,
mountpoint: P,
mount_type: MountType,
) -> Result<(), Error> {
tokio::fs::create_dir_all(mountpoint.as_ref()).await?;
let mut cmd = tokio::process::Command::new("mount");
cmd.arg("-t")
.arg("efivarfs")
.arg("efivarfs")
.arg(mountpoint.as_ref());
if mount_type == ReadOnly {
cmd.arg("-o").arg("ro");
}
cmd.invoke(crate::ErrorKind::Filesystem).await?;
Ok(())
fn mount_type(&self) -> Option<impl AsRef<str>> {
Some("efivarfs")
}
async fn source(&self) -> Result<Option<impl AsRef<Path>>, Error> {
Ok(Some("efivarfs"))
}
async fn source_hash(
&self,

Some files were not shown because too many files have changed in this diff Show More