From 8f0bdcd1720939b9135242ee53628c8e69cdfee0 Mon Sep 17 00:00:00 2001 From: Jade <2364004+Blu-J@users.noreply.github.com> Date: Wed, 17 Jul 2024 15:46:27 -0600 Subject: [PATCH] Fix/backups (#2659) * fix master build (#2639) * feat: Change ts to use rsync Chore: Update the ts to use types over interface * feat: Get the rust and the js to do a backup * Wip: Got the backup working? * fix permissions * remove trixie list * update tokio to fix timer bug * fix error handling on backup * wip * remove idmap * run restore before init, and init with own version on restore --------- Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Co-authored-by: Aiden McClelland --- .../src/Adapters/Systems/SystemForStartOs.ts | 12 +- container-runtime/src/Interfaces/System.ts | 2 +- core/Cargo.lock | 354 +++++++++--------- core/startos/Cargo.toml | 3 +- core/startos/src/backup/backup_bulk.rs | 2 +- core/startos/src/backup/restore.rs | 2 +- core/startos/src/bins/startd.rs | 2 + core/startos/src/disk/mount/backup.rs | 26 +- .../src/disk/mount/filesystem/backupfs.rs | 21 +- core/startos/src/service/mod.rs | 76 ++-- .../src/service/persistent_container.rs | 12 +- core/startos/src/service/service_map.rs | 43 +-- core/startos/src/service/transition/backup.rs | 68 ++-- core/startos/src/service/transition/mod.rs | 5 +- core/startos/src/sound.rs | 8 +- core/startos/src/util/mod.rs | 2 +- image-recipe/build.sh | 3 + sdk/lib/backup/Backups.ts | 165 ++++---- sdk/lib/backup/setupBackups.ts | 6 +- sdk/lib/config/configTypes.ts | 1 - sdk/lib/manifest/ManifestTypes.ts | 4 +- sdk/lib/types.ts | 7 +- .../ui/src/app/services/api/api.fixures.ts | 1 + 23 files changed, 445 insertions(+), 380 deletions(-) diff --git a/container-runtime/src/Adapters/Systems/SystemForStartOs.ts b/container-runtime/src/Adapters/Systems/SystemForStartOs.ts index e7f09952a..84bf57302 100644 --- a/container-runtime/src/Adapters/Systems/SystemForStartOs.ts +++ b/container-runtime/src/Adapters/Systems/SystemForStartOs.ts @@ -7,6 +7,7 @@ import { RpcResult, matchRpcResult } from "../RpcListener" import { duration } from "../../Models/Duration" import { T } from "@start9labs/start-sdk" import { MainEffects } from "@start9labs/start-sdk/cjs/lib/StartSdk" +import { Volume } from "../../Models/Volume" export const STARTOS_JS_LOCATION = "/usr/lib/startos/package/index.js" export class SystemForStartOs implements System { private onTerm: (() => Promise) | undefined @@ -151,8 +152,17 @@ export class SystemForStartOs implements System { return this.abi.getConfig({ effects }) } case "/backup/create": + return this.abi.createBackup({ + effects, + pathMaker: ((options) => + new Volume(options.volume, options.path).path) as T.PathMaker, + }) case "/backup/restore": - throw new Error("this should be called with the init/unit") + return this.abi.restoreBackup({ + effects, + pathMaker: ((options) => + new Volume(options.volume, options.path).path) as T.PathMaker, + }) case "/actions/metadata": { return this.abi.actionsMetadata({ effects }) } diff --git a/container-runtime/src/Interfaces/System.ts b/container-runtime/src/Interfaces/System.ts index 986288411..58e045356 100644 --- a/container-runtime/src/Interfaces/System.ts +++ b/container-runtime/src/Interfaces/System.ts @@ -5,7 +5,7 @@ import { hostSystemStartOs } from "../Adapters/HostSystemStartOs" export type ExecuteResult = | { ok: unknown } | { err: { code: number; message: string } } -export interface System { +export type System = { // init(effects: Effects): Promise // exit(effects: Effects): Promise // start(effects: Effects): Promise diff --git a/core/Cargo.lock b/core/Cargo.lock index 9f7face93..1072f4203 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -231,18 +231,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -306,7 +306,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "itoa", "matchit", "memchr", @@ -333,9 +333,9 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", "itoa", "matchit", @@ -385,7 +385,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -405,9 +405,9 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", "pin-project-lite", "tokio", @@ -532,7 +532,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.68", + "syn 2.0.71", "which", ] @@ -609,9 +609,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.1" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +checksum = "e9ec96fe9a81b5e365f9db71fe00edc4fe4ca2cc7dcb7861f0603012a7caa210" dependencies = [ "arrayref", "arrayvec 0.7.4", @@ -619,7 +619,7 @@ dependencies = [ "cfg-if", "constant_time_eq", "memmap2", - "rayon", + "rayon-core", ] [[package]] @@ -675,9 +675,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" [[package]] name = "cache-padded" @@ -687,13 +687,12 @@ checksum = "981520c98f422fcc584dc1a95c334e6953900b9106bc47a9839b81790009eb21" [[package]] name = "cc" -version = "1.0.101" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac367972e516d45567c7eafc73d24e1c193dcf200a8d94e9db7b3d38b349572d" +checksum = "324c74f2155653c90b04f25b2a47a8a631360cb908f92a772695f430c7e31052" dependencies = [ "jobserver", "libc", - "once_cell", ] [[package]] @@ -729,7 +728,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -800,9 +799,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.7" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" +checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" dependencies = [ "clap_builder", "clap_derive", @@ -810,9 +809,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.7" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" +checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" dependencies = [ "anstream", "anstyle", @@ -822,14 +821,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.5" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" +checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -1237,14 +1236,14 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] name = "darling" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ "darling_core", "darling_macro", @@ -1252,27 +1251,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] name = "darling_macro" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -1303,7 +1302,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -1320,13 +1319,13 @@ dependencies = [ [[package]] name = "der_derive" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe87ce4529967e0ba1dcf8450bab64d97dfd5010a6256187ffe2e43e6f0e049" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -1349,7 +1348,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -1571,7 +1570,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -1829,7 +1828,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -2143,9 +2142,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http 1.1.0", @@ -2160,7 +2159,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -2184,9 +2183,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.29" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", @@ -2208,16 +2207,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", "futures-util", "h2 0.4.5", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -2235,9 +2234,9 @@ checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", - "rustls 0.23.10", + "rustls 0.23.11", "rustls-pki-types", "tokio", "tokio-rustls", @@ -2250,7 +2249,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.29", + "hyper 0.14.30", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -2264,7 +2263,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", "native-tls", "tokio", @@ -2274,16 +2273,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", - "http-body 1.0.0", - "hyper 1.3.1", + "http-body 1.0.1", + "hyper 1.4.1", "pin-project-lite", "socket2", "tokio", @@ -2783,7 +2782,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" dependencies = [ "cfg-if", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -2837,9 +2836,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lzma-sys" @@ -3230,7 +3229,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -3262,9 +3261,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssh-keys" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9939566c441a6542e87c74310d4ac713c17679c76f296932e732f405bc25e3d" +checksum = "abb830a82898b2ac17c9620ddce839ac3b34b9cb8a1a037cbdbfb9841c756c3e" dependencies = [ "base64 0.21.7", "byteorder", @@ -3296,7 +3295,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -3395,9 +3394,9 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.2", + "redox_syscall 0.5.3", "smallvec", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -3474,9 +3473,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" +checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" dependencies = [ "memchr", "thiserror", @@ -3485,9 +3484,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" +checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" dependencies = [ "pest", "pest_generator", @@ -3495,22 +3494,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" +checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] name = "pest_meta" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" +checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" dependencies = [ "once_cell", "pest", @@ -3559,7 +3558,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -3632,7 +3631,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -3704,7 +3703,7 @@ checksum = "6ff7ff745a347b87471d859a377a9a404361e7efc2a971d73424a6d183c0fc77" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -3727,7 +3726,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -3871,16 +3870,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - [[package]] name = "rayon-core" version = "1.12.1" @@ -3917,9 +3906,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ "bitflags 2.6.0", ] @@ -3994,9 +3983,9 @@ dependencies = [ "futures-util", "h2 0.4.5", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-rustls", "hyper-tls", "hyper-util", @@ -4192,15 +4181,15 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.10" +version = "0.23.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402" +checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" dependencies = [ "aws-lc-rs", "log", "once_cell", "rustls-pki-types", - "rustls-webpki 0.102.4", + "rustls-webpki 0.102.5", "subtle", "zeroize", ] @@ -4242,9 +4231,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.4" +version = "0.102.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" dependencies = [ "aws-lc-rs", "ring", @@ -4342,9 +4331,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.6.0", "core-foundation", @@ -4355,9 +4344,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" dependencies = [ "core-foundation-sys", "libc", @@ -4374,9 +4363,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] @@ -4400,20 +4389,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] name = "serde_json" -version = "1.0.118" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "indexmap 2.2.6", "itoa", @@ -4454,9 +4443,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" +checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" dependencies = [ "base64 0.22.1", "chrono", @@ -4472,14 +4461,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" +checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -4906,7 +4895,7 @@ dependencies = [ "quote", "regex-syntax 0.6.29", "strsim 0.10.0", - "syn 2.0.68", + "syn 2.0.71", "unicode-width", ] @@ -5020,6 +5009,7 @@ dependencies = [ "nix 0.29.0", "nom 7.1.3", "num", + "num_cpus", "num_enum", "once_cell", "openssh-keys", @@ -5140,9 +5130,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.68" +version = "2.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462" dependencies = [ "proc-macro2", "quote", @@ -5224,9 +5214,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.3" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -5243,22 +5233,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.61" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "f2675633b1499176c2dff06b0856a27976a8f9d436737b4cf4f312d4d91d8bbb" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.61" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "d20468752b09f49e909e55a5d338caa8bedf615594e9d80bc4c565d30faf798c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -5324,9 +5314,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -5339,9 +5329,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.0" +version = "1.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" dependencies = [ "backtrace", "bytes", @@ -5375,7 +5365,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -5394,7 +5384,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.10", + "rustls 0.23.11", "rustls-pki-types", "tokio", ] @@ -5497,7 +5487,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.14", + "toml_edit 0.22.15", ] [[package]] @@ -5535,9 +5525,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.14" +version = "0.22.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" dependencies = [ "indexmap 2.2.6", "serde", @@ -5560,7 +5550,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-timeout", "percent-encoding", "pin-project", @@ -5644,7 +5634,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -5796,7 +5786,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", "termcolor", ] @@ -5856,7 +5846,7 @@ checksum = "1f718dfaf347dcb5b983bfc87608144b0bad87970aebcbea5ce44d2a30c08e63" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -5975,9 +5965,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "getrandom 0.2.15", ] @@ -6067,7 +6057,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", "wasm-bindgen-shared", ] @@ -6101,7 +6091,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6200,7 +6190,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -6218,7 +6208,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -6238,18 +6228,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -6260,9 +6250,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -6272,9 +6262,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -6284,15 +6274,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -6302,9 +6292,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -6314,9 +6304,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -6326,9 +6316,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -6338,9 +6328,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" @@ -6440,22 +6430,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] @@ -6475,32 +6465,32 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.71", ] [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.11+zstd.1.5.6" +version = "2.0.12+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75652c55c0b6f3e6f12eb786fe1bc960396bf05a1eb3bf1f3691c3610ac2e6d4" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" dependencies = [ "cc", "pkg-config", diff --git a/core/startos/Cargo.toml b/core/startos/Cargo.toml index 842ef44ef..2695c7813 100644 --- a/core/startos/Cargo.toml +++ b/core/startos/Cargo.toml @@ -128,6 +128,7 @@ nix = { version = "0.29.0", features = ["user", "process", "signal", "fs"] } nom = "7.1.3" num = "0.4.1" num_enum = "0.7.0" +num_cpus = "1.16.0" once_cell = "1.19.0" openssh-keys = "0.6.2" openssl = { version = "0.10.57", features = ["vendored"] } @@ -170,7 +171,7 @@ sscanf = "0.4.1" ssh-key = { version = "0.6.2", features = ["ed25519"] } tar = "0.4.40" thiserror = "1.0.49" -tokio = { version = "1.38.0", features = ["full"] } +tokio = { version = "1.38.1", features = ["full"] } tokio-rustls = "0.26.0" tokio-socks = "0.5.1" tokio-stream = { version = "0.1.14", features = ["io-util", "sync", "net"] } diff --git a/core/startos/src/backup/backup_bulk.rs b/core/startos/src/backup/backup_bulk.rs index a52c97f9b..b4419e88e 100644 --- a/core/startos/src/backup/backup_bulk.rs +++ b/core/startos/src/backup/backup_bulk.rs @@ -260,7 +260,7 @@ async fn perform_backup( for id in package_ids { if let Some(service) = &*ctx.services.get(id).await { let backup_result = service - .backup(backup_guard.package_backup(id)) + .backup(backup_guard.package_backup(id).await?) .await .err() .map(|e| e.to_string()); diff --git a/core/startos/src/backup/restore.rs b/core/startos/src/backup/restore.rs index 23e0c8ac1..28d70653f 100644 --- a/core/startos/src/backup/restore.rs +++ b/core/startos/src/backup/restore.rs @@ -158,7 +158,7 @@ async fn restore_packages( let backup_guard = Arc::new(backup_guard); let mut tasks = BTreeMap::new(); for id in ids { - let backup_dir = backup_guard.clone().package_backup(&id); + let backup_dir = backup_guard.clone().package_backup(&id).await?; let s9pk_path = backup_dir.path().join(&id).with_extension("s9pk"); let task = ctx .services diff --git a/core/startos/src/bins/startd.rs b/core/startos/src/bins/startd.rs index 7576c41e9..d383f3091 100644 --- a/core/startos/src/bins/startd.rs +++ b/core/startos/src/bins/startd.rs @@ -1,3 +1,4 @@ +use std::cmp::max; use std::ffi::OsString; use std::net::{Ipv6Addr, SocketAddr}; use std::sync::Arc; @@ -136,6 +137,7 @@ pub fn main(args: impl IntoIterator) { let res = { let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(max(4, num_cpus::get())) .enable_all() .build() .expect("failed to initialize runtime"); diff --git a/core/startos/src/disk/mount/backup.rs b/core/startos/src/disk/mount/backup.rs index 142301a74..8f45b0d4f 100644 --- a/core/startos/src/disk/mount/backup.rs +++ b/core/startos/src/disk/mount/backup.rs @@ -106,8 +106,11 @@ impl BackupMountGuard { ) })?; } - let encrypted_guard = - TmpMountGuard::mount(&BackupFS::new(&crypt_path, &enc_key), ReadWrite).await?; + let encrypted_guard = TmpMountGuard::mount( + &BackupFS::new(&crypt_path, &enc_key, vec![(100000, 65536)]), + ReadWrite, + ) + .await?; let metadata_path = encrypted_guard.path().join("metadata.json"); let metadata: BackupInfo = if tokio::fs::metadata(&metadata_path).await.is_ok() { @@ -148,8 +151,23 @@ impl BackupMountGuard { } #[instrument(skip_all)] - pub fn package_backup(self: &Arc, id: &PackageId) -> SubPath> { - SubPath::new(self.clone(), id) + pub async fn package_backup( + self: &Arc, + id: &PackageId, + ) -> Result>, Error> { + let package_guard = SubPath::new(self.clone(), id); + let package_path = package_guard.path(); + if tokio::fs::metadata(&package_path).await.is_err() { + tokio::fs::create_dir_all(&package_path) + .await + .with_ctx(|_| { + ( + crate::ErrorKind::Filesystem, + package_path.display().to_string(), + ) + })?; + } + Ok(package_guard) } #[instrument(skip_all)] diff --git a/core/startos/src/disk/mount/filesystem/backupfs.rs b/core/startos/src/disk/mount/filesystem/backupfs.rs index 9ef258f34..254abde20 100644 --- a/core/startos/src/disk/mount/filesystem/backupfs.rs +++ b/core/startos/src/disk/mount/filesystem/backupfs.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::fmt::{self, Display}; use std::os::unix::ffi::OsStrExt; use std::path::Path; @@ -12,10 +13,15 @@ use crate::prelude::*; pub struct BackupFS, Password: fmt::Display> { data_dir: DataDir, password: Password, + idmapped_root: Vec<(u32, u32)>, } impl, Password: fmt::Display> BackupFS { - pub fn new(data_dir: DataDir, password: Password) -> Self { - BackupFS { data_dir, password } + pub fn new(data_dir: DataDir, password: Password, idmapped_root: Vec<(u32, u32)>) -> Self { + BackupFS { + data_dir, + password, + idmapped_root, + } } } impl + Send + Sync, Password: fmt::Display + Send + Sync> FileSystem @@ -26,9 +32,16 @@ impl + Send + Sync, Password: fmt::Display + Send + Sync> F } fn mount_options(&self) -> impl IntoIterator { [ - format!("password={}", self.password), - format!("file-size-padding=0.05"), + Cow::Owned(format!("password={}", self.password)), + Cow::Borrowed("file-size-padding=0.05"), + Cow::Borrowed("allow_other"), ] + .into_iter() + .chain( + self.idmapped_root + .iter() + .map(|(root, range)| Cow::Owned(format!("idmapped-root={root}:{range}"))), + ) } async fn source(&self) -> Result>, Error> { Ok(Some(&self.data_dir)) diff --git a/core/startos/src/service/mod.rs b/core/startos/src/service/mod.rs index 581d2ac0d..d78df6c79 100644 --- a/core/startos/src/service/mod.rs +++ b/core/startos/src/service/mod.rs @@ -34,6 +34,7 @@ use crate::util::actor::concurrent::ConcurrentActor; use crate::util::actor::Actor; use crate::util::io::create_file; use crate::util::serde::Pem; +use crate::util::Never; use crate::volume::data_dir; mod action; @@ -220,12 +221,13 @@ impl Service { tracing::error!("Error opening s9pk for install: {e}"); tracing::debug!("{e:?}") }) { - if let Ok(service) = Self::install(ctx.clone(), s9pk, None, None) - .await - .map_err(|e| { - tracing::error!("Error installing service: {e}"); - tracing::debug!("{e:?}") - }) + if let Ok(service) = + Self::install(ctx.clone(), s9pk, None, None::, None) + .await + .map_err(|e| { + tracing::error!("Error installing service: {e}"); + tracing::debug!("{e:?}") + }) { return Ok(Some(service)); } @@ -257,6 +259,7 @@ impl Service { ctx.clone(), s9pk, Some(s.as_manifest().as_version().de()?), + None::, None, ) .await @@ -334,13 +337,35 @@ impl Service { pub async fn install( ctx: RpcContext, s9pk: S9pk, - src_version: Option, + mut src_version: Option, + recovery_source: Option, progress: Option, ) -> Result { let manifest = s9pk.as_manifest().clone(); let developer_key = s9pk.as_archive().signer(); let icon = s9pk.icon_data_url().await?; let service = Self::new(ctx.clone(), s9pk, StartStop::Stop).await?; + if let Some(recovery_source) = recovery_source { + service + .actor + .send( + Guid::new(), + transition::restore::Restore { + path: recovery_source.path().to_path_buf(), + }, + ) + .await??; + recovery_source.unmount().await?; + src_version = Some( + service + .seed + .persistent_container + .s9pk + .as_manifest() + .version + .clone(), + ); + } service .seed .persistent_container @@ -382,26 +407,6 @@ impl Service { Ok(service) } - pub async fn restore( - ctx: RpcContext, - s9pk: S9pk, - backup_source: impl GenericMountGuard, - progress: Option, - ) -> Result { - let service = Service::install(ctx.clone(), s9pk, None, progress).await?; - - service - .actor - .send( - Guid::new(), - transition::restore::Restore { - path: backup_source.path().to_path_buf(), - }, - ) - .await??; - Ok(service) - } - #[instrument(skip_all)] pub async fn backup(&self, guard: impl GenericMountGuard) -> Result<(), Error> { let id = &self.seed.id; @@ -417,10 +422,11 @@ impl Service { .send( Guid::new(), transition::backup::Backup { - path: guard.path().to_path_buf(), + path: guard.path().join("data"), }, ) - .await??; + .await?? + .await?; Ok(()) } @@ -505,13 +511,21 @@ impl Actor for ServiceActor { } (Some(TransitionKind::Restarting), _, _) => MainStatus::Restarting, (Some(TransitionKind::Restoring), _, _) => MainStatus::Restoring, - (Some(TransitionKind::BackingUp), _, Some(status)) => { + (Some(TransitionKind::BackingUp), StartStop::Stop, Some(status)) => { + seed.persistent_container.stop().await?; MainStatus::BackingUp { started: Some(status.started), health: status.health.clone(), } } - (Some(TransitionKind::BackingUp), _, None) => MainStatus::BackingUp { + (Some(TransitionKind::BackingUp), StartStop::Start, _) => { + seed.persistent_container.start().await?; + MainStatus::BackingUp { + started: None, + health: OrdMap::new(), + } + } + (Some(TransitionKind::BackingUp), _, _) => MainStatus::BackingUp { started: None, health: OrdMap::new(), }, diff --git a/core/startos/src/service/persistent_container.rs b/core/startos/src/service/persistent_container.rs index e0b31ea97..11cc237d3 100644 --- a/core/startos/src/service/persistent_container.rs +++ b/core/startos/src/service/persistent_container.rs @@ -1,5 +1,5 @@ use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::{Arc, Weak}; use std::time::Duration; @@ -277,7 +277,7 @@ impl PersistentContainer { backup_path: impl AsRef, mount_type: MountType, ) -> Result { - let backup_path: PathBuf = backup_path.as_ref().to_path_buf(); + let backup_path = backup_path.as_ref(); let mountpoint = self .lxc_container .get() @@ -295,14 +295,14 @@ impl PersistentContainer { .arg(mountpoint.as_os_str()) .invoke(ErrorKind::Filesystem) .await?; - let bind = Bind::new(&backup_path); - let mount_guard = MountGuard::mount(&bind, &mountpoint, mount_type).await; + tokio::fs::create_dir_all(backup_path).await?; Command::new("chown") .arg("100000:100000") - .arg(backup_path.as_os_str()) + .arg(backup_path) .invoke(ErrorKind::Filesystem) .await?; - mount_guard + let bind = Bind::new(backup_path); + MountGuard::mount(&bind, &mountpoint, mount_type).await } #[instrument(skip_all)] diff --git a/core/startos/src/service/service_map.rs b/core/startos/src/service/service_map.rs index af10a065c..90223216c 100644 --- a/core/startos/src/service/service_map.rs +++ b/core/startos/src/service/service_map.rs @@ -265,35 +265,20 @@ impl ServiceMap { } else { None }; - if let Some(recovery_source) = recovery_source { - *service = Some( - Service::restore( - ctx, - s9pk, - recovery_source, - Some(InstallProgressHandles { - finalization_progress, - progress, - }), - ) - .await? - .into(), - ); - } else { - *service = Some( - Service::install( - ctx, - s9pk, - prev, - Some(InstallProgressHandles { - finalization_progress, - progress, - }), - ) - .await? - .into(), - ); - } + *service = Some( + Service::install( + ctx, + s9pk, + prev, + recovery_source, + Some(InstallProgressHandles { + finalization_progress, + progress, + }), + ) + .await? + .into(), + ); drop(service); sync_progress_task.await.map_err(|_| { diff --git a/core/startos/src/service/transition/backup.rs b/core/startos/src/service/transition/backup.rs index f7591f0d9..d8606f534 100644 --- a/core/startos/src/service/transition/backup.rs +++ b/core/startos/src/service/transition/backup.rs @@ -1,5 +1,7 @@ use std::path::PathBuf; +use std::sync::Arc; +use futures::future::BoxFuture; use futures::FutureExt; use models::ProcedureName; @@ -19,7 +21,7 @@ pub(in crate::service) struct Backup { pub path: PathBuf, } impl Handler for ServiceActor { - type Response = Result<(), Error>; + type Response = Result>, Error>; fn conflicts_with(_: &Backup) -> ConflictBuilder { ConflictBuilder::everything() .except::() @@ -37,43 +39,31 @@ impl Handler for ServiceActor { let path = backup.path.clone(); let seed = self.0.clone(); - let state = self.0.persistent_container.state.clone(); - let transition = RemoteCancellable::new( - async move { - temp.stop(); + let transition = RemoteCancellable::new(async move { + temp.stop(); + current + .wait_for(|s| s.running_status.is_none()) + .await + .with_kind(ErrorKind::Unknown)?; + + let backup_guard = seed + .persistent_container + .mount_backup(path, ReadWrite) + .await?; + seed.persistent_container + .execute(id, ProcedureName::CreateBackup, Value::Null, None) + .await?; + backup_guard.unmount(true).await?; + + if temp.restore().is_start() { current - .wait_for(|s| s.running_status.is_none()) + .wait_for(|s| s.running_status.is_some()) .await .with_kind(ErrorKind::Unknown)?; - - let backup_guard = seed - .persistent_container - .mount_backup(path, ReadWrite) - .await?; - seed.persistent_container - .execute(id, ProcedureName::CreateBackup, Value::Null, None) - .await?; - backup_guard.unmount(true).await?; - - if temp.restore().is_start() { - current - .wait_for(|s| s.running_status.is_some()) - .await - .with_kind(ErrorKind::Unknown)?; - } - drop(temp); - state.send_modify(|s| { - s.transition_state.take(); - }); - Ok::<_, Error>(()) } - .map(|x| { - if let Err(err) = dbg!(x) { - tracing::debug!("{:?}", err); - tracing::warn!("{}", err); - } - }), - ); + drop(temp); + Ok::<_, Arc>(()) + }); let cancel_handle = transition.cancellation_handle(); let transition = transition.shared(); let job_transition = transition.clone(); @@ -92,9 +82,11 @@ impl Handler for ServiceActor { if let Some(t) = old { t.abort().await; } - match transition.await { - None => Err(Error::new(eyre!("Backup canceled"), ErrorKind::Unknown)), - Some(x) => Ok(x), - } + Ok(transition + .map(|r| { + r.ok_or_else(|| Error::new(eyre!("Backup canceled"), ErrorKind::Cancelled))? + .map_err(|e| e.clone_output()) + }) + .boxed()) } } diff --git a/core/startos/src/service/transition/mod.rs b/core/startos/src/service/transition/mod.rs index 7b7f10f2a..a6a41073b 100644 --- a/core/startos/src/service/transition/mod.rs +++ b/core/startos/src/service/transition/mod.rs @@ -79,7 +79,10 @@ impl TempDesiredRestore { } impl Drop for TempDesiredRestore { fn drop(&mut self) { - self.0.send_modify(|s| s.temp_desired_state = None); + self.0.send_modify(|s| { + s.temp_desired_state.take(); + s.transition_state.take(); + }); } } // impl Deref for TempDesiredState { diff --git a/core/startos/src/sound.rs b/core/startos/src/sound.rs index 8dc78357c..8cedd78ce 100644 --- a/core/startos/src/sound.rs +++ b/core/startos/src/sound.rs @@ -10,12 +10,12 @@ use crate::util::{FileLock, Invoke}; use crate::{Error, ErrorKind}; lazy_static::lazy_static! { - static ref SEMITONE_K: f64 = 2f64.powf(1f64 / 12f64); - static ref A_4: f64 = 440f64; - static ref C_0: f64 = *A_4 / SEMITONE_K.powf(9f64) / 2f64.powf(4f64); + static ref SEMITONE_K: f64 = 2f64.powf(1.0 / 12.0); + static ref A_4: f64 = 440.0; + static ref C_0: f64 = *A_4 / SEMITONE_K.powf(9.0) / 2_f64.powf(4.0); } -pub const SOUND_LOCK_FILE: &str = "/etc/embassy/sound.lock"; +pub const SOUND_LOCK_FILE: &str = "/run/startos/sound.lock"; struct SoundInterface { guard: Option, diff --git a/core/startos/src/util/mod.rs b/core/startos/src/util/mod.rs index 0c9e5c5f9..f591eaaf3 100644 --- a/core/startos/src/util/mod.rs +++ b/core/startos/src/util/mod.rs @@ -555,7 +555,7 @@ impl T, T> Drop for GeneralGuard { } } -pub struct FileLock(OwnedMutexGuard<()>, Option>); +pub struct FileLock(#[allow(unused)] OwnedMutexGuard<()>, Option>); impl Drop for FileLock { fn drop(&mut self) { if let Some(fd_lock) = self.1.take() { diff --git a/image-recipe/build.sh b/image-recipe/build.sh index 5ec500ce3..5635e94f3 100755 --- a/image-recipe/build.sh +++ b/image-recipe/build.sh @@ -166,6 +166,9 @@ fi curl -fsSL https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc > config/archives/tor.key echo "deb [arch=${IB_TARGET_ARCH} signed-by=/etc/apt/trusted.gpg.d/tor.key.gpg] https://deb.torproject.org/torproject.org ${IB_SUITE} main" > config/archives/tor.list +curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o config/archives/docker.key +echo "deb [arch=${IB_TARGET_ARCH} signed-by=/etc/apt/trusted.gpg.d/docker.key.gpg] https://download.docker.com/linux/debian ${IB_SUITE} stable" > config/archives/docker.list + # Dependencies ## Base dependencies diff --git a/sdk/lib/backup/Backups.ts b/sdk/lib/backup/Backups.ts index 20099c86d..6ffe74b70 100644 --- a/sdk/lib/backup/Backups.ts +++ b/sdk/lib/backup/Backups.ts @@ -1,6 +1,10 @@ +import { recursive } from "ts-matches" import { SDKManifest } from "../manifest/ManifestTypes" import * as T from "../types" +import * as child_process from "child_process" +import { promises as fsPromises } from "fs" + export type BACKUP = "BACKUP" export const DEFAULT_OPTIONS: T.BackupOptions = { delete: true, @@ -91,58 +95,22 @@ export class Backups { ) return this } - build() { + build(pathMaker: T.PathMaker) { const createBackup: T.ExpectedExports.createBackup = async ({ effects, }) => { - // const previousItems = ( - // await effects - // .readDir({ - // volumeId: Backups.BACKUP, - // path: ".", - // }) - // .catch(() => []) - // ).map((x) => `${x}`) - // const backupPaths = this.backupSet - // .filter((x) => x.dstVolume === Backups.BACKUP) - // .map((x) => x.dstPath) - // .map((x) => x.replace(/\.\/([^]*)\//, "$1")) - // const filteredItems = previousItems.filter( - // (x) => backupPaths.indexOf(x) === -1, - // ) - // for (const itemToRemove of filteredItems) { - // effects.console.error(`Trying to remove ${itemToRemove}`) - // await effects - // .removeDir({ - // volumeId: Backups.BACKUP, - // path: itemToRemove, - // }) - // .catch(() => - // effects.removeFile({ - // volumeId: Backups.BACKUP, - // path: itemToRemove, - // }), - // ) - // .catch(() => { - // console.warn(`Failed to remove ${itemToRemove} from backup volume`) - // }) - // } for (const item of this.backupSet) { - // if (notEmptyPath(item.dstPath)) { - // await effects.createDir({ - // volumeId: item.dstVolume, - // path: item.dstPath, - // }) - // } - // await effects - // .runRsync({ - // ...item, - // options: { - // ...this.options, - // ...item.options, - // }, - // }) - // .wait() + const rsyncResults = await runRsync( + { + dstPath: item.dstPath, + dstVolume: item.dstVolume, + options: { ...this.options, ...item.options }, + srcPath: item.srcPath, + srcVolume: item.srcVolume, + }, + pathMaker, + ) + await rsyncResults.wait() } return } @@ -150,26 +118,17 @@ export class Backups { effects, }) => { for (const item of this.backupSet) { - // if (notEmptyPath(item.srcPath)) { - // await new Promise((resolve, reject) => fs.mkdir(items.src)).createDir( - // { - // volumeId: item.srcVolume, - // path: item.srcPath, - // }, - // ) - // } - // await effects - // .runRsync({ - // options: { - // ...this.options, - // ...item.options, - // }, - // srcVolume: item.dstVolume, - // dstVolume: item.srcVolume, - // srcPath: item.dstPath, - // dstPath: item.srcPath, - // }) - // .wait() + const rsyncResults = await runRsync( + { + dstPath: item.dstPath, + dstVolume: item.dstVolume, + options: { ...this.options, ...item.options }, + srcPath: item.srcPath, + srcVolume: item.srcVolume, + }, + pathMaker, + ) + await rsyncResults.wait() } return } @@ -179,3 +138,73 @@ export class Backups { function notEmptyPath(file: string) { return ["", ".", "./"].indexOf(file) === -1 } +async function runRsync( + rsyncOptions: { + srcVolume: string + dstVolume: string + srcPath: string + dstPath: string + options: T.BackupOptions + }, + pathMaker: T.PathMaker, +): Promise<{ + id: () => Promise + wait: () => Promise + progress: () => Promise +}> { + const { srcVolume, dstVolume, srcPath, dstPath, options } = rsyncOptions + + const command = "rsync" + const args: string[] = [] + if (options.delete) { + args.push("--delete") + } + if (options.force) { + args.push("--force") + } + if (options.ignoreExisting) { + args.push("--ignore-existing") + } + for (const exclude of options.exclude) { + args.push(`--exclude=${exclude}`) + } + args.push("-actAXH") + args.push("--info=progress2") + args.push("--no-inc-recursive") + args.push(pathMaker({ volume: srcVolume, path: srcPath })) + args.push(pathMaker({ volume: dstVolume, path: dstPath })) + const spawned = child_process.spawn(command, args, { detached: true }) + let percentage = 0.0 + spawned.stdout.on("data", (data: unknown) => { + const lines = String(data).replace("\r", "\n").split("\n") + for (const line of lines) { + const parsed = /$([0-9.]+)%/.exec(line)?.[1] + if (!parsed) continue + percentage = Number.parseFloat(parsed) + } + }) + + spawned.stderr.on("data", (data: unknown) => { + console.error(String(data)) + }) + + const id = async () => { + const pid = spawned.pid + if (pid === undefined) { + throw new Error("rsync process has no pid") + } + return String(pid) + } + const waitPromise = new Promise((resolve, reject) => { + spawned.on("exit", (code: any) => { + if (code === 0) { + resolve(null) + } else { + reject(new Error(`rsync exited with code ${code}`)) + } + }) + }) + const wait = () => waitPromise + const progress = () => Promise.resolve(percentage) + return { id, wait, progress } +} diff --git a/sdk/lib/backup/setupBackups.ts b/sdk/lib/backup/setupBackups.ts index af2d08410..69d1bf7df 100644 --- a/sdk/lib/backup/setupBackups.ts +++ b/sdk/lib/backup/setupBackups.ts @@ -1,6 +1,6 @@ import { Backups } from "./Backups" import { SDKManifest } from "../manifest/ManifestTypes" -import { ExpectedExports } from "../types" +import { ExpectedExports, PathMaker } from "../types" import { _ } from "../util" export type SetupBackupsParams = Array< @@ -27,14 +27,14 @@ export function setupBackups( get createBackup() { return (async (options) => { for (const backup of backups) { - await backup.build().createBackup(options) + await backup.build(options.pathMaker).createBackup(options) } }) as ExpectedExports.createBackup }, get restoreBackup() { return (async (options) => { for (const backup of backups) { - await backup.build().restoreBackup(options) + await backup.build(options.pathMaker).restoreBackup(options) } }) as ExpectedExports.restoreBackup }, diff --git a/sdk/lib/config/configTypes.ts b/sdk/lib/config/configTypes.ts index 14d857433..0179e531e 100644 --- a/sdk/lib/config/configTypes.ts +++ b/sdk/lib/config/configTypes.ts @@ -240,7 +240,6 @@ export type ListValueSpecText = { inputmode: "text" | "email" | "tel" | "url" placeholder: string | null } - export type ListValueSpecObject = { type: "object" /** this is a mapped type of the config object at this level, replacing the object's values with specs on those values */ diff --git a/sdk/lib/manifest/ManifestTypes.ts b/sdk/lib/manifest/ManifestTypes.ts index c820930c8..8e6b572d3 100644 --- a/sdk/lib/manifest/ManifestTypes.ts +++ b/sdk/lib/manifest/ManifestTypes.ts @@ -1,7 +1,7 @@ import { ValidEmVer } from "../emverLite/mod" import { ActionMetadata, ImageConfig, ImageId } from "../types" -export interface Container { +export type Container = { /** This should be pointing to a docker container name */ image: string /** These should match the manifest data volumes */ @@ -72,7 +72,7 @@ export type SDKManifest = { readonly dependencies: Readonly> } -export interface ManifestDependency { +export type ManifestDependency = { /** * A human readable explanation on what the dependency is used for */ diff --git a/sdk/lib/types.ts b/sdk/lib/types.ts index 1f1245adc..686cb4750 100644 --- a/sdk/lib/types.ts +++ b/sdk/lib/types.ts @@ -26,6 +26,7 @@ export * from "./osBindings" export { SDKManifest } from "./manifest/ManifestTypes" export { HealthReceipt } from "./health/HealthReceipt" +export type PathMaker = (options: { volume: string; path: string }) => string export type ExportedAction = (options: { effects: Effects input?: Record @@ -43,10 +44,14 @@ export namespace ExpectedExports { // /** These are how we make sure the our dependency configurations are valid and if not how to fix them. */ // export type dependencies = Dependencies; /** For backing up service data though the startOS UI */ - export type createBackup = (options: { effects: Effects }) => Promise + export type createBackup = (options: { + effects: Effects + pathMaker: PathMaker + }) => Promise /** For restoring service data that was previously backed up using the startOS UI create backup flow. Backup restores are also triggered via the startOS UI, or doing a system restore flow during setup. */ export type restoreBackup = (options: { effects: Effects + pathMaker: PathMaker }) => Promise // /** Health checks are used to determine if the service is working properly after starting diff --git a/web/projects/ui/src/app/services/api/api.fixures.ts b/web/projects/ui/src/app/services/api/api.fixures.ts index a1f81fd95..c1f98dcd4 100644 --- a/web/projects/ui/src/app/services/api/api.fixures.ts +++ b/web/projects/ui/src/app/services/api/api.fixures.ts @@ -22,6 +22,7 @@ export module Mock { headline: 'Our biggest release ever.', releaseNotes: { '0.3.6': 'Some **Markdown** release _notes_ for 0.3.6', + '0.3.5.2': 'Some **Markdown** release _notes_ for 0.3.5.2', '0.3.5.1': 'Some **Markdown** release _notes_ for 0.3.5.1', '0.3.4.4': 'Some **Markdown** release _notes_ for 0.3.4.4', '0.3.4.3': 'Some **Markdown** release _notes_ for 0.3.4.3',