Start9/start-os
2
0
Fork 0
mirror of https://github.com/Start9Labs/start-os.git synced 2026-03-26 10:21:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix/backups (#2659)

Browse Source 
* fix master build (#2639)

* feat: Change ts to use rsync
Chore: Update the ts to use types over interface

* feat: Get the rust and the js to do a backup

* Wip: Got the backup working?

* fix permissions

* remove trixie list

* update tokio to fix timer bug

* fix error handling on backup

* wip

* remove idmap

* run restore before init, and init with own version on restore

---------

Co-authored-by: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com>
Co-authored-by: Aiden McClelland <me@drbonez.dev>
This commit is contained in:
Jade
2024-07-17 15:46:27 -06:00
committed by GitHub
parent 95611e9c4b
commit 8f0bdcd172
23 changed files with 445 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

354
core/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

3
core/startos/Cargo.toml
View File

@@ -128,6 +128,7 @@ nix = { version = "0.29.0", features = ["user", "process", "signal", "fs"] }
nom = "7.1.3"
num = "0.4.1"
num_enum = "0.7.0"
num_cpus = "1.16.0"
once_cell = "1.19.0"
openssh-keys = "0.6.2"
openssl = { version = "0.10.57", features = ["vendored"] }
@@ -170,7 +171,7 @@ sscanf = "0.4.1"
ssh-key = { version = "0.6.2", features = ["ed25519"] }
tar = "0.4.40"
thiserror = "1.0.49"
tokio = { version = "1.38.0", features = ["full"] }
tokio = { version = "1.38.1", features = ["full"] }
tokio-rustls = "0.26.0"
tokio-socks = "0.5.1"
tokio-stream = { version = "0.1.14", features = ["io-util", "sync", "net"] }

2
core/startos/src/backup/backup_bulk.rs
View File

@@ -260,7 +260,7 @@ async fn perform_backup(
for id in package_ids {
if let Some(service) = &*ctx.services.get(id).await {
let backup_result = service
.backup(backup_guard.package_backup(id))
.backup(backup_guard.package_backup(id).await?)
.await
.err()
.map(|e| e.to_string());

2
core/startos/src/backup/restore.rs
View File

@@ -158,7 +158,7 @@ async fn restore_packages(
let backup_guard = Arc::new(backup_guard);
let mut tasks = BTreeMap::new();
for id in ids {
let backup_dir = backup_guard.clone().package_backup(&id);
let backup_dir = backup_guard.clone().package_backup(&id).await?;
let s9pk_path = backup_dir.path().join(&id).with_extension("s9pk");
let task = ctx
.services

2
core/startos/src/bins/startd.rs
View File

@@ -1,3 +1,4 @@
use std::cmp::max;
use std::ffi::OsString;
use std::net::{Ipv6Addr, SocketAddr};
use std::sync::Arc;
@@ -136,6 +137,7 @@ pub fn main(args: impl IntoIterator<Item = OsString>) {
let res = {
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(max(4, num_cpus::get()))
.enable_all()
.build()
.expect("failed to initialize runtime");

26
core/startos/src/disk/mount/backup.rs
View File

@@ -106,8 +106,11 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
)
})?;
}
let encrypted_guard =
TmpMountGuard::mount(&BackupFS::new(&crypt_path, &enc_key), ReadWrite).await?;
let encrypted_guard = TmpMountGuard::mount(
&BackupFS::new(&crypt_path, &enc_key, vec![(100000, 65536)]),
ReadWrite,
)
.await?;
let metadata_path = encrypted_guard.path().join("metadata.json");
let metadata: BackupInfo = if tokio::fs::metadata(&metadata_path).await.is_ok() {
@@ -148,8 +151,23 @@ impl<G: GenericMountGuard> BackupMountGuard<G> {
}
#[instrument(skip_all)]
pub fn package_backup(self: &Arc<Self>, id: &PackageId) -> SubPath<Arc<Self>> {
SubPath::new(self.clone(), id)
pub async fn package_backup(
self: &Arc<Self>,
id: &PackageId,
) -> Result<SubPath<Arc<Self>>, Error> {
let package_guard = SubPath::new(self.clone(), id);
let package_path = package_guard.path();
if tokio::fs::metadata(&package_path).await.is_err() {
tokio::fs::create_dir_all(&package_path)
.await
.with_ctx(|_| {
(
crate::ErrorKind::Filesystem,
package_path.display().to_string(),
)
})?;
}
Ok(package_guard)
}
#[instrument(skip_all)]

21
core/startos/src/disk/mount/filesystem/backupfs.rs
View File

@@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::fmt::{self, Display};
use std::os::unix::ffi::OsStrExt;
use std::path::Path;
@@ -12,10 +13,15 @@ use crate::prelude::*;
pub struct BackupFS<DataDir: AsRef<Path>, Password: fmt::Display> {
data_dir: DataDir,
password: Password,
idmapped_root: Vec<(u32, u32)>,
}
impl<DataDir: AsRef<Path>, Password: fmt::Display> BackupFS<DataDir, Password> {
pub fn new(data_dir: DataDir, password: Password) -> Self {
BackupFS { data_dir, password }
pub fn new(data_dir: DataDir, password: Password, idmapped_root: Vec<(u32, u32)>) -> Self {
BackupFS {
data_dir,
password,
idmapped_root,
}
}
}
impl<DataDir: AsRef<Path> + Send + Sync, Password: fmt::Display + Send + Sync> FileSystem
@@ -26,9 +32,16 @@ impl<DataDir: AsRef<Path> + Send + Sync, Password: fmt::Display + Send + Sync> F
}
fn mount_options(&self) -> impl IntoIterator<Item = impl Display> {
[
format!("password={}", self.password),
format!("file-size-padding=0.05"),
Cow::Owned(format!("password={}", self.password)),
Cow::Borrowed("file-size-padding=0.05"),
Cow::Borrowed("allow_other"),
]
.into_iter()
.chain(
self.idmapped_root
.iter()
.map(|(root, range)| Cow::Owned(format!("idmapped-root={root}:{range}"))),
)
}
async fn source(&self) -> Result<Option<impl AsRef<Path>>, Error> {
Ok(Some(&self.data_dir))

76
core/startos/src/service/mod.rs
View File

@@ -34,6 +34,7 @@ use crate::util::actor::concurrent::ConcurrentActor;
use crate::util::actor::Actor;
use crate::util::io::create_file;
use crate::util::serde::Pem;
use crate::util::Never;
use crate::volume::data_dir;
mod action;
@@ -220,12 +221,13 @@ impl Service {
tracing::error!("Error opening s9pk for install: {e}");
tracing::debug!("{e:?}")
}) {
if let Ok(service) = Self::install(ctx.clone(), s9pk, None, None)
.await
.map_err(|e| {
tracing::error!("Error installing service: {e}");
tracing::debug!("{e:?}")
})
if let Ok(service) =
Self::install(ctx.clone(), s9pk, None, None::<Never>, None)
.await
.map_err(|e| {
tracing::error!("Error installing service: {e}");
tracing::debug!("{e:?}")
})
{
return Ok(Some(service));
}
@@ -257,6 +259,7 @@ impl Service {
ctx.clone(),
s9pk,
Some(s.as_manifest().as_version().de()?),
None::<Never>,
None,
)
.await
@@ -334,13 +337,35 @@ impl Service {
pub async fn install(
ctx: RpcContext,
s9pk: S9pk,
src_version: Option<models::VersionString>,
mut src_version: Option<models::VersionString>,
recovery_source: Option<impl GenericMountGuard>,
progress: Option<InstallProgressHandles>,
) -> Result<ServiceRef, Error> {
let manifest = s9pk.as_manifest().clone();
let developer_key = s9pk.as_archive().signer();
let icon = s9pk.icon_data_url().await?;
let service = Self::new(ctx.clone(), s9pk, StartStop::Stop).await?;
if let Some(recovery_source) = recovery_source {
service
.actor
.send(
Guid::new(),
transition::restore::Restore {
path: recovery_source.path().to_path_buf(),
},
)
.await??;
recovery_source.unmount().await?;
src_version = Some(
service
.seed
.persistent_container
.s9pk
.as_manifest()
.version
.clone(),
);
}
service
.seed
.persistent_container
@@ -382,26 +407,6 @@ impl Service {
Ok(service)
}
pub async fn restore(
ctx: RpcContext,
s9pk: S9pk,
backup_source: impl GenericMountGuard,
progress: Option<InstallProgressHandles>,
) -> Result<ServiceRef, Error> {
let service = Service::install(ctx.clone(), s9pk, None, progress).await?;
service
.actor
.send(
Guid::new(),
transition::restore::Restore {
path: backup_source.path().to_path_buf(),
},
)
.await??;
Ok(service)
}
#[instrument(skip_all)]
pub async fn backup(&self, guard: impl GenericMountGuard) -> Result<(), Error> {
let id = &self.seed.id;
@@ -417,10 +422,11 @@ impl Service {
.send(
Guid::new(),
transition::backup::Backup {
path: guard.path().to_path_buf(),
path: guard.path().join("data"),
},
)
.await??;
.await??
.await?;
Ok(())
}
@@ -505,13 +511,21 @@ impl Actor for ServiceActor {
}
(Some(TransitionKind::Restarting), _, _) => MainStatus::Restarting,
(Some(TransitionKind::Restoring), _, _) => MainStatus::Restoring,
(Some(TransitionKind::BackingUp), _, Some(status)) => {
(Some(TransitionKind::BackingUp), StartStop::Stop, Some(status)) => {
seed.persistent_container.stop().await?;
MainStatus::BackingUp {
started: Some(status.started),
health: status.health.clone(),
}
}
(Some(TransitionKind::BackingUp), _, None) => MainStatus::BackingUp {
(Some(TransitionKind::BackingUp), StartStop::Start, _) => {
seed.persistent_container.start().await?;
MainStatus::BackingUp {
started: None,
health: OrdMap::new(),
}
}
(Some(TransitionKind::BackingUp), _, _) => MainStatus::BackingUp {
started: None,
health: OrdMap::new(),
},

12
core/startos/src/service/persistent_container.rs
View File

@@ -1,5 +1,5 @@
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::path::Path;
use std::sync::{Arc, Weak};
use std::time::Duration;
@@ -277,7 +277,7 @@ impl PersistentContainer {
backup_path: impl AsRef<Path>,
mount_type: MountType,
) -> Result<MountGuard, Error> {
let backup_path: PathBuf = backup_path.as_ref().to_path_buf();
let backup_path = backup_path.as_ref();
let mountpoint = self
.lxc_container
.get()
@@ -295,14 +295,14 @@ impl PersistentContainer {
.arg(mountpoint.as_os_str())
.invoke(ErrorKind::Filesystem)
.await?;
let bind = Bind::new(&backup_path);
let mount_guard = MountGuard::mount(&bind, &mountpoint, mount_type).await;
tokio::fs::create_dir_all(backup_path).await?;
Command::new("chown")
.arg("100000:100000")
.arg(backup_path.as_os_str())
.arg(backup_path)
.invoke(ErrorKind::Filesystem)
.await?;
mount_guard
let bind = Bind::new(backup_path);
MountGuard::mount(&bind, &mountpoint, mount_type).await
}
#[instrument(skip_all)]

43
core/startos/src/service/service_map.rs
View File

@@ -265,35 +265,20 @@ impl ServiceMap {
} else {
None
};
if let Some(recovery_source) = recovery_source {
*service = Some(
Service::restore(
ctx,
s9pk,
recovery_source,
Some(InstallProgressHandles {
finalization_progress,
progress,
}),
)
.await?
.into(),
);
} else {
*service = Some(
Service::install(
ctx,
s9pk,
prev,
Some(InstallProgressHandles {
finalization_progress,
progress,
}),
)
.await?
.into(),
);
}
*service = Some(
Service::install(
ctx,
s9pk,
prev,
recovery_source,
Some(InstallProgressHandles {
finalization_progress,
progress,
}),
)
.await?
.into(),
);
drop(service);
sync_progress_task.await.map_err(|_| {

68
core/startos/src/service/transition/backup.rs
View File

@@ -1,5 +1,7 @@
use std::path::PathBuf;
use std::sync::Arc;
use futures::future::BoxFuture;
use futures::FutureExt;
use models::ProcedureName;
@@ -19,7 +21,7 @@ pub(in crate::service) struct Backup {
pub path: PathBuf,
}
impl Handler<Backup> for ServiceActor {
type Response = Result<(), Error>;
type Response = Result<BoxFuture<'static, Result<(), Error>>, Error>;
fn conflicts_with(_: &Backup) -> ConflictBuilder<Self> {
ConflictBuilder::everything()
.except::<GetConfig>()
@@ -37,43 +39,31 @@ impl Handler<Backup> for ServiceActor {
let path = backup.path.clone();
let seed = self.0.clone();
let state = self.0.persistent_container.state.clone();
let transition = RemoteCancellable::new(
async move {
temp.stop();
let transition = RemoteCancellable::new(async move {
temp.stop();
current
.wait_for(|s| s.running_status.is_none())
.await
.with_kind(ErrorKind::Unknown)?;
let backup_guard = seed
.persistent_container
.mount_backup(path, ReadWrite)
.await?;
seed.persistent_container
.execute(id, ProcedureName::CreateBackup, Value::Null, None)
.await?;
backup_guard.unmount(true).await?;
if temp.restore().is_start() {
current
.wait_for(|s| s.running_status.is_none())
.wait_for(|s| s.running_status.is_some())
.await
.with_kind(ErrorKind::Unknown)?;
let backup_guard = seed
.persistent_container
.mount_backup(path, ReadWrite)
.await?;
seed.persistent_container
.execute(id, ProcedureName::CreateBackup, Value::Null, None)
.await?;
backup_guard.unmount(true).await?;
if temp.restore().is_start() {
current
.wait_for(|s| s.running_status.is_some())
.await
.with_kind(ErrorKind::Unknown)?;
}
drop(temp);
state.send_modify(|s| {
s.transition_state.take();
});
Ok::<_, Error>(())
}
.map(|x| {
if let Err(err) = dbg!(x) {
tracing::debug!("{:?}", err);
tracing::warn!("{}", err);
}
}),
);
drop(temp);
Ok::<_, Arc<Error>>(())
});
let cancel_handle = transition.cancellation_handle();
let transition = transition.shared();
let job_transition = transition.clone();
@@ -92,9 +82,11 @@ impl Handler<Backup> for ServiceActor {
if let Some(t) = old {
t.abort().await;
}
match transition.await {
None => Err(Error::new(eyre!("Backup canceled"), ErrorKind::Unknown)),
Some(x) => Ok(x),
}
Ok(transition
.map(|r| {
r.ok_or_else(|| Error::new(eyre!("Backup canceled"), ErrorKind::Cancelled))?
.map_err(|e| e.clone_output())
})
.boxed())
}
}

5
core/startos/src/service/transition/mod.rs
View File

@@ -79,7 +79,10 @@ impl TempDesiredRestore {
}
impl Drop for TempDesiredRestore {
fn drop(&mut self) {
self.0.send_modify(|s| s.temp_desired_state = None);
self.0.send_modify(|s| {
s.temp_desired_state.take();
s.transition_state.take();
});
}
}
// impl Deref for TempDesiredState {

8
core/startos/src/sound.rs
View File

@@ -10,12 +10,12 @@ use crate::util::{FileLock, Invoke};
use crate::{Error, ErrorKind};
lazy_static::lazy_static! {
static ref SEMITONE_K: f64 = 2f64.powf(1f64 / 12f64);
static ref A_4: f64 = 440f64;
static ref C_0: f64 = *A_4 / SEMITONE_K.powf(9f64) / 2f64.powf(4f64);
static ref SEMITONE_K: f64 = 2f64.powf(1.0 / 12.0);
static ref A_4: f64 = 440.0;
static ref C_0: f64 = *A_4 / SEMITONE_K.powf(9.0) / 2_f64.powf(4.0);
}
pub const SOUND_LOCK_FILE: &str = "/etc/embassy/sound.lock";
pub const SOUND_LOCK_FILE: &str = "/run/startos/sound.lock";
struct SoundInterface {
guard: Option<FileLock>,

2
core/startos/src/util/mod.rs
View File

@@ -555,7 +555,7 @@ impl<F: FnOnce() -> T, T> Drop for GeneralGuard<F, T> {
}
}
pub struct FileLock(OwnedMutexGuard<()>, Option<FdLock<File>>);
pub struct FileLock(#[allow(unused)] OwnedMutexGuard<()>, Option<FdLock<File>>);
impl Drop for FileLock {
fn drop(&mut self) {
if let Some(fd_lock) = self.1.take() {