From 22b273b145ff04717ea177ca5e6f1f9d907b8f9a Mon Sep 17 00:00:00 2001 From: Aiden McClelland <3732071+dr-bonez@users.noreply.github.com> Date: Thu, 10 Nov 2022 10:20:26 -0700 Subject: [PATCH] fix migration to support portable fatties (#1935) * load docker images directly from s9pk to ensure fatties can be loaded across platform * don't migrate tmpdir * init after package data transfer * set default rsync options --- backend/src/init.rs | 4 +-- backend/src/install/cleanup.rs | 12 +------ backend/src/install/mod.rs | 63 ++++++++++++---------------------- backend/src/setup.rs | 5 ++- backend/src/update/mod.rs | 1 + libs/helpers/src/lib.rs | 4 +++ libs/helpers/src/rsync.rs | 11 +++++- 7 files changed, 44 insertions(+), 56 deletions(-) diff --git a/backend/src/init.rs b/backend/src/init.rs index 797415c51..43e3bdd35 100644 --- a/backend/src/init.rs +++ b/backend/src/init.rs @@ -10,7 +10,7 @@ use tokio::process::Command; use crate::context::rpc::RpcContextConfig; use crate::db::model::ServerStatus; -use crate::install::PKG_DOCKER_DIR; +use crate::install::PKG_ARCHIVE_DIR; use crate::sound::CIRCLE_OF_5THS_SHORT; use crate::util::Invoke; use crate::Error; @@ -292,7 +292,7 @@ pub async fn init(cfg: &RpcContextConfig) -> Result { tracing::info!("Loaded System Docker Images"); tracing::info!("Loading Package Docker Images"); - crate::install::load_images(cfg.datadir().join(PKG_DOCKER_DIR)).await?; + crate::install::load_images(cfg.datadir().join(PKG_ARCHIVE_DIR)).await?; tracing::info!("Loaded Package Docker Images"); } diff --git a/backend/src/install/cleanup.rs b/backend/src/install/cleanup.rs index bb815a891..c65187361 100644 --- a/backend/src/install/cleanup.rs +++ b/backend/src/install/cleanup.rs @@ -5,7 +5,7 @@ use patch_db::{DbHandle, LockReceipt, LockTargetId, LockType, PatchDbHandle, Ver use sqlx::{Executor, Postgres}; use tracing::instrument; -use super::{PKG_ARCHIVE_DIR, PKG_DOCKER_DIR}; +use super::PKG_ARCHIVE_DIR; use crate::config::{not_found, ConfigReceipts}; use crate::context::RpcContext; use crate::db::model::{ @@ -145,16 +145,6 @@ pub async fn cleanup(ctx: &RpcContext, id: &PackageId, version: &Version) -> Res .await .apply(|res| errors.handle(res)); } - let docker_path = ctx - .datadir - .join(PKG_DOCKER_DIR) - .join(id) - .join(version.as_str()); - if tokio::fs::metadata(&docker_path).await.is_ok() { - tokio::fs::remove_dir_all(&docker_path) - .await - .apply(|res| errors.handle(res)); - } let assets_path = asset_dir(&ctx.datadir, id, version); if tokio::fs::metadata(&assets_path).await.is_ok() { tokio::fs::remove_dir_all(&assets_path) diff --git a/backend/src/install/mod.rs b/backend/src/install/mod.rs index acda3dcc1..7913e4c20 100644 --- a/backend/src/install/mod.rs +++ b/backend/src/install/mod.rs @@ -56,7 +56,6 @@ pub mod update; pub const PKG_ARCHIVE_DIR: &str = "package-data/archive"; pub const PKG_PUBLIC_DIR: &str = "package-data/public"; -pub const PKG_DOCKER_DIR: &str = "package-data/docker"; pub const PKG_WASM_DIR: &str = "package-data/wasm"; #[command(display(display_serializable))] @@ -1014,44 +1013,11 @@ pub async fn install_s9pk( tracing::info!("Install {}@{}: Unpacking Docker Images", pkg_id, version); progress .track_read_during(progress_model.clone(), &ctx.db, || async { - let image_tar_dir = ctx - .datadir - .join(PKG_DOCKER_DIR) - .join(pkg_id) - .join(version.as_str()); - if tokio::fs::metadata(&image_tar_dir).await.is_err() { - tokio::fs::create_dir_all(&image_tar_dir) - .await - .with_ctx(|_| { - ( - crate::ErrorKind::Filesystem, - image_tar_dir.display().to_string(), - ) - })?; - } - let image_tar_path = image_tar_dir.join("image.tar"); - let mut tee = Command::new("tee") - .arg(&image_tar_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .spawn()?; let mut load = Command::new("docker") .arg("load") .stdin(Stdio::piped()) .stderr(Stdio::piped()) .spawn()?; - let tee_in = tee.stdin.take().ok_or_else(|| { - Error::new( - eyre!("Could not write to stdin of tee"), - crate::ErrorKind::Docker, - ) - })?; - let mut tee_out = tee.stdout.take().ok_or_else(|| { - Error::new( - eyre!("Could not read from stdout of tee"), - crate::ErrorKind::Docker, - ) - })?; let load_in = load.stdin.take().ok_or_else(|| { Error::new( eyre!("Could not write to stdin of docker load"), @@ -1059,10 +1025,7 @@ pub async fn install_s9pk( ) })?; let mut docker_rdr = rdr.docker_images().await?; - tokio::try_join!( - copy_and_shutdown(&mut docker_rdr, tee_in), - copy_and_shutdown(&mut tee_out, load_in), - )?; + copy_and_shutdown(&mut docker_rdr, load_in).await?; let res = load.wait_with_output().await?; if !res.status.success() { Err(Error::new( @@ -1435,7 +1398,9 @@ pub fn load_images<'a, P: AsRef + 'a + Send + Sync>( .try_for_each(|entry| async move { let m = entry.metadata().await?; if m.is_file() { - if entry.path().extension().and_then(|ext| ext.to_str()) == Some("tar") { + let path = entry.path(); + let ext = path.extension().and_then(|ext| ext.to_str()); + if ext == Some("tar") || ext == Some("s9pk") { let mut load = Command::new("docker") .arg("load") .stdin(Stdio::piped()) @@ -1447,8 +1412,24 @@ pub fn load_images<'a, P: AsRef + 'a + Send + Sync>( crate::ErrorKind::Docker, ) })?; - let mut docker_rdr = File::open(&entry.path()).await?; - copy_and_shutdown(&mut docker_rdr, load_in).await?; + match ext { + Some("tar") => { + copy_and_shutdown(&mut File::open(&path).await?, load_in) + .await? + } + Some("s9pk") => { + copy_and_shutdown( + &mut S9pkReader::open(&path, false) + .await? + .docker_images() + .await?, + load_in, + ) + .await? + } + _ => unreachable!(), + }; + let res = load.wait_with_output().await?; if !res.status.success() { Err(Error::new( diff --git a/backend/src/setup.rs b/backend/src/setup.rs index c4a74cecf..e8373363a 100644 --- a/backend/src/setup.rs +++ b/backend/src/setup.rs @@ -81,7 +81,6 @@ async fn setup_init( ctx: &SetupContext, password: Option, ) -> Result<(Hostname, OnionAddressV3, X509), Error> { - init(&RpcContextConfig::load(ctx.config_path.clone()).await?).await?; let secrets = ctx.secret_store().await?; let db = ctx.db(&secrets).await?; let mut secrets_handle = secrets.acquire().await?; @@ -159,6 +158,7 @@ pub async fn attach( )); } let (hostname, tor_addr, root_ca) = setup_init(&ctx, password).await?; + init(&RpcContextConfig::load(ctx.config_path.clone()).await?).await?; let setup_result = SetupResult { tor_address: format!("http://{}", tor_addr), lan_address: hostname.lan_address(), @@ -410,6 +410,7 @@ pub async fn execute_inner( delete: true, force: true, ignore_existing: false, + exclude: Vec::new(), }, )? .wait() @@ -429,6 +430,7 @@ pub async fn execute_inner( delete: true, force: true, ignore_existing: false, + exclude: vec!["tmp".to_owned()], }, )?; *ctx.recovery_status.write().await = Some(Ok(RecoveryStatus { @@ -448,6 +450,7 @@ pub async fn execute_inner( })); } package_data_transfer.wait().await?; + init(&RpcContextConfig::load(ctx.config_path.clone()).await?).await?; Ok::<_, Error>(()) } .and_then(|_| async { diff --git a/backend/src/update/mod.rs b/backend/src/update/mod.rs index 5c9435606..b28a775b5 100644 --- a/backend/src/update/mod.rs +++ b/backend/src/update/mod.rs @@ -320,6 +320,7 @@ async fn sync_boot() -> Result<(), Error> { delete: false, force: false, ignore_existing: true, + exclude: Vec::new(), }, )? .wait() diff --git a/libs/helpers/src/lib.rs b/libs/helpers/src/lib.rs index 30336f258..9dcd370a2 100644 --- a/libs/helpers/src/lib.rs +++ b/libs/helpers/src/lib.rs @@ -16,6 +16,10 @@ pub use byte_replacement_reader::*; pub use rsync::*; pub use script_dir::*; +pub fn const_true() -> bool { + true +} + pub fn to_tmp_path(path: impl AsRef) -> Result { let path = path.as_ref(); if let (Some(parent), Some(file_name)) = diff --git a/libs/helpers/src/rsync.rs b/libs/helpers/src/rsync.rs index 805138d0d..40a76eb6e 100644 --- a/libs/helpers/src/rsync.rs +++ b/libs/helpers/src/rsync.rs @@ -1,7 +1,7 @@ use color_eyre::eyre::eyre; use std::path::Path; -use crate::{ByteReplacementReader, NonDetachingJoinHandle}; +use crate::{const_true, ByteReplacementReader, NonDetachingJoinHandle}; use models::{Error, ErrorKind}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; use tokio::process::{Child, Command}; @@ -11,9 +11,14 @@ use tokio_stream::wrappers::WatchStream; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "camelCase")] pub struct RsyncOptions { + #[serde(default = "const_true")] pub delete: bool, + #[serde(default = "const_true")] pub force: bool, + #[serde(default)] pub ignore_existing: bool, + #[serde(default)] + pub exclude: Vec, } impl Default for RsyncOptions { fn default() -> Self { @@ -21,6 +26,7 @@ impl Default for RsyncOptions { delete: true, force: true, ignore_existing: false, + exclude: Vec::new(), } } } @@ -47,6 +53,9 @@ impl Rsync { if options.ignore_existing { cmd.arg("--ignore-existing"); } + for exclude in options.exclude { + cmd.arg(format!("--exclude={}", exclude)); + } let mut command = cmd .arg("-ac") .arg("--info=progress2")