diff --git a/backend/src/context/rpc.rs b/backend/src/context/rpc.rs index dadef1d8e..c2394fea9 100644 --- a/backend/src/context/rpc.rs +++ b/backend/src/context/rpc.rs @@ -41,6 +41,8 @@ use crate::{Error, ErrorKind, ResultExt}; #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct RpcContextConfig { + pub migration_batch_rows: Option, + pub migration_prefetch_rows: Option, pub bind_rpc: Option, pub bind_ws: Option, pub bind_static: Option, @@ -100,14 +102,12 @@ impl RpcContextConfig { .with_kind(crate::ErrorKind::Database)?; let old_db_path = self.datadir().join("main/secrets.db"); if tokio::fs::metadata(&old_db_path).await.is_ok() { - let mut res = Ok(()); - for _ in 0..5 { - res = pgloader(&old_db_path).await; - if res.is_ok() { - break; - } - } - res? + pgloader( + &old_db_path, + self.migration_batch_rows.unwrap_or(25000), + self.migration_prefetch_rows.unwrap_or(100_000), + ) + .await?; } Ok(secret_store) } diff --git a/backend/src/context/setup.rs b/backend/src/context/setup.rs index 5b19f9e95..005ee471b 100644 --- a/backend/src/context/setup.rs +++ b/backend/src/context/setup.rs @@ -36,6 +36,8 @@ pub struct SetupResult { #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct SetupContextConfig { + pub migration_batch_rows: Option, + pub migration_prefetch_rows: Option, pub bind_rpc: Option, pub datadir: Option, } @@ -64,6 +66,8 @@ impl SetupContextConfig { pub struct SetupContextSeed { pub config_path: Option, + pub migration_batch_rows: usize, + pub migration_prefetch_rows: usize, pub bind_rpc: SocketAddr, pub shutdown: Sender<()>, pub datadir: PathBuf, @@ -92,6 +96,8 @@ impl SetupContext { let datadir = cfg.datadir().to_owned(); Ok(Self(Arc::new(SetupContextSeed { config_path: path.as_ref().map(|p| p.as_ref().to_owned()), + migration_batch_rows: cfg.migration_batch_rows.unwrap_or(25000), + migration_prefetch_rows: cfg.migration_prefetch_rows.unwrap_or(100_000), bind_rpc: cfg.bind_rpc.unwrap_or(([127, 0, 0, 1], 5959).into()), shutdown, datadir, @@ -141,14 +147,12 @@ impl SetupContext { .with_kind(crate::ErrorKind::Database)?; let old_db_path = self.datadir.join("main/secrets.db"); if tokio::fs::metadata(&old_db_path).await.is_ok() { - let mut res = Ok(()); - for _ in 0..5 { - res = pgloader(&old_db_path).await; - if res.is_ok() { - break; - } - } - res? + pgloader( + &old_db_path, + self.migration_batch_rows, + self.migration_prefetch_rows, + ) + .await?; } Ok(secret_store) } diff --git a/backend/src/init.rs b/backend/src/init.rs index 8c2d5efc7..5d18dd4fc 100644 --- a/backend/src/init.rs +++ b/backend/src/init.rs @@ -75,15 +75,25 @@ impl InitReceipts { } } -pub async fn pgloader(old_db_path: impl AsRef) -> Result<(), Error> { +pub async fn pgloader( + old_db_path: impl AsRef, + batch_rows: usize, + prefetch_rows: usize, +) -> Result<(), Error> { tokio::fs::write( "/etc/embassy/migrate.load", format!( include_str!("migrate.load"), - sqlite_path = old_db_path.as_ref().display() + sqlite_path = old_db_path.as_ref().display(), + batch_rows = batch_rows, + prefetch_rows = prefetch_rows ), ) .await?; + match tokio::fs::remove_dir_all("/tmp/pgloader").await { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + a => a, + }?; tracing::info!("Running pgloader"); let out = Command::new("pgloader") .arg("-v") diff --git a/backend/src/migrate.load b/backend/src/migrate.load index e36d9698a..16e7e7c05 100644 --- a/backend/src/migrate.load +++ b/backend/src/migrate.load @@ -2,6 +2,6 @@ load database from sqlite://{sqlite_path} into postgresql://root@unix:/var/run/postgresql:5432/secrets - with include no drop, truncate, reset sequences, data only + with include no drop, truncate, reset sequences, data only, workers = 1, concurrency = 1, max parallel create index = 1, batch rows = {batch_rows}, prefetch rows = {prefetch_rows} - excluding table names like '_sqlx_migrations'; + excluding table names like '_sqlx_migrations', 'notifications';