From 7ffb462355cbeb81fddab65a55f057c04b48e052 Mon Sep 17 00:00:00 2001 From: Matt Hill Date: Sun, 22 Mar 2026 19:49:58 -0600 Subject: [PATCH] better smtp and backups for postgres and mysql --- sdk/CHANGELOG.md | 8 + .../lib/actions/input/inputSpecConstants.ts | 13 + sdk/package/lib/StartSdk.ts | 12 + sdk/package/lib/backup/Backups.ts | 396 ++++++++++++++++++ sdk/package/lib/index.ts | 1 + sdk/package/package-lock.json | 4 +- sdk/package/package.json | 2 +- 7 files changed, 433 insertions(+), 3 deletions(-) diff --git a/sdk/CHANGELOG.md b/sdk/CHANGELOG.md index 3285ef34f..ef30ea57c 100644 --- a/sdk/CHANGELOG.md +++ b/sdk/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.4.0-beta.64 (2026-03-22) + +### Added + +- `Backups.withPgDump()`: dump-based PostgreSQL backup using `pg_dump`/`pg_restore`, replacing raw volume rsync of PG data directories +- `Backups.withMysqlDump()`: dump-based MySQL/MariaDB backup using `mysqldump`/`mysql` +- Password configs accept `string | (() => string | Promise)` for deferred resolution during restore + ## 0.4.0-beta.63 — StartOS v0.4.0-alpha.22 (2026-03-22) ### Fixed diff --git a/sdk/base/lib/actions/input/inputSpecConstants.ts b/sdk/base/lib/actions/input/inputSpecConstants.ts index 8fe4eb098..633d8e338 100644 --- a/sdk/base/lib/actions/input/inputSpecConstants.ts +++ b/sdk/base/lib/actions/input/inputSpecConstants.ts @@ -267,3 +267,16 @@ export const smtpShape: z.ZodType = z }), ]) .catch({ selection: 'disabled' as const, value: {} }) as any + +/** + * Convert a stored SmtpSelection to a value suitable for prefilling smtpInputSpec. + * + * The stored type (SmtpSelection from smtpShape) uses flat unions for provider/security + * selection, while the input spec (smtpInputSpec) uses distributed discriminated unions + * (UnionRes). These are structurally incompatible in TypeScript's type system, even + * though the runtime values are identical. This function bridges the two types so that + * service code doesn't need `as any`. + */ +export function smtpPrefill(smtp: SmtpSelection | null | undefined): any { + return smtp || undefined +} diff --git a/sdk/package/lib/StartSdk.ts b/sdk/package/lib/StartSdk.ts index 9df7164fa..f70e775bf 100644 --- a/sdk/package/lib/StartSdk.ts +++ b/sdk/package/lib/StartSdk.ts @@ -712,6 +712,18 @@ export class StartSdk { * @param options - Partial sync options to override defaults */ withOptions: Backups.withOptions, + /** + * Create a Backups configuration that uses pg_dump/pg_restore instead of + * rsyncing the raw PostgreSQL data directory. Chain `.addVolume()` to include + * additional volumes in the backup. + */ + withPgDump: Backups.withPgDump, + /** + * Create a Backups configuration that uses mysqldump/mysql instead of + * rsyncing the raw MySQL/MariaDB data directory. Chain `.addVolume()` to + * include additional volumes in the backup. + */ + withMysqlDump: Backups.withMysqlDump, }, InputSpec: { /** diff --git a/sdk/package/lib/backup/Backups.ts b/sdk/package/lib/backup/Backups.ts index e137c657e..335c8482f 100644 --- a/sdk/package/lib/backup/Backups.ts +++ b/sdk/package/lib/backup/Backups.ts @@ -3,6 +3,63 @@ import * as child_process from 'child_process' import * as fs from 'fs/promises' import { Affine, asError } from '../util' import { InitKind, InitScript } from '../../../base/lib/inits' +import { SubContainerRc, execFile } from '../util/SubContainer' +import { Mounts } from '../mainFn/Mounts' + +const BACKUP_HOST_PATH = '/media/startos/backup' +const BACKUP_CONTAINER_MOUNT = '/backup-target' + +/** A password value, or a function that returns one. Functions are resolved lazily (only during restore). */ +export type LazyPassword = string | (() => string | Promise) + +async function resolvePassword(pw: LazyPassword): Promise { + return typeof pw === 'function' ? pw() : pw +} + +/** Configuration for PostgreSQL dump-based backup */ +export type PgDumpConfig = { + /** Image ID of the PostgreSQL container (e.g. 'postgres') */ + imageId: keyof M['images'] & T.ImageId + /** Volume ID containing the PostgreSQL data directory */ + dbVolume: M['volumes'][number] + /** Path to PGDATA within the container (e.g. '/var/lib/postgresql/data') */ + pgdata: string + /** PostgreSQL database name to dump */ + database: string + /** PostgreSQL user */ + user: string + /** PostgreSQL password (for restore). Can be a string or a function that returns one — functions are resolved lazily after volumes are restored. */ + password: LazyPassword + /** Additional initdb arguments (e.g. ['--data-checksums']) */ + initdbArgs?: string[] +} + +/** Configuration for MySQL/MariaDB dump-based backup */ +export type MysqlDumpConfig = { + /** Image ID of the MySQL/MariaDB container (e.g. 'mysql', 'mariadb') */ + imageId: keyof M['images'] & T.ImageId + /** Volume ID containing the MySQL data directory */ + dbVolume: M['volumes'][number] + /** Path to MySQL data directory within the container (typically '/var/lib/mysql') */ + datadir: string + /** MySQL database name to dump */ + database: string + /** MySQL user for dump operations */ + user: string + /** MySQL password. Can be a string or a function that returns one — functions are resolved lazily after volumes are restored. */ + password: LazyPassword + /** Database engine: 'mysql' uses --initialize-insecure, 'mariadb' uses mysql_install_db */ + engine: 'mysql' | 'mariadb' + /** Custom readiness check command (default: ['mysqladmin', 'ping', ...]) */ + readyCommand?: string[] +} + +/** Bind-mount the backup target into a SubContainer's rootfs */ +async function mountBackupTarget(rootfs: string) { + const target = `${rootfs}${BACKUP_CONTAINER_MOUNT}` + await fs.mkdir(target, { recursive: true }) + await execFile('mount', ['--rbind', BACKUP_HOST_PATH, target]) +} /** Default rsync options used for backup and restore operations */ export const DEFAULT_OPTIONS: T.SyncOptions = { @@ -79,6 +136,345 @@ export class Backups implements InitScript { return new Backups({ ...DEFAULT_OPTIONS, ...options }) } + /** + * Configure PostgreSQL dump-based backup for a volume. + * + * Instead of rsyncing the raw PostgreSQL data directory (which is slow and error-prone), + * this uses `pg_dump` to create a logical dump before backup and `pg_restore` to rebuild + * the database after restore. + * + * The dump file is written directly to the backup target — no data duplication on disk. + * + * @returns A configured Backups instance with pre/post hooks. Chain `.addVolume()` or + * `.addSync()` to include additional volumes/paths in the backup. + */ + static withPgDump( + config: PgDumpConfig, + ): Backups { + const { + imageId, + dbVolume, + pgdata, + database, + user, + password, + initdbArgs = [], + } = config + const dumpFile = `${BACKUP_CONTAINER_MOUNT}/${database}-db.dump` + const pgMountpoint = pgdata.replace(/\/data$/, '') || pgdata + + function dbMounts() { + return Mounts.of().mountVolume({ + volumeId: dbVolume, + mountpoint: pgMountpoint, + readonly: false, + subpath: null, + }) + } + + async function startPg( + sub: { + exec(cmd: string[], opts?: any): Promise<{ exitCode: number | null }> + execFail( + cmd: string[], + opts?: any, + timeout?: number | null, + ): Promise + }, + label: string, + ) { + await sub.exec(['rm', '-f', `${pgdata}/postmaster.pid`], { + user: 'postgres', + }) + await sub.exec(['mkdir', '-p', '/var/run/postgresql'], { + user: 'root', + }) + await sub.exec(['chown', 'postgres:postgres', '/var/run/postgresql'], { + user: 'root', + }) + console.log(`[${label}] starting postgres`) + await sub.execFail( + ['pg_ctl', 'start', '-D', pgdata, '-o', '-c listen_addresses='], + { user: 'postgres' }, + ) + for (let i = 0; i < 60; i++) { + const { exitCode } = await sub.exec(['pg_isready', '-U', user], { + user: 'postgres', + }) + if (exitCode === 0) { + console.log(`[${label}] postgres is ready`) + return + } + await new Promise((r) => setTimeout(r, 1000)) + } + throw new Error('PostgreSQL failed to become ready within 60 seconds') + } + + return new Backups() + .setPreBackup(async (effects) => { + await SubContainerRc.withTemp( + effects, + { imageId }, + dbMounts() as any, + 'pg-dump', + async (sub) => { + console.log('[pg-dump] mounting backup target') + await mountBackupTarget(sub.rootfs) + await startPg(sub, 'pg-dump') + console.log('[pg-dump] dumping database') + await sub.execFail( + ['pg_dump', '-U', user, '-Fc', '-f', dumpFile, database], + { user: 'postgres' }, + null, + ) + console.log('[pg-dump] stopping postgres') + await sub.execFail(['pg_ctl', 'stop', '-D', pgdata, '-w'], { + user: 'postgres', + }) + console.log('[pg-dump] complete') + }, + ) + }) + .setPostRestore(async (effects) => { + const resolvedPassword = await resolvePassword(password) + await SubContainerRc.withTemp( + effects, + { imageId }, + dbMounts() as any, + 'pg-restore', + async (sub) => { + await mountBackupTarget(sub.rootfs) + await sub.execFail( + ['initdb', '-D', pgdata, '-U', user, ...initdbArgs], + { user: 'postgres' }, + ) + await startPg(sub, 'pg-restore') + await sub.execFail(['createdb', '-U', user, database], { + user: 'postgres', + }) + await sub.execFail( + [ + 'pg_restore', + '-U', + user, + '-d', + database, + '--no-owner', + dumpFile, + ], + { user: 'postgres' }, + null, + ) + await sub.execFail( + [ + 'psql', + '-U', + user, + '-d', + database, + '-c', + `ALTER USER ${user} WITH PASSWORD '${resolvedPassword}'`, + ], + { user: 'postgres' }, + ) + await sub.execFail(['pg_ctl', 'stop', '-D', pgdata, '-w'], { + user: 'postgres', + }) + }, + ) + }) + } + + /** + * Configure MySQL/MariaDB dump-based backup for a volume. + * + * Instead of rsyncing the raw MySQL data directory (which is slow and error-prone), + * this uses `mysqldump` to create a logical dump before backup and `mysql` to restore + * the database after restore. + * + * The dump file is stored temporarily in `dumpVolume` during backup and cleaned up afterward. + * + * @returns A configured Backups instance with pre/post hooks. Chain `.addVolume()` or + * `.addSync()` to include additional volumes/paths in the backup. + */ + static withMysqlDump( + config: MysqlDumpConfig, + ): Backups { + const { + imageId, + dbVolume, + datadir, + database, + user, + password, + engine, + readyCommand, + } = config + const dumpFile = `${BACKUP_CONTAINER_MOUNT}/${database}-db.dump` + + function dbMounts() { + return Mounts.of().mountVolume({ + volumeId: dbVolume, + mountpoint: datadir, + readonly: false, + subpath: null, + }) + } + + async function waitForMysql( + sub: { exec(cmd: string[]): Promise<{ exitCode: number | null }> }, + cmd: string[], + ) { + for (let i = 0; i < 30; i++) { + const { exitCode } = await sub.exec(cmd) + if (exitCode === 0) return + await new Promise((r) => setTimeout(r, 1000)) + } + throw new Error('MySQL/MariaDB failed to become ready within 30 seconds') + } + + return new Backups() + .setPreBackup(async (effects) => { + const pw = await resolvePassword(password) + const readyCmd = readyCommand || [ + 'mysqladmin', + 'ping', + '-u', + user, + `-p${pw}`, + '--silent', + ] + await SubContainerRc.withTemp( + effects, + { imageId }, + dbMounts() as any, + 'mysql-dump', + async (sub) => { + await mountBackupTarget(sub.rootfs) + await sub.exec(['mkdir', '-p', '/var/run/mysqld'], { + user: 'root', + }) + await sub.exec(['chown', 'mysql:mysql', '/var/run/mysqld'], { + user: 'root', + }) + if (engine === 'mysql') { + await sub.execFail(['chown', '-R', 'mysql:mysql', datadir], { + user: 'root', + }) + } + await sub.execFail( + [ + 'mysqld', + '--user=mysql', + `--datadir=${datadir}`, + '--skip-networking', + '--daemonize', + ], + { user: 'root' }, + null, + ) + await waitForMysql(sub, readyCmd) + await sub.execFail( + [ + 'mysqldump', + '-u', + user, + `-p${pw}`, + '--single-transaction', + `--result-file=${dumpFile}`, + database, + ], + { user: 'root' }, + null, + ) + await sub.execFail( + ['mysqladmin', '-u', user, `-p${pw}`, 'shutdown'], + { user: 'root' }, + ) + }, + ) + }) + .setPostRestore(async (effects) => { + const pw = await resolvePassword(password) + await SubContainerRc.withTemp( + effects, + { imageId }, + dbMounts() as any, + 'mysql-restore', + async (sub) => { + await mountBackupTarget(sub.rootfs) + await sub.exec(['mkdir', '-p', '/var/run/mysqld'], { + user: 'root', + }) + await sub.exec(['chown', 'mysql:mysql', '/var/run/mysqld'], { + user: 'root', + }) + // Initialize fresh data directory + if (engine === 'mariadb') { + await sub.execFail( + ['mysql_install_db', '--user=mysql', `--datadir=${datadir}`], + { user: 'root' }, + ) + } else { + await sub.execFail( + [ + 'mysqld', + '--initialize-insecure', + '--user=mysql', + `--datadir=${datadir}`, + ], + { user: 'root' }, + ) + } + await sub.execFail( + [ + 'mysqld', + '--user=mysql', + `--datadir=${datadir}`, + '--skip-networking', + '--daemonize', + ], + { user: 'root' }, + null, + ) + // After fresh init, root has no password + await waitForMysql(sub, [ + 'mysqladmin', + 'ping', + '-u', + 'root', + '--silent', + ]) + // Create database, user, and set password + await sub.execFail( + [ + 'mysql', + '-u', + 'root', + '-e', + `CREATE DATABASE IF NOT EXISTS \`${database}\`; CREATE USER IF NOT EXISTS '${user}'@'localhost' IDENTIFIED BY '${pw}'; GRANT ALL ON \`${database}\`.* TO '${user}'@'localhost'; ALTER USER 'root'@'localhost' IDENTIFIED BY '${pw}'; FLUSH PRIVILEGES;`, + ], + { user: 'root' }, + ) + // Restore from dump + await sub.execFail( + [ + 'sh', + '-c', + `mysql -u root -p'${pw}' \`${database}\` < ${dumpFile}`, + ], + { user: 'root' }, + null, + ) + await sub.execFail( + ['mysqladmin', '-u', 'root', `-p${password}`, 'shutdown'], + { user: 'root' }, + ) + }, + ) + }) + } + /** * Override the default rsync options for both backup and restore. * @param options - Partial rsync options to merge with current defaults diff --git a/sdk/package/lib/index.ts b/sdk/package/lib/index.ts index 06f99b915..879aa978b 100644 --- a/sdk/package/lib/index.ts +++ b/sdk/package/lib/index.ts @@ -32,6 +32,7 @@ export { setupManifest, buildManifest } from './manifest/setupManifest' export { FileHelper } from './util/fileHelper' export { smtpShape, + smtpPrefill, type SmtpSelection, } from '../../base/lib/actions/input/inputSpecConstants' diff --git a/sdk/package/package-lock.json b/sdk/package/package-lock.json index fa173dd37..a86d64a29 100644 --- a/sdk/package/package-lock.json +++ b/sdk/package/package-lock.json @@ -1,12 +1,12 @@ { "name": "@start9labs/start-sdk", - "version": "0.4.0-beta.63", + "version": "0.4.0-beta.64", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@start9labs/start-sdk", - "version": "0.4.0-beta.63", + "version": "0.4.0-beta.64", "license": "MIT", "dependencies": { "@iarna/toml": "^3.0.0", diff --git a/sdk/package/package.json b/sdk/package/package.json index 0870f5259..4680a58db 100644 --- a/sdk/package/package.json +++ b/sdk/package/package.json @@ -1,6 +1,6 @@ { "name": "@start9labs/start-sdk", - "version": "0.4.0-beta.63", + "version": "0.4.0-beta.64", "description": "Software development kit to facilitate packaging services for StartOS", "main": "./package/lib/index.js", "types": "./package/lib/index.d.ts",