better smtp and backups for postgres and mysql

sdk/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.4.0-beta.64 (2026-03-22)
+
+### Added
+
+- `Backups.withPgDump()`: dump-based PostgreSQL backup using `pg_dump`/`pg_restore`, replacing raw volume rsync of PG data directories
+- `Backups.withMysqlDump()`: dump-based MySQL/MariaDB backup using `mysqldump`/`mysql`
+- Password configs accept `string | (() => string | Promise<string>)` for deferred resolution during restore
+
 ## 0.4.0-beta.63 — StartOS v0.4.0-alpha.22 (2026-03-22)
 
 ### Fixed

sdk/base/lib/actions/input/inputSpecConstants.ts

@@ -267,3 +267,16 @@ export const smtpShape: z.ZodType<SmtpSelection> = z
     }),
   ])
   .catch({ selection: 'disabled' as const, value: {} }) as any
+
+/**
+ * Convert a stored SmtpSelection to a value suitable for prefilling smtpInputSpec.
+ *
+ * The stored type (SmtpSelection from smtpShape) uses flat unions for provider/security
+ * selection, while the input spec (smtpInputSpec) uses distributed discriminated unions
+ * (UnionRes). These are structurally incompatible in TypeScript's type system, even
+ * though the runtime values are identical. This function bridges the two types so that
+ * service code doesn't need `as any`.
+ */
+export function smtpPrefill(smtp: SmtpSelection | null | undefined): any {
+  return smtp || undefined
+}

sdk/package/lib/StartSdk.ts

@@ -712,6 +712,18 @@ export class StartSdk<Manifest extends T.SDKManifest> {
          * @param options - Partial sync options to override defaults
          */
         withOptions: Backups.withOptions<Manifest>,
+        /**
+         * Create a Backups configuration that uses pg_dump/pg_restore instead of
+         * rsyncing the raw PostgreSQL data directory. Chain `.addVolume()` to include
+         * additional volumes in the backup.
+         */
+        withPgDump: Backups.withPgDump<Manifest>,
+        /**
+         * Create a Backups configuration that uses mysqldump/mysql instead of
+         * rsyncing the raw MySQL/MariaDB data directory. Chain `.addVolume()` to
+         * include additional volumes in the backup.
+         */
+        withMysqlDump: Backups.withMysqlDump<Manifest>,
       },
       InputSpec: {
         /**

sdk/package/lib/backup/Backups.ts

@@ -3,6 +3,63 @@ import * as child_process from 'child_process'
 import * as fs from 'fs/promises'
 import { Affine, asError } from '../util'
 import { InitKind, InitScript } from '../../../base/lib/inits'
+import { SubContainerRc, execFile } from '../util/SubContainer'
+import { Mounts } from '../mainFn/Mounts'
+
+const BACKUP_HOST_PATH = '/media/startos/backup'
+const BACKUP_CONTAINER_MOUNT = '/backup-target'
+
+/** A password value, or a function that returns one. Functions are resolved lazily (only during restore). */
+export type LazyPassword = string | (() => string | Promise<string>)
+
+async function resolvePassword(pw: LazyPassword): Promise<string> {
+  return typeof pw === 'function' ? pw() : pw
+}
+
+/** Configuration for PostgreSQL dump-based backup */
+export type PgDumpConfig<M extends T.SDKManifest> = {
+  /** Image ID of the PostgreSQL container (e.g. 'postgres') */
+  imageId: keyof M['images'] & T.ImageId
+  /** Volume ID containing the PostgreSQL data directory */
+  dbVolume: M['volumes'][number]
+  /** Path to PGDATA within the container (e.g. '/var/lib/postgresql/data') */
+  pgdata: string
+  /** PostgreSQL database name to dump */
+  database: string
+  /** PostgreSQL user */
+  user: string
+  /** PostgreSQL password (for restore). Can be a string or a function that returns one — functions are resolved lazily after volumes are restored. */
+  password: LazyPassword
+  /** Additional initdb arguments (e.g. ['--data-checksums']) */
+  initdbArgs?: string[]
+}
+
+/** Configuration for MySQL/MariaDB dump-based backup */
+export type MysqlDumpConfig<M extends T.SDKManifest> = {
+  /** Image ID of the MySQL/MariaDB container (e.g. 'mysql', 'mariadb') */
+  imageId: keyof M['images'] & T.ImageId
+  /** Volume ID containing the MySQL data directory */
+  dbVolume: M['volumes'][number]
+  /** Path to MySQL data directory within the container (typically '/var/lib/mysql') */
+  datadir: string
+  /** MySQL database name to dump */
+  database: string
+  /** MySQL user for dump operations */
+  user: string
+  /** MySQL password. Can be a string or a function that returns one — functions are resolved lazily after volumes are restored. */
+  password: LazyPassword
+  /** Database engine: 'mysql' uses --initialize-insecure, 'mariadb' uses mysql_install_db */
+  engine: 'mysql' | 'mariadb'
+  /** Custom readiness check command (default: ['mysqladmin', 'ping', ...]) */
+  readyCommand?: string[]
+}
+
+/** Bind-mount the backup target into a SubContainer's rootfs */
+async function mountBackupTarget(rootfs: string) {
+  const target = `${rootfs}${BACKUP_CONTAINER_MOUNT}`
+  await fs.mkdir(target, { recursive: true })
+  await execFile('mount', ['--rbind', BACKUP_HOST_PATH, target])
+}
 
 /** Default rsync options used for backup and restore operations */
 export const DEFAULT_OPTIONS: T.SyncOptions = {
@@ -79,6 +136,345 @@ export class Backups<M extends T.SDKManifest> implements InitScript {
     return new Backups<M>({ ...DEFAULT_OPTIONS, ...options })
   }
 
+  /**
+   * Configure PostgreSQL dump-based backup for a volume.
+   *
+   * Instead of rsyncing the raw PostgreSQL data directory (which is slow and error-prone),
+   * this uses `pg_dump` to create a logical dump before backup and `pg_restore` to rebuild
+   * the database after restore.
+   *
+   * The dump file is written directly to the backup target — no data duplication on disk.
+   *
+   * @returns A configured Backups instance with pre/post hooks. Chain `.addVolume()` or
+   * `.addSync()` to include additional volumes/paths in the backup.
+   */
+  static withPgDump<M extends T.SDKManifest = never>(
+    config: PgDumpConfig<M>,
+  ): Backups<M> {
+    const {
+      imageId,
+      dbVolume,
+      pgdata,
+      database,
+      user,
+      password,
+      initdbArgs = [],
+    } = config
+    const dumpFile = `${BACKUP_CONTAINER_MOUNT}/${database}-db.dump`
+    const pgMountpoint = pgdata.replace(/\/data$/, '') || pgdata
+
+    function dbMounts() {
+      return Mounts.of<M>().mountVolume({
+        volumeId: dbVolume,
+        mountpoint: pgMountpoint,
+        readonly: false,
+        subpath: null,
+      })
+    }
+
+    async function startPg(
+      sub: {
+        exec(cmd: string[], opts?: any): Promise<{ exitCode: number | null }>
+        execFail(
+          cmd: string[],
+          opts?: any,
+          timeout?: number | null,
+        ): Promise<any>
+      },
+      label: string,
+    ) {
+      await sub.exec(['rm', '-f', `${pgdata}/postmaster.pid`], {
+        user: 'postgres',
+      })
+      await sub.exec(['mkdir', '-p', '/var/run/postgresql'], {
+        user: 'root',
+      })
+      await sub.exec(['chown', 'postgres:postgres', '/var/run/postgresql'], {
+        user: 'root',
+      })
+      console.log(`[${label}] starting postgres`)
+      await sub.execFail(
+        ['pg_ctl', 'start', '-D', pgdata, '-o', '-c listen_addresses='],
+        { user: 'postgres' },
+      )
+      for (let i = 0; i < 60; i++) {
+        const { exitCode } = await sub.exec(['pg_isready', '-U', user], {
+          user: 'postgres',
+        })
+        if (exitCode === 0) {
+          console.log(`[${label}] postgres is ready`)
+          return
+        }
+        await new Promise((r) => setTimeout(r, 1000))
+      }
+      throw new Error('PostgreSQL failed to become ready within 60 seconds')
+    }
+
+    return new Backups<M>()
+      .setPreBackup(async (effects) => {
+        await SubContainerRc.withTemp<M, void, BackupEffects>(
+          effects,
+          { imageId },
+          dbMounts() as any,
+          'pg-dump',
+          async (sub) => {
+            console.log('[pg-dump] mounting backup target')
+            await mountBackupTarget(sub.rootfs)
+            await startPg(sub, 'pg-dump')
+            console.log('[pg-dump] dumping database')
+            await sub.execFail(
+              ['pg_dump', '-U', user, '-Fc', '-f', dumpFile, database],
+              { user: 'postgres' },
+              null,
+            )
+            console.log('[pg-dump] stopping postgres')
+            await sub.execFail(['pg_ctl', 'stop', '-D', pgdata, '-w'], {
+              user: 'postgres',
+            })
+            console.log('[pg-dump] complete')
+          },
+        )
+      })
+      .setPostRestore(async (effects) => {
+        const resolvedPassword = await resolvePassword(password)
+        await SubContainerRc.withTemp<M, void, BackupEffects>(
+          effects,
+          { imageId },
+          dbMounts() as any,
+          'pg-restore',
+          async (sub) => {
+            await mountBackupTarget(sub.rootfs)
+            await sub.execFail(
+              ['initdb', '-D', pgdata, '-U', user, ...initdbArgs],
+              { user: 'postgres' },
+            )
+            await startPg(sub, 'pg-restore')
+            await sub.execFail(['createdb', '-U', user, database], {
+              user: 'postgres',
+            })
+            await sub.execFail(
+              [
+                'pg_restore',
+                '-U',
+                user,
+                '-d',
+                database,
+                '--no-owner',
+                dumpFile,
+              ],
+              { user: 'postgres' },
+              null,
+            )
+            await sub.execFail(
+              [
+                'psql',
+                '-U',
+                user,
+                '-d',
+                database,
+                '-c',
+                `ALTER USER ${user} WITH PASSWORD '${resolvedPassword}'`,
+              ],
+              { user: 'postgres' },
+            )
+            await sub.execFail(['pg_ctl', 'stop', '-D', pgdata, '-w'], {
+              user: 'postgres',
+            })
+          },
+        )
+      })
+  }
+
+  /**
+   * Configure MySQL/MariaDB dump-based backup for a volume.
+   *
+   * Instead of rsyncing the raw MySQL data directory (which is slow and error-prone),
+   * this uses `mysqldump` to create a logical dump before backup and `mysql` to restore
+   * the database after restore.
+   *
+   * The dump file is stored temporarily in `dumpVolume` during backup and cleaned up afterward.
+   *
+   * @returns A configured Backups instance with pre/post hooks. Chain `.addVolume()` or
+   * `.addSync()` to include additional volumes/paths in the backup.
+   */
+  static withMysqlDump<M extends T.SDKManifest = never>(
+    config: MysqlDumpConfig<M>,
+  ): Backups<M> {
+    const {
+      imageId,
+      dbVolume,
+      datadir,
+      database,
+      user,
+      password,
+      engine,
+      readyCommand,
+    } = config
+    const dumpFile = `${BACKUP_CONTAINER_MOUNT}/${database}-db.dump`
+
+    function dbMounts() {
+      return Mounts.of<M>().mountVolume({
+        volumeId: dbVolume,
+        mountpoint: datadir,
+        readonly: false,
+        subpath: null,
+      })
+    }
+
+    async function waitForMysql(
+      sub: { exec(cmd: string[]): Promise<{ exitCode: number | null }> },
+      cmd: string[],
+    ) {
+      for (let i = 0; i < 30; i++) {
+        const { exitCode } = await sub.exec(cmd)
+        if (exitCode === 0) return
+        await new Promise((r) => setTimeout(r, 1000))
+      }
+      throw new Error('MySQL/MariaDB failed to become ready within 30 seconds')
+    }
+
+    return new Backups<M>()
+      .setPreBackup(async (effects) => {
+        const pw = await resolvePassword(password)
+        const readyCmd = readyCommand || [
+          'mysqladmin',
+          'ping',
+          '-u',
+          user,
+          `-p${pw}`,
+          '--silent',
+        ]
+        await SubContainerRc.withTemp<M, void, BackupEffects>(
+          effects,
+          { imageId },
+          dbMounts() as any,
+          'mysql-dump',
+          async (sub) => {
+            await mountBackupTarget(sub.rootfs)
+            await sub.exec(['mkdir', '-p', '/var/run/mysqld'], {
+              user: 'root',
+            })
+            await sub.exec(['chown', 'mysql:mysql', '/var/run/mysqld'], {
+              user: 'root',
+            })
+            if (engine === 'mysql') {
+              await sub.execFail(['chown', '-R', 'mysql:mysql', datadir], {
+                user: 'root',
+              })
+            }
+            await sub.execFail(
+              [
+                'mysqld',
+                '--user=mysql',
+                `--datadir=${datadir}`,
+                '--skip-networking',
+                '--daemonize',
+              ],
+              { user: 'root' },
+              null,
+            )
+            await waitForMysql(sub, readyCmd)
+            await sub.execFail(
+              [
+                'mysqldump',
+                '-u',
+                user,
+                `-p${pw}`,
+                '--single-transaction',
+                `--result-file=${dumpFile}`,
+                database,
+              ],
+              { user: 'root' },
+              null,
+            )
+            await sub.execFail(
+              ['mysqladmin', '-u', user, `-p${pw}`, 'shutdown'],
+              { user: 'root' },
+            )
+          },
+        )
+      })
+      .setPostRestore(async (effects) => {
+        const pw = await resolvePassword(password)
+        await SubContainerRc.withTemp<M, void, BackupEffects>(
+          effects,
+          { imageId },
+          dbMounts() as any,
+          'mysql-restore',
+          async (sub) => {
+            await mountBackupTarget(sub.rootfs)
+            await sub.exec(['mkdir', '-p', '/var/run/mysqld'], {
+              user: 'root',
+            })
+            await sub.exec(['chown', 'mysql:mysql', '/var/run/mysqld'], {
+              user: 'root',
+            })
+            // Initialize fresh data directory
+            if (engine === 'mariadb') {
+              await sub.execFail(
+                ['mysql_install_db', '--user=mysql', `--datadir=${datadir}`],
+                { user: 'root' },
+              )
+            } else {
+              await sub.execFail(
+                [
+                  'mysqld',
+                  '--initialize-insecure',
+                  '--user=mysql',
+                  `--datadir=${datadir}`,
+                ],
+                { user: 'root' },
+              )
+            }
+            await sub.execFail(
+              [
+                'mysqld',
+                '--user=mysql',
+                `--datadir=${datadir}`,
+                '--skip-networking',
+                '--daemonize',
+              ],
+              { user: 'root' },
+              null,
+            )
+            // After fresh init, root has no password
+            await waitForMysql(sub, [
+              'mysqladmin',
+              'ping',
+              '-u',
+              'root',
+              '--silent',
+            ])
+            // Create database, user, and set password
+            await sub.execFail(
+              [
+                'mysql',
+                '-u',
+                'root',
+                '-e',
+                `CREATE DATABASE IF NOT EXISTS \`${database}\`; CREATE USER IF NOT EXISTS '${user}'@'localhost' IDENTIFIED BY '${pw}'; GRANT ALL ON \`${database}\`.* TO '${user}'@'localhost'; ALTER USER 'root'@'localhost' IDENTIFIED BY '${pw}'; FLUSH PRIVILEGES;`,
+              ],
+              { user: 'root' },
+            )
+            // Restore from dump
+            await sub.execFail(
+              [
+                'sh',
+                '-c',
+                `mysql -u root -p'${pw}' \`${database}\` < ${dumpFile}`,
+              ],
+              { user: 'root' },
+              null,
+            )
+            await sub.execFail(
+              ['mysqladmin', '-u', 'root', `-p${password}`, 'shutdown'],
+              { user: 'root' },
+            )
+          },
+        )
+      })
+  }
+
   /**
    * Override the default rsync options for both backup and restore.
    * @param options - Partial rsync options to merge with current defaults

sdk/package/lib/index.ts

@@ -32,6 +32,7 @@ export { setupManifest, buildManifest } from './manifest/setupManifest'
 export { FileHelper } from './util/fileHelper'
 export {
   smtpShape,
+  smtpPrefill,
   type SmtpSelection,
 } from '../../base/lib/actions/input/inputSpecConstants'
 

sdk/package/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-beta.63",
+  "version": "0.4.0-beta.64",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@start9labs/start-sdk",
-      "version": "0.4.0-beta.63",
+      "version": "0.4.0-beta.64",
       "license": "MIT",
       "dependencies": {
         "@iarna/toml": "^3.0.0",

sdk/package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-beta.63",
+  "version": "0.4.0-beta.64",
   "description": "Software development kit to facilitate packaging services for StartOS",
   "main": "./package/lib/index.js",
   "types": "./package/lib/index.d.ts",