diff --git a/container-runtime/package-lock.json b/container-runtime/package-lock.json index ecbcdb415..3b6564041 100644 --- a/container-runtime/package-lock.json +++ b/container-runtime/package-lock.json @@ -37,7 +37,7 @@ }, "../sdk/dist": { "name": "@start9labs/start-sdk", - "version": "0.4.0-beta.17", + "version": "0.4.0-beta.18", "license": "MIT", "dependencies": { "@iarna/toml": "^3.0.0", diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts index e151aa2a9..c48891ff2 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts @@ -12,11 +12,14 @@ import { import { Mounts } from "@start9labs/start-sdk/package/lib/mainFn/Mounts" import { Manifest } from "@start9labs/start-sdk/base/lib/osBindings" import { BackupEffects } from "@start9labs/start-sdk/package/lib/backup/Backups" +import { Drop } from "@start9labs/start-sdk/package/lib/util" export const exec = promisify(cp.exec) export const execFile = promisify(cp.execFile) -export class DockerProcedureContainer { - private constructor(private readonly subcontainer: ExecSpawnable) {} +export class DockerProcedureContainer extends Drop { + private constructor(private readonly subcontainer: ExecSpawnable) { + super() + } static async of( effects: T.Effects, @@ -61,10 +64,20 @@ export class DockerProcedureContainer { const volumeMount = volumes[mount] if (volumeMount.type === "data") { await subcontainer.mount( - Mounts.of().addVolume(mount, null, mounts[mount], false), + Mounts.of().addVolume({ + volumeId: mount, + subpath: null, + mountpoint: mounts[mount], + readonly: false, + }), ) } else if (volumeMount.type === "assets") { - await subcontainer.mount(Mounts.of().addAssets(mount, mounts[mount])) + await subcontainer.mount( + Mounts.of().addAssets({ + subpath: mount, + mountpoint: mounts[mount], + }), + ) } else if (volumeMount.type === "certificate") { const hostnames = [ `${packageId}.embassy`, @@ -105,7 +118,12 @@ export class DockerProcedureContainer { }, }) } else if (volumeMount.type === "backup") { - await subcontainer.mount(Mounts.of().addBackups(null, mounts[mount])) + await subcontainer.mount( + Mounts.of().addBackups({ + subpath: null, + mountpoint: mounts[mount], + }), + ) } } } @@ -146,7 +164,11 @@ export class DockerProcedureContainer { } } - async spawn(commands: string[]): Promise { - return await this.subcontainer.spawn(commands) + // async spawn(commands: string[]): Promise { + // return await this.subcontainer.spawn(commands) + // } + + onDrop(): void { + this.subcontainer.destroy?.() } } diff --git a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts index 88e9d7ae6..0dc6dac24 100644 --- a/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts +++ b/container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts @@ -169,12 +169,12 @@ export const polyfillEffects = ( { imageId: manifest.main.image }, commands, { - mounts: Mounts.of().addVolume( - input.volumeId, - null, - "/drive", - false, - ), + mounts: Mounts.of().addVolume({ + volumeId: input.volumeId, + subpath: null, + mountpoint: "/drive", + readonly: false, + }), }, commands.join(" "), ) @@ -206,12 +206,12 @@ export const polyfillEffects = ( { imageId: manifest.main.image }, commands, { - mounts: Mounts.of().addVolume( - input.volumeId, - null, - "/drive", - false, - ), + mounts: Mounts.of().addVolume({ + volumeId: input.volumeId, + subpath: null, + mountpoint: "/drive", + readonly: false, + }), }, commands.join(" "), ) diff --git a/core/startos/src/service/effects/dependency.rs b/core/startos/src/service/effects/dependency.rs index eaeab0c9f..e2b84be90 100644 --- a/core/startos/src/service/effects/dependency.rs +++ b/core/startos/src/service/effects/dependency.rs @@ -23,6 +23,14 @@ use crate::util::Invoke; use crate::volume::data_dir; use crate::DATA_DIR; +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export)] +#[serde(rename_all = "camelCase")] +pub enum FileType { + File, + Directory, +} + #[derive(Debug, Clone, Serialize, Deserialize, TS)] #[ts(export)] #[serde(rename_all = "camelCase")] @@ -31,6 +39,8 @@ pub struct MountTarget { volume_id: VolumeId, subpath: Option, readonly: bool, + #[ts(optional)] + filetype: Option, } #[derive(Debug, Clone, Serialize, Deserialize, TS)] #[ts(export)] @@ -49,6 +59,7 @@ pub async fn mount( volume_id, subpath, readonly, + filetype, }, }: MountParams, ) -> Result<(), Error> { @@ -56,9 +67,7 @@ pub async fn mount( let subpath = subpath.unwrap_or_default(); let subpath = subpath.strip_prefix("/").unwrap_or(&subpath); let source = data_dir(DATA_DIR, &package_id, &volume_id).join(subpath); - if tokio::fs::metadata(&source).await.is_err() { - tokio::fs::create_dir_all(&source).await?; - } + let from_meta = tokio::fs::metadata(&source).await.ok(); let location = location.strip_prefix("/").unwrap_or(&location); let mountpoint = context .seed @@ -68,6 +77,38 @@ pub async fn mount( .or_not_found("lxc container")? .rootfs_dir() .join(location); + let to_meta = tokio::fs::metadata(&mountpoint).await.ok(); + + if matches!(filetype, Some(FileType::File)) + || (filetype.is_none() && from_meta.as_ref().map_or(false, |m| m.is_file())) + { + if to_meta.as_ref().map_or(false, |m| m.is_dir()) { + tokio::fs::remove_dir(&mountpoint).await?; + } + if from_meta.is_none() { + if let Some(parent) = source.parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::write(&source, "").await?; + } + if to_meta.is_none() { + if let Some(parent) = mountpoint.parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::write(&mountpoint, "").await?; + } + } else { + if to_meta.as_ref().map_or(false, |m| m.is_file()) { + tokio::fs::remove_file(&mountpoint).await?; + } + if from_meta.is_none() { + tokio::fs::create_dir_all(&source).await?; + } + if to_meta.is_none() { + tokio::fs::create_dir_all(&mountpoint).await?; + } + } + tokio::fs::create_dir_all(&mountpoint).await?; Command::new("chown") .arg("100000:100000") diff --git a/sdk/base/lib/osBindings/FileType.ts b/sdk/base/lib/osBindings/FileType.ts new file mode 100644 index 000000000..82b9ca474 --- /dev/null +++ b/sdk/base/lib/osBindings/FileType.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type FileType = "file" | "directory" diff --git a/sdk/base/lib/osBindings/MountTarget.ts b/sdk/base/lib/osBindings/MountTarget.ts index bbee5453b..e208383e3 100644 --- a/sdk/base/lib/osBindings/MountTarget.ts +++ b/sdk/base/lib/osBindings/MountTarget.ts @@ -1,4 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { FileType } from "./FileType" import type { PackageId } from "./PackageId" import type { VolumeId } from "./VolumeId" @@ -7,4 +8,5 @@ export type MountTarget = { volumeId: VolumeId subpath: string | null readonly: boolean + filetype?: FileType } diff --git a/sdk/base/lib/osBindings/index.ts b/sdk/base/lib/osBindings/index.ts index 9162d46d0..bc9413ba5 100644 --- a/sdk/base/lib/osBindings/index.ts +++ b/sdk/base/lib/osBindings/index.ts @@ -78,6 +78,7 @@ export { EncryptedWire } from "./EncryptedWire" export { ExportActionParams } from "./ExportActionParams" export { ExportServiceInterfaceParams } from "./ExportServiceInterfaceParams" export { ExposeForDependentsParams } from "./ExposeForDependentsParams" +export { FileType } from "./FileType" export { ForgetInterfaceParams } from "./ForgetInterfaceParams" export { FullIndex } from "./FullIndex" export { FullProgress } from "./FullProgress" diff --git a/sdk/package/lib/mainFn/Mounts.ts b/sdk/package/lib/mainFn/Mounts.ts index 49d6b914e..e9d4dcd60 100644 --- a/sdk/package/lib/mainFn/Mounts.ts +++ b/sdk/package/lib/mainFn/Mounts.ts @@ -3,31 +3,39 @@ import { MountOptions } from "../util/SubContainer" type MountArray = { mountpoint: string; options: MountOptions }[] +type SharedOptions = { + /** The path within the resource to mount. Use `null` to mount the entire resource */ + subpath: string | null + /** Where to mount the resource. e.g. /data */ + mountpoint: string + /** Whether to mount this as a file or directory */ + type?: "file" | "directory" +} + +type VolumeOpts = { + /** The ID of the volume to mount. Must be one of the volume IDs defined in the manifest */ + volumeId: Manifest["volumes"][number] + /** Whether or not the resource should be readonly for this subcontainer */ + readonly: boolean +} & SharedOptions + +type DependencyOpts = { + /** The ID of the dependency */ + dependencyId: Manifest["id"] + /** The ID of the volume to mount. Must be one of the volume IDs defined in the manifest of the dependency */ + volumeId: Manifest["volumes"][number] + /** Whether or not the resource should be readonly for this subcontainer */ + readonly: boolean +} & SharedOptions + export class Mounts< Manifest extends T.SDKManifest, - Backups extends { - subpath: string | null - mountpoint: string - } = never, + Backups extends SharedOptions = never, > { private constructor( - readonly volumes: { - id: Manifest["volumes"][number] - subpath: string | null - mountpoint: string - readonly: boolean - }[], - readonly assets: { - subpath: string | null - mountpoint: string - }[], - readonly dependencies: { - dependencyId: string - volumeId: string - subpath: string | null - mountpoint: string - readonly: boolean - }[], + readonly volumes: VolumeOpts[], + readonly assets: SharedOptions[], + readonly dependencies: DependencyOpts[], readonly backups: Backups[], ) {} @@ -35,82 +43,36 @@ export class Mounts< return new Mounts([], [], [], []) } - addVolume( - /** The ID of the volume to mount. Must be one of the volume IDs defined in the manifest */ - id: Manifest["volumes"][number], - /** The path within the volume to mount. Use `null` to mount the entire volume */ - subpath: string | null, - /** Where to mount the volume. e.g. /data */ - mountpoint: string, - /** Whether or not the volume should be readonly for this daemon */ - readonly: boolean, - ) { + addVolume(options: VolumeOpts) { return new Mounts( - [ - ...this.volumes, - { - id, - subpath, - mountpoint, - readonly, - }, - ], + [...this.volumes, options], [...this.assets], [...this.dependencies], [...this.backups], ) } - addAssets( - /** The path within the asset directory to mount. Use `null` to mount the entire volume */ - subpath: string | null, - /** Where to mount the asset. e.g. /asset */ - mountpoint: string, - ) { + addAssets(options: SharedOptions) { return new Mounts( [...this.volumes], - [ - ...this.assets, - { - subpath, - mountpoint, - }, - ], + [...this.assets, options], [...this.dependencies], [...this.backups], ) } addDependency( - /** The ID of the dependency service */ - dependencyId: keyof Manifest["dependencies"] & string, - /** The ID of the volume belonging to the dependency service to mount */ - volumeId: DependencyManifest["volumes"][number], - /** The path within the dependency's volume to mount. Use `null` to mount the entire volume */ - subpath: string | null, - /** Where to mount the dependency's volume. e.g. /service-id */ - mountpoint: string, - /** Whether or not the volume should be readonly for this daemon */ - readonly: boolean, + options: DependencyOpts, ) { return new Mounts( [...this.volumes], [...this.assets], - [ - ...this.dependencies, - { - dependencyId, - volumeId, - subpath, - mountpoint, - readonly, - }, - ], + [...this.dependencies, options], [...this.backups], ) } - addBackups(subpath: string | null, mountpoint: string) { + addBackups(options: SharedOptions) { return new Mounts< Manifest, { @@ -121,7 +83,7 @@ export class Mounts< [...this.volumes], [...this.assets], [...this.dependencies], - [...this.backups, { subpath, mountpoint }], + [...this.backups, options], ) } @@ -144,9 +106,10 @@ export class Mounts< mountpoint: v.mountpoint, options: { type: "volume", - id: v.id, + volumeId: v.volumeId, subpath: v.subpath, readonly: v.readonly, + filetype: v.type, }, })), ) @@ -156,6 +119,7 @@ export class Mounts< options: { type: "assets", subpath: a.subpath, + filetype: a.type, }, })), ) @@ -168,12 +132,13 @@ export class Mounts< volumeId: d.volumeId, subpath: d.subpath, readonly: d.readonly, + filetype: d.type, }, })), ) } } -const a = Mounts.of().addBackups(null, "") +const a = Mounts.of().addBackups({ subpath: null, mountpoint: "" }) // @ts-expect-error const m: Mounts = a diff --git a/sdk/package/lib/util/SubContainer.ts b/sdk/package/lib/util/SubContainer.ts index cd25e1df6..08a13c453 100644 --- a/sdk/package/lib/util/SubContainer.ts +++ b/sdk/package/lib/util/SubContainer.ts @@ -24,6 +24,37 @@ export type ExecOptions = { const TIMES_TO_WAIT_FOR_PROC = 100 +async function prepBind( + from: string | null, + to: string, + type?: "file" | "directory", +) { + const fromMeta = from ? await fs.stat(from).catch((_) => null) : null + const toMeta = await fs.stat(to).catch((_) => null) + + if (type === "file" || (!type && from && fromMeta?.isFile())) { + if (toMeta && toMeta.isDirectory()) await fs.rmdir(to, { recursive: false }) + if (from && !fromMeta) { + await fs.mkdir(from.replace(/\/[^\/]*\/?$/, ""), { recursive: true }) + await fs.writeFile(from, "") + } + if (!toMeta) { + await fs.mkdir(to.replace(/\/[^\/]*\/?$/, ""), { recursive: true }) + await fs.writeFile(to, "") + } + } else { + if (toMeta && toMeta.isFile() && !toMeta.size) await fs.rm(to) + if (from && !fromMeta) await fs.mkdir(from, { recursive: true }) + if (!toMeta) await fs.mkdir(to, { recursive: true }) + } +} + +async function bind(from: string, to: string, type?: "file" | "directory") { + await prepBind(from, to, type) + + await execFile("mount", ["--bind", from, to]) +} + /** * This is the type that is going to describe what an subcontainer could do. The main point of the * subcontainer is to have commands that run in a chrooted environment. This is useful for running @@ -211,11 +242,9 @@ export class SubContainer< ? options.subpath : `/${options.subpath}` : "/" - const from = `/media/startos/volumes/${options.id}${subpath}` + const from = `/media/startos/volumes/${options.volumeId}${subpath}` - await fs.mkdir(from, { recursive: true }) - await fs.mkdir(path, { recursive: true }) - await execFile("mount", ["--bind", from, path]) + await bind(from, path, mount.options.filetype) } else if (options.type === "assets") { const subpath = options.subpath ? options.subpath.startsWith("/") @@ -224,10 +253,9 @@ export class SubContainer< : "/" const from = `/media/startos/assets/${subpath}` - await fs.mkdir(from, { recursive: true }) - await fs.mkdir(path, { recursive: true }) - await execFile("mount", ["--bind", from, path]) + await bind(from, path, mount.options.filetype) } else if (options.type === "pointer") { + await prepBind(null, path, options.filetype) await this.effects.mount({ location: path, target: options }) } else if (options.type === "backup") { const subpath = options.subpath @@ -237,9 +265,7 @@ export class SubContainer< : "/" const from = `/media/startos/backup${subpath}` - await fs.mkdir(from, { recursive: true }) - await fs.mkdir(path, { recursive: true }) - await execFile("mount", ["--bind", from, path]) + await bind(from, path, mount.options.filetype) } else { throw new Error(`unknown type ${(options as any).type}`) } @@ -560,14 +586,16 @@ export type MountOptions = export type MountOptionsVolume = { type: "volume" - id: string + volumeId: string subpath: string | null readonly: boolean + filetype?: "file" | "directory" } export type MountOptionsAssets = { type: "assets" subpath: string | null + filetype?: "file" | "directory" } export type MountOptionsPointer = { @@ -576,11 +604,13 @@ export type MountOptionsPointer = { volumeId: string subpath: string | null readonly: boolean + filetype?: "file" | "directory" } export type MountOptionsBackup = { type: "backup" subpath: string | null + filetype?: "file" | "directory" } function wait(time: number) { return new Promise((resolve) => setTimeout(resolve, time)) diff --git a/sdk/package/lib/util/fileHelper.ts b/sdk/package/lib/util/fileHelper.ts index 58729cb1d..9ba00777d 100644 --- a/sdk/package/lib/util/fileHelper.ts +++ b/sdk/package/lib/util/fileHelper.ts @@ -329,6 +329,36 @@ export class FileHelper { ) } + /** + * Create a File Helper for a text file + */ + static string(path: string): FileHelper + static string( + path: string, + shape: Validator, + ): FileHelper + static string( + path: string, + shape: Validator, + transformers: Transformers, + ): FileHelper + static string( + path: string, + shape?: Validator, + transformers?: Transformers, + ) { + return FileHelper.rawTransformed( + path, + (inData) => inData, + (inString) => inString, + (data) => + (shape || (matches.string as Validator)).unsafeCast( + data, + ), + transformers, + ) + } + /** * Create a File Helper for a .json file. */ diff --git a/sdk/package/package-lock.json b/sdk/package/package-lock.json index 227b1e450..b014885ce 100644 --- a/sdk/package/package-lock.json +++ b/sdk/package/package-lock.json @@ -1,12 +1,12 @@ { "name": "@start9labs/start-sdk", - "version": "0.4.0-beta.17", + "version": "0.4.0-beta.18", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@start9labs/start-sdk", - "version": "0.4.0-beta.17", + "version": "0.4.0-beta.18", "license": "MIT", "dependencies": { "@iarna/toml": "^3.0.0", diff --git a/sdk/package/package.json b/sdk/package/package.json index 320d465c1..db2528f03 100644 --- a/sdk/package/package.json +++ b/sdk/package/package.json @@ -1,6 +1,6 @@ { "name": "@start9labs/start-sdk", - "version": "0.4.0-beta.17", + "version": "0.4.0-beta.18", "description": "Software development kit to facilitate packaging services for StartOS", "main": "./package/lib/index.js", "types": "./package/lib/index.d.ts",