allow mounting files directly (#2931)

* allow mounting files directly

* fixes from testing

* more fixes

container-runtime/package-lock.json

@@ -37,7 +37,7 @@
     },
     "../sdk/dist": {
       "name": "@start9labs/start-sdk",
-      "version": "0.4.0-beta.17",
+      "version": "0.4.0-beta.18",
       "license": "MIT",
       "dependencies": {
         "@iarna/toml": "^3.0.0",

container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts

@@ -12,11 +12,14 @@ import {
 import { Mounts } from "@start9labs/start-sdk/package/lib/mainFn/Mounts"
 import { Manifest } from "@start9labs/start-sdk/base/lib/osBindings"
 import { BackupEffects } from "@start9labs/start-sdk/package/lib/backup/Backups"
+import { Drop } from "@start9labs/start-sdk/package/lib/util"
 export const exec = promisify(cp.exec)
 export const execFile = promisify(cp.execFile)
 
-export class DockerProcedureContainer {
-  private constructor(private readonly subcontainer: ExecSpawnable) {}
+export class DockerProcedureContainer extends Drop {
+  private constructor(private readonly subcontainer: ExecSpawnable) {
+    super()
+  }
 
   static async of(
     effects: T.Effects,
@@ -61,10 +64,20 @@ export class DockerProcedureContainer {
         const volumeMount = volumes[mount]
         if (volumeMount.type === "data") {
           await subcontainer.mount(
-            Mounts.of().addVolume(mount, null, mounts[mount], false),
+            Mounts.of().addVolume({
+              volumeId: mount,
+              subpath: null,
+              mountpoint: mounts[mount],
+              readonly: false,
+            }),
           )
         } else if (volumeMount.type === "assets") {
-          await subcontainer.mount(Mounts.of().addAssets(mount, mounts[mount]))
+          await subcontainer.mount(
+            Mounts.of().addAssets({
+              subpath: mount,
+              mountpoint: mounts[mount],
+            }),
+          )
         } else if (volumeMount.type === "certificate") {
           const hostnames = [
             `${packageId}.embassy`,
@@ -105,7 +118,12 @@ export class DockerProcedureContainer {
             },
           })
         } else if (volumeMount.type === "backup") {
-          await subcontainer.mount(Mounts.of().addBackups(null, mounts[mount]))
+          await subcontainer.mount(
+            Mounts.of().addBackups({
+              subpath: null,
+              mountpoint: mounts[mount],
+            }),
+          )
         }
       }
     }
@@ -146,7 +164,11 @@ export class DockerProcedureContainer {
     }
   }
 
-  async spawn(commands: string[]): Promise<cp.ChildProcess> {
-    return await this.subcontainer.spawn(commands)
+  // async spawn(commands: string[]): Promise<cp.ChildProcess> {
+  //   return await this.subcontainer.spawn(commands)
+  // }
+
+  onDrop(): void {
+    this.subcontainer.destroy?.()
   }
 }

container-runtime/src/Adapters/Systems/SystemForEmbassy/polyfillEffects.ts

@@ -169,12 +169,12 @@ export const polyfillEffects = (
           { imageId: manifest.main.image },
           commands,
           {
-            mounts: Mounts.of().addVolume(
-              input.volumeId,
-              null,
-              "/drive",
-              false,
-            ),
+            mounts: Mounts.of().addVolume({
+              volumeId: input.volumeId,
+              subpath: null,
+              mountpoint: "/drive",
+              readonly: false,
+            }),
           },
           commands.join(" "),
         )
@@ -206,12 +206,12 @@ export const polyfillEffects = (
           { imageId: manifest.main.image },
           commands,
           {
-            mounts: Mounts.of().addVolume(
-              input.volumeId,
-              null,
-              "/drive",
-              false,
-            ),
+            mounts: Mounts.of().addVolume({
+              volumeId: input.volumeId,
+              subpath: null,
+              mountpoint: "/drive",
+              readonly: false,
+            }),
           },
           commands.join(" "),
         )

core/startos/src/service/effects/dependency.rs

@@ -23,6 +23,14 @@ use crate::util::Invoke;
 use crate::volume::data_dir;
 use crate::DATA_DIR;
 
+#[derive(Debug, Clone, Serialize, Deserialize, TS)]
+#[ts(export)]
+#[serde(rename_all = "camelCase")]
+pub enum FileType {
+    File,
+    Directory,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, TS)]
 #[ts(export)]
 #[serde(rename_all = "camelCase")]
@@ -31,6 +39,8 @@ pub struct MountTarget {
     volume_id: VolumeId,
     subpath: Option<PathBuf>,
     readonly: bool,
+    #[ts(optional)]
+    filetype: Option<FileType>,
 }
 #[derive(Debug, Clone, Serialize, Deserialize, TS)]
 #[ts(export)]
@@ -49,6 +59,7 @@ pub async fn mount(
                 volume_id,
                 subpath,
                 readonly,
+                filetype,
             },
     }: MountParams,
 ) -> Result<(), Error> {
@@ -56,9 +67,7 @@ pub async fn mount(
     let subpath = subpath.unwrap_or_default();
     let subpath = subpath.strip_prefix("/").unwrap_or(&subpath);
     let source = data_dir(DATA_DIR, &package_id, &volume_id).join(subpath);
-    if tokio::fs::metadata(&source).await.is_err() {
-        tokio::fs::create_dir_all(&source).await?;
-    }
+    let from_meta = tokio::fs::metadata(&source).await.ok();
     let location = location.strip_prefix("/").unwrap_or(&location);
     let mountpoint = context
         .seed
@@ -68,6 +77,38 @@ pub async fn mount(
         .or_not_found("lxc container")?
         .rootfs_dir()
         .join(location);
+    let to_meta = tokio::fs::metadata(&mountpoint).await.ok();
+
+    if matches!(filetype, Some(FileType::File))
+        || (filetype.is_none() && from_meta.as_ref().map_or(false, |m| m.is_file()))
+    {
+        if to_meta.as_ref().map_or(false, |m| m.is_dir()) {
+            tokio::fs::remove_dir(&mountpoint).await?;
+        }
+        if from_meta.is_none() {
+            if let Some(parent) = source.parent() {
+                tokio::fs::create_dir_all(parent).await?;
+            }
+            tokio::fs::write(&source, "").await?;
+        }
+        if to_meta.is_none() {
+            if let Some(parent) = mountpoint.parent() {
+                tokio::fs::create_dir_all(parent).await?;
+            }
+            tokio::fs::write(&mountpoint, "").await?;
+        }
+    } else {
+        if to_meta.as_ref().map_or(false, |m| m.is_file()) {
+            tokio::fs::remove_file(&mountpoint).await?;
+        }
+        if from_meta.is_none() {
+            tokio::fs::create_dir_all(&source).await?;
+        }
+        if to_meta.is_none() {
+            tokio::fs::create_dir_all(&mountpoint).await?;
+        }
+    }
+
     tokio::fs::create_dir_all(&mountpoint).await?;
     Command::new("chown")
         .arg("100000:100000")

sdk/base/lib/osBindings/FileType.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export type FileType = "file" | "directory"

sdk/base/lib/osBindings/MountTarget.ts

@@ -1,4 +1,5 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { FileType } from "./FileType"
 import type { PackageId } from "./PackageId"
 import type { VolumeId } from "./VolumeId"
 
@@ -7,4 +8,5 @@ export type MountTarget = {
   volumeId: VolumeId
   subpath: string | null
   readonly: boolean
+  filetype?: FileType
 }

sdk/base/lib/osBindings/index.ts

@@ -78,6 +78,7 @@ export { EncryptedWire } from "./EncryptedWire"
 export { ExportActionParams } from "./ExportActionParams"
 export { ExportServiceInterfaceParams } from "./ExportServiceInterfaceParams"
 export { ExposeForDependentsParams } from "./ExposeForDependentsParams"
+export { FileType } from "./FileType"
 export { ForgetInterfaceParams } from "./ForgetInterfaceParams"
 export { FullIndex } from "./FullIndex"
 export { FullProgress } from "./FullProgress"

sdk/package/lib/mainFn/Mounts.ts

@@ -3,31 +3,39 @@ import { MountOptions } from "../util/SubContainer"
 
 type MountArray = { mountpoint: string; options: MountOptions }[]
 
+type SharedOptions = {
+  /** The path within the resource to mount. Use `null` to mount the entire resource */
+  subpath: string | null
+  /** Where to mount the resource. e.g. /data */
+  mountpoint: string
+  /** Whether to mount this as a file or directory */
+  type?: "file" | "directory"
+}
+
+type VolumeOpts<Manifest extends T.SDKManifest> = {
+  /** The ID of the volume to mount. Must be one of the volume IDs defined in the manifest */
+  volumeId: Manifest["volumes"][number]
+  /** Whether or not the resource should be readonly for this subcontainer */
+  readonly: boolean
+} & SharedOptions
+
+type DependencyOpts<Manifest extends T.SDKManifest> = {
+  /** The ID of the dependency */
+  dependencyId: Manifest["id"]
+  /** The ID of the volume to mount. Must be one of the volume IDs defined in the manifest of the dependency */
+  volumeId: Manifest["volumes"][number]
+  /** Whether or not the resource should be readonly for this subcontainer */
+  readonly: boolean
+} & SharedOptions
+
 export class Mounts<
   Manifest extends T.SDKManifest,
-  Backups extends {
-    subpath: string | null
-    mountpoint: string
-  } = never,
+  Backups extends SharedOptions = never,
 > {
   private constructor(
-    readonly volumes: {
-      id: Manifest["volumes"][number]
-      subpath: string | null
-      mountpoint: string
-      readonly: boolean
-    }[],
-    readonly assets: {
-      subpath: string | null
-      mountpoint: string
-    }[],
-    readonly dependencies: {
-      dependencyId: string
-      volumeId: string
-      subpath: string | null
-      mountpoint: string
-      readonly: boolean
-    }[],
+    readonly volumes: VolumeOpts<Manifest>[],
+    readonly assets: SharedOptions[],
+    readonly dependencies: DependencyOpts<T.SDKManifest>[],
     readonly backups: Backups[],
   ) {}
 
@@ -35,82 +43,36 @@ export class Mounts<
     return new Mounts<Manifest>([], [], [], [])
   }
 
-  addVolume(
-    /** The ID of the volume to mount. Must be one of the volume IDs defined in the manifest */
-    id: Manifest["volumes"][number],
-    /** The path within the volume to mount. Use `null` to mount the entire volume */
-    subpath: string | null,
-    /** Where to mount the volume. e.g. /data */
-    mountpoint: string,
-    /** Whether or not the volume should be readonly for this daemon */
-    readonly: boolean,
-  ) {
+  addVolume(options: VolumeOpts<Manifest>) {
     return new Mounts<Manifest, Backups>(
-      [
-        ...this.volumes,
-        {
-          id,
-          subpath,
-          mountpoint,
-          readonly,
-        },
-      ],
+      [...this.volumes, options],
       [...this.assets],
       [...this.dependencies],
       [...this.backups],
     )
   }
 
-  addAssets(
-    /** The path within the asset directory to mount. Use `null` to mount the entire volume */
-    subpath: string | null,
-    /** Where to mount the asset. e.g. /asset */
-    mountpoint: string,
-  ) {
+  addAssets(options: SharedOptions) {
     return new Mounts<Manifest, Backups>(
       [...this.volumes],
-      [
-        ...this.assets,
-        {
-          subpath,
-          mountpoint,
-        },
-      ],
+      [...this.assets, options],
       [...this.dependencies],
       [...this.backups],
     )
   }
 
   addDependency<DependencyManifest extends T.SDKManifest>(
-    /** The ID of the dependency service */
-    dependencyId: keyof Manifest["dependencies"] & string,
-    /** The ID of the volume belonging to the dependency service to mount */
-    volumeId: DependencyManifest["volumes"][number],
-    /** The path within the dependency's volume to mount. Use `null` to mount the entire volume */
-    subpath: string | null,
-    /** Where to mount the dependency's volume. e.g. /service-id */
-    mountpoint: string,
-    /** Whether or not the volume should be readonly for this daemon */
-    readonly: boolean,
+    options: DependencyOpts<DependencyManifest>,
   ) {
     return new Mounts<Manifest, Backups>(
       [...this.volumes],
       [...this.assets],
-      [
-        ...this.dependencies,
-        {
-          dependencyId,
-          volumeId,
-          subpath,
-          mountpoint,
-          readonly,
-        },
-      ],
+      [...this.dependencies, options],
       [...this.backups],
     )
   }
 
-  addBackups(subpath: string | null, mountpoint: string) {
+  addBackups(options: SharedOptions) {
     return new Mounts<
       Manifest,
       {
@@ -121,7 +83,7 @@ export class Mounts<
       [...this.volumes],
       [...this.assets],
       [...this.dependencies],
-      [...this.backups, { subpath, mountpoint }],
+      [...this.backups, options],
     )
   }
 
@@ -144,9 +106,10 @@ export class Mounts<
           mountpoint: v.mountpoint,
           options: {
             type: "volume",
-            id: v.id,
+            volumeId: v.volumeId,
             subpath: v.subpath,
             readonly: v.readonly,
+            filetype: v.type,
           },
         })),
       )
@@ -156,6 +119,7 @@ export class Mounts<
           options: {
             type: "assets",
             subpath: a.subpath,
+            filetype: a.type,
           },
         })),
       )
@@ -168,12 +132,13 @@ export class Mounts<
             volumeId: d.volumeId,
             subpath: d.subpath,
             readonly: d.readonly,
+            filetype: d.type,
           },
         })),
       )
   }
 }
 
-const a = Mounts.of().addBackups(null, "")
+const a = Mounts.of().addBackups({ subpath: null, mountpoint: "" })
 // @ts-expect-error
 const m: Mounts<T.SDKManifest, never> = a

sdk/package/lib/util/SubContainer.ts

@@ -24,6 +24,37 @@ export type ExecOptions = {
 
 const TIMES_TO_WAIT_FOR_PROC = 100
 
+async function prepBind(
+  from: string | null,
+  to: string,
+  type?: "file" | "directory",
+) {
+  const fromMeta = from ? await fs.stat(from).catch((_) => null) : null
+  const toMeta = await fs.stat(to).catch((_) => null)
+
+  if (type === "file" || (!type && from && fromMeta?.isFile())) {
+    if (toMeta && toMeta.isDirectory()) await fs.rmdir(to, { recursive: false })
+    if (from && !fromMeta) {
+      await fs.mkdir(from.replace(/\/[^\/]*\/?$/, ""), { recursive: true })
+      await fs.writeFile(from, "")
+    }
+    if (!toMeta) {
+      await fs.mkdir(to.replace(/\/[^\/]*\/?$/, ""), { recursive: true })
+      await fs.writeFile(to, "")
+    }
+  } else {
+    if (toMeta && toMeta.isFile() && !toMeta.size) await fs.rm(to)
+    if (from && !fromMeta) await fs.mkdir(from, { recursive: true })
+    if (!toMeta) await fs.mkdir(to, { recursive: true })
+  }
+}
+
+async function bind(from: string, to: string, type?: "file" | "directory") {
+  await prepBind(from, to, type)
+
+  await execFile("mount", ["--bind", from, to])
+}
+
 /**
  * This is the type that is going to describe what an subcontainer could do. The main point of the
  * subcontainer is to have commands that run in a chrooted environment. This is useful for running
@@ -211,11 +242,9 @@ export class SubContainer<
             ? options.subpath
             : `/${options.subpath}`
           : "/"
-        const from = `/media/startos/volumes/${options.id}${subpath}`
+        const from = `/media/startos/volumes/${options.volumeId}${subpath}`
 
-        await fs.mkdir(from, { recursive: true })
-        await fs.mkdir(path, { recursive: true })
-        await execFile("mount", ["--bind", from, path])
+        await bind(from, path, mount.options.filetype)
       } else if (options.type === "assets") {
         const subpath = options.subpath
           ? options.subpath.startsWith("/")
@@ -224,10 +253,9 @@ export class SubContainer<
           : "/"
         const from = `/media/startos/assets/${subpath}`
 
-        await fs.mkdir(from, { recursive: true })
-        await fs.mkdir(path, { recursive: true })
-        await execFile("mount", ["--bind", from, path])
+        await bind(from, path, mount.options.filetype)
       } else if (options.type === "pointer") {
+        await prepBind(null, path, options.filetype)
         await this.effects.mount({ location: path, target: options })
       } else if (options.type === "backup") {
         const subpath = options.subpath
@@ -237,9 +265,7 @@ export class SubContainer<
           : "/"
         const from = `/media/startos/backup${subpath}`
 
-        await fs.mkdir(from, { recursive: true })
-        await fs.mkdir(path, { recursive: true })
-        await execFile("mount", ["--bind", from, path])
+        await bind(from, path, mount.options.filetype)
       } else {
         throw new Error(`unknown type ${(options as any).type}`)
       }
@@ -560,14 +586,16 @@ export type MountOptions =
 
 export type MountOptionsVolume = {
   type: "volume"
-  id: string
+  volumeId: string
   subpath: string | null
   readonly: boolean
+  filetype?: "file" | "directory"
 }
 
 export type MountOptionsAssets = {
   type: "assets"
   subpath: string | null
+  filetype?: "file" | "directory"
 }
 
 export type MountOptionsPointer = {
@@ -576,11 +604,13 @@ export type MountOptionsPointer = {
   volumeId: string
   subpath: string | null
   readonly: boolean
+  filetype?: "file" | "directory"
 }
 
 export type MountOptionsBackup = {
   type: "backup"
   subpath: string | null
+  filetype?: "file" | "directory"
 }
 function wait(time: number) {
   return new Promise((resolve) => setTimeout(resolve, time))

sdk/package/lib/util/fileHelper.ts

@@ -329,6 +329,36 @@ export class FileHelper<A> {
     )
   }
 
+  /**
+   * Create a File Helper for a text file
+   */
+  static string(path: string): FileHelper<string>
+  static string<A extends string>(
+    path: string,
+    shape: Validator<string, A>,
+  ): FileHelper<A>
+  static string<A extends Transformed, Transformed = string>(
+    path: string,
+    shape: Validator<Transformed, A>,
+    transformers: Transformers<string, Transformed>,
+  ): FileHelper<A>
+  static string<A extends Transformed, Transformed = string>(
+    path: string,
+    shape?: Validator<Transformed, A>,
+    transformers?: Transformers<string, Transformed>,
+  ) {
+    return FileHelper.rawTransformed<A, string, Transformed>(
+      path,
+      (inData) => inData,
+      (inString) => inString,
+      (data) =>
+        (shape || (matches.string as Validator<Transformed, A>)).unsafeCast(
+          data,
+        ),
+      transformers,
+    )
+  }
+
   /**
    * Create a File Helper for a .json file.
    */

sdk/package/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-beta.17",
+  "version": "0.4.0-beta.18",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@start9labs/start-sdk",
-      "version": "0.4.0-beta.17",
+      "version": "0.4.0-beta.18",
       "license": "MIT",
       "dependencies": {
         "@iarna/toml": "^3.0.0",

sdk/package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@start9labs/start-sdk",
-  "version": "0.4.0-beta.17",
+  "version": "0.4.0-beta.18",
   "description": "Software development kit to facilitate packaging services for StartOS",
   "main": "./package/lib/index.js",
   "types": "./package/lib/index.d.ts",