Feature/UI sideload (#2658)

* ui sideloading

* remove subtlecrypto import

* fix parser

* misc fixes

* allow docker pull during compat conversion

sdk/lib/emverLite/mod.ts

@@ -2,9 +2,9 @@ import * as matches from "ts-matches"
 
 const starSub = /((\d+\.)*\d+)\.\*/
 // prettier-ignore
-export type ValidEmVer = `${number}${`.${number}` | ""}${`.${number}` | ""}${`-${string}` | ""}`;
+export type ValidEmVer = string;
 // prettier-ignore
-export type ValidEmVerRange = `${'>=' | '<='| '<' | '>' | ''}${'^' | '~' | ''}${number | '*'}${`.${number | '*'}` | ""}${`.${number | '*'}` | ""}${`-${string}` | ""}`;
+export type ValidEmVerRange = string;
 
 function incrementLastNumber(list: number[]) {
   const newList = [...list]

sdk/lib/index.browser.ts

@@ -1,6 +1,7 @@
 export { EmVer } from "./emverLite/mod"
 export { setupManifest } from "./manifest/setupManifest"
 export { setupExposeStore } from "./store/setupExposeStore"
+export { S9pk } from "./s9pk"
 export * as config from "./config"
 export * as CB from "./config/builder"
 export * as CT from "./config/configTypes"

sdk/lib/index.ts

@@ -6,6 +6,7 @@ export { setupManifest } from "./manifest/setupManifest"
 export { FileHelper } from "./util/fileHelper"
 export { setupExposeStore } from "./store/setupExposeStore"
 export { pathBuilder } from "./store/PathBuilder"
+export { S9pk } from "./s9pk"
 
 export * as actions from "./actions"
 export * as backup from "./backup"

sdk/lib/osBindings/GetPackageParams.ts

@@ -1,10 +1,11 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { PackageDetailLevel } from "./PackageDetailLevel"
 import type { PackageId } from "./PackageId"
+import type { Version } from "./Version"
 
 export type GetPackageParams = {
   id: PackageId | null
   version: string | null
-  sourceVersion: string | null
+  sourceVersion: Version | null
   otherVersions: PackageDetailLevel | null
 }

sdk/lib/osBindings/Manifest.ts

@@ -28,6 +28,6 @@ export type Manifest = {
   dependencies: Dependencies
   hardwareRequirements: HardwareRequirements
   gitHash: string | null
-  osVersion: Version
+  osVersion: string
   hasConfig: boolean
 }

sdk/lib/osBindings/PackageVersionInfo.ts

@@ -4,7 +4,6 @@ import type { Description } from "./Description"
 import type { HardwareRequirements } from "./HardwareRequirements"
 import type { MerkleArchiveCommitment } from "./MerkleArchiveCommitment"
 import type { RegistryAsset } from "./RegistryAsset"
-import type { Version } from "./Version"
 
 export type PackageVersionInfo = {
   title: string
@@ -17,7 +16,7 @@ export type PackageVersionInfo = {
   upstreamRepo: string
   supportSite: string
   marketingSite: string
-  osVersion: Version
+  osVersion: string
   hardwareRequirements: HardwareRequirements
   sourceVersion: string | null
   s9pk: RegistryAsset<MerkleArchiveCommitment>

sdk/lib/osBindings/ServerInfo.ts

@@ -2,7 +2,6 @@
 import type { Governor } from "./Governor"
 import type { IpInfo } from "./IpInfo"
 import type { ServerStatus } from "./ServerStatus"
-import type { Version } from "./Version"
 import type { WifiInfo } from "./WifiInfo"
 
 export type ServerInfo = {
@@ -10,7 +9,7 @@ export type ServerInfo = {
   platform: string
   id: string
   hostname: string
-  version: Version
+  version: string
   lastBackup: string | null
   eosVersionCompat: string
   lanAddress: string

sdk/lib/s9pk/index.ts

@@ -0,0 +1,67 @@
+import { DataUrl, Manifest, MerkleArchiveCommitment } from "../osBindings"
+import { ArrayBufferReader, MerkleArchive } from "./merkleArchive"
+import mime from "mime"
+
+const magicAndVersion = new Uint8Array([59, 59, 2])
+
+export function compare(a: Uint8Array, b: Uint8Array) {
+  if (a.length !== b.length) return false
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) return false
+  }
+  return true
+}
+
+export class S9pk {
+  private constructor(
+    readonly manifest: Manifest,
+    readonly archive: MerkleArchive,
+    readonly size: number,
+  ) {}
+  static async deserialize(
+    source: Blob,
+    commitment: MerkleArchiveCommitment | null,
+  ): Promise<S9pk> {
+    const header = new ArrayBufferReader(
+      await source
+        .slice(0, magicAndVersion.length + MerkleArchive.headerSize)
+        .arrayBuffer(),
+    )
+    const magicVersion = new Uint8Array(header.next(magicAndVersion.length))
+    if (!compare(magicVersion, magicAndVersion)) {
+      throw new Error("Invalid Magic or Unexpected Version")
+    }
+
+    const archive = await MerkleArchive.deserialize(
+      source,
+      "s9pk",
+      header,
+      commitment,
+    )
+
+    const manifest = JSON.parse(
+      new TextDecoder().decode(
+        await archive.contents
+          .getPath(["manifest.json"])
+          ?.verifiedFileContents(),
+      ),
+    )
+
+    return new S9pk(manifest, archive, source.length)
+  }
+  async icon(): Promise<DataUrl> {
+    const iconName = Object.keys(this.archive.contents.contents).find(
+      (name) =>
+        name.startsWith("icon.") && mime.getType(name)?.startsWith("image/"),
+    )
+    if (!iconName) {
+      throw new Error("no icon found in archive")
+    }
+    return (
+      `data:${mime.getType(iconName)};base64,` +
+      Buffer.from(
+        await this.archive.contents.getPath([iconName])!.verifiedFileContents(),
+      ).toString("base64")
+    )
+  }
+}

sdk/lib/s9pk/merkleArchive/directoryContents.ts

@@ -0,0 +1,80 @@
+import { ArrayBufferReader, Entry } from "."
+import { blake3 } from "@noble/hashes/blake3"
+import { serializeVarint } from "./varint"
+import { FileContents } from "./fileContents"
+import { compare } from ".."
+
+export class DirectoryContents {
+  static readonly headerSize =
+    8 + // position: u64 BE
+    8 // size: u64 BE
+  private constructor(readonly contents: { [name: string]: Entry }) {}
+  static async deserialize(
+    source: Blob,
+    header: ArrayBufferReader,
+    sighash: Uint8Array,
+    maxSize: bigint,
+  ): Promise<DirectoryContents> {
+    const position = header.nextU64()
+    const size = header.nextU64()
+    if (size > maxSize) {
+      throw new Error("size is greater than signed")
+    }
+
+    const tocReader = new ArrayBufferReader(
+      await source
+        .slice(Number(position), Number(position + size))
+        .arrayBuffer(),
+    )
+    const len = tocReader.nextVarint()
+    const entries: { [name: string]: Entry } = {}
+    for (let i = 0; i < len; i++) {
+      const name = tocReader.nextVarstring()
+      const entry = await Entry.deserialize(source, tocReader)
+      entries[name] = entry
+    }
+
+    const res = new DirectoryContents(entries)
+
+    if (!compare(res.sighash(), sighash)) {
+      throw new Error("hash sum does not match")
+    }
+
+    return res
+  }
+  sighash(): Uint8Array {
+    const hasher = blake3.create({})
+    const names = Object.keys(this.contents).sort()
+    hasher.update(new Uint8Array(serializeVarint(names.length)))
+    for (const name of names) {
+      const entry = this.contents[name]
+      const nameBuf = new TextEncoder().encode(name)
+      hasher.update(new Uint8Array(serializeVarint(nameBuf.length)))
+      hasher.update(nameBuf)
+      hasher.update(new Uint8Array(entry.hash))
+      const sizeBuf = new Uint8Array(8)
+      new DataView(sizeBuf.buffer).setBigUint64(0, entry.size)
+      hasher.update(sizeBuf)
+      hasher.update(new Uint8Array([0]))
+    }
+
+    return hasher.digest()
+  }
+  getPath(path: string[]): Entry | null {
+    if (path.length === 0) {
+      return null
+    }
+    const next = this.contents[path[0]]
+    const rest = path.slice(1)
+    if (next === undefined) {
+      return null
+    }
+    if (rest.length === 0) {
+      return next
+    }
+    if (next.contents instanceof DirectoryContents) {
+      return next.contents.getPath(rest)
+    }
+    return null
+  }
+}

sdk/lib/s9pk/merkleArchive/fileContents.ts

@@ -0,0 +1,24 @@
+import { blake3 } from "@noble/hashes/blake3"
+import { ArrayBufferReader } from "."
+import { compare } from ".."
+
+export class FileContents {
+  private constructor(readonly contents: Blob) {}
+  static deserialize(
+    source: Blob,
+    header: ArrayBufferReader,
+    size: bigint,
+  ): FileContents {
+    const position = header.nextU64()
+    return new FileContents(
+      source.slice(Number(position), Number(position + size)),
+    )
+  }
+  async verified(hash: Uint8Array): Promise<ArrayBuffer> {
+    const res = await this.contents.arrayBuffer()
+    if (!compare(hash, blake3(new Uint8Array(res)))) {
+      throw new Error("hash sum mismatch")
+    }
+    return res
+  }
+}

sdk/lib/s9pk/merkleArchive/index.ts

@@ -0,0 +1,167 @@
+import { MerkleArchiveCommitment } from "../../osBindings"
+import { DirectoryContents } from "./directoryContents"
+import { FileContents } from "./fileContents"
+import { ed25519ph } from "@noble/curves/ed25519"
+import { sha512 } from "@noble/hashes/sha2"
+import { VarIntProcessor } from "./varint"
+import { compare } from ".."
+
+const maxVarstringLen = 1024 * 1024
+
+export type Signer = {
+  pubkey: Uint8Array
+  signature: Uint8Array
+  maxSize: bigint
+  context: string
+}
+
+export class ArrayBufferReader {
+  constructor(private buffer: ArrayBuffer) {}
+  next(length: number): ArrayBuffer {
+    const res = this.buffer.slice(0, length)
+    this.buffer = this.buffer.slice(length)
+    return res
+  }
+  nextU64(): bigint {
+    return new DataView(this.next(8)).getBigUint64(0)
+  }
+  nextVarint(): number {
+    const p = new VarIntProcessor()
+    while (!p.finished()) {
+      p.push(new Uint8Array(this.buffer.slice(0, 1))[0])
+      this.buffer = this.buffer.slice(1)
+    }
+    const res = p.decode()
+    if (res === null) {
+      throw new Error("Reached EOF")
+    }
+    return res
+  }
+  nextVarstring(): string {
+    const len = Math.min(this.nextVarint(), maxVarstringLen)
+    return new TextDecoder().decode(this.next(len))
+  }
+}
+
+export class MerkleArchive {
+  static readonly headerSize =
+    32 + // pubkey
+    64 + // signature
+    32 + // sighash
+    8 + // size
+    DirectoryContents.headerSize
+  private constructor(
+    readonly signer: Signer,
+    readonly contents: DirectoryContents,
+  ) {}
+  static async deserialize(
+    source: Blob,
+    context: string,
+    header: ArrayBufferReader,
+    commitment: MerkleArchiveCommitment | null,
+  ): Promise<MerkleArchive> {
+    const pubkey = new Uint8Array(header.next(32))
+    const signature = new Uint8Array(header.next(64))
+    const sighash = new Uint8Array(header.next(32))
+    const rootMaxSizeBytes = header.next(8)
+    const maxSize = new DataView(rootMaxSizeBytes).getBigUint64(0)
+
+    if (
+      !ed25519ph.verify(
+        signature,
+        new Uint8Array(
+          await new Blob([sighash, rootMaxSizeBytes]).arrayBuffer(),
+        ),
+        pubkey,
+        {
+          context: new TextEncoder().encode(context),
+          zip215: true,
+        },
+      )
+    ) {
+      throw new Error("signature verification failed")
+    }
+
+    if (commitment) {
+      if (
+        !compare(
+          sighash,
+          new Uint8Array(Buffer.from(commitment.rootSighash, "base64").buffer),
+        )
+      ) {
+        throw new Error("merkle root mismatch")
+      }
+      if (maxSize > commitment.rootMaxsize) {
+        throw new Error("root directory max size too large")
+      }
+    } else if (maxSize > 1024 * 1024) {
+      throw new Error(
+        "root directory max size over 1MiB, cancelling download in case of DOS attack",
+      )
+    }
+
+    const contents = await DirectoryContents.deserialize(
+      source,
+      header,
+      sighash,
+      maxSize,
+    )
+
+    return new MerkleArchive(
+      {
+        pubkey,
+        signature,
+        maxSize,
+        context,
+      },
+      contents,
+    )
+  }
+}
+
+export class Entry {
+  private constructor(
+    readonly hash: Uint8Array,
+    readonly size: bigint,
+    readonly contents: EntryContents,
+  ) {}
+  static async deserialize(
+    source: Blob,
+    header: ArrayBufferReader,
+  ): Promise<Entry> {
+    const hash = new Uint8Array(header.next(32))
+    const size = header.nextU64()
+    const contents = await deserializeEntryContents(source, header, hash, size)
+
+    return new Entry(new Uint8Array(hash), size, contents)
+  }
+  async verifiedFileContents(): Promise<ArrayBuffer> {
+    if (!this.contents) {
+      throw new Error("file is missing from archive")
+    }
+    if (!(this.contents instanceof FileContents)) {
+      throw new Error("is not a regular file")
+    }
+    return this.contents.verified(this.hash)
+  }
+}
+
+export type EntryContents = null | FileContents | DirectoryContents
+async function deserializeEntryContents(
+  source: Blob,
+  header: ArrayBufferReader,
+  hash: Uint8Array,
+  size: bigint,
+): Promise<EntryContents> {
+  const typeId = new Uint8Array(header.next(1))[0]
+  switch (typeId) {
+    case 0:
+      return null
+    case 1:
+      return FileContents.deserialize(source, header, size)
+    case 2:
+      return DirectoryContents.deserialize(source, header, hash, size)
+    default:
+      throw new Error(`Unknown type id ${typeId} found in MerkleArchive`)
+  }
+}

sdk/lib/s9pk/merkleArchive/varint.ts

@@ -0,0 +1,62 @@
+const msb = 0x80
+const dropMsb = 0x7f
+const maxSize = Math.floor((8 * 8 + 7) / 7)
+
+export class VarIntProcessor {
+  private buf: Uint8Array
+  private i: number
+  constructor() {
+    this.buf = new Uint8Array(maxSize)
+    this.i = 0
+  }
+  push(b: number) {
+    if (this.i >= maxSize) {
+      throw new Error("Unterminated varint")
+    }
+    this.buf[this.i] = b
+    this.i += 1
+  }
+  finished(): boolean {
+    return this.i > 0 && (this.buf[this.i - 1] & msb) === 0
+  }
+  decode(): number | null {
+    let result = 0
+    let shift = 0
+    let success = false
+    for (let i = 0; i < this.i; i++) {
+      const b = this.buf[i]
+      const msbDropped = b & dropMsb
+      result |= msbDropped << shift
+      shift += 7
+
+      if ((b & msb) == 0 || shift > 9 * 7) {
+        success = (b & msb) === 0
+        break
+      }
+    }
+
+    if (success) {
+      return result
+    } else {
+      console.error(this.buf)
+      return null
+    }
+  }
+}
+
+export function serializeVarint(int: number): ArrayBuffer {
+  const buf = new Uint8Array(maxSize)
+  let n = int
+  let i = 0
+
+  while (n >= msb) {
+    buf[i] = msb | n
+    i += 1
+    n >>= 7
+  }
+
+  buf[i] = n
+  i += 1
+
+  return buf.slice(0, i).buffer
+}

sdk/lib/test/emverList.test.ts

@@ -8,18 +8,14 @@ describe("EmVer", () => {
         checker.check("1.2")
         checker.check("1.2.3")
         checker.check("1.2.3.4")
-        // @ts-expect-error
         checker.check("1.2.3.4.5")
-        // @ts-expect-error
         checker.check("1.2.3.4.5.6")
         expect(checker.check("1")).toEqual(true)
         expect(checker.check("1.2")).toEqual(true)
         expect(checker.check("1.2.3.4")).toEqual(true)
       })
       test("rangeOf('*') invalid", () => {
-        // @ts-expect-error
         expect(() => checker.check("a")).toThrow()
-        // @ts-expect-error
         expect(() => checker.check("")).toThrow()
         expect(() => checker.check("1..3")).toThrow()
       })
@@ -31,7 +27,6 @@ describe("EmVer", () => {
         expect(checker.check("2-beta123")).toEqual(true)
         expect(checker.check("2")).toEqual(true)
         expect(checker.check("1.2.3.5")).toEqual(true)
-        // @ts-expect-error
         expect(checker.check("1.2.3.4.1")).toEqual(true)
       })
 
@@ -58,7 +53,6 @@ describe("EmVer", () => {
       test(`rangeOf(">=1.2.3.4") valid`, () => {
         expect(checker.check("2")).toEqual(true)
         expect(checker.check("1.2.3.5")).toEqual(true)
-        // @ts-expect-error
         expect(checker.check("1.2.3.4.1")).toEqual(true)
         expect(checker.check("1.2.3.4")).toEqual(true)
       })
@@ -73,7 +67,6 @@ describe("EmVer", () => {
       test(`rangeOf("<1.2.3.4") invalid`, () => {
         expect(checker.check("2")).toEqual(false)
         expect(checker.check("1.2.3.5")).toEqual(false)
-        // @ts-expect-error
         expect(checker.check("1.2.3.4.1")).toEqual(false)
         expect(checker.check("1.2.3.4")).toEqual(false)
       })
@@ -88,7 +81,6 @@ describe("EmVer", () => {
       test(`rangeOf("<=1.2.3.4") invalid`, () => {
         expect(checker.check("2")).toEqual(false)
         expect(checker.check("1.2.3.5")).toEqual(false)
-        // @ts-expect-error
         expect(checker.check("1.2.3.4.1")).toEqual(false)
       })
 
@@ -196,7 +188,6 @@ describe("EmVer", () => {
       test(`rangeOf("!>1.2.3.4") invalid`, () => {
         expect(checker.check("2")).toEqual(false)
         expect(checker.check("1.2.3.5")).toEqual(false)
-        // @ts-expect-error
         expect(checker.check("1.2.3.4.1")).toEqual(false)
       })
 

sdk/lib/util/fileHelper.ts

@@ -1,7 +1,7 @@
 import * as matches from "ts-matches"
 import * as YAML from "yaml"
 import * as TOML from "@iarna/toml"
-import _ from "lodash"
+import merge from "lodash.merge"
 import * as T from "../types"
 import * as fs from "node:fs/promises"
 
@@ -82,7 +82,7 @@ export class FileHelper<A> {
 
   async merge(data: A, effects: T.Effects) {
     const fileData = (await this.read(effects).catch(() => ({}))) || {}
-    const mergeData = _.merge({}, fileData, data)
+    const mergeData = merge({}, fileData, data)
     return await this.write(mergeData, effects)
   }
   /**