Feature/UI sideload (#2658)

* ui sideloading

* remove subtlecrypto import

* fix parser

* misc fixes

* allow docker pull during compat conversion

web/package-lock.json

@@ -23,6 +23,8 @@
         "@ng-web-apis/common": "^2.0.0",
         "@ng-web-apis/mutation-observer": "^2.0.0",
         "@ng-web-apis/resize-observer": "^2.0.0",
+        "@noble/curves": "^1.4.0",
+        "@noble/hashes": "^1.4.0",
         "@start9labs/argon2": "^0.2.2",
         "@start9labs/emver": "^0.1.5",
         "@start9labs/start-sdk": "file:../sdk/dist",
@@ -35,6 +37,7 @@
         "angular-svg-round-progressbar": "^9.0.0",
         "ansi-to-html": "^0.7.2",
         "base64-js": "^1.5.1",
+        "buffer": "^6.0.3",
         "cbor": "npm:@jprochazk/cbor@^0.4.9",
         "cbor-web": "^8.1.0",
         "core-js": "^3.21.1",
@@ -45,6 +48,7 @@
         "jose": "^4.9.0",
         "js-yaml": "^4.1.0",
         "marked": "^4.0.0",
+        "mime": "^4.0.3",
         "monaco-editor": "^0.33.0",
         "mustache": "^4.2.0",
         "ng-qrcode": "^7.0.0",
@@ -53,7 +57,7 @@
         "pbkdf2": "^3.1.2",
         "rxjs": "^7.8.1",
         "swiper": "^8.2.4",
-        "ts-matches": "^5.2.1",
+        "ts-matches": "^5.5.1",
         "tslib": "^2.3.0",
         "uuid": "^8.3.2",
         "zone.js": "^0.11.5"
@@ -1978,14 +1982,17 @@
       "license": "MIT",
       "dependencies": {
         "@iarna/toml": "^2.2.5",
+        "@noble/curves": "^1.4.0",
+        "@noble/hashes": "^1.4.0",
         "isomorphic-fetch": "^3.0.0",
-        "lodash": "^4.17.21",
-        "ts-matches": "^5.4.1",
+        "lodash.merge": "^4.6.2",
+        "mime": "^4.0.3",
+        "ts-matches": "^5.5.1",
         "yaml": "^2.2.2"
       },
       "devDependencies": {
         "@types/jest": "^29.4.0",
-        "@types/lodash": "^4.17.5",
+        "@types/lodash.merge": "^4.6.2",
         "jest": "^29.4.3",
         "prettier": "^3.2.5",
         "ts-jest": "^29.0.5",
@@ -5111,6 +5118,28 @@
         "@ng-web-apis/common": ">=2.0.0"
       }
     },
+    "node_modules/@noble/curves": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.4.0.tgz",
+      "integrity": "sha512-p+4cb332SFCrReJkCYe8Xzm0OWi4Jji5jVdIZRL/PmacmDkFNw6MrrV+gGpiPxLHbV+zKFRywUWbaseT+tZRXg==",
+      "dependencies": {
+        "@noble/hashes": "1.4.0"
+      },
+      "funding": {
+        "url": "https://paulmillr.com/funding/"
+      }
+    },
+    "node_modules/@noble/hashes": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.4.0.tgz",
+      "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==",
+      "engines": {
+        "node": ">= 16"
+      },
+      "funding": {
+        "url": "https://paulmillr.com/funding/"
+      }
+    },
     "node_modules/@nodelib/fs.scandir": {
       "version": "2.1.5",
       "devOptional": true,
@@ -9954,6 +9983,19 @@
         "node": ">=6"
       }
     },
+    "node_modules/less/node_modules/mime": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
+      "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==",
+      "dev": true,
+      "optional": true,
+      "bin": {
+        "mime": "cli.js"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/less/node_modules/pify": {
       "version": "4.0.1",
       "dev": true,
@@ -10551,14 +10593,17 @@
       }
     },
     "node_modules/mime": {
-      "version": "1.6.0",
-      "dev": true,
-      "license": "MIT",
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-4.0.3.tgz",
+      "integrity": "sha512-KgUb15Oorc0NEKPbvfa0wRU+PItIEZmiv+pyAO2i0oTIVTJhlzMclU7w4RXWQrSOVH5ax/p/CkIO7KI4OyFJTQ==",
+      "funding": [
+        "https://github.com/sponsors/broofa"
+      ],
       "bin": {
-        "mime": "cli.js"
+        "mime": "bin/cli.js"
       },
       "engines": {
-        "node": ">=4"
+        "node": ">=16"
       }
     },
     "node_modules/mime-db": {
@@ -13644,6 +13689,18 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/send/node_modules/mime": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
+      "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==",
+      "dev": true,
+      "bin": {
+        "mime": "cli.js"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/send/node_modules/ms": {
       "version": "2.1.3",
       "dev": true,
@@ -14617,8 +14674,9 @@
       }
     },
     "node_modules/ts-matches": {
-      "version": "v5.2.1",
-      "license": "MIT"
+      "version": "5.5.1",
+      "resolved": "https://registry.npmjs.org/ts-matches/-/ts-matches-5.5.1.tgz",
+      "integrity": "sha512-UFYaKgfqlg9FROK7bdpYqFwG1CJvP4kOJdjXuWoqxo9jCmANoDw1GxkSCpJgoTeIiSTaTH5Qr1klSspb8c+ydg=="
     },
     "node_modules/ts-morph": {
       "version": "10.0.2",

web/package.json

@@ -46,6 +46,8 @@
     "@ng-web-apis/common": "^2.0.0",
     "@ng-web-apis/mutation-observer": "^2.0.0",
     "@ng-web-apis/resize-observer": "^2.0.0",
+    "@noble/curves": "^1.4.0",
+    "@noble/hashes": "^1.4.0",
     "@start9labs/argon2": "^0.2.2",
     "@start9labs/emver": "^0.1.5",
     "@start9labs/start-sdk": "file:../sdk/dist",
@@ -58,6 +60,7 @@
     "angular-svg-round-progressbar": "^9.0.0",
     "ansi-to-html": "^0.7.2",
     "base64-js": "^1.5.1",
+    "buffer": "^6.0.3",
     "cbor": "npm:@jprochazk/cbor@^0.4.9",
     "cbor-web": "^8.1.0",
     "core-js": "^3.21.1",
@@ -68,6 +71,7 @@
     "jose": "^4.9.0",
     "js-yaml": "^4.1.0",
     "marked": "^4.0.0",
+    "mime": "^4.0.3",
     "monaco-editor": "^0.33.0",
     "mustache": "^4.2.0",
     "ng-qrcode": "^7.0.0",
@@ -76,7 +80,7 @@
     "pbkdf2": "^3.1.2",
     "rxjs": "^7.8.1",
     "swiper": "^8.2.4",
-    "ts-matches": "^5.2.1",
+    "ts-matches": "^5.5.1",
     "tslib": "^2.3.0",
     "uuid": "^8.3.2",
     "zone.js": "^0.11.5"

web/projects/ui/src/app/pages/server-routes/sideload/sideload.page.ts

@@ -4,14 +4,15 @@ import { ApiService } from 'src/app/services/api/embassy-api.service'
 import { ConfigService } from 'src/app/services/config.service'
 import cbor from 'cbor'
 import { ErrorToastService } from '@start9labs/shared'
-import { T } from '@start9labs/start-sdk'
+import { S9pk, T } from '@start9labs/start-sdk'
 
 interface Positions {
   [key: string]: [bigint, bigint] // [position, length]
 }
 
 const MAGIC = new Uint8Array([59, 59])
-const VERSION = new Uint8Array([1])
+const VERSION_1 = new Uint8Array([1])
+const VERSION_2 = new Uint8Array([2])
 
 @Component({
   selector: 'sideload',
@@ -64,11 +65,36 @@ export class SideloadPage {
   async validateS9pk(file: File) {
     const magic = new Uint8Array(await blobToBuffer(file.slice(0, 2)))
     const version = new Uint8Array(await blobToBuffer(file.slice(2, 3)))
-    if (compare(magic, MAGIC) && compare(version, VERSION)) {
-      await this.parseS9pk(file)
-      return {
-        invalid: false,
-        message: 'A valid package file has been detected!',
+    if (compare(magic, MAGIC)) {
+      try {
+        if (compare(version, VERSION_1)) {
+          await this.parseS9pkV1(file)
+          return {
+            invalid: false,
+            message: 'A valid package file has been detected!',
+          }
+        } else if (compare(version, VERSION_2)) {
+          await this.parseS9pkV2(file)
+          return {
+            invalid: false,
+            message: 'A valid package file has been detected!',
+          }
+        } else {
+          console.error(version)
+          return {
+            invalid: true,
+            message: 'Invalid package file',
+          }
+        }
+      } catch (e) {
+        console.error(e)
+        return {
+          invalid: true,
+          message:
+            e instanceof Error
+              ? `Invalid package file: ${e.message}`
+              : 'Invalid package file',
+        }
       }
     } else {
       return {
@@ -91,12 +117,9 @@ export class SideloadPage {
     })
     await loader.present()
     try {
-      const guid = await this.api.sideloadPackage({
-        manifest: this.toUpload.manifest!,
-        icon: this.toUpload.icon!,
-      })
+      const res = await this.api.sideloadPackage()
       this.api
-        .uploadPackage(guid, this.toUpload.file!)
+        .uploadPackage(res.upload, this.toUpload.file!)
         .catch(e => console.error(e))
 
       this.navCtrl.navigateRoot('/services')
@@ -108,7 +131,7 @@ export class SideloadPage {
     }
   }
 
-  async parseS9pk(file: File) {
+  async parseS9pkV1(file: File) {
     const positions: Positions = {}
     // magic=2bytes, version=1bytes, pubkey=32bytes, signature=64bytes, toc_length=4bytes = 103byte is starting point
     let start = 103
@@ -122,6 +145,12 @@ export class SideloadPage {
     await this.getIcon(positions, file)
   }
 
+  async parseS9pkV2(file: File) {
+    const s9pk = await S9pk.deserialize(file, null)
+    this.toUpload.manifest = s9pk.manifest
+    this.toUpload.icon = await s9pk.icon()
+  }
+
   async getManifest(positions: Positions, file: Blob) {
     const data = await blobToBuffer(
       file.slice(
@@ -225,6 +254,7 @@ async function readBlobToArrayBuffer(
 }
 
 function compare(a: Uint8Array, b: Uint8Array) {
+  if (a.length !== b.length) return false
   for (let i = 0; i < a.length; i++) {
     if (a[i] !== b[i]) return false
   }

web/projects/ui/src/app/services/api/api.types.ts

@@ -273,7 +273,10 @@ export module RR {
     manifest: T.Manifest
     icon: string // base64
   }
-  export type SideloadPacakgeRes = string //guid
+  export type SideloadPackageRes = {
+    upload: string // guid
+    progress: string // guid
+  }
 
   // marketplace
 

web/projects/ui/src/app/services/api/embassy-api.service.ts

@@ -243,7 +243,5 @@ export abstract class ApiService {
     params: RR.DryConfigureDependencyReq,
   ): Promise<RR.DryConfigureDependencyRes>
 
-  abstract sideloadPackage(
-    params: RR.SideloadPackageReq,
-  ): Promise<RR.SideloadPacakgeRes>
+  abstract sideloadPackage(): Promise<RR.SideloadPackageRes>
 }

web/projects/ui/src/app/services/api/embassy-live-api.service.ts

@@ -29,7 +29,7 @@ export class LiveApiService extends ApiService {
     @Inject(PATCH_CACHE) private readonly cache$: Observable<Dump<DataModel>>,
   ) {
     super()
-    ; (window as any).rpcClient = this
+    ;(window as any).rpcClient = this
   }
 
   // for getting static files: ex icons, instructions, licenses
@@ -460,12 +460,10 @@ export class LiveApiService extends ApiService {
     })
   }
 
-  async sideloadPackage(
-    params: RR.SideloadPackageReq,
-  ): Promise<RR.SideloadPacakgeRes> {
+  async sideloadPackage(): Promise<RR.SideloadPackageRes> {
     return this.rpcRequest({
       method: 'package.sideload',
-      params,
+      params: {},
     })
   }
 

web/projects/ui/src/app/services/api/embassy-mock-api.service.ts

@@ -1062,11 +1062,12 @@ export class MockApiService extends ApiService {
     }
   }
 
-  async sideloadPackage(
-    params: RR.SideloadPackageReq,
-  ): Promise<RR.SideloadPacakgeRes> {
+  async sideloadPackage(): Promise<RR.SideloadPackageRes> {
     await pauseFor(2000)
-    return '4120e092-05ab-4de2-9fbd-c3f1f4b1df9e' // no significance, randomly generated
+    return {
+      upload: '4120e092-05ab-4de2-9fbd-c3f1f4b1df9e', // no significance, randomly generated
+      progress: '5120e092-05ab-4de2-9fbd-c3f1f4b1df9e', // no significance, randomly generated
+    }
   }
 
   private async initProgress(): Promise<T.FullProgress> {

web/projects/ui/src/polyfills.ts

@@ -52,8 +52,11 @@
  *
  */
 
-(window as any).global = window
-; (window as any).process = { env: { DEBUG: undefined }, browser: true }
+;(window as any).global = window
+;(window as any).process = { env: { DEBUG: undefined }, browser: true }
+
+import { Buffer } from 'buffer'
+window.Buffer = Buffer
 
 import './zone-flags'