Feature/sideload (#1520)

* base styling and action placeholders for package sideload

* apparently didnt add new folder

* wip

* parse manifest and icon from s9pk to upload

* wip handle s9pk upload

* adjust types, finalize actions, cleanup

* clean up and fix data clearing and response

* include rest rpc in proxy conf sample

* address feedback to use shorthand falsy coercion

* update copy and invalid package file ux

* do not wait package upload, instead show install progress

* fix proxy for rest rpc

rename sideload package page titles

frontend/projects/ui/src/app/pages/server-routes/server-routing.module.ts

@@ -67,6 +67,11 @@ const routes: Routes = [
     loadChildren: () =>
       import('./sessions/sessions.module').then(m => m.SessionsPageModule),
   },
+  {
+    path: 'sideload',
+    loadChildren: () =>
+      import('./sideload/sideload.module').then(m => m.SideloadPageModule),
+  },
   {
     path: 'specs',
     loadChildren: () =>

frontend/projects/ui/src/app/pages/server-routes/server-show/server-show.page.ts

@@ -331,6 +331,17 @@ export class ServerShowPage {
         detail: true,
         disabled: of(false),
       },
+      {
+        title: 'Manually install a service',
+        description: `Install a service by drag n' drop`,
+        icon: 'push-outline',
+        action: () =>
+          this.navCtrl.navigateForward(['sideload'], {
+            relativeTo: this.route,
+          }),
+        detail: true,
+        disabled: of(false),
+      },
       {
         title: 'Marketplace Settings',
         description: 'Add or remove marketplaces',

frontend/projects/ui/src/app/pages/server-routes/sideload/dnd.directive.ts

@@ -0,0 +1,39 @@
+import {
+  Directive,
+  ElementRef,
+  EventEmitter,
+  HostBinding,
+  HostListener,
+  Output,
+} from '@angular/core'
+import { DomSanitizer } from '@angular/platform-browser'
+
+@Directive({
+  selector: '[appDnd]',
+})
+export class DragNDropDirective {
+  @Output() onFileDropped: EventEmitter<any> = new EventEmitter()
+
+  @HostBinding('style.background') private background = 'rgba(24, 24, 24, 0.5)'
+
+  constructor(el: ElementRef, private sanitizer: DomSanitizer) {}
+
+  @HostListener('dragover', ['$event']) public onDragOver(evt: DragEvent) {
+    evt.preventDefault()
+    evt.stopPropagation()
+    this.background = '#6a937b3c'
+  }
+
+  @HostListener('dragleave', ['$event']) public onDragLeave(evt: DragEvent) {
+    evt.preventDefault()
+    evt.stopPropagation()
+    this.background = 'rgba(24, 24, 24, 0.5)'
+  }
+
+  @HostListener('drop', ['$event']) public onDrop(evt: DragEvent) {
+    evt.preventDefault()
+    evt.stopPropagation()
+    this.background = ' rgba(24, 24, 24, 0.5)'
+    this.onFileDropped.emit(evt)
+  }
+}

frontend/projects/ui/src/app/pages/server-routes/sideload/sideload.module.ts

@@ -0,0 +1,26 @@
+import { NgModule } from '@angular/core'
+import { CommonModule } from '@angular/common'
+import { IonicModule } from '@ionic/angular'
+import { SideloadPage } from './sideload.page'
+import { Routes, RouterModule } from '@angular/router'
+import { EmverPipesModule, SharedPipesModule } from '@start9labs/shared'
+import { DragNDropDirective } from './dnd.directive'
+
+const routes: Routes = [
+  {
+    path: '',
+    component: SideloadPage,
+  },
+]
+
+@NgModule({
+  imports: [
+    CommonModule,
+    IonicModule,
+    RouterModule.forChild(routes),
+    SharedPipesModule,
+    EmverPipesModule,
+  ],
+  declarations: [SideloadPage, DragNDropDirective],
+})
+export class SideloadPageModule {}

frontend/projects/ui/src/app/pages/server-routes/sideload/sideload.page.html

@@ -0,0 +1,94 @@
+<ion-header>
+  <ion-toolbar>
+    <ion-buttons slot="start">
+      <ion-back-button defaultHref="embassy"></ion-back-button>
+    </ion-buttons>
+    <ion-title>Manually install a service</ion-title>
+  </ion-toolbar>
+</ion-header>
+
+<ion-content class="ion-text-center">
+  <!-- file upload -->
+  <div
+    *ngIf="!toUpload.file"
+    class="drop-area"
+    [class.drop-area_mobile]="isMobile"
+    appDnd
+    (onFileDropped)="handleFileDrop($event)"
+  >
+    <ion-icon
+      name="cloud-upload-outline"
+      color="dark"
+      style="font-size: 42px"
+    ></ion-icon>
+    <h4>To install a service manually, upload the s9pk here</h4>
+    <p *ngIf="onTor">
+      <ion-text color="success"
+        >Tip: switch to LAN for faster uploads.</ion-text
+      >
+    </p>
+    <br />
+    <ion-button color="primary" type="file">
+      <label for="upload-photo">Browse</label>
+      <input
+        type="file"
+        style="position: absolute; opacity: 0; height: 100%"
+        id="upload-photo"
+        (change)="handleFileInput($event)"
+      />
+    </ion-button>
+  </div>
+  <!-- file uploaded -->
+  <div class="drop-area_filled" *ngIf="toUpload.file">
+    <div class="inline" *ngIf="valid; else invalid">
+      <ion-icon name="checkmark-circle-outline" color="success"></ion-icon>
+      <h4>{{ message }}</h4>
+    </div>
+    <ng-template #invalid>
+      <div class="area">
+        <div class="inline">
+          <ion-icon
+            *ngIf="!valid"
+            name="close-circle-outline"
+            color="danger"
+          ></ion-icon>
+          <h4><ion-text color="danger">{{ message }}</ion-text></h4>
+        </div>
+        <ion-button color="primary" (click)="clearToUpload()">
+          Try again
+        </ion-button>
+      </div>
+    </ng-template>
+    <br />
+    <div *ngIf="valid">
+      <div *ngIf="toUpload.manifest " class="service-card">
+        <div class="row row_end">
+          <ion-button
+            style="
+              --background-hover: transparent;
+              --padding-end: 0px;
+              --padding-start: 0px;
+            "
+            fill="clear"
+            size="small"
+            (click)="clearToUpload()"
+          >
+            <ion-icon slot="icon-only" name="close" color="danger"></ion-icon>
+          </ion-button>
+        </div>
+        <div class="row">
+          <img
+            *ngIf="toUpload.icon"
+            [alt]="toUpload.manifest.title + ' Icon'"
+            [src]="toUpload.icon | trustUrl"
+          />
+          <h2>{{ toUpload.manifest.title }}</h2>
+          <p>{{ toUpload.manifest.version | displayEmver }}</p>
+        </div>
+      </div>
+      <ion-button color="primary" (click)="handleUpload()">
+        Upload & Install
+      </ion-button>
+    </div>
+  </div>
+</ion-content>

frontend/projects/ui/src/app/pages/server-routes/sideload/sideload.page.scss

@@ -0,0 +1,90 @@
+.inline {
+  * {
+    vertical-align: initial;
+    padding-right: 5px;
+  }
+}
+
+.area {
+  flex-direction: column;
+  justify-content: center;
+}
+
+.drop-area {
+  display: flex;
+  background-color: rgba(24, 24, 24, 0.5);
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+  border-style: dashed;
+  border-width: 2px;
+  border-color: var(--ion-color-dark);
+  color: var(--ion-color-dark);
+  border-radius: 5px;
+  margin: 60px;
+  padding: 30px;
+  min-height: 600px;
+
+  &_filled {
+    display: flex;
+    background-color: rgba(24, 24, 24, 0.5);
+    flex-direction: column;
+    justify-content: center;
+    align-items: center;
+    border-style: solid;
+    border-width: 2px;
+    border-color: var(--ion-color-dark);
+    color: var(--ion-color-dark);
+    border-radius: 5px;
+    margin: 60px;
+    padding: 30px;
+    min-height: 600px;
+
+  }
+
+  &_mobile {
+    border-width: 0px !important;
+  }
+
+  ion-input {
+    color: var(--ion-color-dark);
+  }
+}
+
+.service-card {
+  background: radial-gradient(var(--ion-color-step-100), transparent);
+  min-width: 200px;
+  max-width: 300px;
+  height: auto;
+  padding: 0;
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  margin: 20px 20px 40px 20px;
+  border-style: solid;
+  border-color: var(--ion-color-step-100);
+  border-radius: 7px;
+  padding: 4px 8px 8px 8px;
+
+  .row {
+    width: auto;
+
+    &_end {
+      align-self: end;
+
+      ion-button {
+        width: 80%;
+      }
+    }
+  }
+
+  img {
+    width: 60px;
+    text-align: center;
+  }
+
+  h2,
+  p {
+    text-align: center;
+  }
+}

frontend/projects/ui/src/app/pages/server-routes/sideload/sideload.page.ts

@@ -0,0 +1,205 @@
+import { Component } from '@angular/core'
+import { isPlatform, LoadingController, NavController } from '@ionic/angular'
+import { ApiService } from 'src/app/services/api/embassy-api.service'
+import { Manifest } from 'src/app/services/patch-db/data-model'
+import { ConfigService } from 'src/app/services/config.service'
+import cbor from 'cbor'
+import { ErrorToastService } from '@start9labs/shared'
+interface Positions {
+  [key: string]: [bigint, bigint] // [position, length]
+}
+
+const MAGIC = new Uint8Array([59, 59])
+const VERSION = new Uint8Array([1])
+@Component({
+  selector: 'sideload',
+  templateUrl: './sideload.page.html',
+  styleUrls: ['./sideload.page.scss'],
+})
+export class SideloadPage {
+  isMobile = isPlatform(window, 'ios') || isPlatform(window, 'android')
+  toUpload: {
+    manifest: Manifest | null
+    icon: string | null
+    file: File | null
+  } = {
+    manifest: null,
+    icon: null,
+    file: null,
+  }
+  onTor = this.config.isTor()
+  valid: boolean
+  message: string
+
+  constructor(
+    private readonly loadingCtrl: LoadingController,
+    private readonly api: ApiService,
+    private readonly navCtrl: NavController,
+    private readonly errToast: ErrorToastService,
+    private readonly config: ConfigService,
+  ) {}
+
+  handleFileDrop(e: any) {
+    const files = e.dataTransfer.files
+    this.setFile(files)
+  }
+
+  handleFileInput(e: any) {
+    const files = e.target.files
+    this.setFile(files)
+  }
+  async setFile(files?: File[]) {
+    const loader = await this.loadingCtrl.create({
+      spinner: 'lines',
+      message: 'Verifying Package',
+      cssClass: 'loader',
+    })
+    await loader.present()
+    if (!files || !files.length) return
+    this.toUpload.file = files[0]
+    // verify valid s9pk
+    const magic = new Uint8Array(
+      await readBlobToArrayBuffer(this.toUpload.file.slice(0, 2)),
+    )
+    const version = new Uint8Array(
+      await readBlobToArrayBuffer(this.toUpload.file.slice(2, 3)),
+    )
+    if (compare(magic, MAGIC) && compare(version, VERSION)) {
+      loader.dismiss()
+      this.valid = true
+      this.message = 'A valid package file has been detected!'
+      await this.parseS9pk(this.toUpload.file)
+    } else {
+      loader.dismiss()
+      this.valid = false
+      this.message = 'Invalid package file'
+    }
+  }
+
+  clearToUpload() {
+    this.toUpload.file = null
+    this.toUpload.manifest = null
+    this.toUpload.icon = null
+  }
+
+  async handleUpload() {
+    const loader = await this.loadingCtrl.create({
+      spinner: 'lines',
+      message: 'Uploading Package',
+      cssClass: 'loader',
+    })
+    await loader.present()
+    try {
+      const guid = await this.api.sideloadPackage({
+        manifest: this.toUpload.manifest!,
+        icon: this.toUpload.icon!,
+      })
+      this.api
+        .uploadPackage(guid, await readBlobToArrayBuffer(this.toUpload.file!))
+        .catch(e => {
+          this.errToast.present(e)
+        })
+    } catch (e: any) {
+      this.errToast.present(e)
+    } finally {
+      loader.dismiss()
+      await this.navCtrl.navigateForward(
+        `/services/${this.toUpload.manifest!.id}`,
+      )
+      this.clearToUpload()
+    }
+  }
+
+  async parseS9pk(file: Blob) {
+    const positions: Positions = {}
+    // magic=2bytes, version=1bytes, pubkey=32bytes, signature=64bytes, toc_length=4bytes = 103byte is starting point
+    let start = 103
+    let end = start + 1 // 104
+    const tocLength = new DataView(
+      await readBlobToArrayBuffer(
+        this.toUpload.file?.slice(99, 103) ?? new Blob(),
+      ),
+    ).getUint32(0, false)
+    await getPositions(start, end, file, positions, tocLength as any)
+
+    await this.getManifest(positions, file)
+    await this.getIcon(positions, file)
+  }
+
+  async getManifest(positions: Positions, file: Blob) {
+    const data = await readBlobToArrayBuffer(
+      file.slice(
+        Number(positions['manifest'][0]),
+        Number(positions['manifest'][0]) + Number(positions['manifest'][1]),
+      ),
+    )
+    this.toUpload.manifest = await cbor.decode(data, true)
+  }
+
+  async getIcon(positions: Positions, file: Blob) {
+    const data = file.slice(
+      Number(positions['icon'][0]),
+      Number(positions['icon'][0]) + Number(positions['icon'][1]),
+    )
+    this.toUpload.icon = await readBlobAsDataURL(data)
+  }
+}
+
+async function getPositions(
+  initialStart: number,
+  initialEnd: number,
+  file: Blob,
+  positions: Positions,
+  tocLength: number,
+) {
+  let start = initialStart
+  let end = initialEnd
+  const titleLength = new Uint8Array(
+    await readBlobToArrayBuffer(file.slice(start, end)),
+  )[0]
+  const tocTitle = await file.slice(end, end + titleLength).text()
+  start = end + titleLength
+  end = start + 8
+  const chapterPosition = new DataView(
+    await readBlobToArrayBuffer(file.slice(start, end)),
+  ).getBigUint64(0, false)
+  start = end
+  end = start + 8
+  const chapterLength = new DataView(
+    await readBlobToArrayBuffer(file.slice(start, end)),
+  ).getBigUint64(0, false)
+
+  positions[tocTitle] = [chapterPosition, chapterLength]
+  start = end
+  end = start + 1
+  if (end <= tocLength + (initialStart - 1)) {
+    await getPositions(start, end, file, positions, tocLength)
+  }
+}
+
+async function readBlobAsDataURL(f: Blob): Promise<string> {
+  const reader = new FileReader()
+  reader.readAsDataURL(f)
+  return new Promise(resolve => {
+    reader.onloadend = () => {
+      resolve(reader.result as string)
+    }
+  })
+}
+
+async function readBlobToArrayBuffer(f: Blob): Promise<ArrayBuffer> {
+  const reader = new FileReader()
+  reader.readAsArrayBuffer(f)
+  return new Promise(resolve => {
+    reader.onloadend = () => {
+      resolve(reader.result as ArrayBuffer)
+    }
+  })
+}
+
+function compare(a: Uint8Array, b: Uint8Array) {
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) return false
+  }
+  return true
+}