Merge pull request #2714 from Start9Labs/sideload-and-backups

fix: implement back sideload and server selection in restoring

web/projects/marketplace/src/pages/show/hero/hero.component.ts

@@ -30,15 +30,9 @@ import { MarketplacePkg, StoreIdentity } from '../../../types'
         <!-- background darkening overlay -->
         <div class="dark-overlay"></div>
         <div class="inner-container-title">
-          <h2 ticker>
-            {{ pkg.title }}
-          </h2>
-          <h3>
-            {{ pkg.version }}
-          </h3>
-          <p>
-            {{ pkg.description.short }}
-          </p>
+          <h2 ticker>{{ pkg.title }}</h2>
+          <h3>{{ pkg.version }}</h3>
+          <p>{{ pkg.description.short }}</p>
         </div>
         <!-- control buttons -->
         <ng-content />
@@ -150,8 +144,8 @@ import { MarketplacePkg, StoreIdentity } from '../../../types'
           position: absolute;
           width: 100%;
           height: 100%;
-          top: 0px;
-          left: 0px;
+          top: 0;
+          left: 0;
           border-radius: 1.5rem;
           background-color: rgb(63 63 70);
           opacity: 0.7;
@@ -164,8 +158,15 @@ import { MarketplacePkg, StoreIdentity } from '../../../types'
   imports: [CommonModule, SharedPipesModule, TickerModule, TuiLet],
 })
 export class MarketplacePackageHeroComponent {
+  // @TODO Matt this used to be MarketplacePkg
   @Input({ required: true })
-  pkg!: MarketplacePkg
+  pkg!: {
+    id: string
+    title: string
+    version: string
+    description: { short: string }
+    icon: string
+  }
 
   private readonly marketplaceService = inject(AbstractMarketplaceService)
   readonly marketplace$ = this.marketplaceService.getSelectedHost$()

web/projects/shared/src/components/server.component.ts

@@ -10,7 +10,7 @@ import { StartOSDiskInfo } from '../types/api'
   template: `
     <tui-icon icon="@tui.save" />
     <span tuiTitle>
-      <strong>{{ server().hostname }}.local</strong>
+      <strong>{{ server().hostname.replace('.local', '') }}.local</strong>
       <span tuiSubtitle>
         <b>StartOS Version</b>
         : {{ server().version }}
@@ -26,7 +26,5 @@ import { StartOSDiskInfo } from '../types/api'
   imports: [DatePipe, TuiIcon, TuiTitle],
 })
 export class ServerComponent {
-  private readonly dialogs = inject(TuiDialogService)
-
   readonly server = input.required<StartOSDiskInfo>()
 }

web/projects/ui/src/app/routes/portal/components/form/form-union/form-union.component.ts

@@ -23,7 +23,7 @@ import { tuiPure } from '@taiga-ui/cdk'
   ],
 })
 export class FormUnionComponent implements OnChanges {
-  @Input()
+  @Input({ required: true })
   spec!: CT.ValueSpecUnion
 
   selectSpec!: CT.ValueSpecSelect

web/projects/ui/src/app/routes/portal/components/header/about.component.ts

@@ -2,7 +2,7 @@ import { TuiCell } from '@taiga-ui/layout'
 import { TuiTitle, TuiButton } from '@taiga-ui/core'
 import { CommonModule } from '@angular/common'
 import { ChangeDetectionStrategy, Component, inject } from '@angular/core'
-import { CopyService, ExverPipesModule } from '@start9labs/shared'
+import { CopyService } from '@start9labs/shared'
 import { PolymorpheusComponent } from '@taiga-ui/polymorpheus'
 import { PatchDB } from 'patch-db-client'
 import { DataModel } from 'src/app/services/patch-db/data-model'
@@ -50,7 +50,7 @@ import { ConfigService } from 'src/app/services/config.service'
   styles: ['[tuiCell] { padding-inline: 0 }'],
   changeDetection: ChangeDetectionStrategy.OnPush,
   standalone: true,
-  imports: [CommonModule, ExverPipesModule, TuiTitle, TuiButton, TuiCell],
+  imports: [CommonModule, TuiTitle, TuiButton, TuiCell],
 })
 export class AboutComponent {
   readonly server$ = inject<PatchDB<DataModel>>(PatchDB).watch$('serverInfo')

web/projects/ui/src/app/routes/portal/routes/service/components/dependency.component.ts

@@ -1,5 +1,4 @@
 import { ChangeDetectionStrategy, Component, Input } from '@angular/core'
-import { ExverPipesModule } from '@start9labs/shared'
 import { TuiIcon } from '@taiga-ui/core'
 import { DependencyInfo } from '../types/dependency-info'
 
@@ -39,7 +38,7 @@ import { DependencyInfo } from '../types/dependency-info'
   ],
   changeDetection: ChangeDetectionStrategy.OnPush,
   standalone: true,
-  imports: [ExverPipesModule, TuiIcon],
+  imports: [TuiIcon],
 })
 export class ServiceDependencyComponent {
   @Input({ required: true, alias: 'serviceDependency' })

web/projects/ui/src/app/routes/portal/routes/system/backups/components/status.component.ts

@@ -21,14 +21,12 @@ import { BackupType } from '../types/backup-type'
   imports: [TuiIcon],
 })
 export class BackupsStatusComponent {
-  private readonly exver = inject(Exver)
-
-  @Input({ required: true }) serverId!: string
+  @Input({ required: true }) hasBackup!: boolean
   @Input({ required: true }) type!: BackupType
-  @Input({ required: true }) target!: BackupTarget
+  @Input({ required: true }) mountable!: boolean
 
   get status() {
-    if (!this.target.mountable) {
+    if (!this.mountable) {
       return {
         icon: '@tui.bar-chart',
         color: 'var(--tui-text-negative)',
@@ -46,28 +44,16 @@ export class BackupsStatusComponent {
       }
     }
 
-    if (this.hasBackup) {
-      return {
-        icon: '@tui.cloud',
-        color: 'var(--tui-text-positive)',
-        text: 'Embassy backup detected',
-      }
-    }
-
-    return {
-      icon: '@tui.cloud-off',
-      color: 'var(--tui-text-negative)',
-      text: 'No Embassy backup',
-    }
-  }
-
-  private get hasBackup(): boolean {
-    return (
-      this.target.startOs[this.serverId] &&
-      this.exver.compareOsVersion(
-        this.target.startOs[this.serverId].version,
-        '0.3.6',
-      ) !== 'less'
-    )
+    return this.hasBackup
+      ? {
+          icon: '@tui.cloud',
+          color: 'var(--tui-text-positive)',
+          text: 'Embassy backup detected',
+        }
+      : {
+          icon: '@tui.cloud-off',
+          color: 'var(--tui-text-negative)',
+          text: 'No Embassy backup',
+        }
   }
 }

web/projects/ui/src/app/routes/portal/routes/system/backups/modals/target.component.ts

@@ -4,7 +4,7 @@ import {
   inject,
   signal,
 } from '@angular/core'
-import { ErrorService } from '@start9labs/shared'
+import { ErrorService, Exver, isEmptyObject } from '@start9labs/shared'
 import {
   TuiButton,
   TuiDialogContext,
@@ -45,8 +45,8 @@ import { DataModel } from 'src/app/services/patch-db/data-model'
               <strong>{{ displayInfo.name }}</strong>
               <backups-status
                 [type]="context.data.type"
-                [target]="target"
-                [serverId]="serverId"
+                [mountable]="target.mountable"
+                [hasBackup]="hasBackup(target)"
               />
               <div [style.color]="'var(--tui-text-secondary'">
                 {{ displayInfo.description }}
@@ -73,6 +73,7 @@ import { DataModel } from 'src/app/services/patch-db/data-model'
   ],
 })
 export class BackupsTargetModal {
+  private readonly exver = inject(Exver)
   private readonly dialogs = inject(TuiDialogService)
   private readonly errorService = inject(ErrorService)
   private readonly api = inject(ApiService)
@@ -106,7 +107,17 @@ export class BackupsTargetModal {
   isDisabled(target: BackupTarget): boolean {
     return (
       !target.mountable ||
-      (this.context.data.type === 'restore' && !target.startOs)
+      (this.context.data.type === 'restore' && !this.hasBackup(target))
+    )
+  }
+
+  hasBackup(target: BackupTarget): boolean {
+    return (
+      target.startOs?.[this.serverId] &&
+      this.exver.compareOsVersion(
+        target.startOs[this.serverId].version,
+        '0.3.6',
+      ) !== 'less'
     )
   }
 

web/projects/ui/src/app/routes/portal/routes/system/backups/services/restore.service.ts

@@ -43,22 +43,17 @@ export class BackupsRestoreService {
     this.dialogs
       .open<BackupTarget>(TARGET, TARGET_RESTORE)
       .pipe(
-        // @TODO Alex implement servers
         switchMap(target =>
           this.dialogs
             .open<StartOSDiskInfo & { id: string }>(SERVERS, {
-              data: { servers: [] },
+              label: 'Select server',
+              data: { servers: Object.values(target.startOs) },
             })
             .pipe(
               switchMap(({ id, passwordHash }) =>
                 this.dialogs.open<string>(PROMPT, PROMPT_OPTIONS).pipe(
                   exhaustMap(password =>
-                    this.getRecoverData(
-                      target.id,
-                      id,
-                      password,
-                      passwordHash || '',
-                    ),
+                    this.getRecoverData(target.id, id, password, passwordHash),
                   ),
                   take(1),
                   switchMap(data =>
@@ -81,10 +76,10 @@ export class BackupsRestoreService {
     targetId: string,
     serverId: string,
     password: string,
-    hash: string,
+    hash: string | null,
   ): Observable<RecoverData> {
     return of(password).pipe(
-      tap(() => argon2.verify(hash, password)),
+      tap(() => argon2.verify(hash || '', password)),
       switchMap(() => {
         const loader = this.loader.open('Decrypting drive...').subscribe()
 

web/projects/ui/src/app/routes/portal/routes/system/metrics/metrics.service.ts

@@ -16,7 +16,6 @@ import { ApiService } from 'src/app/services/api/embassy-api.service'
 export class MetricsService extends Observable<ServerMetrics> {
   private readonly api = inject(ApiService)
 
-  // @TODO Alex do we need to use defer? I am unsure when this is necessary.
   private readonly metrics$ = defer(() =>
     this.api.followServerMetrics({}),
   ).pipe(

web/projects/ui/src/app/routes/portal/routes/system/sideload/package.component.ts

@@ -1,4 +1,3 @@
-import { TuiLet } from '@taiga-ui/cdk'
 import { CommonModule } from '@angular/common'
 import { Component, inject, Input } from '@angular/core'
 import { Router, RouterLink } from '@angular/router'
@@ -7,20 +6,21 @@ import {
   AdditionalModule,
   MarketplaceDependenciesComponent,
   MarketplacePackageHeroComponent,
-  MarketplacePkg,
 } from '@start9labs/marketplace'
 import {
-  Exver,
   ErrorService,
+  Exver,
   LoadingService,
   SharedPipesModule,
 } from '@start9labs/shared'
+import { T } from '@start9labs/start-sdk'
+import { TuiLet } from '@taiga-ui/cdk'
 import { TuiAlertService, TuiButton } from '@taiga-ui/core'
 import { PatchDB } from 'patch-db-client'
 import { combineLatest, map } from 'rxjs'
-import { DataModel } from 'src/app/services/patch-db/data-model'
 import { ApiService } from 'src/app/services/api/embassy-api.service'
 import { ClientStorageService } from 'src/app/services/client-storage.service'
+import { DataModel } from 'src/app/services/patch-db/data-model'
 import { getManifest } from 'src/app/utils/get-package-data'
 
 @Component({
@@ -33,24 +33,26 @@ import { getManifest } from 'src/app/utils/get-package-data'
         [pkg]="package"
       >
         <div class="inner-container">
-          <a
-            *ngIf="button !== null && button !== 'Install'"
-            tuiButton
-            appearance="tertiary-solid"
-            [routerLink]="'/portal/service/' + package.id"
-          >
-            View installed
-          </a>
-          <button *ngIf="button" tuiButton (click)="upload()">
-            {{ button }}
-          </button>
+          @if (button !== null && button !== 'Install') {
+            <a
+              tuiButton
+              appearance="tertiary-solid"
+              [routerLink]="'/portal/service/' + package.id"
+            >
+              View installed
+            </a>
+          }
+          @if (button) {
+            <button tuiButton (click)="upload()">{{ button }}</button>
+          }
         </div>
       </marketplace-package-hero>
-      <marketplace-about [pkg]="package" />
-      @if (!(package.dependencyMetadata | empty)) {
-        <marketplace-dependencies [pkg]="package" (open)="open($event)" />
-      }
-      <marketplace-additional [pkg]="package" />
+      <!-- @TODO Matt do we want this here? How do we turn s9pk into MarketplacePkg? -->
+      <!--      <marketplace-about [pkg]="package" />-->
+      <!--      @if (!(package.dependencyMetadata | empty)) {-->
+      <!--        <marketplace-dependencies [pkg]="package" (open)="open($event)" />-->
+      <!--      }-->
+      <!--      <marketplace-additional [pkg]="package" />-->
     </div>
   `,
   styles: [
@@ -125,7 +127,7 @@ export class SideloadPackageComponent {
   )
 
   @Input({ required: true })
-  package!: MarketplacePkg
+  package!: T.Manifest & { icon: string }
 
   @Input({ required: true })
   file!: File

web/projects/ui/src/app/routes/portal/routes/system/sideload/sideload.component.ts

@@ -1,38 +1,39 @@
-import { CommonModule } from '@angular/common'
-import { ChangeDetectionStrategy, Component, inject } from '@angular/core'
+import {
+  ChangeDetectionStrategy,
+  ChangeDetectorRef,
+  Component,
+  inject,
+  signal,
+} from '@angular/core'
 import { FormsModule } from '@angular/forms'
-import { MarketplacePkg } from '@start9labs/marketplace'
+import { T } from '@start9labs/start-sdk'
+import { tuiIsString } from '@taiga-ui/cdk'
 import { TuiButton, TuiLink } from '@taiga-ui/core'
 import {
   TuiAvatar,
   TuiFiles,
   tuiInputFilesOptionsProvider,
 } from '@taiga-ui/kit'
-import { Subject } from 'rxjs'
 import { ConfigService } from 'src/app/services/config.service'
 import { SideloadPackageComponent } from './package.component'
+import { parseS9pk } from './sideload.utils'
 
 @Component({
   template: `
-    <ng-container *ngIf="refresh$ | async"></ng-container>
-    <sideload-package
-      *ngIf="package && file; else upload"
-      [package]="package"
-      [file]="file"
-    >
-      <button
-        tuiIconButton
-        appearance="secondary"
-        iconStart="@tui.x"
-        [style.border-radius.%]="100"
-        [style.float]="'right'"
-        (click)="clear()"
-        class="justify-self-end"
-      >
-        Close
-      </button>
-    </sideload-package>
-    <ng-template #upload>
+    @if (file && package()) {
+      <sideload-package [package]="package()!" [file]="file!">
+        <button
+          tuiIconButton
+          appearance="secondary"
+          iconStart="@tui.x"
+          [style.border-radius.%]="100"
+          [style.justify-self]="'end'"
+          (click)="clear()"
+        >
+          Close
+        </button>
+      </sideload-package>
+    } @else {
       <label tuiInputFiles (click)="clear()">
         <input
           tuiInputFiles
@@ -41,26 +42,25 @@ import { SideloadPackageComponent } from './package.component'
           (ngModelChange)="onFile($event)"
         />
         <ng-template>
-          <div *ngIf="invalid; else valid">
-            <tui-avatar appearance="secondary" src="@tui.circle-x" />
-            <p [style.color]="'var(--tui-text-negative)'">
-              Invalid package file
-            </p>
-            <button tuiButton>Try again</button>
-          </div>
-          <ng-template #valid>
+          @if (error()) {
+            <div>
+              <tui-avatar appearance="secondary" src="@tui.circle-x" />
+              <p class="g-error">{{ error() }}</p>
+              <button tuiButton>Try again</button>
+            </div>
+          } @else {
             <div>
               <tui-avatar appearance="secondary" src="@tui.cloud-upload" />
               <p>Upload .s9pk package file</p>
-              <p *ngIf="isTor" [style.color]="'var(--tui-text-positive)'">
-                Tip: switch to LAN for faster uploads
-              </p>
+              @if (isTor) {
+                <p class="g-success">Tip: switch to LAN for faster uploads</p>
+              }
               <button tuiButton>Upload</button>
             </div>
-          </ng-template>
+          }
         </ng-template>
       </label>
-    </ng-template>
+    }
   `,
   host: { class: 'g-page', '[style.padding-top.rem]': '2' },
   styles: [
@@ -80,7 +80,6 @@ import { SideloadPackageComponent } from './package.component'
   providers: [tuiInputFilesOptionsProvider({ maxFileSize: Infinity })],
   standalone: true,
   imports: [
-    CommonModule,
     FormsModule,
     TuiFiles,
     TuiLink,
@@ -90,28 +89,26 @@ import { SideloadPackageComponent } from './package.component'
   ],
 })
 export default class SideloadComponent {
-  readonly refresh$ = new Subject<void>()
+  private readonly cdr = inject(ChangeDetectorRef)
   readonly isTor = inject(ConfigService).isTor()
 
-  invalid = false
   file: File | null = null
-  package: MarketplacePkg | null = null
+  readonly package = signal<(T.Manifest & { icon: string }) | null>(null)
+  readonly error = signal('')
 
   clear() {
-    this.invalid = false
     this.file = null
-    this.package = null
+    this.package.set(null)
+    this.error.set('')
   }
 
-  // @TODO Alex refactor sideload
   async onFile(file: File | null) {
-    // if (!file || !(await validateS9pk(file))) {
-    //   this.invalid = true
-    // } else {
-    //   this.package = await parseS9pk(file)
-    //   this.file = file
-    // }
+    const parsed = file ? await parseS9pk(file) : ''
 
-    this.refresh$.next()
+    this.file = file
+    this.package.set(tuiIsString(parsed) ? null : parsed)
+    this.error.set(tuiIsString(parsed) ? parsed : '')
+    // @TODO Alex figure out why it is needed even though we use signals
+    this.cdr.markForCheck()
   }
 }

web/projects/ui/src/app/routes/portal/routes/system/sideload/sideload.utils.ts

@@ -0,0 +1,160 @@
+import { S9pk, T } from '@start9labs/start-sdk'
+import cbor from 'cbor'
+
+const MAGIC = new Uint8Array([59, 59])
+const VERSION_1 = new Uint8Array([1])
+const VERSION_2 = new Uint8Array([2])
+
+interface Positions {
+  [key: string]: [bigint, bigint] // [position, length]
+}
+
+export async function parseS9pk(
+  file: File,
+): Promise<(T.Manifest & { icon: string }) | string> {
+  const magic = new Uint8Array(await blobToBuffer(file.slice(0, 2)))
+  const version = new Uint8Array(await blobToBuffer(file.slice(2, 3)))
+
+  if (compare(magic, MAGIC)) {
+    try {
+      if (compare(version, VERSION_1)) {
+        return await parseS9pkV1(file)
+      } else if (compare(version, VERSION_2)) {
+        const s9pk = await S9pk.deserialize(file, null)
+
+        return {
+          ...s9pk.manifest,
+          icon: await s9pk.icon(),
+        }
+      } else {
+        console.error(version)
+
+        return 'Invalid package file'
+      }
+    } catch (e) {
+      console.error(e)
+
+      return e instanceof Error
+        ? `Invalid package file: ${e.message}`
+        : 'Invalid package file'
+    }
+  }
+
+  return 'Invalid package file'
+}
+
+async function parseS9pkV1(file: File) {
+  const positions: Positions = {}
+  // magic=2bytes, version=1bytes, pubkey=32bytes, signature=64bytes, toc_length=4bytes = 103byte is starting point
+  let start = 103
+  let end = start + 1 // 104
+  const tocLength = new DataView(
+    await blobToBuffer(file.slice(99, 103) ?? new Blob()),
+  ).getUint32(0, false)
+  await getPositions(start, end, file, positions, tocLength as any)
+
+  const data = await blobToBuffer(
+    file.slice(
+      Number(positions['manifest'][0]),
+      Number(positions['manifest'][0]) + Number(positions['manifest'][1]),
+    ),
+  )
+
+  return {
+    ...(await cbor.decode(data, true)),
+    icon: await blobToDataURL(
+      file.slice(
+        Number(positions['icon'][0]),
+        Number(positions['icon'][0]) + Number(positions['icon'][1]),
+        '',
+      ),
+    ),
+  }
+}
+
+async function getPositions(
+  initialStart: number,
+  initialEnd: number,
+  file: Blob,
+  positions: Positions,
+  tocLength: number,
+) {
+  let start = initialStart
+  let end = initialEnd
+  const titleLength = new Uint8Array(
+    await blobToBuffer(file.slice(start, end)),
+  )[0]
+  const tocTitle = await file.slice(end, end + titleLength).text()
+  start = end + titleLength
+  end = start + 8
+  const chapterPosition = new DataView(
+    await blobToBuffer(file.slice(start, end)),
+  ).getBigUint64(0, false)
+  start = end
+  end = start + 8
+  const chapterLength = new DataView(
+    await blobToBuffer(file.slice(start, end)),
+  ).getBigUint64(0, false)
+
+  positions[tocTitle] = [chapterPosition, chapterLength]
+  start = end
+  end = start + 1
+  if (end <= tocLength + (initialStart - 1)) {
+    await getPositions(start, end, file, positions, tocLength)
+  }
+}
+
+async function readBlobAsDataURL(
+  f: Blob | File,
+): Promise<string | ArrayBuffer | null> {
+  const reader = new FileReader()
+  return new Promise((resolve, reject) => {
+    reader.onloadend = () => {
+      resolve(reader.result)
+    }
+    reader.readAsDataURL(f)
+    reader.onerror = _ => reject(new Error('error reading blob'))
+  })
+}
+
+async function blobToDataURL(data: Blob | File): Promise<string> {
+  const res = await readBlobAsDataURL(data)
+  if (res instanceof ArrayBuffer) {
+    throw new Error('readBlobAsDataURL response should not be an array buffer')
+  }
+  if (res == null) {
+    throw new Error('readBlobAsDataURL response should not be null')
+  }
+  if (typeof res === 'string') return res
+  throw new Error('no possible blob to data url resolution found')
+}
+
+async function blobToBuffer(data: Blob | File): Promise<ArrayBuffer> {
+  const res = await readBlobToArrayBuffer(data)
+  if (res instanceof String) {
+    throw new Error('readBlobToArrayBuffer response should not be a string')
+  }
+  if (res == null) {
+    throw new Error('readBlobToArrayBuffer response should not be null')
+  }
+  if (res instanceof ArrayBuffer) return res
+  throw new Error('no possible blob to array buffer resolution found')
+}
+
+async function readBlobToArrayBuffer(
+  f: Blob | File,
+): Promise<string | ArrayBuffer | null> {
+  const reader = new FileReader()
+
+  return new Promise((resolve, reject) => {
+    reader.onloadend = () => {
+      resolve(reader.result)
+    }
+    reader.readAsArrayBuffer(f)
+    reader.onerror = _ => reject(new Error('error reading blob'))
+  })
+}
+
+function compare(a: Uint8Array, b: Uint8Array) {
+  return a.length === b.length && a.every((value, index) => value === b[index])
+}

web/projects/ui/src/app/services/api/api.fixures.ts

@@ -1014,7 +1014,7 @@ export module Mock {
         hostname: 'smb://192.169.10.0',
         path: '/Desktop/embassy-backups',
         username: 'TestUser',
-        mountable: false,
+        mountable: true,
         startOs: {
           abcdefgh: {
             hostname: 'adjective-noun.local',
@@ -1032,7 +1032,7 @@ export module Mock {
         name: 'Dropbox 1',
         provider: 'dropbox',
         path: '/Home/backups',
-        mountable: true,
+        mountable: false,
         startOs: {},
       },
       {