follow sideload progress (#2718)

* follow sideload progress

* small bugfix

* shareReplay with no refcount false

* don't wrap sideload progress in RPCResult

* dont present toast

---------

Co-authored-by: Aiden McClelland <me@drbonez.dev>

core/startos/src/install/mod.rs

@@ -17,6 +17,7 @@ use rpc_toolkit::HandlerArgs;
 use rustyline_async::ReadlineEvent;
 use serde::{Deserialize, Serialize};
 use tokio::sync::oneshot;
+use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
 use tracing::instrument;
 use ts_rs::TS;
 
@@ -188,7 +189,7 @@ pub async fn sideload(
     SideloadParams { session }: SideloadParams,
 ) -> Result<SideloadResponse, Error> {
     let (upload, file) = upload(&ctx, session.clone()).await?;
-    let (err_send, err_recv) = oneshot::channel();
+    let (err_send, err_recv) = oneshot::channel::<Error>();
     let progress = Guid::new();
     let progress_tracker = FullProgressTracker::new();
     let mut progress_listener = progress_tracker.stream(Some(Duration::from_millis(200)));
@@ -202,12 +203,14 @@ pub async fn sideload(
                     use axum::extract::ws::Message;
                     async move {
                         if let Err(e) = async {
-                            type RpcResponse = rpc_toolkit::yajrc::RpcResponse::<GenericRpcMethod<&'static str, (), FullProgress>>;
+                            type RpcResponse = rpc_toolkit::yajrc::RpcResponse<
+                                GenericRpcMethod<&'static str, (), FullProgress>,
+                            >;
                             tokio::select! {
                                 res = async {
                                     while let Some(progress) = progress_listener.next().await {
                                         ws.send(Message::Text(
-                                            serde_json::to_string(&RpcResponse::from_result::<RpcError>(Ok(progress)))
+                                            serde_json::to_string(&progress)
                                                 .with_kind(ErrorKind::Serialization)?,
                                         ))
                                         .await
@@ -217,12 +220,8 @@ pub async fn sideload(
                                 } => res?,
                                 err = err_recv => {
                                     if let Ok(e) = err {
-                                        ws.send(Message::Text(
+                                        ws.close_result(Err::<&str, _>(e.clone_output())).await?;
-                                            serde_json::to_string(&RpcResponse::from_result::<RpcError>(Err(e)))
+                                        return Err(e)
-                                            .with_kind(ErrorKind::Serialization)?,
-                                        ))
-                                        .await
-                                        .with_kind(ErrorKind::Network)?;
                                     }
                                 }
                             }
@@ -260,7 +259,7 @@ pub async fn sideload(
         }
         .await
         {
-            let _ = err_send.send(RpcError::from(e.clone_output()));
+            let _ = err_send.send(e.clone_output());
             tracing::error!("Error sideloading package: {e}");
             tracing::debug!("{e:?}");
         }
@@ -407,19 +406,21 @@ pub async fn cli_install(
 
                 let mut progress = FullProgress::new();
 
-                type RpcResponse = rpc_toolkit::yajrc::RpcResponse<
-                    GenericRpcMethod<&'static str, (), FullProgress>,
-                >;
-
                 loop {
                     tokio::select! {
                         msg = ws.next() => {
                             if let Some(msg) = msg {
-                                if let Message::Text(t) = msg.with_kind(ErrorKind::Network)? {
+                                match msg.with_kind(ErrorKind::Network)? {
-                                    progress =
+                                    Message::Text(t) => {
-                                        serde_json::from_str::<RpcResponse>(&t)
+                                        progress =
-                                            .with_kind(ErrorKind::Deserialization)?.result?;
+                                            serde_json::from_str::<FullProgress>(&t)
-                                    bar.update(&progress);
+                                                .with_kind(ErrorKind::Deserialization)?;
+                                        bar.update(&progress);
+                                    }
+                                    Message::Close(Some(c)) if c.code != CloseCode::Normal => {
+                                        return Err(Error::new(eyre!("{}", c.reason), ErrorKind::Network))
+                                    }
+                                    _ => (),
                                 }
                             } else {
                                 break;

web/projects/ui/src/app/pages/apps-routes/app-show/app-show.module.ts

@@ -56,5 +56,6 @@ const routes: Routes = [
     StatusComponentModule,
     SharedPipesModule,
   ],
+  exports: [AppShowProgressComponent],
 })
 export class AppShowPageModule {}

web/projects/ui/src/app/pages/init/init.service.ts

@@ -29,9 +29,7 @@ export class InitService extends Observable<MappedProgress> {
     from(this.api.initGetProgress()),
   ).pipe(
     switchMap(({ guid, progress }) =>
-      this.api
+      this.api.openWebsocket$<T.FullProgress>(guid).pipe(startWith(progress)),
-        .openWebsocket$<T.FullProgress>(guid, {})
-        .pipe(startWith(progress)),
     ),
     map(({ phases, overall }) => {
       return {

web/projects/ui/src/app/pages/init/logs/logs.service.ts

@@ -38,7 +38,7 @@ export class LogsService extends Observable<readonly string[]> {
   private readonly log$ = defer(() =>
     this.api.initFollowLogs({ boot: 0 }),
   ).pipe(
-    switchMap(({ guid }) => this.api.openWebsocket$<Log>(guid, {})),
+    switchMap(({ guid }) => this.api.openWebsocket$<Log>(guid)),
     bufferTime(500),
     filter(logs => !!logs.length),
     map(convertAnsi),

web/projects/ui/src/app/pages/server-routes/sideload/sideload.module.ts

@@ -5,6 +5,7 @@ import { SideloadPage } from './sideload.page'
 import { Routes, RouterModule } from '@angular/router'
 import { ExverPipesModule, SharedPipesModule } from '@start9labs/shared'
 import { DragNDropDirective } from './dnd.directive'
+import { InstallingProgressPipeModule } from 'src/app/pipes/install-progress/install-progress.module'
 
 const routes: Routes = [
   {
@@ -20,6 +21,7 @@ const routes: Routes = [
     RouterModule.forChild(routes),
     SharedPipesModule,
     ExverPipesModule,
+    InstallingProgressPipeModule,
   ],
   declarations: [SideloadPage, DragNDropDirective],
 })

web/projects/ui/src/app/pages/server-routes/sideload/sideload.page.html

@@ -7,92 +7,121 @@
   </ion-toolbar>
 </ion-header>
 
-<ion-content class="ion-text-center with-widgets">
+<ion-content class="ion-text-center ion-padding with-widgets">
+  <ng-container *ngIf="progress$ | async as progress; else noProgress">
+    <ng-container *ngFor="let phase of progress.phases">
+      <p>
+        {{ phase.name }}
+        <span *ngIf="phase.progress | installingProgress as progress">
+          : {{ progress }}%
+        </span>
+      </p>
+      <ion-progress-bar
+        [type]="
+          phase.progress === false ||
+          (phase.progress !== true &&
+            phase.progress !== null &&
+            !phase.progress.total)
+            ? 'indeterminate'
+            : 'determinate'
+        "
+        [color]="phase.progress === true ? 'success' : 'secondary'"
+        [value]="(phase.progress | installingProgress) / 100"
+      ></ion-progress-bar>
+    </ng-container>
+  </ng-container>
+
   <!-- file upload -->
-  <div
+  <ng-template #noProgress>
-    *ngIf="!toUpload.file; else fileUploaded"
+    <div
-    class="drop-area"
+      *ngIf="!toUpload.file; else fileReady"
-    [class.drop-area_mobile]="isMobile"
+      class="drop-area"
-    appDnd
+      [class.drop-area_mobile]="isMobile"
-    (onFileDropped)="handleFileDrop($event)"
+      appDnd
-  >
+      (onFileDropped)="handleFileDrop($event)"
-    <ion-icon
+    >
-      name="cloud-upload-outline"
+      <ion-icon
-      color="dark"
+        name="cloud-upload-outline"
-      style="font-size: 42px"
+        color="dark"
-    ></ion-icon>
+        style="font-size: 42px"
-    <h4>Upload .s9pk package file</h4>
+      ></ion-icon>
-    <p *ngIf="onTor">
+      <h4>Upload .s9pk package file</h4>
-      <ion-text color="success">
+      <p *ngIf="onTor">
-        Tip: switch to LAN for faster uploads.
+        <ion-text color="success">
-      </ion-text>
+          Tip: switch to LAN for faster uploads.
-    </p>
+        </ion-text>
-    <ion-button color="primary" type="file" class="ion-margin-top">
+      </p>
-      <label for="upload-photo">Browse</label>
+      <ion-button color="primary" type="file" class="ion-margin-top">
-      <input
+        <label for="upload-photo">Browse</label>
-        type="file"
+        <input
-        style="position: absolute; opacity: 0; height: 100%"
+          type="file"
-        id="upload-photo"
+          style="position: absolute; opacity: 0; height: 100%"
-        (change)="handleFileInput($event)"
+          id="upload-photo"
-      />
+          (change)="handleFileInput($event)"
-    </ion-button>
+        />
-  </div>
+      </ion-button>
-  <!-- file uploaded -->
+    </div>
-  <ng-template #fileUploaded>
+    <!-- file uploaded -->
-    <div class="drop-area_filled">
+    <ng-template #fileReady>
-      <h4>
+      <div class="drop-area_filled">
-        <ion-icon
+        <h4>
-          *ngIf="uploadState?.invalid"
+          <ion-icon
-          name="close-circle-outline"
+            *ngIf="uploadState?.invalid"
-          color="danger"
+            name="close-circle-outline"
-          class="inline"
+            color="danger"
-        ></ion-icon>
+            class="inline"
-        <ion-icon
+          ></ion-icon>
-          *ngIf="!uploadState?.invalid"
+          <ion-icon
-          class="inline"
+            *ngIf="!uploadState?.invalid"
-          name="checkmark-circle-outline"
+            class="inline"
-          color="success"
+            name="checkmark-circle-outline"
-        ></ion-icon>
+            color="success"
-        {{ uploadState?.message }}
+          ></ion-icon>
-      </h4>
+          {{ uploadState?.message }}
-      <div class="box" *ngIf="toUpload.icon && toUpload.manifest">
+        </h4>
-        <div class="card">
+        <div class="box" *ngIf="toUpload.icon && toUpload.manifest">
-          <div class="row row_end">
+          <div class="card">
-            <ion-button
+            <div class="row row_end">
-              style="
+              <ion-button
-                --background-hover: transparent;
+                style="
-                --padding-end: 0px;
+                  --background-hover: transparent;
-                --padding-start: 0px;
+                  --padding-end: 0px;
-              "
+                  --padding-start: 0px;
-              fill="clear"
+                "
-              size="small"
+                fill="clear"
-              (click)="clearToUpload()"
+                size="small"
-            >
+                (click)="clearToUpload()"
-              <ion-icon slot="icon-only" name="close" color="danger"></ion-icon>
+              >
-            </ion-button>
+                <ion-icon
-          </div>
+                  slot="icon-only"
-          <div class="row">
+                  name="close"
-            <img
+                  color="danger"
-              [alt]="toUpload.manifest.title + ' Icon'"
+                ></ion-icon>
-              [src]="toUpload.icon | trustUrl"
+              </ion-button>
-            />
+            </div>
-            <h2>{{ toUpload.manifest.title }}</h2>
+            <div class="row">
-            <p>{{ toUpload.manifest.version }}</p>
+              <img
+                [alt]="toUpload.manifest.title + ' Icon'"
+                [src]="toUpload.icon | trustUrl"
+              />
+              <h2>{{ toUpload.manifest.title }}</h2>
+              <p>{{ toUpload.manifest.version }}</p>
+            </div>
           </div>
         </div>
-      </div>
+        <ion-button
-      <ion-button
+          *ngIf="!toUpload.icon && !toUpload.manifest; else uploadButton"
-        *ngIf="!toUpload.icon && !toUpload.manifest; else uploadButton"
+          color="primary"
-        color="primary"
+          (click)="clearToUpload()"
-        (click)="clearToUpload()"
+        >
-      >
+          Try again
-        Try again
-      </ion-button>
-      <ng-template #uploadButton>
-        <ion-button color="primary" (click)="handleUpload()">
-          Upload & Install
         </ion-button>
-      </ng-template>
+        <ng-template #uploadButton>
-    </div>
+          <ion-button color="primary" (click)="handleUpload()">
+            Upload & Install
+          </ion-button>
+        </ng-template>
+      </div>
+    </ng-template>
   </ng-template>
 </ion-content>

web/projects/ui/src/app/pages/server-routes/sideload/sideload.page.ts

@@ -1,10 +1,12 @@
 import { Component } from '@angular/core'
-import { isPlatform, NavController } from '@ionic/angular'
+import { isPlatform } from '@ionic/angular'
 import { ErrorService, LoadingService } from '@start9labs/shared'
-import { S9pk, T } from '@start9labs/start-sdk'
+import { S9pk } from '@start9labs/start-sdk'
 import cbor from 'cbor'
 import { ApiService } from 'src/app/services/api/embassy-api.service'
 import { ConfigService } from 'src/app/services/config.service'
+import { SideloadService } from './sideload.service'
+import { firstValueFrom } from 'rxjs'
 
 interface Positions {
   [key: string]: [bigint, bigint] // [position, length]
@@ -36,12 +38,14 @@ export class SideloadPage {
     message: string
   }
 
+  readonly progress$ = this.sideloadService.progress$
+
   constructor(
     private readonly loader: LoadingService,
     private readonly api: ApiService,
-    private readonly navCtrl: NavController,
     private readonly errorService: ErrorService,
     private readonly config: ConfigService,
+    private readonly sideloadService: SideloadService,
   ) {}
 
   handleFileDrop(e: any) {
@@ -111,15 +115,15 @@ export class SideloadPage {
   }
 
   async handleUpload() {
-    const loader = this.loader.open('Uploading package').subscribe()
+    const loader = this.loader.open('Starting upload').subscribe()
 
     try {
       const res = await this.api.sideloadPackage()
+      this.sideloadService.followProgress(res.progress)
       this.api
         .uploadPackage(res.upload, this.toUpload.file!)
         .catch(e => console.error(e))
-
+      await firstValueFrom(this.sideloadService.websocketConnected$)
-      this.navCtrl.navigateRoot('/services')
     } catch (e: any) {
       this.errorService.handleError(e)
     } finally {

web/projects/ui/src/app/pages/server-routes/sideload/sideload.service.ts

@@ -0,0 +1,32 @@
+import { Injectable } from '@angular/core'
+import { T } from '@start9labs/start-sdk'
+import { endWith, ReplaySubject, shareReplay, Subject, switchMap } from 'rxjs'
+import { ApiService } from 'src/app/services/api/embassy-api.service'
+
+@Injectable({
+  providedIn: 'root',
+})
+export class SideloadService {
+  private readonly guid$ = new Subject<string>()
+
+  readonly websocketConnected$ = new ReplaySubject()
+
+  readonly progress$ = this.guid$.pipe(
+    switchMap(guid =>
+      this.api
+        .openWebsocket$<T.FullProgress>(guid, {
+          openObserver: {
+            next: () => this.websocketConnected$.next(''),
+          },
+        })
+        .pipe(endWith(null)),
+    ),
+    shareReplay(1),
+  )
+
+  constructor(private readonly api: ApiService) {}
+
+  followProgress(guid: string) {
+    this.guid$.next(guid)
+  }
+}

web/projects/ui/src/app/services/api/embassy-api.service.ts

@@ -12,7 +12,7 @@ export abstract class ApiService {
   // http
 
   // for sideloading packages
-  abstract uploadPackage(guid: string, body: Blob): Promise<string>
+  abstract uploadPackage(guid: string, body: Blob): Promise<void>
 
   // for getting static files: ex icons, instructions, licenses
   abstract getStaticProxy(
@@ -29,7 +29,7 @@ export abstract class ApiService {
 
   abstract openWebsocket$<T>(
     guid: string,
-    config: RR.WebsocketConfig<T>,
+    config?: RR.WebsocketConfig<T>,
   ): Observable<T>
 
   // state

web/projects/ui/src/app/services/api/embassy-live-api.service.ts

@@ -43,12 +43,11 @@ export class LiveApiService extends ApiService {
 
   // for sideloading packages
 
-  async uploadPackage(guid: string, body: Blob): Promise<string> {
+  async uploadPackage(guid: string, body: Blob): Promise<void> {
-    return this.httpRequest({
+    await this.httpRequest({
       method: Method.POST,
       body,
       url: `/rest/rpc/${guid}`,
-      responseType: 'text',
     })
   }
 
@@ -86,7 +85,7 @@ export class LiveApiService extends ApiService {
 
   openWebsocket$<T>(
     guid: string,
-    config: RR.WebsocketConfig<T>,
+    config: RR.WebsocketConfig<T> = {},
   ): Observable<T> {
     const { location } = this.document.defaultView!
     const protocol = location.protocol === 'http:' ? 'ws' : 'wss'

web/projects/ui/src/app/services/api/embassy-mock-api.service.ts

@@ -81,9 +81,8 @@ export class MockApiService extends ApiService {
       .subscribe()
   }
 
-  async uploadPackage(guid: string, body: Blob): Promise<string> {
+  async uploadPackage(guid: string, body: Blob): Promise<void> {
     await pauseFor(2000)
-    return 'success'
   }
 
   async getStaticProxy(
@@ -106,7 +105,7 @@ export class MockApiService extends ApiService {
 
   openWebsocket$<T>(
     guid: string,
-    config: RR.WebsocketConfig<T>,
+    config: RR.WebsocketConfig<T> = {},
   ): Observable<T> {
     if (guid === 'db-guid') {
       return this.mockWsSource$.pipe<any>(
@@ -125,6 +124,11 @@ export class MockApiService extends ApiService {
       return from(this.initProgress()).pipe(
         startWith(PROGRESS),
       ) as Observable<T>
+    } else if (guid === 'sideload-progress-guid') {
+      config.openObserver?.next(new Event(''))
+      return from(this.initProgress()).pipe(
+        startWith(PROGRESS),
+      ) as Observable<T>
     } else {
       throw new Error('invalid guid type')
     }
@@ -1079,8 +1083,8 @@ export class MockApiService extends ApiService {
   async sideloadPackage(): Promise<RR.SideloadPackageRes> {
     await pauseFor(2000)
     return {
-      upload: '4120e092-05ab-4de2-9fbd-c3f1f4b1df9e', // no significance, randomly generated
+      upload: 'sideload-upload-guid', // no significance, randomly generated
-      progress: '5120e092-05ab-4de2-9fbd-c3f1f4b1df9e', // no significance, randomly generated
+      progress: 'sideload-progress-guid', // no significance, randomly generated
     }
   }
 

web/projects/ui/src/app/services/marketplace.service.ts

@@ -285,7 +285,12 @@ export class MarketplaceService implements AbstractMarketplaceService {
       this.api.getRegistryPackage(url, id, version ? `=${version}` : null),
     ).pipe(
       map(pkgInfo =>
-        this.convertToMarketplacePkg(id, version, flavor, pkgInfo),
+        this.convertToMarketplacePkg(
+          id,
+          version === '*' ? null : version,
+          flavor,
+          pkgInfo,
+        ),
       ),
     )
   }

web/projects/ui/src/app/services/patch-db/patch-db-source.ts

@@ -33,7 +33,7 @@ export class PatchDbSource extends Observable<Update<DataModel>[]> {
   private readonly stream$ = inject(AuthService).isVerified$.pipe(
     switchMap(verified => (verified ? this.api.subscribeToPatchDB({}) : EMPTY)),
     switchMap(({ dump, guid }) =>
-      this.api.openWebsocket$<Revision>(guid, {}).pipe(
+      this.api.openWebsocket$<Revision>(guid).pipe(
         bufferTime(250),
         filter(revisions => !!revisions.length),
         startWith([dump]),