Update Marketplace (#2742)

* update abstract marketplace for usage accuracy andrename store to registry

* use new abstract functions

* fix(marketplace): get rid of `AbstractMarketplaceService`

* bump shared marketplace lib

* update marketplace to use query params for registry url; comment out updates page - will be refactored

* cleanup

* cleanup duplicate

* cleanup unused imports

* rework setting registry url when loading marketplace

* cleanup marketplace service

* fix background

---------

Co-authored-by: Matt Hill <mattnine@protonmail.com>
Co-authored-by: waterplea <alexander@inkin.ru>
Co-authored-by: Matt Hill <MattDHill@users.noreply.github.com>

web/projects/ui/src/app/services/badge.service.ts

@@ -1,5 +1,4 @@
 import { inject, Injectable } from '@angular/core'
-import { AbstractMarketplaceService } from '@start9labs/marketplace'
 import { Exver } from '@start9labs/shared'
 import { PatchDB } from 'patch-db-client'
 import {
@@ -10,7 +9,6 @@ import {
   map,
   Observable,
   pairwise,
-  shareReplay,
   startWith,
   switchMap,
 } from 'rxjs'
@@ -32,9 +30,7 @@ export class BadgeService {
     this.patch.watch$('serverInfo', 'ntpSynced'),
     inject(EOSService).updateAvailable$,
   ]).pipe(map(([synced, update]) => Number(!synced) + Number(update)))
-  private readonly marketplace = inject(
-    AbstractMarketplaceService,
-  ) as MarketplaceService
+  private readonly marketplaceService = inject(MarketplaceService)
 
   private readonly local$ = inject(ConnectionService).pipe(
     filter(Boolean),
@@ -58,35 +54,35 @@ export class BadgeService {
     ),
   )
 
-  private readonly updates$ = combineLatest([
-    this.marketplace.getMarketplace$(true),
-    this.local$,
-  ]).pipe(
-    map(
-      ([marketplace, local]) =>
-        Object.entries(marketplace).reduce(
-          (list, [_, store]) =>
-            store?.packages.reduce(
-              (result, { id, version }) =>
-                local[id] &&
-                this.exver.compareExver(
-                  version,
-                  getManifest(local[id]).version,
-                ) === 1
-                  ? result.add(id)
-                  : result,
-              list,
-            ) || list,
-          new Set<string>(),
-        ).size,
-    ),
-    shareReplay(1),
-  )
+  // private readonly updates$ = combineLatest([
+  //   this.marketplaceService.getMarketplace$(true),
+  //   this.local$,
+  // ]).pipe(
+  //   map(
+  //     ([marketplace, local]) =>
+  //       Object.entries(marketplace).reduce(
+  //         (list, [_, store]) =>
+  //           store?.packages.reduce(
+  //             (result, { id, version }) =>
+  //               local[id] &&
+  //               this.exver.compareExver(
+  //                 version,
+  //                 getManifest(local[id]).version,
+  //               ) === 1
+  //                 ? result.add(id)
+  //                 : result,
+  //             list,
+  //           ) || list,
+  //         new Set<string>(),
+  //       ).size,
+  //   ),
+  //   shareReplay(1),
+  // )
 
   getCount(id: string): Observable<number> {
     switch (id) {
-      case '/portal/system/updates':
-        return this.updates$
+      // case '/portal/system/updates':
+      //   return this.updates$
       case '/portal/system/settings':
         return this.settings$
       case '/portal/system/notifications':

web/projects/ui/src/app/services/marketplace.service.ts

@@ -1,42 +1,65 @@
 import { Injectable } from '@angular/core'
 import {
-  AbstractMarketplaceService,
-  Marketplace,
   StoreIdentity,
   MarketplacePkg,
   GetPackageRes,
-  StoreData,
+  StoreDataWithUrl,
 } from '@start9labs/marketplace'
 import { PatchDB } from 'patch-db-client'
 import {
   BehaviorSubject,
   catchError,
   combineLatest,
-  distinctUntilKeyChanged,
   filter,
   from,
   map,
-  mergeMap,
   Observable,
   of,
-  scan,
-  pairwise,
   shareReplay,
-  startWith,
   switchMap,
-  take,
+  distinctUntilChanged,
+  ReplaySubject,
   tap,
 } from 'rxjs'
 import { RR } from 'src/app/services/api/api.types'
 import { ApiService } from 'src/app/services/api/embassy-api.service'
 import { DataModel, UIStore } from 'src/app/services/patch-db/data-model'
 import { ConfigService } from './config.service'
-import { Exver, sameUrl } from '@start9labs/shared'
+import { Exver } from '@start9labs/shared'
 import { ClientStorageService } from './client-storage.service'
 import { T } from '@start9labs/start-sdk'
 
-@Injectable()
-export class MarketplaceService implements AbstractMarketplaceService {
+@Injectable({
+  providedIn: 'root',
+})
+export class MarketplaceService {
+  private readonly registryUrlSubject$ = new ReplaySubject<string>(1)
+  private readonly registryUrl$ = this.registryUrlSubject$.pipe(
+    distinctUntilChanged(),
+  )
+
+  private readonly registry$: Observable<StoreDataWithUrl> =
+    this.registryUrl$.pipe(
+      switchMap(url => this.fetchRegistry$(url)),
+      filter(Boolean),
+      // @TODO is updateStoreName needed?
+      map(registry => {
+        registry.info.categories = {
+          all: {
+            name: 'All',
+            description: {
+              short: 'All registry packages',
+              long: 'An unfiltered list of all packages available on this registry.',
+            },
+          },
+          ...registry.info.categories,
+        }
+
+        return registry
+      }),
+      shareReplay(1),
+    )
+
   private readonly knownHosts$: Observable<StoreIdentity[]> = this.patch
     .watch$('ui', 'marketplace', 'knownHosts')
     .pipe(
@@ -69,71 +92,6 @@ export class MarketplaceService implements AbstractMarketplaceService {
       ),
     )
 
-  private readonly selectedHost$: Observable<StoreIdentity> = this.patch
-    .watch$('ui', 'marketplace')
-    .pipe(
-      distinctUntilKeyChanged('selectedUrl'),
-      map(({ selectedUrl: url, knownHosts: hosts }) =>
-        toStoreIdentity(url, hosts[url]),
-      ),
-      shareReplay(1),
-    )
-
-  private readonly marketplace$ = this.knownHosts$.pipe(
-    startWith<StoreIdentity[]>([]),
-    pairwise(),
-    mergeMap(([prev, curr]) =>
-      curr.filter(c => !prev.find(p => sameUrl(c.url, p.url))),
-    ),
-    mergeMap(({ url, name }) =>
-      this.fetchStore$(url).pipe(
-        tap(data => {
-          if (data?.info.name) this.updateStoreName(url, name, data.info.name)
-        }),
-        map<StoreData | null, [string, StoreData | null]>(data => [url, data]),
-        startWith<[string, StoreData | null]>([url, null]),
-      ),
-    ),
-    scan<[string, StoreData | null], Record<string, StoreData | null>>(
-      (requests, [url, store]) => {
-        requests[url] = store
-
-        return requests
-      },
-      {},
-    ),
-    shareReplay(1),
-  )
-
-  private readonly filteredMarketplace$ = combineLatest([
-    this.clientStorageService.showDevTools$,
-    this.marketplace$,
-  ]).pipe(
-    map(([devMode, marketplace]) =>
-      Object.entries(marketplace).reduce(
-        (filtered, [url, store]) =>
-          !devMode && (url.includes('alpha') || url.includes('beta'))
-            ? filtered
-            : {
-                [url]: store,
-                ...filtered,
-              },
-        {} as Marketplace,
-      ),
-    ),
-  )
-
-  private readonly selectedStore$: Observable<StoreData> =
-    this.selectedHost$.pipe(
-      switchMap(({ url }) =>
-        this.marketplace$.pipe(
-          map(m => m[url]),
-          filter(Boolean),
-          take(1),
-        ),
-      ),
-    )
-
   private readonly requestErrors$ = new BehaviorSubject<string[]>([])
 
   constructor(
@@ -149,33 +107,17 @@ export class MarketplaceService implements AbstractMarketplaceService {
     return filtered ? this.filteredKnownHosts$ : this.knownHosts$
   }
 
-  getSelectedHost$(): Observable<StoreIdentity> {
-    return this.selectedHost$
+  getRegistryUrl$() {
+    return this.registryUrl$
   }
 
-  getMarketplace$(filtered = false): Observable<Marketplace> {
-    // option to filter out hosts containing 'alpha' or 'beta' substrings in registryURL
-    return filtered ? this.filteredMarketplace$ : this.marketplace$
+  setRegistryUrl(url: string | null) {
+    const registryUrl = url || this.config.marketplace.start9
+    this.registryUrlSubject$.next(registryUrl)
   }
 
-  getSelectedStore$(): Observable<StoreData> {
-    return this.selectedStore$
-  }
-
-  getSelectedStoreWithCategories$() {
-    return this.selectedHost$.pipe(
-      switchMap(({ url }) =>
-        this.marketplace$.pipe(
-          map(m => m[url]),
-          filter(Boolean),
-          map(({ info, packages }) => ({
-            url,
-            info,
-            packages,
-          })),
-        ),
-      ),
-    )
+  getRegistry$(): Observable<StoreDataWithUrl> {
+    return this.registry$
   }
 
   getPackage$(
@@ -184,47 +126,20 @@ export class MarketplaceService implements AbstractMarketplaceService {
     flavor: string | null,
     registryUrl?: string,
   ): Observable<MarketplacePkg> {
-    return this.selectedHost$.pipe(
-      switchMap(selected =>
-        this.marketplace$.pipe(
-          switchMap(m => {
-            const url = registryUrl || selected.url
-            const pkg = m[url]?.packages.find(
-              p =>
-                p.id === id &&
-                p.flavor === flavor &&
-                (!version || this.exver.compareExver(p.version, version) === 0),
-            )
-
-            return pkg ? of(pkg) : this.fetchPackage$(url, id, version, flavor)
-          }),
-        ),
-      ),
+    return this.registry$.pipe(
+      switchMap(registry => {
+        const url = registryUrl || registry.url
+        const pkg = registry.packages.find(
+          p =>
+            p.id === id &&
+            p.flavor === flavor &&
+            (!version || this.exver.compareExver(p.version, version) === 0),
+        )
+        return pkg ? of(pkg) : this.fetchPackage$(url, id, version, flavor)
+      }),
     )
   }
 
-  // UI only
-  readonly updateErrors: Record<string, string> = {}
-  readonly updateQueue: Record<string, boolean> = {}
-
-  getRequestErrors$(): Observable<string[]> {
-    return this.requestErrors$
-  }
-
-  async installPackage(
-    id: string,
-    version: string,
-    url: string,
-  ): Promise<void> {
-    const params: RR.InstallPackageReq = {
-      id,
-      version,
-      registry: url,
-    }
-
-    await this.api.installPackage(params)
-  }
-
   fetchInfo$(url: string): Observable<T.RegistryInfo> {
     return from(this.api.getRegistryInfo(url)).pipe(
       map(info => ({
@@ -232,7 +147,10 @@ export class MarketplaceService implements AbstractMarketplaceService {
         categories: {
           all: {
             name: 'All',
-            description: { short: 'All services', long: 'All services' },
+            description: {
+              short: 'All services',
+              long: 'An unfiltered list of all services available on this registry.',
+            },
           },
           ...info.categories,
         },
@@ -240,16 +158,17 @@ export class MarketplaceService implements AbstractMarketplaceService {
     )
   }
 
-  fetchStatic$(
+  getStatic$(
     pkg: MarketplacePkg,
     type: 'LICENSE.md' | 'instructions.md',
   ): Observable<string> {
     return from(this.api.getStaticProxy(pkg, type))
   }
 
-  private fetchStore$(url: string): Observable<StoreData | null> {
+  private fetchRegistry$(url: string): Observable<StoreDataWithUrl | null> {
+    console.log('FETCHING REGISTRY: ', url)
     return combineLatest([this.fetchInfo$(url), this.fetchPackages$(url)]).pipe(
-      map(([info, packages]) => ({ info, packages })),
+      map(([info, packages]) => ({ info, packages, url })),
       catchError(e => {
         console.error(e)
         this.requestErrors$.next(this.requestErrors$.value.concat(url))
@@ -275,7 +194,22 @@ export class MarketplaceService implements AbstractMarketplaceService {
     )
   }
 
-  convertToMarketplacePkg(
+  private fetchPackage$(
+    url: string,
+    id: string,
+    version: string | null,
+    flavor: string | null,
+  ): Observable<MarketplacePkg> {
+    return from(
+      this.api.getRegistryPackage(url, id, version ? `=${version}` : null),
+    ).pipe(
+      map(pkgInfo =>
+        this.convertToMarketplacePkg(id, version, flavor, pkgInfo),
+      ),
+    )
+  }
+
+  private convertToMarketplacePkg(
     id: string,
     version: string | null | undefined,
     flavor: string | null,
@@ -297,26 +231,6 @@ export class MarketplaceService implements AbstractMarketplaceService {
         }
   }
 
-  private fetchPackage$(
-    url: string,
-    id: string,
-    version: string | null,
-    flavor: string | null,
-  ): Observable<MarketplacePkg> {
-    return from(
-      this.api.getRegistryPackage(url, id, version ? `=${version}` : null),
-    ).pipe(
-      map(pkgInfo =>
-        this.convertToMarketplacePkg(
-          id,
-          version === '*' ? null : version,
-          flavor,
-          pkgInfo,
-        ),
-      ),
-    )
-  }
-
   private async updateStoreName(
     url: string,
     oldName: string | undefined,
@@ -329,6 +243,28 @@ export class MarketplaceService implements AbstractMarketplaceService {
       )
     }
   }
+
+  // UI only
+  readonly updateErrors: Record<string, string> = {}
+  readonly updateQueue: Record<string, boolean> = {}
+
+  getRequestErrors$(): Observable<string[]> {
+    return this.requestErrors$
+  }
+
+  async installPackage(
+    id: string,
+    version: string,
+    url: string,
+  ): Promise<void> {
+    const params: RR.InstallPackageReq = {
+      id,
+      version,
+      registry: url,
+    }
+
+    await this.api.installPackage(params)
+  }
 }
 
 function toStoreIdentity(url: string, uiStore: UIStore): StoreIdentity {

web/projects/ui/src/app/services/patch-data.service.ts

@@ -1,14 +1,12 @@
-import { Inject, Injectable } from '@angular/core'
-import { AbstractMarketplaceService } from '@start9labs/marketplace'
+import { Injectable } from '@angular/core'
 import { TuiDialogService } from '@taiga-ui/core'
-import { filter, share, switchMap, take, Observable, map } from 'rxjs'
+import { filter, share, switchMap, Observable, map } from 'rxjs'
 import { PatchDB } from 'patch-db-client'
 import { DataModel } from 'src/app/services/patch-db/data-model'
 import { EOSService } from 'src/app/services/eos.service'
 import { WelcomeComponent } from 'src/app/components/welcome.component'
 import { ConfigService } from 'src/app/services/config.service'
 import { ApiService } from 'src/app/services/api/embassy-api.service'
-import { MarketplaceService } from 'src/app/services/marketplace.service'
 import { ConnectionService } from 'src/app/services/connection.service'
 import { PolymorpheusComponent } from '@taiga-ui/polymorpheus'
 import { LocalStorageBootstrap } from './patch-db/local-storage-bootstrap'
@@ -40,8 +38,6 @@ export class PatchDataService extends Observable<void> {
     private readonly config: ConfigService,
     private readonly dialogs: TuiDialogService,
     private readonly embassyApi: ApiService,
-    @Inject(AbstractMarketplaceService)
-    private readonly marketplaceService: MarketplaceService,
     private readonly connection$: ConnectionService,
     private readonly bootstrapper: LocalStorageBootstrap,
   ) {
@@ -50,7 +46,7 @@ export class PatchDataService extends Observable<void> {
 
   private checkForUpdates(): void {
     this.eosService.loadEos()
-    this.marketplaceService.getMarketplace$().pipe(take(1)).subscribe()
+    // this.marketplaceService.getMarketplace$().pipe(take(1)).subscribe()
   }
 
   private showEosWelcome(ackVersion: string) {