Auto poll (#947)

* switch to polling automatically

* connected conditional

* better logs

* refactor

* remove old diagnosing connection code

* continue refactor

* bluj edits

* polling working without refresh

* pr review

* bluj comments addressed

* fix: Build for ui

Co-authored-by: Drew Ansbacher <drew.ansbacher@gmail.com>

* Delete settings.json

* lint fix

Co-authored-by: Drew Ansbacher <drew.ansbacher@spiredigital.com>
Co-authored-by: J M <dragondef@gmail.com>
Co-authored-by: Drew Ansbacher <drew.ansbacher@gmail.com>
Co-authored-by: J M <2364004+Blu-J@users.noreply.github.com>

diagnostic-ui/package-lock.json

@@ -5,6 +5,7 @@
   "requires": true,
   "packages": {
     "": {
+      "name": "diagnostic-ui",
       "version": "0.3.0",
       "dependencies": {
         "@angular/animations": "^12.2.5",

setup-wizard/package-lock.json

@@ -5,6 +5,7 @@
   "requires": true,
   "packages": {
     "": {
+      "name": "setup-wizard",
       "version": "0.0.1",
       "dependencies": {
         "@angular/common": "^12.2.1",

system-images/compat/Cargo.lock

@@ -216,6 +216,15 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitmaps"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2"
+dependencies = [
+ "typenum",
+]
+
 [[package]]
 name = "bitvec"
 version = "0.19.5"
@@ -1461,6 +1470,20 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "imbl"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "543682c9082b25e63d03b5acbd65ad111fd49dd93e70843e5175db4ff81d606b"
+dependencies = [
+ "bitmaps",
+ "rand_core 0.6.3",
+ "rand_xoshiro",
+ "sized-chunks",
+ "typenum",
+ "version_check",
+]
+
 [[package]]
 name = "indenter"
 version = "0.3.3"
@@ -2105,6 +2128,7 @@ dependencies = [
  "async-trait",
  "fd-lock-rs",
  "futures",
+ "imbl",
  "json-patch",
  "json-ptr",
  "lazy_static",
@@ -2525,6 +2549,15 @@ dependencies = [
  "rand_core 0.6.3",
 ]
 
+[[package]]
+name = "rand_xoshiro"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
+dependencies = [
+ "rand_core 0.6.3",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.1.57"
@@ -3078,6 +3111,16 @@ version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "533494a8f9b724d33625ab53c6c4800f7cc445895924a8ef649222dcb76e938b"
 
+[[package]]
+name = "sized-chunks"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e"
+dependencies = [
+ "bitmaps",
+ "typenum",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.4"

ui/src/app/app.component.ts

@@ -88,16 +88,15 @@ export class AppComponent {
   async init () {
     await this.storage.create()
     await this.authService.init()
-    await this.patch.init()
 
     this.router.initialNavigation()
 
     // watch auth
     this.authService.watch$()
-    .subscribe(auth => {
+    .subscribe(async auth => {
       // VERIFIED
       if (auth === AuthState.VERIFIED) {
-        this.patch.start()
+        await this.patch.start()
 
         this.showMenu = true
         // if on the login screen, route to dashboard
@@ -196,7 +195,7 @@ export class AppComponent {
     .subscribe(async connectionFailure => {
       if (connectionFailure === ConnectionFailure.None) {
         if (this.offlineToast) {
-          this.offlineToast.dismiss()
+          await this.offlineToast.dismiss()
           this.offlineToast = undefined
         }
       } else {
@@ -206,16 +205,13 @@ export class AppComponent {
           case ConnectionFailure.Network:
             message = 'Phone or computer has no network connection.'
             break
-          case ConnectionFailure.Diagnosing:
-            message = new IonicSafeString('Running network diagnostics <ion-spinner style="padding: 0; margin: 0" name="dots"></ion-spinner>')
-            break
           case ConnectionFailure.Tor:
             message = 'Browser unable to connect over Tor.'
-            link = 'https://docs.start9.com/support/FAQ/setup-faq.html#tor-failure'
+            link = 'https://docs.start9.com/support/FAQ/troubleshooting.html#tor-failure'
             break
           case ConnectionFailure.Lan:
             message = 'Embassy not found on Local Area Network.'
-            link = 'https://docs.start9.com/support/FAQ/setup-faq.html#lan-failure'
+            link = 'https://docs.start9.com/support/FAQ/troubleshooting.html#lan-failure'
             break
         }
         await this.presentToastOffline(message, link)

ui/src/app/app.module.ts

@@ -3,7 +3,7 @@ import { BrowserModule } from '@angular/platform-browser'
 import { RouteReuseStrategy } from '@angular/router'
 import { IonicModule, IonicRouteStrategy, IonNav } from '@ionic/angular'
 import { Drivers } from '@ionic/storage'
-import { IonicStorageModule } from '@ionic/storage-angular'
+import { IonicStorageModule, Storage } from '@ionic/storage-angular'
 import { HttpClientModule } from '@angular/common/http'
 import { AppComponent } from './app.component'
 import { AppRoutingModule } from './app-routing.module'
@@ -48,8 +48,10 @@ import { GlobalErrorHandler } from './services/global-error-handler.service'
   providers: [
     FormBuilder,
     IonNav,
-    Storage,
+    {
-    { provide: RouteReuseStrategy, useClass: IonicRouteStrategy },
+      provide: RouteReuseStrategy,
+      useClass: IonicRouteStrategy,
+    },
     {
       provide: ApiService,
       useFactory: ApiServiceFactory,
@@ -58,9 +60,12 @@ import { GlobalErrorHandler } from './services/global-error-handler.service'
     {
       provide: PatchDbService,
       useFactory: PatchDbServiceFactory,
-      deps: [ConfigService, ApiService, LocalStorageBootstrap, AuthService],
+      deps: [ConfigService, ApiService, LocalStorageBootstrap, AuthService, Storage],
+    },
+    {
+      provide: ErrorHandler,
+      useClass: GlobalErrorHandler,
     },
-    { provide: ErrorHandler, useClass: GlobalErrorHandler},
   ],
   bootstrap: [AppComponent],
   schemas: [ CUSTOM_ELEMENTS_SCHEMA ],

ui/src/app/pages/login/login.page.ts

@@ -52,6 +52,7 @@ export class LoginPage {
         password: this.password,
         metadata: { platforms: getPlatforms() },
       })
+
       this.authService.setVerified()
       this.password = ''
     } catch (e) {

ui/src/app/services/api/embassy-api.service.ts

@@ -1,20 +1,20 @@
 import { Subject, Observable } from 'rxjs'
-import { Http, Update, Operation, Revision, Source, Store } from 'patch-db-client'
+import { Http, Update, Operation, Revision, Source, Store, RPCResponse } from 'patch-db-client'
 import { RR } from './api.types'
 import { DataModel } from 'src/app/services/patch-db/data-model'
 import { RequestError } from '../http.service'
+import { map } from 'rxjs/operators'
 
 export abstract class ApiService implements Source<DataModel>, Http<DataModel> {
   protected readonly sync$ = new Subject<Update<DataModel>>()
 
   /** PatchDb Source interface. Post/Patch requests provide a source of patches to the db. */
   // sequenceStream '_' is not used by the live api, but is overridden by the mock
-  watch$ (_?: Store<DataModel>): Observable<Update<DataModel>> {
+  watch$ (_?: Store<DataModel>): Observable<RPCResponse<Update<DataModel>>> {
-    return this.sync$.asObservable()
+    return this.sync$.asObservable().pipe(map( result => ({ result,
+      jsonrpc: '2.0'})))
   }
 
-  connectionMade$ = new Subject<void>()
-
   // for getting static files: ex icons, instructions, licenses
   abstract getStatic (url: string): Promise<string>
 
@@ -209,7 +209,6 @@ export abstract class ApiService implements Source<DataModel>, Http<DataModel> {
           throw e
         })
         .then(({ response, revision }) => {
-          this.connectionMade$.next()
           if (revision) this.sync$.next(revision)
           return response
         })

ui/src/app/services/api/embassy-mock-api.service.ts

@@ -59,7 +59,11 @@ export class MockApiService extends ApiService {
 
   async login (params: RR.LoginReq): Promise<RR.loginRes> {
     await pauseFor(2000)
-    this.mockPatch$.next({ id: 1, value: mockPatchData, expireId: null })
+
+    setTimeout(() => {
+      this.mockPatch$.next({ id: 1, value: mockPatchData, expireId: null })
+    }, 2000)
+
     return null
   }
 

ui/src/app/services/connection.service.ts

@@ -1,7 +1,6 @@
 import { Injectable } from '@angular/core'
 import { BehaviorSubject, combineLatest, fromEvent, merge, Subscription } from 'rxjs'
 import { PatchConnection, PatchDbService } from './patch-db/patch-db.service'
-import { HttpService, Method } from './http.service'
 import { distinctUntilChanged } from 'rxjs/operators'
 import { ConfigService } from './config.service'
 
@@ -13,7 +12,6 @@ export class ConnectionService {
   private readonly connectionFailure$ = new BehaviorSubject<ConnectionFailure>(ConnectionFailure.None)
 
   constructor (
-    private readonly httpService: HttpService,
     private readonly configService: ConfigService,
     private readonly patch: PatchDbService,
   ) { }
@@ -46,10 +44,10 @@ export class ConnectionService {
       ),
     ])
     .subscribe(async ([network, patchConnection, progress]) => {
-      if (patchConnection !== PatchConnection.Disconnected) {
+      if (!network) {
-        this.connectionFailure$.next(ConnectionFailure.None)
-      } else if (!network) {
         this.connectionFailure$.next(ConnectionFailure.Network)
+      } else if (patchConnection !== PatchConnection.Disconnected) {
+        this.connectionFailure$.next(ConnectionFailure.None)
       } else if (!!progress && progress.downloaded === progress.size) {
         this.connectionFailure$.next(ConnectionFailure.None)
       } else if (!this.configService.isTor()) {
@@ -64,7 +62,6 @@ export class ConnectionService {
 
 export enum ConnectionFailure {
   None = 'none',
-  Diagnosing = 'diagnosing',
   Network = 'network',
   Tor = 'tor',
   Lan = 'lan',

ui/src/app/services/patch-db/patch-db.factory.ts

@@ -1,4 +1,4 @@
-import { MockSource, PollSource, Source, WebsocketSource } from 'patch-db-client'
+import { MockSource, PollSource, WebsocketSource } from 'patch-db-client'
 import { ConfigService } from 'src/app/services/config.service'
 import { DataModel } from './data-model'
 import { LocalStorageBootstrap } from './local-storage-bootstrap'
@@ -8,29 +8,47 @@ import { AuthService } from '../auth.service'
 import { MockApiService } from '../api/embassy-mock-api.service'
 import { filter } from 'rxjs/operators'
 import { exists } from 'src/app/util/misc.util'
+import { Storage } from '@ionic/storage-angular'
 
 export function PatchDbServiceFactory (
   config: ConfigService,
   embassyApi: ApiService,
   bootstrapper: LocalStorageBootstrap,
   auth: AuthService,
+  storage: Storage,
 ): PatchDbService {
-
+  const {
-  const { mocks, patchDb: { poll }, supportsWebSockets } = config
+    mocks,
-
+    patchDb: { poll },
-  let source: Source<DataModel>
+  } = config
 
   if (mocks.enabled) {
-    source = new MockSource((embassyApi as MockApiService).mockPatch$.pipe(filter(exists)))
+    const source = new MockSource<DataModel>(
+      (embassyApi as MockApiService).mockPatch$.pipe(filter(exists)),
+    )
+    return new PatchDbService(
+      source,
+      source,
+      embassyApi,
+      bootstrapper,
+      auth,
+      storage,
+    )
   } else {
-    if (!supportsWebSockets) {
+    const protocol = window.location.protocol === 'http:' ? 'ws' : 'wss'
-      source = new PollSource({ ...poll }, embassyApi)
+    const host = window.location.host
-    } else {
+    const wsSource = new WebsocketSource<DataModel>(
-      const protocol = window.location.protocol === 'http:' ? 'ws' : 'wss'
+      `${protocol}://${host}/ws/db`,
-      const host = window.location.host
+    )
-      source = new WebsocketSource(`${protocol}://${host}/ws/db`)
+    const pollSource = new PollSource<DataModel>({ ...poll }, embassyApi)
-    }
-  }
 
-  return new PatchDbService(source, embassyApi, bootstrapper, auth)
+    return new PatchDbService(
-}
+      wsSource,
+      pollSource,
+      embassyApi,
+      bootstrapper,
+      auth,
+      storage,
+    )
+  }
+}

ui/src/app/services/patch-db/patch-db.service.ts

@@ -1,7 +1,15 @@
 import { Inject, Injectable, InjectionToken } from '@angular/core'
+import { Storage } from '@ionic/storage-angular'
 import { Bootstrapper, PatchDB, Source, Store } from 'patch-db-client'
 import { BehaviorSubject, Observable, of, Subscription } from 'rxjs'
-import { catchError, debounceTime, finalize, map, tap } from 'rxjs/operators'
+import {
+  catchError,
+  debounceTime,
+  finalize,
+  mergeMap,
+  tap,
+  withLatestFrom,
+} from 'rxjs/operators'
 import { pauseFor } from 'src/app/util/misc.util'
 import { ApiService } from '../api/embassy-api.service'
 import { AuthService } from '../auth.service'
@@ -11,6 +19,7 @@ export const PATCH_HTTP = new InjectionToken<Source<DataModel>>('')
 export const PATCH_SOURCE = new InjectionToken<Source<DataModel>>('')
 export const BOOTSTRAPPER = new InjectionToken<Bootstrapper<DataModel>>('')
 export const AUTH = new InjectionToken<AuthService>('')
+export const STORAGE = new InjectionToken<Storage>('')
 
 export enum PatchConnection {
   Initializing = 'initializing',
@@ -22,90 +31,153 @@ export enum PatchConnection {
   providedIn: 'root',
 })
 export class PatchDbService {
+  private readonly WS_SUCCESS = 'wsSuccess'
   private patchConnection$ = new BehaviorSubject(PatchConnection.Initializing)
+  private wsSuccess$ = new BehaviorSubject(false)
+  private polling$ = new BehaviorSubject(false)
   private patchDb: PatchDB<DataModel>
-  private patchSub: Subscription
+  private subs: Subscription[] = []
-  data: DataModel
+  private sources$: BehaviorSubject<Source<DataModel>[]> = new BehaviorSubject([
+    this.wsSource,
+  ])
 
-  getData () { return this.patchDb.store.cache.data }
+  data: DataModel
+  errors = 0
+
+  getData () {
+    return this.patchDb.store.cache.data
+  }
 
   constructor (
-    @Inject(PATCH_SOURCE) private readonly source: Source<DataModel>,
+    @Inject(PATCH_SOURCE) private readonly wsSource: Source<DataModel>,
+    @Inject(PATCH_SOURCE) private readonly pollSource: Source<DataModel>,
     @Inject(PATCH_HTTP) private readonly http: ApiService,
-    @Inject(BOOTSTRAPPER) private readonly bootstrapper: Bootstrapper<DataModel>,
+    @Inject(BOOTSTRAPPER)
+    private readonly bootstrapper: Bootstrapper<DataModel>,
     @Inject(AUTH) private readonly auth: AuthService,
+    @Inject(STORAGE) private readonly storage: Storage,
   ) { }
 
   async init (): Promise<void> {
     const cache = await this.bootstrapper.init()
-    this.patchDb = new PatchDB([this.source, this.http], this.http, cache)
+    this.sources$.next([this.wsSource, this.http])
+
+    this.patchDb = new PatchDB(this.sources$, this.http, cache)
+
+    this.patchConnection$.next(PatchConnection.Initializing)
     this.data = this.patchDb.store.cache.data
   }
 
-  start (): void {
+  async start (): Promise<void> {
-    console.log(this.patchSub ? 'restarting patch-db' : 'starting patch-db')
+    await this.init()
 
-    // make sure everything is stopped before initializing
+    this.subs.push(
-    if (this.patchSub) {
+      // Connection Error
-      this.patchSub.unsubscribe()
+      this.patchDb.connectionError$
-      this.patchSub = undefined
+        .pipe(
-    }
+          debounceTime(420),
+          withLatestFrom(this.polling$),
+          mergeMap(async ([e, polling]) => {
+            if (polling) {
+              console.log('patchDB: POLLING FAILED', e)
+              this.patchConnection$.next(PatchConnection.Disconnected)
+              await pauseFor(2000)
+              this.sources$.next([this.pollSource, this.http])
+              return
+            }
 
-    this.patchSub = this.patchDb.sync$()
+            console.log('patchDB: WEBSOCKET FAILED', e)
-    .pipe(
+            this.polling$.next(true)
-      debounceTime(400),
+            this.sources$.next([this.pollSource, this.http])
-      tap(cache => {
+          }),
-        this.patchConnection$.next(PatchConnection.Connected)
+        )
-        this.bootstrapper.update(cache)
+        .subscribe({
-      }),
+          complete: () => {
+            console.warn('patchDB: SYNC COMPLETE')
+          },
+        }),
+
+      // RPC ERROR
+      this.patchDb.rpcError$
+        .pipe(
+          tap(({ error }) => {
+            if (error.code === 34) {
+              console.log('patchDB: Unauthorized. Logging out.')
+              this.auth.setUnverified()
+            }
+          }),
+        )
+        .subscribe({
+          complete: () => {
+            console.warn('patchDB: SYNC COMPLETE')
+          },
+        }),
+
+      // GOOD CONNECTION
+      this.patchDb.cache$
+        .pipe(
+          debounceTime(420),
+          withLatestFrom(this.patchConnection$, this.wsSuccess$, this.polling$),
+          tap(async ([cache, connection, wsSuccess, polling]) => {
+            this.bootstrapper.update(cache)
+
+            if (connection === PatchConnection.Initializing) {
+              console.log(
+                polling
+                  ? 'patchDB: POLL CONNECTED'
+                  : 'patchDB: WEBSOCKET CONNECTED',
+              )
+              this.patchConnection$.next(PatchConnection.Connected)
+              if (!wsSuccess && !polling) {
+                console.log('patchDB: WEBSOCKET SUCCESS')
+                this.storage.set(this.WS_SUCCESS, 'true')
+                this.wsSuccess$.next(true)
+              }
+            } else if (
+              connection === PatchConnection.Disconnected &&
+              wsSuccess
+            ) {
+              console.log('patchDB: SWITCHING BACK TO WEBSOCKETS')
+              this.patchConnection$.next(PatchConnection.Initializing)
+              this.polling$.next(false)
+              this.sources$.next([this.wsSource, this.http])
+            }
+          }),
+        )
+        .subscribe({
+          complete: () => {
+            console.warn('patchDB: SYNC COMPLETE')
+          },
+        }),
     )
-    .subscribe({
-      error: async e => {
-        console.error('patch-db SYNC ERROR', e)
-        this.patchConnection$.next(PatchConnection.Disconnected)
-        if (e.code === 34) {
-          this.auth.setUnverified()
-        } else {
-          await pauseFor(4000)
-          this.start()
-        }
-      },
-      complete: () => {
-        console.warn('patch-db SYNC COMPLETE')
-      },
-    })
   }
 
   stop (): void {
-    console.log('stopping patch-db')
+    if (this.patchDb) {
-    this.patchConnection$.next(PatchConnection.Initializing)
+      console.log('patchDB: STOPPING')
-    if (this.patchSub) {
+      this.patchConnection$.next(PatchConnection.Initializing)
-      this.patchSub.unsubscribe()
+      this.patchDb.store.reset()
-      this.patchSub = undefined
     }
-  }
+    this.subs.forEach((x) => x.unsubscribe())
-
+    this.subs = []
-  connected$ (): Observable<boolean> {
-    return this.patchConnection$
-    .pipe(
-      map(status => status === PatchConnection.Connected),
-    )
   }
 
   watchPatchConnection$ (): Observable<PatchConnection> {
     return this.patchConnection$.asObservable()
   }
 
-  watch$: Store<DataModel>['watch$'] = (...args: (string | number)[]): Observable<DataModel> => {
+  watch$: Store<DataModel>['watch$'] = (
-    console.log('WATCHING', ...args)
+    ...args: (string | number)[],
-    return this.patchDb.store.watch$(...(args as []))
+  ): Observable<DataModel> => {
-    .pipe(
+    const argsString = '/' + args.join('/')
-      tap(data => console.log('NEW VALUE', data, ...args)),
+    console.log('patchDB: WATCHING ', argsString)
-      catchError(e => {
+    return this.patchDb.store.watch$(...(args as [])).pipe(
-        console.error('Error watching patch-db', e)
+      tap((data) => console.log('patchDB: NEW VALUE', argsString, data)),
+      catchError((e) => {
+        console.error('patchDB: WATCH ERROR', e)
         return of(e.message)
       }),
-      finalize(() => console.log('UNSUBSCRIBING', ...args)),
+      finalize(() => console.log('patchDB: UNSUBSCRIBING', argsString)),
     )
   }
 }