statically type server metrics and use websocket (#2124)

Co-authored-by: Matt Hill <matthewonthemoon@gmail.com>
This commit is contained in:
Aiden McClelland
2023-05-09 15:05:42 -06:00
committed by Aiden McClelland
parent 873f2b2814
commit 8313dfaeb9
14 changed files with 303 additions and 405 deletions

View File

@@ -1,26 +0,0 @@
import { NgModule } from '@angular/core'
import { CommonModule } from '@angular/common'
import { Routes, RouterModule } from '@angular/router'
import { IonicModule } from '@ionic/angular'
import { AppMetricsPage } from './app-metrics.page'
import { SharedPipesModule } from '@start9labs/shared'
import { SkeletonListComponentModule } from 'src/app/components/skeleton-list/skeleton-list.component.module'
const routes: Routes = [
{
path: '',
component: AppMetricsPage,
},
]
@NgModule({
imports: [
CommonModule,
IonicModule,
RouterModule.forChild(routes),
SharedPipesModule,
SkeletonListComponentModule,
],
declarations: [AppMetricsPage],
})
export class AppMetricsPageModule {}

View File

@@ -1,25 +0,0 @@
<ion-header>
<ion-toolbar>
<ion-buttons slot="start">
<ion-back-button [defaultHref]="'/services/' + pkgId"></ion-back-button>
</ion-buttons>
<ion-title>Monitor</ion-title>
<ion-title slot="end"
><ion-spinner name="dots" class="fader"></ion-spinner
></ion-title>
</ion-toolbar>
</ion-header>
<ion-content class="ion-padding with-widgets">
<skeleton-list *ngIf="loading"></skeleton-list>
<ion-item-group *ngIf="!loading">
<ion-item *ngFor="let metric of metrics | keyvalue : asIsOrder">
<ion-label>{{ metric.key }}</ion-label>
<ion-note *ngIf="metric.value" slot="end" class="metric-note">
<ion-text style="color: white"
>{{ metric.value.value }} {{ metric.value.unit }}</ion-text
>
</ion-note>
</ion-item>
</ion-item-group>
</ion-content>

View File

@@ -1,3 +0,0 @@
.metric-note {
font-size: 16px;
}

View File

@@ -1,59 +0,0 @@
import { Component } from '@angular/core'
import { ActivatedRoute } from '@angular/router'
import { Metric } from 'src/app/services/api/api.types'
import { ApiService } from 'src/app/services/api/embassy-api.service'
import { pauseFor, ErrorToastService, getPkgId } from '@start9labs/shared'
@Component({
selector: 'app-metrics',
templateUrl: './app-metrics.page.html',
styleUrls: ['./app-metrics.page.scss'],
})
export class AppMetricsPage {
loading = true
readonly pkgId = getPkgId(this.route)
going = false
metrics?: Metric
constructor(
private readonly route: ActivatedRoute,
private readonly errToast: ErrorToastService,
private readonly embassyApi: ApiService,
) {}
ngOnInit() {
this.startDaemon()
}
ngOnDestroy() {
this.stopDaemon()
}
async startDaemon(): Promise<void> {
this.going = true
while (this.going) {
const startTime = Date.now()
await this.getMetrics()
await pauseFor(Math.max(4000 - (Date.now() - startTime), 0))
}
}
stopDaemon() {
this.going = false
}
async getMetrics(): Promise<void> {
try {
this.metrics = await this.embassyApi.getPkgMetrics({ id: this.pkgId })
} catch (e: any) {
this.errToast.present(e)
this.stopDaemon()
} finally {
this.loading = false
}
}
asIsOrder(a: any, b: any) {
return 0
}
}

View File

@@ -36,13 +36,6 @@ const routes: Routes = [
loadChildren: () =>
import('./app-logs/app-logs.module').then(m => m.AppLogsPageModule),
},
{
path: ':pkgId/metrics',
loadChildren: () =>
import('./app-metrics/app-metrics.module').then(
m => m.AppMetricsPageModule,
),
},
{
path: ':pkgId/properties',
loadChildren: () =>

View File

@@ -4,58 +4,121 @@
<ion-back-button defaultHref="system"></ion-back-button>
</ion-buttons>
<ion-title>Monitor</ion-title>
<ion-title slot="end"
><ion-spinner name="dots" class="fader"></ion-spinner
></ion-title>
</ion-toolbar>
</ion-header>
<ion-content class="ion-padding with-widgets">
<skeleton-list *ngIf="loading" [groups]="2"></skeleton-list>
<ion-item-group *ngIf="serverData$ | async as serverData; else loading">
<p *ngIf="websocketFail" class="loading-dots ion-text-center">
<ion-text color="warning">Websocket Failed. Reconnecting</ion-text>
</p>
<div id="metricSection">
<ng-container *ngIf="!loading">
<ion-item-group>
<ion-item-divider>Time</ion-item-divider>
<ion-item>
<ion-label>System Time</ion-label>
<ion-note slot="end" class="metric-note">
<ion-text style="color: white"
>{{ systemTime$ | async | date:'MMMM d, y, h:mm a z':'UTC'
}}</ion-text
>
</ion-note>
</ion-item>
<ion-item>
<ion-label>System Uptime</ion-label>
<ion-note
*ngIf="systemUptime$ | async as uptime"
slot="end"
class="metric-note"
>
<ion-text style="color: white">
<b>{{ uptime.days }}</b> Days, <b>{{ uptime.hours }}</b> Hours,
<b>{{ uptime.minutes }}</b> Minutes
</ion-text>
</ion-note>
</ion-item>
</ion-item-group>
<ion-item-group
*ngFor="let metricGroup of metrics | keyvalue : asIsOrder"
>
<ion-item-divider>{{ metricGroup.key }}</ion-item-divider>
<ion-item
*ngFor="let metric of metricGroup.value | keyvalue : asIsOrder"
>
<ion-label>{{ metric.key }}</ion-label>
<ion-note *ngIf="metric.value" slot="end" class="metric-note">
<ion-text style="color: white"
>{{ metric.value.value }} {{ metric.value.unit }}</ion-text
>
</ion-note>
</ion-item>
</ion-item-group>
<ng-container *ngIf="serverData[0] as timeInfo">
<ion-item-divider>System Time</ion-item-divider>
<ion-item>
<ion-label>Current Time (UTC)</ion-label>
<h6 slot="end">
{{ timeInfo.systemCurrentTime | date : 'MMMM d, y, h:mm:ss a' : 'UTC'
}}
</h6>
</ion-item>
<ion-item>
<ion-label>Start Time (UTC)</ion-label>
<h6 slot="end">
{{ timeInfo.systemStartTime | date : 'MMMM d, y, h:mm:ss a' : 'UTC' }}
</h6>
</ion-item>
<ion-item>
<ion-label>Uptime</ion-label>
<h6 *ngIf="timeInfo.systemUptime as uptime" slot="end">
<b>{{ uptime.days }}</b> Days, <b>{{ uptime.hours }}</b> Hours,
<b>{{ uptime.minutes }}</b> Minutes,
<b>{{ uptime.seconds }}</b> Seconds
</h6>
</ion-item>
</ng-container>
</div>
<ng-container *ngIf="serverData[1] as metrics">
<!-- General -->
<ion-item-divider>General</ion-item-divider>
<ion-item>
<ion-label>Temperature</ion-label>
<h6 slot="end">{{ metrics.general.temperature }} °C</h6>
</ion-item>
<!-- Memory -->
<ion-item-divider>Memory</ion-item-divider>
<ion-item>
<ion-label>Percentage Used</ion-label>
<h6 slot="end">{{ metrics.memory['percentage-used'] }} %</h6>
</ion-item>
<ion-item>
<ion-label>Total</ion-label>
<h6 slot="end">{{ metrics.memory.total }} MiB</h6>
</ion-item>
<ion-item>
<ion-label>Available</ion-label>
<h6 slot="end">{{ metrics.memory.available }} MiB</h6>
</ion-item>
<ion-item>
<ion-label>Used</ion-label>
<h6 slot="end">{{ metrics.memory.used }} MiB</h6>
</ion-item>
<ion-item>
<ion-label>Swap Total</ion-label>
<h6 slot="end">{{ metrics.memory['swap-total'] }} MiB</h6>
</ion-item>
<ion-item>
<ion-label>Swap Free</ion-label>
<h6 slot="end">{{ metrics.memory['swap-free'] }} MiB</h6>
</ion-item>
<ion-item>
<ion-label>Swap Used</ion-label>
<h6 slot="end">{{ metrics.memory['swap-used'] }} MiB</h6>
</ion-item>
<!-- CPU -->
<ion-item-divider>CPU</ion-item-divider>
<ion-item>
<ion-label>User Space</ion-label>
<h6 slot="end">{{ metrics.cpu['user-space'] }} %</h6>
</ion-item>
<ion-item>
<ion-label>Kernel Space</ion-label>
<h6 slot="end">{{ metrics.cpu['kernel-space'] }} %</h6>
</ion-item>
<ion-item>
<ion-label>I/O Wait</ion-label>
<h6 slot="end">{{ metrics.cpu['io-wait'] }} %</h6>
</ion-item>
<ion-item>
<ion-label>Idle</ion-label>
<h6 slot="end">{{ metrics.cpu.idle }} %</h6>
</ion-item>
<ion-item>
<ion-label>Usage</ion-label>
<h6 slot="end">{{ metrics.cpu.usage }} %</h6>
</ion-item>
<!-- Disk -->
<ion-item-divider>Disk</ion-item-divider>
<ion-item>
<ion-label>Size</ion-label>
<h6 slot="end">{{ metrics.disk.size }} GB</h6>
</ion-item>
<ion-item>
<ion-label>Used</ion-label>
<h6 slot="end">{{ metrics.disk.used }} GB</h6>
</ion-item>
<ion-item>
<ion-label>Percentage Used</ion-label>
<h6 slot="end">{{ metrics.disk['percentage-used'] }} %</h6>
</ion-item>
<ion-item>
<ion-label>Available</ion-label>
<h6 slot="end">{{ metrics.disk.available }} GB</h6>
</ion-item>
</ng-container>
</ion-item-group>
<ng-template #loading>
<skeleton-list [groups]="3"></skeleton-list>
</ng-template>
</ion-content>

View File

@@ -1,3 +1,3 @@
.metric-note {
font-size: 16px;
ion-note {
font-size: medium;
}

View File

@@ -1,8 +1,17 @@
import { Component } from '@angular/core'
import { Metrics } from 'src/app/services/api/api.types'
import { ApiService } from 'src/app/services/api/embassy-api.service'
import { TimeService } from 'src/app/services/time-service'
import { pauseFor, ErrorToastService } from '@start9labs/shared'
import { TimeInfo, TimeService } from 'src/app/services/time-service'
import {
catchError,
combineLatest,
filter,
from,
Observable,
startWith,
switchMap,
} from 'rxjs'
import { ConnectionService } from 'src/app/services/connection.service'
@Component({
selector: 'server-metrics',
@@ -10,63 +19,43 @@ import { pauseFor, ErrorToastService } from '@start9labs/shared'
styleUrls: ['./server-metrics.page.scss'],
})
export class ServerMetricsPage {
loading = true
going = false
metrics: Metrics = {}
websocketFail = false
readonly systemTime$ = this.timeService.systemTime$
readonly systemUptime$ = this.timeService.systemUptime$
readonly serverData$ = this.getServerData$()
constructor(
private readonly errToast: ErrorToastService,
private readonly embassyApi: ApiService,
private readonly timeService: TimeService,
private readonly api: ApiService,
readonly timeService: TimeService,
private readonly connectionService: ConnectionService,
) {}
async ngOnInit() {
await this.getMetrics()
let headersCount = 0
let rowsCount = 0
Object.values(this.metrics).forEach(groupVal => {
headersCount++
Object.keys(groupVal).forEach(_ => {
rowsCount++
})
})
const height = headersCount * 54 + rowsCount * 50 + 24 // extra 24 for room at the bottom
const elem = document.getElementById('metricSection')
if (elem) elem.style.height = `${height}px`
this.startDaemon()
this.loading = false
private getServerData$(): Observable<[TimeInfo, Metrics]> {
return combineLatest([
this.timeService.getTimeInfo$(),
this.getMetrics$(),
]).pipe(
catchError(() => {
this.websocketFail = true
return this.connectionService.connected$.pipe(
filter(Boolean),
switchMap(() => this.getServerData$()),
)
}),
)
}
ngOnDestroy() {
this.stopDaemon()
}
private async startDaemon(): Promise<void> {
this.going = true
while (this.going) {
const startTime = Date.now()
await this.getMetrics()
await pauseFor(4000 - Math.max(Date.now() - startTime, 0))
}
}
private stopDaemon() {
this.going = false
}
private async getMetrics(): Promise<void> {
try {
this.metrics = await this.embassyApi.getServerMetrics({})
} catch (e: any) {
this.errToast.present(e)
this.stopDaemon()
}
}
asIsOrder(a: any, b: any) {
return 0
private getMetrics$(): Observable<Metrics> {
return from(this.api.getServerMetrics({})).pipe(
switchMap(({ metrics, guid }) =>
this.api
.openMetricsWebsocket$({
url: `/rpc/${guid}`,
openObserver: {
next: () => (this.websocketFail = false),
},
})
.pipe(startWith(metrics)),
),
)
}
}

View File

@@ -5,7 +5,13 @@ import {
PackageState,
ServerStatusInfo,
} from 'src/app/services/patch-db/data-model'
import { Metric, RR, NotificationLevel, ServerNotifications } from './api.types'
import {
RR,
NotificationLevel,
ServerNotifications,
Metrics,
} from './api.types'
import { BTC_ICON, LND_ICON, PROXY_ICON } from './api-icons'
import {
DependencyMetadata,
@@ -365,112 +371,36 @@ export module Mock {
},
]
export function getServerMetrics() {
export function getMetrics(): Metrics {
return {
Group1: {
Metric1: {
value: Math.random(),
unit: 'mi/b',
},
Metric2: {
value: Math.random(),
unit: '%',
},
Metric3: {
value: 10.1,
unit: '%',
},
general: {
temperature: (Math.random() * 100).toFixed(1),
},
Group2: {
Hmmmm1: {
value: 22.2,
unit: 'mi/b',
},
Hmmmm2: {
value: 50,
unit: '%',
},
Hmmmm3: {
value: 10.1,
unit: '%',
},
memory: {
'percentage-used': '20',
total: (Math.random() * 100).toFixed(2),
available: '18000',
used: '4000',
'swap-total': '1000',
'swap-free': Math.random().toFixed(2),
'swap-used': '0',
},
Group3: {
Hmmmm1: {
value: Math.random(),
unit: 'mi/b',
},
Hmmmm2: {
value: 50,
unit: '%',
},
Hmmmm3: {
value: 10.1,
unit: '%',
},
cpu: {
'user-space': '100',
'kernel-space': '50',
'io-wait': String(Math.random() * 50),
idle: '80',
usage: '30',
},
Group4: {
Hmmmm1: {
value: Math.random(),
unit: 'mi/b',
},
Hmmmm2: {
value: 50,
unit: '%',
},
Hmmmm3: {
value: 10.1,
unit: '%',
},
},
Group5: {
Hmmmm1: {
value: Math.random(),
unit: 'mi/b',
},
Hmmmm2: {
value: 50,
unit: '%',
},
Hmmmm3: {
value: 10.1,
unit: '%',
},
Hmmmm4: {
value: Math.random(),
unit: 'mi/b',
},
Hmmmm5: {
value: 50,
unit: '%',
},
Hmmmm6: {
value: 10.1,
unit: '%',
},
disk: {
size: '1000',
used: '900',
available: '100',
'percentage-used': '90',
},
}
}
export function getAppMetrics() {
const metr: Metric = {
Metric1: {
value: Math.random(),
unit: 'mi/b',
},
Metric2: {
value: Math.random(),
unit: '%',
},
Metric3: {
value: 10.1,
unit: '%',
},
}
return metr
}
export const ServerLogs: Log[] = [
{
timestamp: '2022-07-28T03:52:54.808769Z',

View File

@@ -47,7 +47,10 @@ export module RR {
}
export type GetServerMetricsReq = {} // server.metrics
export type GetServerMetricsRes = Metrics
export type GetServerMetricsRes = {
guid: string
metrics: Metrics
}
export type UpdateServerReq = { 'marketplace-url': string } // server.update
export type UpdateServerRes = 'updating' | 'no-updates'
@@ -233,9 +236,6 @@ export module RR {
export type FollowPackageLogsReq = FollowServerLogsReq & { id: string } // package.logs.follow
export type FollowPackageLogsRes = FollowServerLogsRes
export type GetPackageMetricsReq = { id: string } // package.metrics
export type GetPackageMetricsRes = Metric
export type InstallPackageReq = {
id: string
'version-spec'?: string
@@ -350,18 +350,30 @@ export interface ActionResponse {
}
export interface Metrics {
[key: string]: {
[key: string]: {
value: string | number | null
unit?: string
}
general: {
temperature: string
}
}
export interface Metric {
[key: string]: {
value: string | number | null
unit?: string
memory: {
'percentage-used': string
total: string
available: string
used: string
'swap-total': string
'swap-free': string
'swap-used': string
}
cpu: {
'user-space': string
'kernel-space': string
'io-wait': string
idle: string
usage: string
}
disk: {
size: string
used: string
available: string
'percentage-used': string
}
}

View File

@@ -1,6 +1,6 @@
import { BehaviorSubject, Observable } from 'rxjs'
import { Update } from 'patch-db-client'
import { RR, Encrypted, BackupTargetType } from './api.types'
import { RR, Encrypted, BackupTargetType, Metrics } from './api.types'
import { DataModel } from 'src/app/services/patch-db/data-model'
import { Log } from '@start9labs/shared'
import { WebSocketSubjectConfig } from 'rxjs/webSocket'
@@ -63,6 +63,10 @@ export abstract class ApiService {
config: WebSocketSubjectConfig<Log>,
): Observable<Log>
abstract openMetricsWebsocket$(
config: WebSocketSubjectConfig<Metrics>,
): Observable<Metrics>
abstract getSystemTime(
params: RR.GetSystemTimeReq,
): Promise<RR.GetSystemTimeRes>
@@ -93,10 +97,6 @@ export abstract class ApiService {
params: RR.GetServerMetricsReq,
): Promise<RR.GetServerMetricsRes>
abstract getPkgMetrics(
params: RR.GetPackageMetricsReq,
): Promise<RR.GetPackageMetricsRes>
abstract updateServer(url?: string): Promise<RR.UpdateServerRes>
abstract restartServer(

View File

@@ -10,7 +10,7 @@ import {
RPCOptions,
} from '@start9labs/shared'
import { ApiService } from './embassy-api.service'
import { BackupTargetType, RR } from './api.types'
import { BackupTargetType, Metrics, RR } from './api.types'
import { parsePropertiesPermissive } from 'src/app/util/properties.util'
import { ConfigService } from '../config.service'
import { webSocket, WebSocketSubjectConfig } from 'rxjs/webSocket'
@@ -126,6 +126,12 @@ export class LiveApiService extends ApiService {
return this.openWebsocket(config)
}
openMetricsWebsocket$(
config: WebSocketSubjectConfig<Metrics>,
): Observable<Metrics> {
return this.openWebsocket(config)
}
async getSystemTime(
params: RR.GetSystemTimeReq,
): Promise<RR.GetSystemTimeRes> {
@@ -400,12 +406,6 @@ export class LiveApiService extends ApiService {
return this.rpcRequest({ method: 'package.logs.follow', params })
}
async getPkgMetrics(
params: RR.GetPackageMetricsReq,
): Promise<RR.GetPackageMetricsRes> {
return this.rpcRequest({ method: 'package.metrics', params })
}
async installPackage(
params: RR.InstallPackageReq,
): Promise<RR.InstallPackageRes> {

View File

@@ -16,7 +16,7 @@ import {
PackageMainStatus,
PackageState,
} from 'src/app/services/patch-db/data-model'
import { BackupTargetType, CifsBackupTarget, RR } from './api.types'
import { BackupTargetType, Metrics, RR } from './api.types'
import { parsePropertiesPermissive } from 'src/app/util/properties.util'
import { Mock } from './api.fixures'
import markdown from 'raw-loader!../../../../../shared/assets/markdown/md-sample.md'
@@ -181,6 +181,19 @@ export class MockApiService extends ApiService {
)
}
openMetricsWebsocket$(
config: WebSocketSubjectConfig<Metrics>,
): Observable<Metrics> {
return interval(2000).pipe(
map((_, index) => {
// mock fire open observer
if (index === 0) config.openObserver?.next(new Event(''))
if (index === 4) throw new Error('HAHAHA')
return Mock.getMetrics()
}),
)
}
async getSystemTime(
params: RR.GetSystemTimeReq,
): Promise<RR.GetSystemTimeRes> {
@@ -268,14 +281,10 @@ export class MockApiService extends ApiService {
params: RR.GetServerMetricsReq,
): Promise<RR.GetServerMetricsRes> {
await pauseFor(2000)
return Mock.getServerMetrics()
}
async getPkgMetrics(
params: RR.GetServerMetricsReq,
): Promise<RR.GetPackageMetricsRes> {
await pauseFor(2000)
return Mock.getAppMetrics()
return {
guid: 'iqudh37um-i38u3-34-a51b-jkhd783ein',
metrics: Mock.getMetrics(),
}
}
async updateServer(url?: string): Promise<RR.UpdateServerRes> {

View File

@@ -1,63 +1,78 @@
import { Injectable } from '@angular/core'
import {
map,
shareReplay,
startWith,
switchMap,
take,
tap,
} from 'rxjs/operators'
import { map, startWith, switchMap } from 'rxjs/operators'
import { PatchDB } from 'patch-db-client'
import { DataModel } from './patch-db/data-model'
import { ApiService } from './api/embassy-api.service'
import { combineLatest, from, timer } from 'rxjs'
import { combineLatest, from, Observable, timer } from 'rxjs'
export interface TimeInfo {
systemStartTime: number
systemCurrentTime: number
systemUptime: {
days: number
hours: number
minutes: number
seconds: number
}
}
@Injectable({
providedIn: 'root',
})
export class TimeService {
private readonly startTimeMs$ = this.patch
private readonly systemStartTime$ = this.patch
.watch$('server-info', 'system-start-time')
.pipe(
take(1),
map(startTime => new Date(startTime).valueOf()),
shareReplay(),
)
readonly systemTime$ = from(this.apiService.getSystemTime({})).pipe(
switchMap(utcStr => {
const dateObj = new Date(utcStr)
const msRemaining = (60 - dateObj.getSeconds()) * 1000
dateObj.setSeconds(0)
const current = dateObj.valueOf()
return timer(msRemaining, 60000).pipe(
map(index => {
const incremented = index + 1
const msToAdd = 60000 * incremented
return current + msToAdd
}),
startWith(current),
)
}),
)
readonly systemUptime$ = combineLatest([
this.startTimeMs$,
this.systemTime$,
]).pipe(
map(([startTime, currentTime]) => {
const ms = currentTime - startTime
const days = Math.floor(ms / (24 * 60 * 60 * 1000))
const daysms = ms % (24 * 60 * 60 * 1000)
const hours = Math.floor(daysms / (60 * 60 * 1000))
const hoursms = ms % (60 * 60 * 1000)
const minutes = Math.floor(hoursms / (60 * 1000))
return { days, hours, minutes }
}),
)
.pipe(map(startTime => new Date(startTime).valueOf()))
constructor(
private readonly patch: PatchDB<DataModel>,
private readonly apiService: ApiService,
) {}
getTimeInfo$(): Observable<TimeInfo> {
return combineLatest([
this.systemStartTime$.pipe(),
this.getSystemCurrentTime$(),
]).pipe(
map(([systemStartTime, systemCurrentTime]) => ({
systemStartTime,
systemCurrentTime,
systemUptime: this.getSystemUptime(systemStartTime, systemCurrentTime),
})),
)
}
private getSystemCurrentTime$() {
return from(this.apiService.getSystemTime({})).pipe(
switchMap(utcStr => {
const dateObj = new Date(utcStr)
const current = dateObj.valueOf()
return timer(0, 1000).pipe(
map(index => {
const incremented = index + 1
const msToAdd = 1000 * incremented
return current + msToAdd
}),
startWith(current),
)
}),
)
}
private getSystemUptime(systemStartTime: number, systemCurrentTime: number) {
const ms = systemCurrentTime - systemStartTime
const days = Math.floor(ms / (24 * 60 * 60 * 1000))
const daysms = ms % (24 * 60 * 60 * 1000)
const hours = Math.floor(daysms / (60 * 60 * 1000))
const hoursms = ms % (60 * 60 * 1000)
const minutes = Math.floor(hoursms / (60 * 1000))
const minutesms = ms % (60 * 1000)
const seconds = Math.floor(minutesms / 1000)
return { days, hours, minutes, seconds }
}
}