pre-download static files, and add api versioning

This commit is contained in:
Aiden McClelland
2022-02-07 23:44:22 -07:00
committed by Aiden McClelland
parent 01d766fce9
commit d3c5648608
8 changed files with 171 additions and 115 deletions

View File

@@ -168,13 +168,6 @@ impl StaticFiles {
icon: format!("/public/package-data/{}/{}/icon.{}", id, version, icon_type),
}
}
pub fn remote(id: &PackageId, version: &Version) -> Self {
StaticFiles {
license: format!("/marketplace/package/license/{}?spec=={}", id, version),
instructions: format!("/marketplace/package/instructions/{}?spec=={}", id, version),
icon: format!("/marketplace/package/icon/{}?spec=={}", id, version),
}
}
}
#[derive(Debug, Deserialize, Serialize, HasModel)]

View File

@@ -42,7 +42,7 @@ use crate::notifications::NotificationLevel;
use crate::s9pk::manifest::{Manifest, PackageId};
use crate::s9pk::reader::S9pkReader;
use crate::status::{MainStatus, Status};
use crate::util::io::copy_and_shutdown;
use crate::util::io::{copy_and_shutdown, response_to_reader};
use crate::util::serde::{display_serializable, IoFormat};
use crate::util::{display_none, AsyncFileExt, Version};
use crate::version::{Current, VersionT};
@@ -101,7 +101,7 @@ pub async fn install(
marketplace_url.unwrap_or_else(|| crate::DEFAULT_MARKETPLACE.parse().unwrap());
let (man_res, s9pk) = tokio::try_join!(
reqwest::get(format!(
"{}/package/manifest/{}?spec={}&eos-version-compat={}&arch={}",
"{}/package/v0/manifest/{}?spec={}&eos-version-compat={}&arch={}",
marketplace_url,
id,
version,
@@ -109,7 +109,7 @@ pub async fn install(
platforms::TARGET_ARCH,
)),
reqwest::get(format!(
"{}/package/{}.s9pk?spec={}&eos-version-compat={}&arch={}",
"{}/package/v0/{}.s9pk?spec={}&eos-version-compat={}&arch={}",
marketplace_url,
id,
version,
@@ -135,8 +135,82 @@ pub async fn install(
));
}
let public_dir_path = ctx
.datadir
.join(PKG_PUBLIC_DIR)
.join(&man.id)
.join(man.version.as_str());
tokio::fs::create_dir_all(&public_dir_path).await?;
let icon_type = man.assets.icon_type();
let (license_res, instructions_res, icon_res) = tokio::join!(
async {
tokio::io::copy(
&mut response_to_reader(
reqwest::get(format!(
"{}/package/v0/license/{}?spec={}&eos-version-compat={}&arch={}",
marketplace_url,
id,
version,
Current::new().compat(),
platforms::TARGET_ARCH,
))
.await?,
),
&mut File::create(public_dir_path.join("LICENSE.md")).await?,
)
.await?;
Ok::<_, color_eyre::eyre::Report>(())
},
async {
tokio::io::copy(
&mut response_to_reader(
reqwest::get(format!(
"{}/package/v0/instructions/{}?spec={}&eos-version-compat={}&arch={}",
marketplace_url,
id,
version,
Current::new().compat(),
platforms::TARGET_ARCH,
))
.await?,
),
&mut File::create(public_dir_path.join("INSTRUCTIONS.md")).await?,
)
.await?;
Ok::<_, color_eyre::eyre::Report>(())
},
async {
tokio::io::copy(
&mut response_to_reader(
reqwest::get(format!(
"{}/package/v0/icon/{}?spec={}&eos-version-compat={}&arch={}",
marketplace_url,
id,
version,
Current::new().compat(),
platforms::TARGET_ARCH,
))
.await?,
),
&mut File::create(public_dir_path.join(format!("icon.{}", icon_type))).await?,
)
.await?;
Ok::<_, color_eyre::eyre::Report>(())
},
);
if let Err(e) = license_res {
tracing::warn!("Failed to pre-download license: {}", e);
}
if let Err(e) = instructions_res {
tracing::warn!("Failed to pre-download instructions: {}", e);
}
if let Err(e) = icon_res {
tracing::warn!("Failed to pre-download icon: {}", e);
}
let progress = InstallProgress::new(s9pk.content_length());
let static_files = StaticFiles::remote(&man.id, &man.version);
let static_files = StaticFiles::local(&man.id, &man.version, icon_type);
let mut db_handle = ctx.db.handle();
let mut tx = db_handle.begin().await?;
let mut pde = crate::db::DatabaseModel::new()
@@ -182,18 +256,7 @@ pub async fn install(
&man,
Some(marketplace_url),
InstallProgress::new(s9pk.content_length()),
tokio_util::io::StreamReader::new(s9pk.bytes_stream().map_err(|e| {
std::io::Error::new(
if e.is_connect() {
std::io::ErrorKind::ConnectionRefused
} else if e.is_timeout() {
std::io::ErrorKind::TimedOut
} else {
std::io::ErrorKind::Other
},
e,
)
})),
response_to_reader(s9pk),
)
.await
{
@@ -626,7 +689,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
Some(local_man)
} else if let Some(marketplace_url) = &marketplace_url {
match reqwest::get(format!(
"{}/package/manifest/{}?spec={}&eos-version-compat={}&arch={}",
"{}/package/v0/manifest/{}?spec={}&eos-version-compat={}&arch={}",
marketplace_url,
dep,
info.version,
@@ -661,7 +724,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
if tokio::fs::metadata(&icon_path).await.is_err() {
tokio::fs::create_dir_all(&dir).await?;
let icon = reqwest::get(format!(
"{}/package/icon/{}?spec={}&eos-version-compat={}&arch={}",
"{}/package/v0/icon/{}?spec={}&eos-version-compat={}&arch={}",
marketplace_url,
dep,
info.version,
@@ -671,22 +734,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
.await
.with_kind(crate::ErrorKind::Registry)?;
let mut dst = File::create(&icon_path).await?;
tokio::io::copy(
&mut tokio_util::io::StreamReader::new(icon.bytes_stream().map_err(|e| {
std::io::Error::new(
if e.is_connect() {
std::io::ErrorKind::ConnectionRefused
} else if e.is_timeout() {
std::io::ErrorKind::TimedOut
} else {
std::io::ErrorKind::Other
},
e,
)
})),
&mut dst,
)
.await?;
tokio::io::copy(&mut response_to_reader(icon), &mut dst).await?;
dst.sync_all().await?;
}
}

View File

@@ -134,7 +134,7 @@ async fn maybe_do_update(
) -> Result<Option<Arc<Revision>>, Error> {
let mut db = ctx.db.handle();
let latest_version = reqwest::get(format!(
"{}/eos/latest?eos-version={}&arch={}",
"{}/eos/v0/latest?eos-version={}&arch={}",
marketplace_url,
Current::new().semver(),
platforms::TARGET_ARCH,
@@ -295,7 +295,7 @@ impl std::fmt::Display for EosUrl {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}/eos/eos.img?spec=={}&eos-version={}&arch={}",
"{}/eos/v0/eos.img?spec=={}&eos-version={}&arch={}",
self.base,
self.version,
Current::new().semver(),

View File

@@ -218,3 +218,18 @@ pub fn dir_size<'a, P: AsRef<Path> + 'a + Send + Sync>(
}
.boxed()
}
pub fn response_to_reader(response: reqwest::Response) -> impl AsyncRead + Unpin {
tokio_util::io::StreamReader::new(response.bytes_stream().map_err(|e| {
std::io::Error::new(
if e.is_connect() {
std::io::ErrorKind::ConnectionRefused
} else if e.is_timeout() {
std::io::ErrorKind::TimedOut
} else {
std::io::ErrorKind::Other
},
e,
)
}))
}

View File

@@ -122,7 +122,7 @@ impl EmbassyLogger {
let log_epoch = Arc::new(AtomicU64::new(rand::random()));
let sharing = Arc::new(AtomicBool::new(share_errors));
let share_dest = match share_dest {
None => "http://registry.privacy34kn4ez3y3nijweec6w4g54i3g54sdv7r5mr6soma3w4begyd.onion/support/error-logs".to_owned(),
None => "http://registry.privacy34kn4ez3y3nijweec6w4g54i3g54sdv7r5mr6soma3w4begyd.onion/support/v0/error-logs".to_owned(),
Some(a) => a.to_string(),
};
let tor_proxy = Client::builder()

View File

@@ -23,14 +23,14 @@ export class MarketplaceService {
} = {}
marketplaceUrl: string
constructor(
constructor (
private readonly api: ApiService,
private readonly emver: Emver,
private readonly patch: PatchDbService,
private readonly config: ConfigService,
) {}
) { }
async init() {
async init () {
this.patch.watch$('ui', 'marketplace').subscribe(marketplace => {
if (!marketplace || !marketplace['selected-id']) {
this.marketplaceUrl = this.config.marketplace.url
@@ -41,7 +41,7 @@ export class MarketplaceService {
})
}
async load(): Promise<void> {
async load (): Promise<void> {
try {
const [data, pkgs] = await Promise.all([
this.getMarketplaceData({}),
@@ -66,7 +66,7 @@ export class MarketplaceService {
}
}
async getUpdates(localPkgs: {
async getUpdates (localPkgs: {
[id: string]: PackageDataEntry
}): Promise<MarketplacePkg[]> {
const id = this.patch.getData().ui.marketplace?.['selected-id']
@@ -96,7 +96,7 @@ export class MarketplaceService {
})
}
async getPkg(id: string, version = '*'): Promise<MarketplacePkg> {
async getPkg (id: string, version = '*'): Promise<MarketplacePkg> {
const pkgs = await this.getMarketplacePkgs({
ids: [{ id, version }],
'eos-version-compat':
@@ -111,11 +111,11 @@ export class MarketplaceService {
}
}
async cacheReleaseNotes(id: string): Promise<void> {
async cacheReleaseNotes (id: string): Promise<void> {
this.releaseNotes[id] = await this.getReleaseNotes({ id })
}
private async getPkgs(
private async getPkgs (
page: number,
perPage: number,
): Promise<MarketplacePkg[]> {
@@ -129,20 +129,20 @@ export class MarketplaceService {
return pkgs
}
async getMarketplaceData(
async getMarketplaceData (
params: RR.GetMarketplaceDataReq,
url?: string,
): Promise<RR.GetMarketplaceDataRes> {
url = url || this.marketplaceUrl
return this.api.marketplaceProxy('/package/data', params, url)
return this.api.marketplaceProxy('/package/v0/data', params, url)
}
async getMarketplacePkgs(
async getMarketplacePkgs (
params: RR.GetMarketplacePackagesReq,
): Promise<RR.GetMarketplacePackagesRes> {
if (params.query) params.category = undefined
return this.api.marketplaceProxy(
'/package/index',
'/package/v0/index',
{
...params,
ids: JSON.stringify(params.ids),
@@ -151,11 +151,11 @@ export class MarketplaceService {
)
}
async getReleaseNotes(
async getReleaseNotes (
params: RR.GetReleaseNotesReq,
): Promise<RR.GetReleaseNotesRes> {
return this.api.marketplaceProxy(
'/package/release-notes',
'/package/v0/release-notes',
params,
this.marketplaceUrl,
)

View File

@@ -9,15 +9,15 @@ import { ConfigService } from '../config.service'
export class LiveApiService extends ApiService {
private marketplaceUrl: string
constructor(
constructor (
private readonly http: HttpService,
private readonly config: ConfigService,
) {
super()
;(window as any).rpcClient = this
; (window as any).rpcClient = this
}
async getStatic(url: string): Promise<string> {
async getStatic (url: string): Promise<string> {
return this.http.httpRequest({
method: Method.GET,
url,
@@ -27,75 +27,75 @@ export class LiveApiService extends ApiService {
// db
async getRevisions(since: number): Promise<RR.GetRevisionsRes> {
async getRevisions (since: number): Promise<RR.GetRevisionsRes> {
return this.http.rpcRequest({ method: 'db.revisions', params: { since } })
}
async getDump(): Promise<RR.GetDumpRes> {
async getDump (): Promise<RR.GetDumpRes> {
return this.http.rpcRequest({ method: 'db.dump' })
}
async setDbValueRaw(params: RR.SetDBValueReq): Promise<RR.SetDBValueRes> {
async setDbValueRaw (params: RR.SetDBValueReq): Promise<RR.SetDBValueRes> {
return this.http.rpcRequest({ method: 'db.put.ui', params })
}
// auth
async login(params: RR.LoginReq): Promise<RR.loginRes> {
async login (params: RR.LoginReq): Promise<RR.loginRes> {
return this.http.rpcRequest({ method: 'auth.login', params })
}
async logout(params: RR.LogoutReq): Promise<RR.LogoutRes> {
async logout (params: RR.LogoutReq): Promise<RR.LogoutRes> {
return this.http.rpcRequest({ method: 'auth.logout', params })
}
async getSessions(params: RR.GetSessionsReq): Promise<RR.GetSessionsRes> {
async getSessions (params: RR.GetSessionsReq): Promise<RR.GetSessionsRes> {
return this.http.rpcRequest({ method: 'auth.session.list', params })
}
async killSessions(params: RR.KillSessionsReq): Promise<RR.KillSessionsRes> {
async killSessions (params: RR.KillSessionsReq): Promise<RR.KillSessionsRes> {
return this.http.rpcRequest({ method: 'auth.session.kill', params })
}
// server
async setShareStatsRaw(
async setShareStatsRaw (
params: RR.SetShareStatsReq,
): Promise<RR.SetShareStatsRes> {
return this.http.rpcRequest({ method: 'server.config.share-stats', params })
}
async getServerLogs(
async getServerLogs (
params: RR.GetServerLogsReq,
): Promise<RR.GetServerLogsRes> {
return this.http.rpcRequest({ method: 'server.logs', params })
}
async getServerMetrics(
async getServerMetrics (
params: RR.GetServerMetricsReq,
): Promise<RR.GetServerMetricsRes> {
return this.http.rpcRequest({ method: 'server.metrics', params })
}
async updateServerRaw(
async updateServerRaw (
params: RR.UpdateServerReq,
): Promise<RR.UpdateServerRes> {
return this.http.rpcRequest({ method: 'server.update', params })
}
async restartServer(
async restartServer (
params: RR.RestartServerReq,
): Promise<RR.RestartServerRes> {
return this.http.rpcRequest({ method: 'server.restart', params })
}
async shutdownServer(
async shutdownServer (
params: RR.ShutdownServerReq,
): Promise<RR.ShutdownServerRes> {
return this.http.rpcRequest({ method: 'server.shutdown', params })
}
async systemRebuild(
async systemRebuild (
params: RR.RestartServerReq,
): Promise<RR.RestartServerRes> {
return this.http.rpcRequest({ method: 'server.rebuild', params })
@@ -103,7 +103,7 @@ export class LiveApiService extends ApiService {
// marketplace URLs
async marketplaceProxy<T>(path: string, params: {}, url: string): Promise<T> {
async marketplaceProxy<T> (path: string, params: {}, url: string): Promise<T> {
const fullURL = `${url}${path}?${new URLSearchParams(params).toString()}`
return this.http.rpcRequest({
method: 'marketplace.get',
@@ -111,11 +111,11 @@ export class LiveApiService extends ApiService {
})
}
async getEos(
async getEos (
params: RR.GetMarketplaceEOSReq,
): Promise<RR.GetMarketplaceEOSRes> {
return this.marketplaceProxy(
'/eos/latest',
'/eos/v0/latest',
params,
this.config.marketplace.url,
)
@@ -128,19 +128,19 @@ export class LiveApiService extends ApiService {
// notification
async getNotificationsRaw(
async getNotificationsRaw (
params: RR.GetNotificationsReq,
): Promise<RR.GetNotificationsRes> {
return this.http.rpcRequest({ method: 'notification.list', params })
}
async deleteNotification(
async deleteNotification (
params: RR.DeleteNotificationReq,
): Promise<RR.DeleteNotificationRes> {
return this.http.rpcRequest({ method: 'notification.delete', params })
}
async deleteAllNotifications(
async deleteAllNotifications (
params: RR.DeleteAllNotificationsReq,
): Promise<RR.DeleteAllNotificationsRes> {
return this.http.rpcRequest({
@@ -151,79 +151,79 @@ export class LiveApiService extends ApiService {
// wifi
async getWifi(
async getWifi (
params: RR.GetWifiReq,
timeout?: number,
): Promise<RR.GetWifiRes> {
return this.http.rpcRequest({ method: 'wifi.get', params, timeout })
}
async setWifiCountry(
async setWifiCountry (
params: RR.SetWifiCountryReq,
): Promise<RR.SetWifiCountryRes> {
return this.http.rpcRequest({ method: 'wifi.country.set', params })
}
async addWifi(params: RR.AddWifiReq): Promise<RR.AddWifiRes> {
async addWifi (params: RR.AddWifiReq): Promise<RR.AddWifiRes> {
return this.http.rpcRequest({ method: 'wifi.add', params })
}
async connectWifi(params: RR.ConnectWifiReq): Promise<RR.ConnectWifiRes> {
async connectWifi (params: RR.ConnectWifiReq): Promise<RR.ConnectWifiRes> {
return this.http.rpcRequest({ method: 'wifi.connect', params })
}
async deleteWifi(params: RR.DeleteWifiReq): Promise<RR.DeleteWifiRes> {
async deleteWifi (params: RR.DeleteWifiReq): Promise<RR.DeleteWifiRes> {
return this.http.rpcRequest({ method: 'wifi.delete', params })
}
// ssh
async getSshKeys(params: RR.GetSSHKeysReq): Promise<RR.GetSSHKeysRes> {
async getSshKeys (params: RR.GetSSHKeysReq): Promise<RR.GetSSHKeysRes> {
return this.http.rpcRequest({ method: 'ssh.list', params })
}
async addSshKey(params: RR.AddSSHKeyReq): Promise<RR.AddSSHKeyRes> {
async addSshKey (params: RR.AddSSHKeyReq): Promise<RR.AddSSHKeyRes> {
return this.http.rpcRequest({ method: 'ssh.add', params })
}
async deleteSshKey(params: RR.DeleteSSHKeyReq): Promise<RR.DeleteSSHKeyRes> {
async deleteSshKey (params: RR.DeleteSSHKeyReq): Promise<RR.DeleteSSHKeyRes> {
return this.http.rpcRequest({ method: 'ssh.delete', params })
}
// backup
async getBackupTargets(
async getBackupTargets (
params: RR.GetBackupTargetsReq,
): Promise<RR.GetBackupTargetsRes> {
return this.http.rpcRequest({ method: 'backup.target.list', params })
}
async addBackupTarget(
async addBackupTarget (
params: RR.AddBackupTargetReq,
): Promise<RR.AddBackupTargetRes> {
params.path = params.path.replace('/\\/g', '/')
return this.http.rpcRequest({ method: 'backup.target.cifs.add', params })
}
async updateBackupTarget(
async updateBackupTarget (
params: RR.UpdateBackupTargetReq,
): Promise<RR.UpdateBackupTargetRes> {
return this.http.rpcRequest({ method: 'backup.target.cifs.update', params })
}
async removeBackupTarget(
async removeBackupTarget (
params: RR.RemoveBackupTargetReq,
): Promise<RR.RemoveBackupTargetRes> {
return this.http.rpcRequest({ method: 'backup.target.cifs.remove', params })
}
async getBackupInfo(
async getBackupInfo (
params: RR.GetBackupInfoReq,
): Promise<RR.GetBackupInfoRes> {
return this.http.rpcRequest({ method: 'backup.target.info', params })
}
async createBackupRaw(
async createBackupRaw (
params: RR.CreateBackupReq,
): Promise<RR.CreateBackupRes> {
return this.http.rpcRequest({ method: 'backup.create', params })
@@ -231,7 +231,7 @@ export class LiveApiService extends ApiService {
// package
async getPackageProperties(
async getPackageProperties (
params: RR.GetPackagePropertiesReq,
): Promise<RR.GetPackagePropertiesRes<2>['data']> {
return this.http
@@ -239,95 +239,95 @@ export class LiveApiService extends ApiService {
.then(parsePropertiesPermissive)
}
async getPackageLogs(
async getPackageLogs (
params: RR.GetPackageLogsReq,
): Promise<RR.GetPackageLogsRes> {
return this.http.rpcRequest({ method: 'package.logs', params })
}
async getPkgMetrics(
async getPkgMetrics (
params: RR.GetPackageMetricsReq,
): Promise<RR.GetPackageMetricsRes> {
return this.http.rpcRequest({ method: 'package.metrics', params })
}
async installPackageRaw(
async installPackageRaw (
params: RR.InstallPackageReq,
): Promise<RR.InstallPackageRes> {
return this.http.rpcRequest({ method: 'package.install', params })
}
async dryUpdatePackage(
async dryUpdatePackage (
params: RR.DryUpdatePackageReq,
): Promise<RR.DryUpdatePackageRes> {
return this.http.rpcRequest({ method: 'package.update.dry', params })
}
async getPackageConfig(
async getPackageConfig (
params: RR.GetPackageConfigReq,
): Promise<RR.GetPackageConfigRes> {
return this.http.rpcRequest({ method: 'package.config.get', params })
}
async drySetPackageConfig(
async drySetPackageConfig (
params: RR.DrySetPackageConfigReq,
): Promise<RR.DrySetPackageConfigRes> {
return this.http.rpcRequest({ method: 'package.config.set.dry', params })
}
async setPackageConfigRaw(
async setPackageConfigRaw (
params: RR.SetPackageConfigReq,
): Promise<RR.SetPackageConfigRes> {
return this.http.rpcRequest({ method: 'package.config.set', params })
}
async restorePackagesRaw(
async restorePackagesRaw (
params: RR.RestorePackagesReq,
): Promise<RR.RestorePackagesRes> {
return this.http.rpcRequest({ method: 'package.backup.restore', params })
}
async executePackageAction(
async executePackageAction (
params: RR.ExecutePackageActionReq,
): Promise<RR.ExecutePackageActionRes> {
return this.http.rpcRequest({ method: 'package.action', params })
}
async startPackageRaw(
async startPackageRaw (
params: RR.StartPackageReq,
): Promise<RR.StartPackageRes> {
return this.http.rpcRequest({ method: 'package.start', params })
}
async dryStopPackage(
async dryStopPackage (
params: RR.DryStopPackageReq,
): Promise<RR.DryStopPackageRes> {
return this.http.rpcRequest({ method: 'package.stop.dry', params })
}
async stopPackageRaw(params: RR.StopPackageReq): Promise<RR.StopPackageRes> {
async stopPackageRaw (params: RR.StopPackageReq): Promise<RR.StopPackageRes> {
return this.http.rpcRequest({ method: 'package.stop', params })
}
async dryUninstallPackage(
async dryUninstallPackage (
params: RR.DryUninstallPackageReq,
): Promise<RR.DryUninstallPackageRes> {
return this.http.rpcRequest({ method: 'package.uninstall.dry', params })
}
async deleteRecoveredPackageRaw(
async deleteRecoveredPackageRaw (
params: RR.DeleteRecoveredPackageReq,
): Promise<RR.DeleteRecoveredPackageRes> {
return this.http.rpcRequest({ method: 'package.delete-recovered', params })
}
async uninstallPackageRaw(
async uninstallPackageRaw (
params: RR.UninstallPackageReq,
): Promise<RR.UninstallPackageRes> {
return this.http.rpcRequest({ method: 'package.uninstall', params })
}
async dryConfigureDependency(
async dryConfigureDependency (
params: RR.DryConfigureDependencyReq,
): Promise<RR.DryConfigureDependencyRes> {
return this.http.rpcRequest({

View File

@@ -191,7 +191,7 @@ export class MockApiService extends ApiService {
async marketplaceProxy (path: string, params: {}, url: string): Promise<any> {
await pauseFor(2000)
if (path === '/package/data') {
if (path === '/package/v0/data') {
return {
name: 'Dark9',
categories: [
@@ -204,9 +204,9 @@ export class MockApiService extends ApiService {
'alt coin',
],
}
} else if (path === '/package/index') {
} else if (path === '/package/v0/index') {
return Mock.MarketplacePkgsList
} else if (path === '/package/release-notes') {
} else if (path === '/package/v0/release-notes') {
return Mock.ReleaseNotes
}
}