pre-download static files, and add api versioning

backend/src/db/model.rs

@@ -168,13 +168,6 @@ impl StaticFiles {
             icon: format!("/public/package-data/{}/{}/icon.{}", id, version, icon_type),
         }
     }
-    pub fn remote(id: &PackageId, version: &Version) -> Self {
-        StaticFiles {
-            license: format!("/marketplace/package/license/{}?spec=={}", id, version),
-            instructions: format!("/marketplace/package/instructions/{}?spec=={}", id, version),
-            icon: format!("/marketplace/package/icon/{}?spec=={}", id, version),
-        }
-    }
 }
 
 #[derive(Debug, Deserialize, Serialize, HasModel)]

backend/src/install/mod.rs

@@ -42,7 +42,7 @@ use crate::notifications::NotificationLevel;
 use crate::s9pk::manifest::{Manifest, PackageId};
 use crate::s9pk::reader::S9pkReader;
 use crate::status::{MainStatus, Status};
-use crate::util::io::copy_and_shutdown;
+use crate::util::io::{copy_and_shutdown, response_to_reader};
 use crate::util::serde::{display_serializable, IoFormat};
 use crate::util::{display_none, AsyncFileExt, Version};
 use crate::version::{Current, VersionT};
@@ -101,7 +101,7 @@ pub async fn install(
         marketplace_url.unwrap_or_else(|| crate::DEFAULT_MARKETPLACE.parse().unwrap());
     let (man_res, s9pk) = tokio::try_join!(
         reqwest::get(format!(
-            "{}/package/manifest/{}?spec={}&eos-version-compat={}&arch={}",
+            "{}/package/v0/manifest/{}?spec={}&eos-version-compat={}&arch={}",
             marketplace_url,
             id,
             version,
@@ -109,7 +109,7 @@ pub async fn install(
             platforms::TARGET_ARCH,
         )),
         reqwest::get(format!(
-            "{}/package/{}.s9pk?spec={}&eos-version-compat={}&arch={}",
+            "{}/package/v0/{}.s9pk?spec={}&eos-version-compat={}&arch={}",
             marketplace_url,
             id,
             version,
@@ -135,8 +135,82 @@ pub async fn install(
         ));
     }
 
+    let public_dir_path = ctx
+        .datadir
+        .join(PKG_PUBLIC_DIR)
+        .join(&man.id)
+        .join(man.version.as_str());
+    tokio::fs::create_dir_all(&public_dir_path).await?;
+
+    let icon_type = man.assets.icon_type();
+    let (license_res, instructions_res, icon_res) = tokio::join!(
+        async {
+            tokio::io::copy(
+                &mut response_to_reader(
+                    reqwest::get(format!(
+                        "{}/package/v0/license/{}?spec={}&eos-version-compat={}&arch={}",
+                        marketplace_url,
+                        id,
+                        version,
+                        Current::new().compat(),
+                        platforms::TARGET_ARCH,
+                    ))
+                    .await?,
+                ),
+                &mut File::create(public_dir_path.join("LICENSE.md")).await?,
+            )
+            .await?;
+            Ok::<_, color_eyre::eyre::Report>(())
+        },
+        async {
+            tokio::io::copy(
+                &mut response_to_reader(
+                    reqwest::get(format!(
+                        "{}/package/v0/instructions/{}?spec={}&eos-version-compat={}&arch={}",
+                        marketplace_url,
+                        id,
+                        version,
+                        Current::new().compat(),
+                        platforms::TARGET_ARCH,
+                    ))
+                    .await?,
+                ),
+                &mut File::create(public_dir_path.join("INSTRUCTIONS.md")).await?,
+            )
+            .await?;
+            Ok::<_, color_eyre::eyre::Report>(())
+        },
+        async {
+            tokio::io::copy(
+                &mut response_to_reader(
+                    reqwest::get(format!(
+                        "{}/package/v0/icon/{}?spec={}&eos-version-compat={}&arch={}",
+                        marketplace_url,
+                        id,
+                        version,
+                        Current::new().compat(),
+                        platforms::TARGET_ARCH,
+                    ))
+                    .await?,
+                ),
+                &mut File::create(public_dir_path.join(format!("icon.{}", icon_type))).await?,
+            )
+            .await?;
+            Ok::<_, color_eyre::eyre::Report>(())
+        },
+    );
+    if let Err(e) = license_res {
+        tracing::warn!("Failed to pre-download license: {}", e);
+    }
+    if let Err(e) = instructions_res {
+        tracing::warn!("Failed to pre-download instructions: {}", e);
+    }
+    if let Err(e) = icon_res {
+        tracing::warn!("Failed to pre-download icon: {}", e);
+    }
+
     let progress = InstallProgress::new(s9pk.content_length());
-    let static_files = StaticFiles::remote(&man.id, &man.version);
+    let static_files = StaticFiles::local(&man.id, &man.version, icon_type);
     let mut db_handle = ctx.db.handle();
     let mut tx = db_handle.begin().await?;
     let mut pde = crate::db::DatabaseModel::new()
@@ -182,18 +256,7 @@ pub async fn install(
             &man,
             Some(marketplace_url),
             InstallProgress::new(s9pk.content_length()),
-            tokio_util::io::StreamReader::new(s9pk.bytes_stream().map_err(|e| {
-                std::io::Error::new(
-                    if e.is_connect() {
-                        std::io::ErrorKind::ConnectionRefused
-                    } else if e.is_timeout() {
-                        std::io::ErrorKind::TimedOut
-                    } else {
-                        std::io::ErrorKind::Other
-                    },
-                    e,
-                )
-            })),
+            response_to_reader(s9pk),
         )
         .await
         {
@@ -626,7 +689,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
             Some(local_man)
         } else if let Some(marketplace_url) = &marketplace_url {
             match reqwest::get(format!(
-                "{}/package/manifest/{}?spec={}&eos-version-compat={}&arch={}",
+                "{}/package/v0/manifest/{}?spec={}&eos-version-compat={}&arch={}",
                 marketplace_url,
                 dep,
                 info.version,
@@ -661,7 +724,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
                 if tokio::fs::metadata(&icon_path).await.is_err() {
                     tokio::fs::create_dir_all(&dir).await?;
                     let icon = reqwest::get(format!(
-                        "{}/package/icon/{}?spec={}&eos-version-compat={}&arch={}",
+                        "{}/package/v0/icon/{}?spec={}&eos-version-compat={}&arch={}",
                         marketplace_url,
                         dep,
                         info.version,
@@ -671,22 +734,7 @@ pub async fn install_s9pk<R: AsyncRead + AsyncSeek + Unpin>(
                     .await
                     .with_kind(crate::ErrorKind::Registry)?;
                     let mut dst = File::create(&icon_path).await?;
-                    tokio::io::copy(
-                        &mut tokio_util::io::StreamReader::new(icon.bytes_stream().map_err(|e| {
-                            std::io::Error::new(
-                                if e.is_connect() {
-                                    std::io::ErrorKind::ConnectionRefused
-                                } else if e.is_timeout() {
-                                    std::io::ErrorKind::TimedOut
-                                } else {
-                                    std::io::ErrorKind::Other
-                                },
-                                e,
-                            )
-                        })),
-                        &mut dst,
-                    )
-                    .await?;
+                    tokio::io::copy(&mut response_to_reader(icon), &mut dst).await?;
                     dst.sync_all().await?;
                 }
             }

backend/src/update/mod.rs

@@ -134,7 +134,7 @@ async fn maybe_do_update(
 ) -> Result<Option<Arc<Revision>>, Error> {
     let mut db = ctx.db.handle();
     let latest_version = reqwest::get(format!(
-        "{}/eos/latest?eos-version={}&arch={}",
+        "{}/eos/v0/latest?eos-version={}&arch={}",
         marketplace_url,
         Current::new().semver(),
         platforms::TARGET_ARCH,
@@ -295,7 +295,7 @@ impl std::fmt::Display for EosUrl {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
-            "{}/eos/eos.img?spec=={}&eos-version={}&arch={}",
+            "{}/eos/v0/eos.img?spec=={}&eos-version={}&arch={}",
             self.base,
             self.version,
             Current::new().semver(),

backend/src/util/io.rs

@@ -218,3 +218,18 @@ pub fn dir_size<'a, P: AsRef<Path> + 'a + Send + Sync>(
     }
     .boxed()
 }
+
+pub fn response_to_reader(response: reqwest::Response) -> impl AsyncRead + Unpin {
+    tokio_util::io::StreamReader::new(response.bytes_stream().map_err(|e| {
+        std::io::Error::new(
+            if e.is_connect() {
+                std::io::ErrorKind::ConnectionRefused
+            } else if e.is_timeout() {
+                std::io::ErrorKind::TimedOut
+            } else {
+                std::io::ErrorKind::Other
+            },
+            e,
+        )
+    }))
+}

backend/src/util/logger.rs

@@ -122,7 +122,7 @@ impl EmbassyLogger {
             let log_epoch = Arc::new(AtomicU64::new(rand::random()));
             let sharing = Arc::new(AtomicBool::new(share_errors));
             let share_dest = match share_dest {
-                None => "http://registry.privacy34kn4ez3y3nijweec6w4g54i3g54sdv7r5mr6soma3w4begyd.onion/support/error-logs".to_owned(),
+                None => "http://registry.privacy34kn4ez3y3nijweec6w4g54i3g54sdv7r5mr6soma3w4begyd.onion/support/v0/error-logs".to_owned(),
                 Some(a) => a.to_string(),
             };
             let tor_proxy = Client::builder()