From 09132db2490e1521dfa1bb26b38be33697de8ce5 Mon Sep 17 00:00:00 2001 From: Lucy Cifferello <12953208+elvece@users.noreply.github.com> Date: Wed, 1 May 2024 01:24:30 -0400 Subject: [PATCH] rework dep metadata with fallback --- src/Database/Queries.hs | 33 ++++++++++++----- src/Handler/Package/V1/Index.hs | 65 ++++++++++++++------------------- 2 files changed, 51 insertions(+), 47 deletions(-) diff --git a/src/Database/Queries.hs b/src/Database/Queries.hs index 685423b..559a79e 100644 --- a/src/Database/Queries.hs +++ b/src/Database/Queries.hs @@ -19,7 +19,7 @@ import Model ( Metric (Metric), PkgDependency (..), PkgRecord (PkgRecord), - VersionRecord (VersionRecord), VersionPlatform (VersionPlatform), EntityField (VersionPlatformPkgId, VersionPlatformVersionNumber, VersionPlatformArch, AdminPkgsPkgId, AdminPkgsAdmin), PkgRecordId, AdminPkgs, AdminId, + VersionRecord (VersionRecord), VersionPlatform (VersionPlatform), EntityField (VersionPlatformPkgId, VersionPlatformVersionNumber, VersionPlatformArch, AdminPkgsPkgId, AdminPkgsAdmin, PkgDependencyDepId), PkgRecordId, AdminPkgs, AdminId, ) import Orphans.Emver () import Startlude ( @@ -73,6 +73,9 @@ import Database.Esqueleto.Experimental ( (==.), (^.), (||.), + isNothing, + (<=.), + limit ) import Database.Persist qualified as P import Database.Persist.Postgresql ( @@ -96,7 +99,7 @@ import Model ( VersionRecordNumber, VersionRecordPkgId, VersionRecordTitle, - VersionRecordUpdatedAt, PkgRecordHidden, VersionPlatformRam, PkgRecordUpdatedAt + VersionRecordUpdatedAt, PkgRecordHidden, VersionPlatformRam, PkgRecordUpdatedAt, VersionRecordCreatedAt ), Key (unPkgRecordKey), PkgCategory, @@ -113,10 +116,8 @@ import Startlude ( snd, sortOn, ($>), - (<$>), Int, + (<$>), Int, listToMaybe, ) -import Database.Esqueleto.Experimental (isNothing) -import Database.Esqueleto.Experimental ((<=.)) serviceQuerySource :: (MonadResource m, MonadIO m) => @@ -182,16 +183,15 @@ getPkgDependencyData :: MonadIO m => PkgId -> Version -> - ReaderT SqlBackend m [(P.Entity PkgDependency, P.Entity PkgRecord)] -getPkgDependencyData pkgId pkgVersion = + ReaderT SqlBackend m [P.Entity PkgRecord] +getPkgDependencyData pkgId pkgVersion = select $ from $ do (pkgDepRecord :& pr) <- from $ table @PkgDependency - `innerJoin` table @PkgRecord `on` (\(v :& p) -> (PkgRecordId === PkgDependencyPkgId) (p :& v)) + `innerJoin` table @PkgRecord `on` (\(pd :& pr) -> (PkgRecordId === PkgDependencyDepId) (pr :& pd)) where_ (pkgDepRecord ^. PkgDependencyPkgId ==. val (PkgRecordKey pkgId)) where_ (pkgDepRecord ^. PkgDependencyPkgVersion ==. val pkgVersion) - pure (pkgDepRecord, pr) - + pure pr (===) :: (PersistEntity val1, PersistEntity val2, P.PersistField typ) => @@ -201,6 +201,19 @@ getPkgDependencyData pkgId pkgVersion = SqlExpr (Value Bool) (===) a' b' (a :& b) = a ^. a' ==. b ^. b' +getLatestVersionRecord :: + MonadIO m => + Key PkgRecord -> + ReaderT SqlBackend m (Maybe VersionRecord) +getLatestVersionRecord pkgId = do + vrs <- select $ do + v <- from $ table @VersionRecord + where_ $ v ^. VersionRecordPkgId ==. val pkgId + orderBy [desc (v ^. VersionRecordCreatedAt)] + limit 1 + pure v + pure $ entityVal <$> listToMaybe vrs + getCategoriesFor :: MonadUnliftIO m => diff --git a/src/Handler/Package/V1/Index.hs b/src/Handler/Package/V1/Index.hs index 7648de5..1f7bda5 100644 --- a/src/Handler/Package/V1/Index.hs +++ b/src/Handler/Package/V1/Index.hs @@ -21,19 +21,18 @@ import Database.Persist.Sql (SqlBackend) import Database.Queries ( collateVersions, getCategoriesFor, - getDependencyVersions, getPkgDataSource, getPkgDependencyData, - serviceQuerySource, + serviceQuerySource, getLatestVersionRecord, ) import Foundation (Handler, Route (InstructionsR, LicenseR), RegistryCtx (appSettings)) import Handler.Package.Api (DependencyRes (..), PackageListRes (..), PackageRes (..)) import Handler.Types.Api (ApiVersion (..)) import Handler.Util (basicRender, parseQueryParam, filterDeprecatedVersions, filterDevices, getPkgArch) import Lib.PkgRepository (PkgRepo, getIcon, getManifest) -import Lib.Types.Core (PkgId) -import Lib.Types.Emver (Version, VersionRange (..), parseRange, satisfies, (<||)) -import Model (Category (..), Key (..), PkgDependency (..), VersionRecord (..), PkgRecord (pkgRecordHidden)) +import Lib.Types.Core (PkgId, PkgId(PkgId)) +import Lib.Types.Emver (Version, Version(Version), VersionRange (..), parseRange, satisfies, (<||)) +import Model (Category (..), Key (..), VersionRecord (..), PkgRecord (..), PkgDependency (..)) import Protolude.Unsafe (unsafeFromJust) import Settings (AppSettings (communityVersion)) import Startlude ( @@ -51,7 +50,6 @@ import Startlude ( Num ((*), (-)), Show, Text, - Traversable (traverse), const, encodeUtf8, filter, @@ -69,7 +67,6 @@ import Startlude ( readMaybe, snd, sortOn, - zipWith, zipWithM, ($), (&&&), @@ -78,8 +75,7 @@ import Startlude ( (<$>), (<&>), (=<<), - (>), - show + (>) ) import UnliftIO (Concurrently (..), mapConcurrently) import Yesod ( @@ -90,7 +86,7 @@ import Yesod ( lookupGetParam, ) import Data.Tuple (fst) -import Database.Persist.Postgresql (entityVal) +import Database.Persist.Postgresql (entityVal, entityKey) import Yesod.Core (getsYesod) import Data.List (head) import Yesod (YesodRequest(reqGetParams)) @@ -99,9 +95,9 @@ import Data.List (last) import Data.Text (isPrefixOf) import Startlude (length) import Control.Monad.Logger (logWarn) -import Data.Bool (not) -import Data.List (null) - +import Data.String.Interpolate.IsString ( + i, + ) data PackageReq = PackageReq { packageReqId :: !PkgId , packageReqVersion :: !VersionRange @@ -139,9 +135,9 @@ getPackageIndexR = do limit' <- fromMaybe 20 <$> getLimitQuery query <- T.strip . fromMaybe "" <$> lookupGetParam "query" let (source, packageRanges) = case pkgIds of - Nothing -> (serviceQuerySource category query pkgArch ram, const Any) + Nothing -> (Database.Queries.serviceQuerySource category query pkgArch ram, const Any) Just packages -> - let s = getPkgDataSource (packageReqId <$> packages) pkgArch ram + let s = Database.Queries.getPkgDataSource (packageReqId <$> packages) pkgArch ram r = fromMaybe None . (flip lookup $ (packageReqId &&& packageReqVersion) <$> packages) in (s, r) filteredPackages <- @@ -149,7 +145,7 @@ getPackageIndexR = do runConduit $ source -- group conduit pipeline by pkg id - .| collateVersions + .| Database.Queries.collateVersions -- filter out versions of apps that are incompatible with the OS predicate .| mapC (second (filter (osPredicate . versionRecordOsVersion . fst))) -- filter hardware device compatability @@ -164,7 +160,7 @@ getPackageIndexR = do -- grab the latest matching version if it exists .| concatMapC (\(a, b) -> (a,b,) <$> (selectLatestVersionFromSpec packageRanges b)) -- construct - .| mapMC (\(a, b, c) -> PackageMetadata a b (versionRecordNumber c) <$> getCategoriesFor a) + .| mapMC (\(a, b, c) -> PackageMetadata a b (versionRecordNumber c) <$> Database.Queries.getCategoriesFor a) -- pages start at 1 for some reason. TODO: make pages start at 0 .| (dropC (limit' * (page - 1)) *> takeC limit') .| sinkList @@ -219,16 +215,20 @@ getPackageDependencies :: ReaderT SqlBackend m (HashMap PkgId DependencyRes) getPackageDependencies PackageMetadata{packageMetadataPkgId = pkg, packageMetadataPkgVersion = pkgVersion} = do - pkgDepInfo' <- getPkgDependencyData pkg pkgVersion - let pkgDepInfo = fmap (\a -> (entityVal $ fst a, entityVal $ snd a)) pkgDepInfo' - pkgDepInfoWithVersions <- traverse getDependencyVersions (fst <$> pkgDepInfo) - let depMetadata = zipWith formatDependencyInfo pkgDepInfo $ filter (not . null) pkgDepInfoWithVersions - lift $ - fmap HM.fromList $ - for depMetadata $ \(depId, title, v, isLocal) -> do - icon <- loadIcon depId v - pure $ (depId, DependencyRes title icon isLocal) - + depPkgRecordEntities <- Database.Queries.getPkgDependencyData pkg pkgVersion + fmap HM.fromList $ + for depPkgRecordEntities $ \(pr) -> do + let depId = unPkgRecordKey $ entityKey pr + let depPkgRecord = entityVal pr + mVersionRecord <- Database.Queries.getLatestVersionRecord $ entityKey pr + case mVersionRecord of + Just VersionRecord{..} -> do + icon <- lift $ loadIcon depId versionRecordNumber + pure $ (depId, DependencyRes versionRecordTitle icon $ pkgRecordHidden depPkgRecord) + Nothing -> do + $logWarn [i|No latest version record found for #{depId} while getting dependency metadata for #{pkg}. Using fallback package.|] + icon <- lift $ loadIcon (PkgId "fallback") $ Version(1,0,0,0) + pure $ (depId, DependencyRes "Unknown" icon $ pkgRecordHidden depPkgRecord) constructPackageListApiRes :: (MonadResource m, MonadReader r m, Has AppSettings r, Has PkgRepo r) => @@ -295,13 +295,4 @@ selectDependencyBestVersion osPredicate pkgDepInfo depVersions = do Just bestVersion -> (pkgId, versionRecordTitle bestVersion, versionRecordNumber bestVersion, isLocal) -- use latest version of dep for metadata info Nothing -> (pkgId, versionRecordTitle latestDepVersion, versionRecordNumber latestDepVersion, isLocal) - else (pkgId, versionRecordTitle latestDepVersion, versionRecordNumber latestDepVersion, isLocal) - -formatDependencyInfo :: (PkgDependency, PkgRecord) -> [VersionRecord] -> (PkgId, Text, Version, Bool) -formatDependencyInfo pkgDepInfo depVersions = do - let pkgDepRecord = fst pkgDepInfo - let isLocal = pkgRecordHidden $ snd pkgDepInfo - let depId = pkgDependencyDepId pkgDepRecord - let pkgId = unPkgRecordKey depId - let latestDepVersion = head $ sortOn (Down . versionRecordNumber) depVersions - (pkgId, versionRecordTitle latestDepVersion, versionRecordNumber latestDepVersion, isLocal) \ No newline at end of file + else (pkgId, versionRecordTitle latestDepVersion, versionRecordNumber latestDepVersion, isLocal) \ No newline at end of file