filter packages for os compatibility before fetching metadata (#117)

* filter packages for os compatibility before fetching metadata

* address PR feedback and add os filtering to get package latest version endpoint

* additional test suite data

* filter empty version sets

* attempt to fix leak

Co-authored-by: Keagan McClelland <keagan.mcclelland@gmail.com>
This commit is contained in:
Lucy C
2022-07-12 15:06:21 -06:00
committed by GitHub
parent 18b951388b
commit e42cd787b4
16 changed files with 742 additions and 91 deletions

View File

@@ -229,8 +229,8 @@ getDependencyVersions pkgDepRecord = do
pure $ entityVal <$> depVers
fetchAllAppVersions :: MonadUnliftIO m => ConnectionPool -> PkgId -> m [VersionRecord]
fetchAllAppVersions appConnPool appId = do
fetchAllPkgVersions :: MonadUnliftIO m => ConnectionPool -> PkgId -> m [VersionRecord]
fetchAllPkgVersions appConnPool appId = do
entityAppVersions <- runSqlPool (P.selectList [VersionRecordPkgId P.==. PkgRecordKey appId] []) appConnPool
pure $ entityVal <$> entityAppVersions

View File

@@ -2,30 +2,50 @@
module Handler.Package.V0.Icon where
import Conduit (awaitForever, (.|))
import Conduit (
awaitForever,
(.|),
)
import Data.String.Interpolate.IsString (
i,
)
import Foundation (Handler)
import Handler.Package.V1.Index (getOsVersionQuery)
import Handler.Util (
fetchCompatiblePkgVersions,
getVersionSpecFromQuery,
orThrow,
versionPriorityFromQueryIsMin,
)
import Lib.Error (S9Error (..))
import Lib.PkgRepository (getBestVersion, getIcon)
import Lib.PkgRepository (
getBestVersion,
getIcon,
)
import Lib.Types.Core (PkgId)
import Network.HTTP.Types (status400)
import Startlude (show, ($))
import Yesod (TypedContent, addHeader, respondSource, sendChunkBS, sendResponseStatus)
import Startlude (
pure,
show,
($),
)
import Yesod (
TypedContent,
addHeader,
respondSource,
sendChunkBS,
sendResponseStatus,
)
getIconsR :: PkgId -> Handler TypedContent
getIconsR pkg = do
osVersion <- getOsVersionQuery
osCompatibleVersions <- fetchCompatiblePkgVersions osVersion pkg
spec <- getVersionSpecFromQuery
preferMin <- versionPriorityFromQueryIsMin
version <-
getBestVersion pkg spec preferMin
(pure $ getBestVersion spec preferMin osCompatibleVersions)
`orThrow` sendResponseStatus status400 (NotFoundE [i|Icon for #{pkg} satisfying #{spec}|])
(ct, len, src) <- getIcon pkg version
addHeader "Content-Length" (show len)

View File

@@ -2,24 +2,53 @@
module Handler.Package.V0.Instructions where
import Conduit (awaitForever, (.|))
import Data.String.Interpolate.IsString (i)
import Conduit (
awaitForever,
(.|),
)
import Data.String.Interpolate.IsString (
i,
)
import Foundation (Handler)
import Handler.Util (getVersionSpecFromQuery, orThrow, versionPriorityFromQueryIsMin)
import Handler.Package.V1.Index (getOsVersionQuery)
import Handler.Util (
fetchCompatiblePkgVersions,
getVersionSpecFromQuery,
orThrow,
versionPriorityFromQueryIsMin,
)
import Lib.Error (S9Error (..))
import Lib.PkgRepository (getBestVersion, getInstructions)
import Lib.PkgRepository (
getBestVersion,
getInstructions,
)
import Lib.Types.Core (PkgId)
import Network.HTTP.Types (status400)
import Startlude (show, ($))
import Yesod (TypedContent, addHeader, respondSource, sendChunkBS, sendResponseStatus, typePlain)
import Startlude (
pure,
show,
($),
)
import Yesod (
TypedContent,
addHeader,
respondSource,
sendChunkBS,
sendResponseStatus,
typePlain,
)
getInstructionsR :: PkgId -> Handler TypedContent
getInstructionsR pkg = do
spec <- getVersionSpecFromQuery
osVersion <- getOsVersionQuery
osCompatibleVersions <- fetchCompatiblePkgVersions osVersion pkg
preferMin <- versionPriorityFromQueryIsMin
version <-
getBestVersion pkg spec preferMin
( pure $
getBestVersion spec preferMin osCompatibleVersions
)
`orThrow` sendResponseStatus status400 (NotFoundE [i|Instructions for #{pkg} satisfying #{spec}|])
(len, src) <- getInstructions pkg version
addHeader "Content-Length" (show len)

View File

@@ -1,19 +1,23 @@
module Handler.Package.V0.Latest where
import Conduit (concatMapC, mapC, runConduit, sinkList, (.|))
import Data.Aeson (ToJSON (..), eitherDecode)
import Data.ByteString.Lazy qualified as LBS
import Data.HashMap.Strict (HashMap)
import Data.HashMap.Strict qualified as HM
import Data.List (lookup)
import Database.Queries (fetchLatestApp)
import Data.List (lookup, sortOn)
import Data.List.NonEmpty.Extra qualified as NE
import Data.Tuple.Extra (second)
import Database.Queries (collateVersions, getPkgDataSource)
import Foundation (Handler)
import Handler.Package.V1.Index (getOsVersionQuery)
import Lib.Error (S9Error (..))
import Lib.Types.Core (PkgId)
import Lib.Types.Emver (Version)
import Model (Key (..), VersionRecord (..))
import Lib.Types.Emver (Version, satisfies)
import Model (VersionRecord (..))
import Network.HTTP.Types (status400)
import Startlude (Either (..), Generic, Maybe (..), Show, catMaybes, encodeUtf8, fst, pure, snd, traverse, ($), (.), (<$>))
import Yesod (Entity (..), ToContent (..), ToTypedContent (..), YesodPersist (runDB), YesodRequest (reqGetParams), getRequest, sendResponseStatus)
import Startlude (Bool (True), Down (Down), Either (..), Generic, Maybe (..), NonEmpty, Show, const, encodeUtf8, filter, flip, headMay, nonEmpty, pure, ($), (.), (<$>), (<&>))
import Yesod (ToContent (..), ToTypedContent (..), YesodPersist (runDB), YesodRequest (reqGetParams), getRequest, sendResponseStatus)
newtype VersionLatestRes = VersionLatestRes (HashMap PkgId (Maybe Version))
@@ -25,24 +29,37 @@ instance ToTypedContent VersionLatestRes where
toTypedContent = toTypedContent . toJSON
-- TODO refactor with conduit
getVersionLatestR :: Handler VersionLatestRes
getVersionLatestR = do
getParameters <- reqGetParams <$> getRequest
osPredicate' <-
getOsVersionQuery <&> \case
Nothing -> const True
Just v -> flip satisfies v
case lookup "ids" getParameters of
Nothing -> sendResponseStatus status400 (InvalidParamsE "get:ids" "<MISSING>")
Just packages -> case eitherDecode $ LBS.fromStrict $ encodeUtf8 packages of
Left _ -> sendResponseStatus status400 (InvalidParamsE "get:ids" packages)
Right p -> do
let packageList = (,Nothing) <$> p
found <- runDB $ traverse fetchLatestApp $ fst <$> packageList
let source = getPkgDataSource p
filteredPackages <-
runDB $
runConduit $
source
-- group conduit pipeline by pkg id
.| collateVersions
-- filter out versions of apps that are incompatible with the OS predicate
.| mapC (second (filter (osPredicate' . versionRecordOsVersion)))
-- prune empty version sets
.| concatMapC (\(pkgId, vs) -> (pkgId,) <$> nonEmpty vs)
-- grab the latest matching version if it exists
.| mapC (\(a, b) -> (a, (Just $ selectLatestVersion b)))
.| sinkList
-- if the requested package does not have available versions, return it as a key with a null value
pure $
VersionLatestRes $
HM.union
( HM.fromList $
( \v ->
(unPkgRecordKey . entityKey $ fst v, Just $ versionRecordNumber $ entityVal $ snd v)
)
<$> catMaybes found
)
$ HM.fromList packageList
HM.union (HM.fromList $ filteredPackages) (HM.fromList packageList)
where
selectLatestVersion :: NonEmpty VersionRecord -> Version
selectLatestVersion vs = NE.head $ (versionRecordNumber <$>) $ NE.sortOn (Down . versionRecordNumber) $ vs

View File

@@ -2,24 +2,51 @@
module Handler.Package.V0.License where
import Conduit (awaitForever, (.|))
import Data.String.Interpolate.IsString (i)
import Conduit (
awaitForever,
(.|),
)
import Data.String.Interpolate.IsString (
i,
)
import Foundation (Handler)
import Handler.Util (getVersionSpecFromQuery, orThrow, versionPriorityFromQueryIsMin)
import Handler.Package.V1.Index (getOsVersionQuery)
import Handler.Util (
fetchCompatiblePkgVersions,
getVersionSpecFromQuery,
orThrow,
versionPriorityFromQueryIsMin,
)
import Lib.Error (S9Error (..))
import Lib.PkgRepository (getBestVersion, getLicense)
import Lib.PkgRepository (
getBestVersion,
getLicense,
)
import Lib.Types.Core (PkgId)
import Network.HTTP.Types (status400)
import Startlude (show, ($))
import Yesod (TypedContent, addHeader, respondSource, sendChunkBS, sendResponseStatus, typePlain)
import Startlude (
pure,
show,
($),
)
import Yesod (
TypedContent,
addHeader,
respondSource,
sendChunkBS,
sendResponseStatus,
typePlain,
)
getLicenseR :: PkgId -> Handler TypedContent
getLicenseR pkg = do
osVersion <- getOsVersionQuery
osCompatibleVersions <- fetchCompatiblePkgVersions osVersion pkg
spec <- getVersionSpecFromQuery
preferMin <- versionPriorityFromQueryIsMin
version <-
getBestVersion pkg spec preferMin
(pure $ getBestVersion spec preferMin osCompatibleVersions)
`orThrow` sendResponseStatus status400 (NotFoundE [i|License for #{pkg} satisfying #{spec}|])
(len, src) <- getLicense pkg version
addHeader "Content-Length" (show len)

View File

@@ -2,26 +2,54 @@
module Handler.Package.V0.Manifest where
import Conduit (awaitForever, (.|))
import Data.String.Interpolate.IsString (i)
import Conduit (
awaitForever,
(.|),
)
import Data.String.Interpolate.IsString (
i,
)
import Foundation (Handler)
import Handler.Util (addPackageHeader, getVersionSpecFromQuery, orThrow, versionPriorityFromQueryIsMin)
import Handler.Package.V1.Index (getOsVersionQuery)
import Handler.Util (
addPackageHeader,
fetchCompatiblePkgVersions,
getVersionSpecFromQuery,
orThrow,
versionPriorityFromQueryIsMin,
)
import Lib.Error (S9Error (..))
import Lib.PkgRepository (getBestVersion, getManifest)
import Lib.PkgRepository (
getBestVersion,
getManifest,
)
import Lib.Types.Core (PkgId)
import Network.HTTP.Types (status404)
import Startlude (show, ($))
import Yesod (TypedContent, addHeader, respondSource, sendChunkBS, sendResponseStatus, typeJson)
import Startlude (
pure,
show,
($),
)
import Yesod (
TypedContent,
addHeader,
respondSource,
sendChunkBS,
sendResponseStatus,
typeJson,
)
getAppManifestR :: PkgId -> Handler TypedContent
getAppManifestR pkg = do
osVersion <- getOsVersionQuery
osCompatibleVersions <- fetchCompatiblePkgVersions osVersion pkg
versionSpec <- getVersionSpecFromQuery
preferMin <- versionPriorityFromQueryIsMin
version <-
getBestVersion pkg versionSpec preferMin
(pure $ getBestVersion versionSpec preferMin osCompatibleVersions)
`orThrow` sendResponseStatus status404 (NotFoundE [i|#{pkg} satisfying #{versionSpec}|])
addPackageHeader pkg version
(len, src) <- getManifest pkg version
addHeader "Content-Length" (show len)
respondSource typeJson $ src .| awaitForever sendChunkBS
respondSource typeJson $ src .| awaitForever sendChunkBS

View File

@@ -2,23 +2,45 @@
module Handler.Package.V0.ReleaseNotes where
import Data.Aeson (ToJSON (..), object, KeyValue((.=)))
import Data.Aeson (
KeyValue ((.=)),
ToJSON (..),
object,
)
import Data.Aeson.Key (fromText)
import Data.HashMap.Strict (HashMap)
import Data.HashMap.Strict qualified as HM
import Database.Queries (fetchAllAppVersions)
import Foundation (Handler, RegistryCtx (..))
import Foundation (Handler)
import Handler.Package.V1.Index (getOsVersionQuery)
import Handler.Util (fetchCompatiblePkgVersions)
import Lib.Types.Core (PkgId)
import Lib.Types.Emver (Version)
import Model (VersionRecord (..))
import Startlude (Down (..), Eq, Show, Text, fst, pure, sortOn, ($), (&&&), (.), (<$>), show)
import Yesod (ToContent (..), ToTypedContent (..), YesodPersist (runDB), getYesod)
import Data.Aeson.Key (fromText)
import Startlude (
Down (..),
Eq,
Show,
Text,
fst,
pure,
show,
sortOn,
($),
(&&&),
(.),
(<$>),
)
import Yesod (
ToContent (..),
ToTypedContent (..),
)
newtype ReleaseNotes = ReleaseNotes {unReleaseNotes :: HashMap Version Text}
deriving (Eq, Show)
instance ToJSON ReleaseNotes where
toJSON ReleaseNotes {..} = object [ version .= value | (key, value) <- HM.toList unReleaseNotes, let version = fromText $ show key]
toJSON ReleaseNotes{..} =
object [version .= value | (key, value) <- HM.toList unReleaseNotes, let version = fromText $ show key]
instance ToContent ReleaseNotes where
toContent = toContent . toJSON
instance ToTypedContent ReleaseNotes where
@@ -27,9 +49,9 @@ instance ToTypedContent ReleaseNotes where
getReleaseNotesR :: PkgId -> Handler ReleaseNotes
getReleaseNotesR pkg = do
appConnPool <- appConnPool <$> getYesod
versionRecords <- runDB $ fetchAllAppVersions appConnPool pkg
pure $ constructReleaseNotesApiRes versionRecords
osVersion <- getOsVersionQuery
osCompatibleVersions <- fetchCompatiblePkgVersions osVersion pkg
pure $ constructReleaseNotesApiRes osCompatibleVersions
where
constructReleaseNotesApiRes :: [VersionRecord] -> ReleaseNotes
constructReleaseNotesApiRes vers = do

View File

@@ -4,30 +4,65 @@
module Handler.Package.V0.S9PK where
import Data.String.Interpolate.IsString (i)
import Data.String.Interpolate.IsString (
i,
)
import Data.Text qualified as T
import Database.Queries (createMetric, fetchAppVersion)
import Database.Queries (
createMetric,
fetchAppVersion,
)
import Foundation (Handler)
import GHC.Show (show)
import Handler.Util (addPackageHeader, getVersionSpecFromQuery, orThrow, versionPriorityFromQueryIsMin)
import Handler.Package.V1.Index (getOsVersionQuery)
import Handler.Util (
addPackageHeader,
fetchCompatiblePkgVersions,
getVersionSpecFromQuery,
orThrow,
versionPriorityFromQueryIsMin,
)
import Lib.Error (S9Error (..))
import Lib.PkgRepository (getBestVersion, getPackage)
import Lib.Types.Core (PkgId (..), S9PK)
import Lib.PkgRepository (
getBestVersion,
getPackage,
)
import Lib.Types.Core (
PkgId (..),
S9PK,
)
import Lib.Types.Emver (Version (..))
import Network.HTTP.Types (status404)
import Startlude (Maybe (..), pure, void, ($), (.), (>>=))
import Startlude (
Maybe (..),
pure,
void,
($),
(.),
(>>=),
)
import System.FilePath (takeBaseName)
import Yesod (Content (..), TypedContent, YesodPersist (runDB), notFound, respond, sendResponseStatus, typeOctet)
import Yesod (
Content (..),
TypedContent,
YesodPersist (runDB),
notFound,
respond,
sendResponseStatus,
typeOctet,
)
import Yesod.Core (logError)
getAppR :: S9PK -> Handler TypedContent
getAppR file = do
let pkg = PkgId . T.pack $ takeBaseName (show file)
osVersion <- getOsVersionQuery
osCompatibleVersions <- fetchCompatiblePkgVersions osVersion pkg
versionSpec <- getVersionSpecFromQuery
preferMin <- versionPriorityFromQueryIsMin
version <-
getBestVersion pkg versionSpec preferMin
(pure $ getBestVersion versionSpec preferMin osCompatibleVersions)
`orThrow` sendResponseStatus status404 (NotFoundE [i|#{pkg} satisfying #{versionSpec}|])
addPackageHeader pkg version
void $ recordMetrics pkg version
@@ -42,8 +77,7 @@ recordMetrics :: PkgId -> Version -> Handler ()
recordMetrics pkg appVersion = do
existingVersion <- runDB $ fetchAppVersion pkg appVersion
case existingVersion of
Nothing ->
do
$logError [i|#{pkg}@#{appVersion} not found in database|]
notFound
Just _ -> runDB $ createMetric pkg appVersion
Nothing -> do
$logError [i|#{pkg}@#{appVersion} not found in database|]
notFound
Just _ -> runDB $ createMetric pkg appVersion

View File

@@ -2,10 +2,18 @@
module Handler.Package.V0.Version where
import Data.Aeson (ToJSON, object, (.=))
import Data.String.Interpolate.IsString (i)
import Data.Aeson (
ToJSON,
object,
(.=),
)
import Data.String.Interpolate.IsString (
i,
)
import Foundation (Handler)
import Handler.Package.V1.Index (getOsVersionQuery)
import Handler.Util (
fetchCompatiblePkgVersions,
getVersionSpecFromQuery,
orThrow,
versionPriorityFromQueryIsMin,
@@ -15,9 +23,24 @@ import Lib.PkgRepository (getBestVersion)
import Lib.Types.Core (PkgId)
import Lib.Types.Emver (Version (..))
import Network.HTTP.Types (status404)
import Startlude (Eq, Maybe, Show, (.), (<$>))
import Yesod (ToContent (..), ToTypedContent, sendResponseStatus)
import Yesod.Core (ToJSON (..), ToTypedContent (..))
import Startlude (
Eq,
Maybe,
Show,
pure,
($),
(.),
(<$>),
)
import Yesod (
ToContent (..),
ToTypedContent,
sendResponseStatus,
)
import Yesod.Core (
ToJSON (..),
ToTypedContent (..),
)
newtype AppVersionRes = AppVersionRes
@@ -38,9 +61,11 @@ instance ToTypedContent (Maybe AppVersionRes) where
getPkgVersionR :: PkgId -> Handler AppVersionRes
getPkgVersionR pkg = do
osVersion <- getOsVersionQuery
osCompatibleVersions <- fetchCompatiblePkgVersions osVersion pkg
spec <- getVersionSpecFromQuery
preferMin <- versionPriorityFromQueryIsMin
AppVersionRes <$> getBestVersion pkg spec preferMin
AppVersionRes <$> (pure $ getBestVersion spec preferMin osCompatibleVersions)
`orThrow` sendResponseStatus
status404
(NotFoundE [i|Version for #{pkg} satisfying #{spec}|])
(NotFoundE [i|Version for #{pkg} satisfying #{spec}|])

View File

@@ -6,19 +6,32 @@ import Control.Monad.Reader.Has (
Has,
MonadReader,
)
import Data.Attoparsec.Text (Parser, parseOnly)
import Data.String.Interpolate.IsString (i)
import Data.Attoparsec.Text (
Parser,
parseOnly,
)
import Data.String.Interpolate.IsString (
i,
)
import Data.Text qualified as T
import Data.Text.Lazy qualified as TL
import Data.Text.Lazy.Builder qualified as TB
import Database.Queries (fetchAllPkgVersions)
import Foundation
import Lib.PkgRepository (PkgRepo, getHash)
import Lib.PkgRepository (
PkgRepo,
getHash,
)
import Lib.Types.Core (PkgId)
import Lib.Types.Emver (
Version,
VersionRange,
satisfies,
)
import Model (
UserActivity (..),
VersionRecord (versionRecordOsVersion),
)
import Model (UserActivity (..))
import Network.HTTP.Types (
Status,
status400,
@@ -31,7 +44,10 @@ import Startlude (
Monoid (..),
Semigroup ((<>)),
Text,
const,
decodeUtf8,
filter,
flip,
fromMaybe,
fst,
getCurrentTime,
@@ -52,6 +68,7 @@ import Yesod (
RenderRoute (..),
TypedContent (..),
YesodPersist (runDB),
getYesod,
insertRecord,
liftHandler,
lookupGetParam,
@@ -106,8 +123,7 @@ queryParamAs k p =
lookupGetParam k >>= \case
Nothing -> pure Nothing
Just x -> case parseOnly p x of
Left e ->
sendResponseText status400 [i|Invalid Request! The query parameter '#{k}' failed to parse: #{e}|]
Left e -> sendResponseText status400 [i|Invalid Request! The query parameter '#{k}' failed to parse: #{e}|]
Right a -> pure (Just a)
@@ -118,3 +134,15 @@ tickleMAU = do
Just sid -> do
now <- liftIO getCurrentTime
void $ liftHandler $ runDB $ insertRecord $ UserActivity now sid
fetchCompatiblePkgVersions :: Maybe VersionRange -> PkgId -> Handler [VersionRecord]
fetchCompatiblePkgVersions osVersion pkg = do
appConnPool <- appConnPool <$> getYesod
versionRecords <- fetchAllPkgVersions appConnPool pkg
pure $ filter (osPredicate osVersion . versionRecordOsVersion) versionRecords
where
osPredicate osV = do
case osV of
Nothing -> const True
Just v -> flip satisfies v

View File

@@ -68,22 +68,24 @@ import Database.Persist.Sql (
import Database.PostgreSQL.Simple (SqlError (sqlState))
import Lib.Error (S9Error (NotFoundE))
import Lib.External.AppMgr qualified as AppMgr
import Lib.Types.Core (
PkgId (..),
)
import Lib.Types.Core (PkgId (..))
import Lib.Types.Emver (
Version,
VersionRange,
parseVersion,
satisfies,
)
import Lib.Types.Manifest (PackageDependency (..), PackageManifest (..))
import Lib.Types.Manifest (
PackageDependency (..),
PackageManifest (..),
)
import Model (
EntityField (EosHashHash, PkgRecordUpdatedAt),
EosHash (EosHash),
Key (PkgRecordKey),
PkgDependency (PkgDependency),
PkgRecord (PkgRecord),
VersionRecord (versionRecordNumber),
)
import Startlude (
Bool (..),
@@ -208,17 +210,16 @@ getVersionsFor pkg = do
else pure []
getViableVersions :: (MonadIO m, MonadReader r m, Has PkgRepo r, MonadLogger m) => PkgId -> VersionRange -> m [Version]
getViableVersions pkg spec = filter (`satisfies` spec) <$> getVersionsFor pkg
getViableVersions :: VersionRange -> [VersionRecord] -> [Version]
getViableVersions spec vrs = filter (`satisfies` spec) (versionRecordNumber <$> vrs)
getBestVersion ::
(MonadIO m, MonadReader r m, Has PkgRepo r, MonadLogger m) =>
PkgId ->
VersionRange ->
Bool ->
m (Maybe Version)
getBestVersion pkg spec preferMin = headMay . sortBy comparator <$> getViableVersions pkg spec
[VersionRecord] ->
(Maybe Version)
getBestVersion spec preferMin vrs = headMay $ sortBy comparator $ getViableVersions spec vrs
where
comparator = if preferMin then compare else compare `on` Down