diff --git a/CHANGELOG.md b/CHANGELOG.md index c78020868..5ee590862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ changes. ### Added +- added `bio` `dRepName` `email` `references` `metadataValid` and `metadataStatus` fields to `drep/list` +- added `metadatavalidationmaxconcurrentrequests` field to the backend config - added `metadata/validate` endpoint [Issue 876](https://github.com/IntersectMBO/govtool/issues/876) - added pagination to `drep/list` [Issue 756](https://github.com/IntersectMBO/govtool/issues/756) - added search query param to the `drep/getVotes` [Issue 640](https://github.com/IntersectMBO/govtool/issues/640) @@ -92,6 +94,7 @@ changes. ### Changed +- `proposal.about` changed to `proposal.abstract` - `drep/info` now returns 4 different tx hashes instead of one latest tx hash [Issue 688](https://github.com/IntersectMBO/govtool/issues/688) - `proposal/list` allows user to search by tx hash [Issue 603](https://github.com/IntersectMBO/govtool/issues/603) - `proposal/list` returns additional data such ass `expiryEpochNo`, `createdEpochNo`, `title`, `about`, `motivation`, diff --git a/govtool/backend/app/Main.hs b/govtool/backend/app/Main.hs index 5ca505210..1385ae1f1 100644 --- a/govtool/backend/app/Main.hs +++ b/govtool/backend/app/Main.hs @@ -8,6 +8,7 @@ module Main where +import Control.Concurrent.QSem (newQSem) import Control.Exception (Exception, SomeException, fromException, throw) @@ -115,7 +116,8 @@ startApp vvaConfig = do dRepVotingPowerCache <- newCache dRepListCache <- newCache networkMetricsCache <- newCache - metadataValidationCache <- newCache + proposalMetadataValidationCache <- newCache + dRepMetadataValidationCache <- newCache return $ CacheEnv { proposalListCache , getProposalCache @@ -127,12 +129,13 @@ startApp vvaConfig = do , dRepVotingPowerCache , dRepListCache , networkMetricsCache - , metadataValidationCache + , proposalMetadataValidationCache + , dRepMetadataValidationCache } connectionPool <- createPool (connectPostgreSQL (encodeUtf8 (dbSyncConnectionString $ getter vvaConfig))) close 1 1 60 vvaTlsManager <- newManager tlsManagerSettings - - let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager} + qsem <- newQSem (metadataValidationMaxConcurrentRequests vvaConfig) + let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager, vvaMetadataQSem=qsem} server' <- mkVVAServer appEnv runSettings settings server' diff --git a/govtool/backend/example-config.json b/govtool/backend/example-config.json index 756e75282..417afb3c9 100644 --- a/govtool/backend/example-config.json +++ b/govtool/backend/example-config.json @@ -11,5 +11,6 @@ "cachedurationseconds": 20, "sentrydsn": "https://username:password@senty.host/id", "metadatavalidationhost": "localhost", - "metadatavalidationport": 3001 + "metadatavalidationport": 3001, + "metadatavalidationmaxconcurrentrequests": 10 } diff --git a/govtool/backend/src/VVA/API.hs b/govtool/backend/src/VVA/API.hs index 6067000cc..f5e29cc63 100644 --- a/govtool/backend/src/VVA/API.hs +++ b/govtool/backend/src/VVA/API.hs @@ -8,8 +8,10 @@ module VVA.API where -import Control.Exception (throw) -import Control.Monad.Except (throwError) +import Control.Concurrent.QSem (waitQSem, signalQSem) +import Control.Concurrent.Async (mapConcurrently) +import Control.Exception (throw, throwIO) +import Control.Monad.Except (throwError, runExceptT) import Control.Monad.Reader import Data.Aeson (Result(Error, Success), fromJSON) import Data.Bool (Bool) @@ -74,8 +76,8 @@ type VVAApi = :<|> "transaction" :> "status" :> Capture "transactionId" HexText :> Get '[JSON] GetTransactionStatusResponse :<|> "throw500" :> Get '[JSON] () :<|> "network" :> "metrics" :> Get '[JSON] GetNetworkMetricsResponse - :<|> "metadata" :> "validate" :> ReqBody '[JSON] MetadataValidationParams :> Post '[JSON] MetadataValidationResponse - + :<|> "proposal" :> "metadata" :> "validate" :> ReqBody '[JSON] MetadataValidationParams :> Post '[JSON] MetadataValidationResponse + :<|> "drep" :> "metadata" :> "validate" :> ReqBody '[JSON] MetadataValidationParams :> Post '[JSON] MetadataValidationResponse server :: App m => ServerT VVAApi m server = drepList :<|> getVotingPower @@ -89,7 +91,8 @@ server = drepList :<|> getTransactionStatus :<|> throw500 :<|> getNetworkMetrics - :<|> validateMetadata + :<|> getProposalMetadataValidationResponse + :<|> getDRepMetadataValidationResponse mapDRepType :: Types.DRepType -> DRepType @@ -101,9 +104,10 @@ mapDRepStatus Types.Retired = Retired mapDRepStatus Types.Active = Active mapDRepStatus Types.Inactive = Inactive -drepRegistrationToDrep :: Types.DRepRegistration -> DRep -drepRegistrationToDrep Types.DRepRegistration {..} = - DRep +drepRegistrationToDrep :: Types.DRepRegistration -> Types.MetadataValidationResult Types.DRepMetadata -> DRep +drepRegistrationToDrep Types.DRepRegistration {..} Types.MetadataValidationResult{..} = + let metadata = metadataValidationResultMetadata + in DRep { dRepDrepId = DRepHash dRepRegistrationDRepHash, dRepView = dRepRegistrationView, dRepUrl = dRepRegistrationUrl, @@ -113,7 +117,13 @@ drepRegistrationToDrep Types.DRepRegistration {..} = dRepStatus = mapDRepStatus dRepRegistrationStatus, dRepType = mapDRepType dRepRegistrationType, dRepLatestTxHash = HexText <$> dRepRegistrationLatestTxHash, - dRepLatestRegistrationDate = dRepRegistrationLatestRegistrationDate + dRepLatestRegistrationDate = dRepRegistrationLatestRegistrationDate, + dRepBio = Types.dRepMetadataBio <$> metadata, + dRepDRepName = Types.dRepMetadataDRepName <$> metadata, + dRepEmail = Types.dRepMetadataEmail <$> metadata, + dRepReferences = maybe [] Types.dRepMetadataReferences metadata, + dRepMetadataStatus = metadataValidationResultStatus, + dRepMetadataValid = metadataValidationResultValid } delegationToResponse :: Types.Delegation -> DelegationResponse @@ -152,8 +162,22 @@ drepList mSearchQuery statuses mSortMode mPage mPageSize = do Just Status -> sortOn $ \Types.DRepRegistration {..} -> dRepRegistrationStatus + appEnv <- ask + qsem <- asks vvaMetadataQSem - let allValidDReps = map drepRegistrationToDrep $ sortDReps $ filterDRepsByQuery $ filterDRepsByStatus dreps + allValidDReps <- liftIO $ mapConcurrently + (\d@Types.DRepRegistration{..} -> + drepRegistrationToDrep d + <$> do + waitQSem qsem + r <- (either throwIO return =<< (runExceptT + $ flip runReaderT appEnv (validateDRepMetadata + (MetadataValidationParams + (fromMaybe "" dRepRegistrationUrl) + $ HexText (fromMaybe "" dRepRegistrationDataHash))))) + signalQSem qsem + return r) + $ sortDReps $ filterDRepsByQuery $ filterDRepsByStatus dreps let page = (fromIntegral $ fromMaybe 0 mPage) :: Int @@ -177,8 +201,10 @@ getVotingPower (unHexText -> dRepId) = do cacheRequest dRepVotingPowerCache dRepId $ DRep.getVotingPower dRepId -proposalToResponse :: Types.Proposal -> MetadataValidationResponse -> ProposalResponse -proposalToResponse Types.Proposal {..} metadataValidationResponse = +proposalToResponse :: Types.Proposal -> Types.MetadataValidationResult Types.ProposalMetadata -> ProposalResponse +proposalToResponse Types.Proposal {..} Types.MetadataValidationResult{..} = + let metadata = metadataValidationResultMetadata + in ProposalResponse { proposalResponseId = pack $ show proposalId, proposalResponseTxHash = HexText proposalTxHash, @@ -191,16 +217,17 @@ proposalToResponse Types.Proposal {..} metadataValidationResponse = proposalResponseCreatedEpochNo = proposalCreatedEpochNo, proposalResponseUrl = proposalUrl, proposalResponseMetadataHash = HexText proposalDocHash, - proposalResponseTitle = proposalTitle, - proposalResponseAbout = proposalAbout, - proposalResponseMotivation = proposalMotivaiton, - proposalResponseRationale = proposalRationale, + proposalResponseTitle = Types.proposalMetadataTitle <$> metadata, + proposalResponseAbstract = Types.proposalMetadataAbstract <$> metadata, + proposalResponseMotivation = Types.proposalMetadataMotivation <$> metadata, + proposalResponseRationale = Types.proposalMetadataRationale <$> metadata, proposalResponseMetadata = GovernanceActionMetadata <$> proposalMetadata, - proposalResponseReferences = GovernanceActionReferences <$> proposalReferences, + proposalResponseReferences = maybe [] Types.proposalMetadataReferences metadata, proposalResponseYesVotes = proposalYesVotes, proposalResponseNoVotes = proposalNoVotes, proposalResponseAbstainVotes = proposalAbstainVotes, - proposalResponseMetadataStatus = Just metadataValidationResponse + proposalResponseMetadataStatus = metadataValidationResultStatus, + proposalResponseMetadataValid = metadataValidationResultValid } voteToResponse :: Types.Vote -> VoteParams @@ -224,9 +251,19 @@ mapSortAndFilterProposals -> [Types.Proposal] -> m [ProposalResponse] mapSortAndFilterProposals selectedTypes sortMode proposals = do + + appEnv <- ask + qsem <- asks vvaMetadataQSem + mappedProposals <- - mapM - (\proposal@Types.Proposal {proposalUrl, proposalDocHash} -> proposalToResponse proposal <$> validateMetadata (MetadataValidationParams proposalUrl $ HexText proposalDocHash)) + liftIO $ mapConcurrently + (\proposal@Types.Proposal {proposalUrl, proposalDocHash} -> + do + waitQSem qsem + r <- either throwIO return =<< (runExceptT + $ flip runReaderT appEnv (proposalToResponse proposal <$> validateProposalMetadata (MetadataValidationParams proposalUrl $ HexText proposalDocHash))) + signalQSem qsem + return r) proposals let filteredProposals = if null selectedTypes @@ -297,7 +334,7 @@ isProposalSearchedFor (Just searchQuery) (ProposalResponse{..}) = fromMaybe Fals let valuesToCheck = catMaybes [ Just govActionId , proposalResponseTitle - , proposalResponseAbout + , proposalResponseAbstract , proposalResponseMotivation , proposalResponseRationale ] @@ -351,8 +388,8 @@ getProposal g@(GovActionId govActionTxHash govActionIndex) mDrepId' = do let mDrepId = unHexText <$> mDrepId' CacheEnv {getProposalCache} <- asks vvaCache proposal@Types.Proposal {proposalUrl, proposalDocHash} <- cacheRequest getProposalCache (unHexText govActionTxHash, govActionIndex) (Proposal.getProposal (unHexText govActionTxHash) govActionIndex) - metadataStatus <- validateMetadata $ MetadataValidationParams proposalUrl $ HexText proposalDocHash - let proposalResponse = proposalToResponse proposal metadataStatus + proposalMetadataValidationResult <- validateProposalMetadata $ MetadataValidationParams proposalUrl $ HexText proposalDocHash + let proposalResponse = proposalToResponse proposal proposalMetadataValidationResult voteResponse <- case mDrepId of Nothing -> return Nothing Just drepId -> do @@ -400,14 +437,34 @@ getNetworkMetrics = do , getNetworkMetricsResponseAlwaysNoConfidenceVotingPower = networkMetricsAlwaysNoConfidenceVotingPower } -validateMetadata :: App m => MetadataValidationParams -> m MetadataValidationResponse -validateMetadata MetadataValidationParams {..} = do - CacheEnv {metadataValidationCache} <- asks vvaCache - result <- cacheRequest metadataValidationCache (metadataValidationParamsUrl, unHexText metadataValidationParamsHash) - $ Metadata.validateMetadata metadataValidationParamsUrl (unHexText metadataValidationParamsHash) - - case fromJSON result of - Error e -> return $ MetadataValidationResponse Nothing False (AnyValue $ Just result) - Success (InternalMetadataValidationResponse {..}) -> return $ MetadataValidationResponse {metadataValidationResponseStatus=internalMetadataValidationResponseStatus, metadataValidationResponseValid=internalMmetadataValidationResponseValid, metadataValidationResponseRaw=AnyValue $ Just result} - - +validateProposalMetadata :: App m => MetadataValidationParams -> m (Types.MetadataValidationResult Types.ProposalMetadata) +validateProposalMetadata MetadataValidationParams {..} = do + CacheEnv {proposalMetadataValidationCache} <- asks vvaCache + cacheRequest proposalMetadataValidationCache (metadataValidationParamsUrl, unHexText metadataValidationParamsHash) + $ Metadata.getProposalMetadataValidationResult metadataValidationParamsUrl (unHexText metadataValidationParamsHash) + +getProposalMetadataValidationResponse :: App m => MetadataValidationParams -> m MetadataValidationResponse +getProposalMetadataValidationResponse params = do + result <- validateProposalMetadata params + case result of + Types.MetadataValidationResult {..} -> do + return $ MetadataValidationResponse + { metadataValidationResponseValid = metadataValidationResultValid + , metadataValidationResponseStatus = metadataValidationResultStatus + } + +validateDRepMetadata :: App m => MetadataValidationParams -> m (Types.MetadataValidationResult Types.DRepMetadata) +validateDRepMetadata MetadataValidationParams {..} = do + CacheEnv {dRepMetadataValidationCache} <- asks vvaCache + cacheRequest dRepMetadataValidationCache (metadataValidationParamsUrl, unHexText metadataValidationParamsHash) + $ Metadata.getDRepMetadataValidationResult metadataValidationParamsUrl (unHexText metadataValidationParamsHash) + +getDRepMetadataValidationResponse :: App m => MetadataValidationParams -> m MetadataValidationResponse +getDRepMetadataValidationResponse params = do + result <- validateDRepMetadata params + case result of + Types.MetadataValidationResult {..} -> do + return $ MetadataValidationResponse + { metadataValidationResponseValid = metadataValidationResultValid + , metadataValidationResponseStatus = metadataValidationResultStatus + } \ No newline at end of file diff --git a/govtool/backend/src/VVA/API/Types.hs b/govtool/backend/src/VVA/API/Types.hs index 760b24857..ccb11e750 100644 --- a/govtool/backend/src/VVA/API/Types.hs +++ b/govtool/backend/src/VVA/API/Types.hs @@ -167,9 +167,8 @@ instance ToSchema InternalMetadataValidationResponse where data MetadataValidationResponse = MetadataValidationResponse - { metadataValidationResponseStatus :: Maybe MetadataValidationStatus + { metadataValidationResponseStatus :: Maybe Text , metadataValidationResponseValid :: Bool - , metadataValidationResponseRaw :: AnyValue } deriving (Generic, Show) @@ -463,15 +462,16 @@ data ProposalResponse , proposalResponseUrl :: Text , proposalResponseMetadataHash :: HexText , proposalResponseTitle :: Maybe Text - , proposalResponseAbout :: Maybe Text + , proposalResponseAbstract :: Maybe Text , proposalResponseMotivation :: Maybe Text , proposalResponseRationale :: Maybe Text , proposalResponseMetadata :: Maybe GovernanceActionMetadata - , proposalResponseReferences :: Maybe GovernanceActionReferences + , proposalResponseReferences :: [Text] , proposalResponseYesVotes :: Integer , proposalResponseNoVotes :: Integer , proposalResponseAbstainVotes :: Integer - , proposalResponseMetadataStatus :: Maybe MetadataValidationResponse + , proposalResponseMetadataStatus :: Maybe Text + , proposalResponseMetadataValid :: Bool } deriving (Generic, Show) @@ -490,15 +490,16 @@ exampleProposalResponse = "{ \"id\": \"proposalId123\"," <> "\"url\": \"https://proposal.metadata.xyz\"," <> "\"metadataHash\": \"9af10e89979e51b8cdc827c963124a1ef4920d1253eef34a1d5cfe76438e3f11\"," <> "\"title\": \"Proposal Title\"," - <> "\"about\": \"Proposal About\"," + <> "\"abstract\": \"Proposal About\"," <> "\"motivation\": \"Proposal Motivation\"," <> "\"rationale\": \"Proposal Rationale\"," <> "\"metadata\": {\"key\": \"value\"}," - <> "\"references\": [{\"uri\": \"google.com\", \"@type\": \"Other\", \"label\": \"example label\"}]," + <> "\"references\": [\"google.com\"]," <> "\"yesVotes\": 0," <> "\"noVotes\": 0," - <> "\"abstainVotes\": 0" - <> "\"metadataStatus\": {\"status\": null, \"valid\": true}}" + <> "\"abstainVotes\": 0," + <> "\"metadataStatus\": \"URL_NOT_FOUND\"," + <> "\"metadataValid\": true}" instance ToSchema ProposalResponse where declareNamedSchema proxy = do @@ -841,6 +842,12 @@ data DRep , dRepType :: DRepType , dRepLatestTxHash :: Maybe HexText , dRepLatestRegistrationDate :: UTCTime + , dRepBio :: Maybe Text + , dRepDRepName :: Maybe Text + , dRepEmail :: Maybe Text + , dRepReferences :: [Text] + , dRepMetadataStatus :: Maybe Text + , dRepMetadataValid :: Bool } deriving (Generic, Show) @@ -858,7 +865,13 @@ exampleDrep = <> "\"status\": \"Active\"," <> "\"type\": \"DRep\"," <> "\"latestTxHash\": \"47c14a128cd024f1b990c839d67720825921ad87ed875def42641ddd2169b39c\"," - <> "\"latestRegistrationDate\": \"1970-01-01T00:00:00Z\"}" + <> "\"latestRegistrationDate\": \"1970-01-01T00:00:00Z\"," + <> "\"bio\": \"DRep Bio\"," + <> "\"dRepName\": \"DRep Name\"," + <> "\"email\": \"google@gmail.com\"," + <> "\"references\": [\"google.com\"]," + <> "\"metadataStatus\": \"URL_NOT_FOUND\"," + <> "\"metadataValid\": true}" -- ToSchema instance for DRep instance ToSchema DRep where diff --git a/govtool/backend/src/VVA/Config.hs b/govtool/backend/src/VVA/Config.hs index c4781681f..bb2ba48d1 100644 --- a/govtool/backend/src/VVA/Config.hs +++ b/govtool/backend/src/VVA/Config.hs @@ -84,6 +84,8 @@ data VVAConfigInternal , vVAConfigInternalMetadataValidationHost :: Text -- | Metadata validation service port , vVAConfigInternalMetadataValidationPort :: Int + -- | Maximum number of concurrent metadata requests + , vVAConfigInternalMetadataValidationMaxConcurrentRequests :: Int } deriving (FromConfig, Generic, Show) @@ -96,7 +98,8 @@ instance DefaultConfig VVAConfigInternal where vVaConfigInternalCacheDurationSeconds = 20, vVAConfigInternalSentrydsn = "https://username:password@senty.host/id", vVAConfigInternalMetadataValidationHost = "localhost", - vVAConfigInternalMetadataValidationPort = 3001 + vVAConfigInternalMetadataValidationPort = 3001, + vVAConfigInternalMetadataValidationMaxConcurrentRequests = 10 } -- | DEX configuration. @@ -116,6 +119,8 @@ data VVAConfig , metadataValidationHost :: Text -- | Metadata validation service port , metadataValidationPort :: Int + -- | Maximum number of concurrent metadata requests + , metadataValidationMaxConcurrentRequests :: Int } deriving (Generic, Show, ToJSON) @@ -157,7 +162,8 @@ convertConfig VVAConfigInternal {..} = cacheDurationSeconds = vVaConfigInternalCacheDurationSeconds, sentryDSN = vVAConfigInternalSentrydsn, metadataValidationHost = vVAConfigInternalMetadataValidationHost, - metadataValidationPort = vVAConfigInternalMetadataValidationPort + metadataValidationPort = vVAConfigInternalMetadataValidationPort, + metadataValidationMaxConcurrentRequests = vVAConfigInternalMetadataValidationMaxConcurrentRequests } -- | Load configuration from a file specified on the command line. Load from diff --git a/govtool/backend/src/VVA/Metadata.hs b/govtool/backend/src/VVA/Metadata.hs index d6033633f..f3cf05e74 100644 --- a/govtool/backend/src/VVA/Metadata.hs +++ b/govtool/backend/src/VVA/Metadata.hs @@ -1,18 +1,24 @@ {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} module VVA.Metadata where +import Prelude hiding (lookup) import Control.Monad.Except (MonadError, throwError) import Control.Monad.Reader +import Control.Exception (try, Exception) -import Data.Aeson (Value, decode, encode, object, (.=)) +import Data.Typeable (Typeable) +import Data.Vector (toList) +import Data.Aeson.KeyMap (lookup) +import Data.Aeson (Value(..), decode, encode, object, (.=)) import Data.Maybe (fromJust) import Data.ByteString (ByteString) import Data.FileEmbed (embedFile) import Data.Has (Has, getter) import Data.String (fromString) -import Data.Text (Text, unpack) +import Data.Text (Text, unpack, pack) import qualified Data.Text.Encoding as Text import Data.Time.Clock @@ -29,17 +35,79 @@ validateMetadata :: (Has VVAConfig r, Has Manager r, MonadReader r m, MonadIO m, MonadError AppError m) => Text -> Text - -> m Value -validateMetadata url hash = do + -> Maybe Text + -> m (Either Text Value) +validateMetadata url hash standard = do metadataHost <- getMetadataValidationHost metadataPort <- getMetadataValidationPort manager <- asks getter - let requestBody = encode $ object ["url" .= unpack url, "hash" .= unpack hash] + let requestBody = encode $ object (["url" .= unpack url, "hash" .= unpack hash] ++ maybe [] (\x -> ["standard" .= unpack x]) standard) initialRequest <- liftIO $ parseRequest (unpack metadataHost <> ":" <> show metadataPort <> "/validate") let request = initialRequest { method = "POST" , requestBody = RequestBodyLBS requestBody , requestHeaders = [("Content-Type", "application/json")] } - response <- liftIO $ httpLbs request manager - return $ fromJust $ decode $ responseBody response + response <- liftIO $ try $ httpLbs request manager + case response of + Left (e :: HttpException) -> return $ Left (pack $ show e) + Right r -> case decode $ responseBody r of + Nothing -> throwError $ InternalError "Failed to validate metadata" + Just x -> return $ Right x + +getProposalMetadataValidationResult :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) => + Text -> + Text -> + m (MetadataValidationResult ProposalMetadata) +getProposalMetadataValidationResult url hash = do + result <- validateMetadata url hash (Just "CIP108") + case result of + Left e -> return $ MetadataValidationResult False (Just e) Nothing + Right (Object r) -> case go r of + Nothing -> throwError $ InternalError "Failed to validate metadata" + Just x -> return x + where + go result = do + (Bool valid) <- lookup "valid" result + let status = case lookup "status" result of + Just (String s) -> Just s + _ -> Nothing + let proposalMetadata = do + (Object m) <- lookup "metadata" result + let abstract = (\(String s) -> s) <$> lookup "abstract" m + let motivation = (\(String s) -> s) <$> lookup "motivation" m + let rationale = (\(String s) -> s) <$> lookup "rationale" m + let title = (\(String s) -> s) <$> lookup "title" m + let references = (\(Array references') -> map (\(String x) -> x) $ toList references') <$> lookup "references" m + ProposalMetadata <$> abstract <*> motivation <*> rationale <*> title <*> references + return $ MetadataValidationResult valid status proposalMetadata + + + +getDRepMetadataValidationResult :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) => + Text -> + Text -> + m (MetadataValidationResult DRepMetadata) +getDRepMetadataValidationResult url hash = do + result <- validateMetadata url hash (Just "CIPQQQ") + case result of + Left e -> return $ MetadataValidationResult False (Just e) Nothing + Right (Object r) -> case go r of + Nothing -> throwError $ InternalError "Failed to validate metadata" + Just x -> return x + where + go result = do + (Bool valid) <- lookup "valid" result + let status = case lookup "status" result of + Just (String s) -> Just s + _ -> Nothing + let proposalMetadata = do + (Object m) <- lookup "metadata" result + let bio = (\(String s) -> s) <$> lookup "bio" m + let dRepName = (\(String s) -> s) <$> lookup "dRepName" m + let email = (\(String s) -> s) <$> lookup "email" m + let references = (\(Array references') -> map (\(String x) -> x) $ toList references') <$> lookup "references" m + DRepMetadata <$> bio <*> dRepName <*> email <*> references + return $ MetadataValidationResult valid status proposalMetadata \ No newline at end of file diff --git a/govtool/backend/src/VVA/Types.hs b/govtool/backend/src/VVA/Types.hs index fba1e2321..701bdda87 100644 --- a/govtool/backend/src/VVA/Types.hs +++ b/govtool/backend/src/VVA/Types.hs @@ -24,6 +24,7 @@ import Database.PostgreSQL.Simple (Connection) import VVA.Cache import VVA.Config import Network.HTTP.Client (Manager) +import Control.Concurrent.QSem type App m = (MonadReader AppEnv m, MonadIO m, MonadFail m, MonadError AppError m) @@ -33,6 +34,7 @@ data AppEnv , vvaCache :: CacheEnv , vvaConnectionPool :: Pool Connection , vvaTlsManager :: Manager + , vvaMetadataQSem :: QSem } instance Has VVAConfig AppEnv where @@ -51,6 +53,11 @@ instance Has Manager AppEnv where getter AppEnv {vvaTlsManager} = vvaTlsManager modifier f a@AppEnv {vvaTlsManager} = a {vvaTlsManager = f vvaTlsManager} +instance Has QSem AppEnv where + getter AppEnv {vvaMetadataQSem} = vvaMetadataQSem + modifier f a@AppEnv {vvaMetadataQSem} = a {vvaMetadataQSem = f vvaMetadataQSem} + + data AppError = ValidationError Text | NotFoundError Text @@ -133,6 +140,34 @@ data Proposal data TransactionStatus = TransactionConfirmed | TransactionUnconfirmed + +data ProposalMetadata = + ProposalMetadata + { proposalMetadataAbstract :: Text + , proposalMetadataMotivation :: Text + , proposalMetadataRationale :: Text + , proposalMetadataTitle :: Text + , proposalMetadataReferences :: [Text] + } deriving (Show) + +data DRepMetadata = + DRepMetadata + { dRepMetadataBio :: Text + , dRepMetadataDRepName :: Text + , dRepMetadataEmail :: Text + , dRepMetadataReferences :: [Text] + } deriving (Show) + +data MetadataValidationResult a = + MetadataValidationResult + { metadataValidationResultValid :: Bool + , metadataValidationResultStatus :: Maybe Text + , metadataValidationResultMetadata :: Maybe a + } deriving (Show) + + + + data CacheEnv = CacheEnv { proposalListCache :: Cache.Cache () [Proposal] @@ -145,7 +180,8 @@ data CacheEnv , dRepVotingPowerCache :: Cache.Cache Text Integer , dRepListCache :: Cache.Cache () [DRepRegistration] , networkMetricsCache :: Cache.Cache () NetworkMetrics - , metadataValidationCache :: Cache.Cache (Text, Text) Value + , proposalMetadataValidationCache :: Cache.Cache (Text, Text) (MetadataValidationResult ProposalMetadata) + , dRepMetadataValidationCache :: Cache.Cache (Text, Text) (MetadataValidationResult DRepMetadata) } data NetworkMetrics @@ -174,4 +210,6 @@ data MetadataValidationStatus = IncorrectFormat | IncorrectJSONLD | IncorrectHash - | UrlNotFound \ No newline at end of file + | UrlNotFound + + diff --git a/govtool/backend/vva-be.cabal b/govtool/backend/vva-be.cabal index be278ec8f..1cea27716 100644 --- a/govtool/backend/vva-be.cabal +++ b/govtool/backend/vva-be.cabal @@ -100,6 +100,8 @@ library , swagger2 , http-client , http-client-tls + , vector + , async exposed-modules: VVA.Config , VVA.CommandLine diff --git a/govtool/metadata-validation/src/app.service.ts b/govtool/metadata-validation/src/app.service.ts index fb31dd4fd..0e197bb5e 100644 --- a/govtool/metadata-validation/src/app.service.ts +++ b/govtool/metadata-validation/src/app.service.ts @@ -6,7 +6,8 @@ import * as blake from 'blakejs'; import { ValidateMetadataDTO } from '@dto'; import { MetadataValidationStatus } from '@enums'; import { canonizeJSON, validateMetadataStandard, parseMetadata } from '@utils'; -import { ValidateMetadataResult } from '@types'; +import { MetadataStandard, ValidateMetadataResult } from '@types'; + @Injectable() export class AppService { @@ -15,11 +16,12 @@ export class AppService { async validateMetadata({ hash, url, - standard, + standard = MetadataStandard.CIP108, }: ValidateMetadataDTO): Promise { let status: MetadataValidationStatus; let metadata: any; try { + console.log(standard); const { data } = await firstValueFrom( this.httpService.get(url).pipe( catchError(() => { diff --git a/scripts/govtool/config/templates/backend-config.json.tpl b/scripts/govtool/config/templates/backend-config.json.tpl index 9ff9f2f11..cab68ebc0 100644 --- a/scripts/govtool/config/templates/backend-config.json.tpl +++ b/scripts/govtool/config/templates/backend-config.json.tpl @@ -11,5 +11,6 @@ "cachedurationseconds": 20, "sentrydsn": "", "metadatavalidationhost": "http://metadata-validation", - "metadatavalidationport": "3000" + "metadatavalidationport": "3000", + "metadatavalidationmaxconcurrentrequests": 10 }