[#1142] fetch metadata from metadata-validation-service

CHANGELOG.md

@@ -24,6 +24,8 @@ changes.
 
 ### Added
 
+- added `bio` `dRepName` `email` `references` `metadataValid` and `metadataStatus` fields to `drep/list`
+- added `metadatavalidationmaxconcurrentrequests` field to the backend config
 - added `metadata/validate` endpoint [Issue 876](https://github.com/IntersectMBO/govtool/issues/876)
 - added pagination to `drep/list` [Issue 756](https://github.com/IntersectMBO/govtool/issues/756)
 - added search query param to the `drep/getVotes` [Issue 640](https://github.com/IntersectMBO/govtool/issues/640)
@@ -92,6 +94,7 @@ changes.
 
 ### Changed
 
+- `proposal.about` changed to `proposal.abstract`
 - `drep/info` now returns 4 different tx hashes instead of one latest tx hash [Issue 688](https://github.com/IntersectMBO/govtool/issues/688)
 - `proposal/list` allows user to search by tx hash [Issue 603](https://github.com/IntersectMBO/govtool/issues/603)
 - `proposal/list` returns additional data such ass `expiryEpochNo`, `createdEpochNo`, `title`, `about`, `motivation`,

govtool/backend/app/Main.hs

@@ -8,6 +8,7 @@
 
 module Main where
 
+import           Control.Concurrent.QSem                (newQSem)
 import           Control.Exception                      (Exception,
                                                          SomeException,
                                                          fromException, throw)
@@ -115,7 +116,8 @@ startApp vvaConfig = do
     dRepVotingPowerCache <- newCache
     dRepListCache <- newCache
     networkMetricsCache <- newCache
-    metadataValidationCache <- newCache
+    proposalMetadataValidationCache <- newCache
+    dRepMetadataValidationCache <- newCache
     return $ CacheEnv
       { proposalListCache
       , getProposalCache
@@ -127,12 +129,13 @@ startApp vvaConfig = do
       , dRepVotingPowerCache
       , dRepListCache
       , networkMetricsCache
-      , metadataValidationCache
+      , proposalMetadataValidationCache
+      , dRepMetadataValidationCache
       }
   connectionPool <- createPool (connectPostgreSQL (encodeUtf8 (dbSyncConnectionString $ getter vvaConfig))) close 1 1 60
   vvaTlsManager <- newManager tlsManagerSettings
-
-  let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager}
+  qsem <- newQSem (metadataValidationMaxConcurrentRequests vvaConfig)
+  let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager, vvaMetadataQSem=qsem}
   server' <- mkVVAServer appEnv
   runSettings settings server'
 

govtool/backend/example-config.json

@@ -11,5 +11,6 @@
     "cachedurationseconds": 20,
     "sentrydsn": "https://username:[email protected]/id",
     "metadatavalidationhost": "localhost",
-    "metadatavalidationport": 3001
+    "metadatavalidationport": 3001,
+    "metadatavalidationmaxconcurrentrequests": 10
 }

govtool/backend/src/VVA/API.hs

@@ -8,8 +8,10 @@
 
 module VVA.API where
 
-import           Control.Exception    (throw)
-import           Control.Monad.Except (throwError)
+import Control.Concurrent.QSem (waitQSem, signalQSem)
+import Control.Concurrent.Async (mapConcurrently)
+import           Control.Exception    (throw, throwIO)
+import           Control.Monad.Except (throwError, runExceptT)
 import           Control.Monad.Reader
 import           Data.Aeson           (Result(Error, Success), fromJSON)
 import           Data.Bool            (Bool)
@@ -74,8 +76,8 @@ type VVAApi =
     :<|> "transaction" :> "status" :> Capture "transactionId" HexText :> Get '[JSON] GetTransactionStatusResponse
     :<|> "throw500" :> Get '[JSON] ()
     :<|> "network" :> "metrics" :> Get '[JSON] GetNetworkMetricsResponse
-    :<|> "metadata" :> "validate" :> ReqBody '[JSON] MetadataValidationParams :> Post '[JSON] MetadataValidationResponse
-
+    :<|> "proposal" :> "metadata" :> "validate" :> ReqBody '[JSON] MetadataValidationParams :> Post '[JSON] MetadataValidationResponse
+    :<|> "drep" :> "metadata" :> "validate" :> ReqBody '[JSON] MetadataValidationParams :> Post '[JSON] MetadataValidationResponse
 server :: App m => ServerT VVAApi m
 server = drepList
     :<|> getVotingPower
@@ -89,7 +91,8 @@ server = drepList
     :<|> getTransactionStatus
     :<|> throw500
     :<|> getNetworkMetrics
-    :<|> validateMetadata
+    :<|> getProposalMetadataValidationResponse
+    :<|> getDRepMetadataValidationResponse
 
 
 mapDRepType :: Types.DRepType -> DRepType
@@ -101,9 +104,10 @@ mapDRepStatus Types.Retired  = Retired
 mapDRepStatus Types.Active   = Active
 mapDRepStatus Types.Inactive = Inactive
 
-drepRegistrationToDrep :: Types.DRepRegistration -> DRep
-drepRegistrationToDrep Types.DRepRegistration {..} =
-  DRep
+drepRegistrationToDrep :: Types.DRepRegistration -> Types.MetadataValidationResult Types.DRepMetadata -> DRep
+drepRegistrationToDrep Types.DRepRegistration {..} Types.MetadataValidationResult{..} =
+  let metadata = metadataValidationResultMetadata
+  in DRep
     { dRepDrepId = DRepHash dRepRegistrationDRepHash,
       dRepView = dRepRegistrationView,
       dRepUrl = dRepRegistrationUrl,
@@ -113,7 +117,13 @@ drepRegistrationToDrep Types.DRepRegistration {..} =
       dRepStatus = mapDRepStatus dRepRegistrationStatus,
       dRepType = mapDRepType dRepRegistrationType,
       dRepLatestTxHash = HexText <$> dRepRegistrationLatestTxHash,
-      dRepLatestRegistrationDate = dRepRegistrationLatestRegistrationDate
+      dRepLatestRegistrationDate = dRepRegistrationLatestRegistrationDate,
+      dRepBio = Types.dRepMetadataBio <$> metadata,
+      dRepDRepName = Types.dRepMetadataDRepName <$> metadata,
+      dRepEmail = Types.dRepMetadataEmail <$> metadata,
+      dRepReferences = maybe [] Types.dRepMetadataReferences metadata,
+      dRepMetadataStatus = metadataValidationResultStatus,
+      dRepMetadataValid = metadataValidationResultValid
     }
 
 delegationToResponse :: Types.Delegation -> DelegationResponse
@@ -152,8 +162,22 @@ drepList mSearchQuery statuses mSortMode mPage mPageSize = do
         Just Status -> sortOn $ \Types.DRepRegistration {..} ->
           dRepRegistrationStatus
 
+  appEnv <- ask
+  qsem <- asks vvaMetadataQSem
 
-  let allValidDReps = map drepRegistrationToDrep $ sortDReps $ filterDRepsByQuery $ filterDRepsByStatus dreps
+  allValidDReps <- liftIO $ mapConcurrently
+    (\d@Types.DRepRegistration{..} ->
+      drepRegistrationToDrep d
+      <$> do
+         waitQSem qsem
+         r <- (either throwIO return =<< (runExceptT
+                    $ flip runReaderT appEnv (validateDRepMetadata
+                      (MetadataValidationParams
+                        (fromMaybe "" dRepRegistrationUrl)
+                        $ HexText (fromMaybe "" dRepRegistrationDataHash)))))
+         signalQSem qsem
+         return r)
+    $ sortDReps $ filterDRepsByQuery $ filterDRepsByStatus dreps
 
 
   let page = (fromIntegral $ fromMaybe 0 mPage) :: Int
@@ -177,8 +201,10 @@ getVotingPower (unHexText -> dRepId) = do
   cacheRequest dRepVotingPowerCache dRepId $ DRep.getVotingPower dRepId
 
 
-proposalToResponse :: Types.Proposal -> MetadataValidationResponse -> ProposalResponse
-proposalToResponse Types.Proposal {..} metadataValidationResponse =
+proposalToResponse :: Types.Proposal -> Types.MetadataValidationResult Types.ProposalMetadata -> ProposalResponse
+proposalToResponse Types.Proposal {..} Types.MetadataValidationResult{..} =
+  let metadata = metadataValidationResultMetadata
+  in
   ProposalResponse
   { proposalResponseId = pack $ show proposalId,
     proposalResponseTxHash = HexText proposalTxHash,
@@ -191,16 +217,17 @@ proposalToResponse Types.Proposal {..} metadataValidationResponse =
     proposalResponseCreatedEpochNo = proposalCreatedEpochNo,
     proposalResponseUrl = proposalUrl,
     proposalResponseMetadataHash = HexText proposalDocHash,
-    proposalResponseTitle = proposalTitle,
-    proposalResponseAbout = proposalAbout,
-    proposalResponseMotivation = proposalMotivaiton,
-    proposalResponseRationale = proposalRationale,
+    proposalResponseTitle = Types.proposalMetadataTitle <$> metadata,
+    proposalResponseAbstract = Types.proposalMetadataAbstract <$> metadata,
+    proposalResponseMotivation = Types.proposalMetadataMotivation <$> metadata,
+    proposalResponseRationale = Types.proposalMetadataRationale <$> metadata,
     proposalResponseMetadata = GovernanceActionMetadata <$> proposalMetadata,
-    proposalResponseReferences = GovernanceActionReferences <$> proposalReferences,
+    proposalResponseReferences = maybe [] Types.proposalMetadataReferences metadata,
     proposalResponseYesVotes = proposalYesVotes,
     proposalResponseNoVotes = proposalNoVotes,
     proposalResponseAbstainVotes = proposalAbstainVotes,
-    proposalResponseMetadataStatus = Just metadataValidationResponse
+    proposalResponseMetadataStatus = metadataValidationResultStatus,
+    proposalResponseMetadataValid = metadataValidationResultValid
   }
 
 voteToResponse :: Types.Vote -> VoteParams
@@ -224,9 +251,19 @@ mapSortAndFilterProposals
   -> [Types.Proposal]
   -> m [ProposalResponse]
 mapSortAndFilterProposals selectedTypes sortMode proposals = do
+
+  appEnv <- ask
+  qsem <- asks vvaMetadataQSem
+
   mappedProposals <-
-        mapM
-          (\proposal@Types.Proposal {proposalUrl, proposalDocHash} -> proposalToResponse proposal <$> validateMetadata (MetadataValidationParams proposalUrl $ HexText proposalDocHash))
+    liftIO $ mapConcurrently
+          (\proposal@Types.Proposal {proposalUrl, proposalDocHash} ->
+                do
+                  waitQSem qsem
+                  r <- either throwIO return =<< (runExceptT
+                    $ flip runReaderT appEnv (proposalToResponse proposal <$> validateProposalMetadata (MetadataValidationParams proposalUrl $ HexText proposalDocHash)))
+                  signalQSem qsem
+                  return r)
           proposals
   let filteredProposals =
         if null selectedTypes
@@ -297,7 +334,7 @@ isProposalSearchedFor (Just searchQuery) (ProposalResponse{..}) = fromMaybe Fals
           let valuesToCheck = catMaybes
                 [ Just govActionId
                 , proposalResponseTitle
-                , proposalResponseAbout
+                , proposalResponseAbstract
                 , proposalResponseMotivation
                 , proposalResponseRationale
                 ]
@@ -351,8 +388,8 @@ getProposal g@(GovActionId govActionTxHash govActionIndex) mDrepId' = do
   let mDrepId = unHexText <$> mDrepId'
   CacheEnv {getProposalCache} <- asks vvaCache
   proposal@Types.Proposal {proposalUrl, proposalDocHash} <- cacheRequest getProposalCache (unHexText govActionTxHash, govActionIndex) (Proposal.getProposal (unHexText govActionTxHash) govActionIndex)
-  metadataStatus <- validateMetadata $ MetadataValidationParams proposalUrl $ HexText proposalDocHash
-  let proposalResponse = proposalToResponse proposal metadataStatus
+  proposalMetadataValidationResult <- validateProposalMetadata $ MetadataValidationParams proposalUrl $ HexText proposalDocHash
+  let proposalResponse = proposalToResponse proposal proposalMetadataValidationResult
   voteResponse <- case mDrepId of
     Nothing -> return Nothing
     Just drepId -> do
@@ -400,14 +437,34 @@ getNetworkMetrics = do
     , getNetworkMetricsResponseAlwaysNoConfidenceVotingPower = networkMetricsAlwaysNoConfidenceVotingPower
     }
 
-validateMetadata :: App m => MetadataValidationParams -> m MetadataValidationResponse
-validateMetadata MetadataValidationParams {..} = do
-  CacheEnv {metadataValidationCache} <- asks vvaCache
-  result <- cacheRequest metadataValidationCache (metadataValidationParamsUrl, unHexText metadataValidationParamsHash)
-      $ Metadata.validateMetadata metadataValidationParamsUrl (unHexText metadataValidationParamsHash)
-
-  case fromJSON result of
-    Error e -> return $ MetadataValidationResponse Nothing False (AnyValue $ Just result)
-    Success (InternalMetadataValidationResponse {..}) -> return $ MetadataValidationResponse {metadataValidationResponseStatus=internalMetadataValidationResponseStatus, metadataValidationResponseValid=internalMmetadataValidationResponseValid, metadataValidationResponseRaw=AnyValue $ Just result}
-
-
+validateProposalMetadata :: App m => MetadataValidationParams -> m (Types.MetadataValidationResult Types.ProposalMetadata)
+validateProposalMetadata MetadataValidationParams {..} = do
+  CacheEnv {proposalMetadataValidationCache} <- asks vvaCache
+  cacheRequest proposalMetadataValidationCache (metadataValidationParamsUrl, unHexText metadataValidationParamsHash)
+      $ Metadata.getProposalMetadataValidationResult metadataValidationParamsUrl (unHexText metadataValidationParamsHash)
+
+getProposalMetadataValidationResponse :: App m => MetadataValidationParams -> m MetadataValidationResponse
+getProposalMetadataValidationResponse params = do
+  result <- validateProposalMetadata params
+  case result of
+    Types.MetadataValidationResult {..} -> do
+      return $ MetadataValidationResponse
+        { metadataValidationResponseValid = metadataValidationResultValid
+        , metadataValidationResponseStatus = metadataValidationResultStatus
+        }
+
+validateDRepMetadata :: App m => MetadataValidationParams -> m (Types.MetadataValidationResult Types.DRepMetadata)
+validateDRepMetadata MetadataValidationParams {..} = do
+  CacheEnv {dRepMetadataValidationCache} <- asks vvaCache
+  cacheRequest dRepMetadataValidationCache (metadataValidationParamsUrl, unHexText metadataValidationParamsHash)
+      $ Metadata.getDRepMetadataValidationResult metadataValidationParamsUrl (unHexText metadataValidationParamsHash)
+
+getDRepMetadataValidationResponse :: App m => MetadataValidationParams -> m MetadataValidationResponse
+getDRepMetadataValidationResponse params = do
+  result <- validateDRepMetadata params
+  case result of
+    Types.MetadataValidationResult {..} -> do
+      return $ MetadataValidationResponse
+        { metadataValidationResponseValid = metadataValidationResultValid
+        , metadataValidationResponseStatus = metadataValidationResultStatus
+        }

govtool/backend/src/VVA/API/Types.hs

@@ -167,9 +167,8 @@ instance ToSchema InternalMetadataValidationResponse where
 
 data MetadataValidationResponse
   = MetadataValidationResponse
-      { metadataValidationResponseStatus :: Maybe MetadataValidationStatus
+      { metadataValidationResponseStatus :: Maybe Text
       , metadataValidationResponseValid :: Bool
-      , metadataValidationResponseRaw :: AnyValue
       }
   deriving (Generic, Show)
 
@@ -463,15 +462,16 @@ data ProposalResponse
       , proposalResponseUrl            :: Text
       , proposalResponseMetadataHash   :: HexText
       , proposalResponseTitle          :: Maybe Text
-      , proposalResponseAbout          :: Maybe Text
+      , proposalResponseAbstract       :: Maybe Text
       , proposalResponseMotivation     :: Maybe Text
       , proposalResponseRationale      :: Maybe Text
       , proposalResponseMetadata       :: Maybe GovernanceActionMetadata
-      , proposalResponseReferences     :: Maybe GovernanceActionReferences
+      , proposalResponseReferences     :: [Text]
       , proposalResponseYesVotes       :: Integer
       , proposalResponseNoVotes        :: Integer
       , proposalResponseAbstainVotes   :: Integer
-      , proposalResponseMetadataStatus :: Maybe MetadataValidationResponse
+      , proposalResponseMetadataStatus :: Maybe Text
+      , proposalResponseMetadataValid  :: Bool
       }
   deriving (Generic, Show)
 
@@ -490,15 +490,16 @@ exampleProposalResponse = "{ \"id\": \"proposalId123\","
                   <> "\"url\": \"https://proposal.metadata.xyz\","
                   <> "\"metadataHash\": \"9af10e89979e51b8cdc827c963124a1ef4920d1253eef34a1d5cfe76438e3f11\","
                   <> "\"title\": \"Proposal Title\","
-                  <> "\"about\": \"Proposal About\","
+                  <> "\"abstract\": \"Proposal About\","
                   <> "\"motivation\": \"Proposal Motivation\","
                   <> "\"rationale\": \"Proposal Rationale\","
                   <> "\"metadata\": {\"key\": \"value\"},"
-                  <> "\"references\": [{\"uri\": \"google.com\", \"@type\": \"Other\", \"label\": \"example label\"}],"
+                  <> "\"references\": [\"google.com\"],"
                   <> "\"yesVotes\": 0,"
                   <> "\"noVotes\": 0,"
-                  <> "\"abstainVotes\": 0"
-                  <> "\"metadataStatus\": {\"status\": null, \"valid\": true}}"
+                  <> "\"abstainVotes\": 0,"
+                  <> "\"metadataStatus\": \"URL_NOT_FOUND\","
+                  <> "\"metadataValid\": true}"
 
 instance ToSchema ProposalResponse where
   declareNamedSchema proxy = do
@@ -841,6 +842,12 @@ data DRep
       , dRepType                   :: DRepType
       , dRepLatestTxHash           :: Maybe HexText
       , dRepLatestRegistrationDate :: UTCTime
+      , dRepBio                    :: Maybe Text
+      , dRepDRepName               :: Maybe Text
+      , dRepEmail                  :: Maybe Text
+      , dRepReferences             :: [Text]
+      , dRepMetadataStatus         :: Maybe Text
+      , dRepMetadataValid          :: Bool
       }
   deriving (Generic, Show)
 
@@ -858,7 +865,13 @@ exampleDrep =
   <> "\"status\": \"Active\","
   <> "\"type\": \"DRep\","
   <> "\"latestTxHash\": \"47c14a128cd024f1b990c839d67720825921ad87ed875def42641ddd2169b39c\","
-  <> "\"latestRegistrationDate\": \"1970-01-01T00:00:00Z\"}"
+  <> "\"latestRegistrationDate\": \"1970-01-01T00:00:00Z\","
+  <> "\"bio\": \"DRep Bio\","
+  <> "\"dRepName\": \"DRep Name\","
+  <> "\"email\": \"[email protected]\","
+  <> "\"references\": [\"google.com\"],"
+  <> "\"metadataStatus\": \"URL_NOT_FOUND\","
+  <> "\"metadataValid\": true}"
 
 -- ToSchema instance for DRep
 instance ToSchema DRep where

govtool/backend/src/VVA/Config.hs

@@ -84,6 +84,8 @@ data VVAConfigInternal
       , vVAConfigInternalMetadataValidationHost :: Text
         -- | Metadata validation service port
       , vVAConfigInternalMetadataValidationPort :: Int
+        -- | Maximum number of concurrent metadata requests
+      , vVAConfigInternalMetadataValidationMaxConcurrentRequests :: Int
       }
   deriving (FromConfig, Generic, Show)
 
@@ -96,7 +98,8 @@ instance DefaultConfig VVAConfigInternal where
         vVaConfigInternalCacheDurationSeconds = 20,
         vVAConfigInternalSentrydsn = "https://username:[email protected]/id",
         vVAConfigInternalMetadataValidationHost = "localhost",
-        vVAConfigInternalMetadataValidationPort = 3001
+        vVAConfigInternalMetadataValidationPort = 3001,
+        vVAConfigInternalMetadataValidationMaxConcurrentRequests = 10
       }
 
 -- | DEX configuration.
@@ -116,6 +119,8 @@ data VVAConfig
       , metadataValidationHost :: Text
         -- | Metadata validation service port
       , metadataValidationPort :: Int
+        -- | Maximum number of concurrent metadata requests
+      , metadataValidationMaxConcurrentRequests :: Int
       }
   deriving (Generic, Show, ToJSON)
 
@@ -157,7 +162,8 @@ convertConfig VVAConfigInternal {..} =
       cacheDurationSeconds = vVaConfigInternalCacheDurationSeconds,
       sentryDSN = vVAConfigInternalSentrydsn,
       metadataValidationHost = vVAConfigInternalMetadataValidationHost,
-      metadataValidationPort = vVAConfigInternalMetadataValidationPort
+      metadataValidationPort = vVAConfigInternalMetadataValidationPort,
+      metadataValidationMaxConcurrentRequests = vVAConfigInternalMetadataValidationMaxConcurrentRequests
     }
 
 -- | Load configuration from a file specified on the command line.  Load from