diff --git a/CHANGELOG.md b/CHANGELOG.md index 048bea0ed..795935c74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ changes. ### Added +- added separate async process that fetches new voting_anchors, validates their metadata using metadata-validation service, and then stores it in Redis database [Issue 1234](https://github.com/IntersectMBO/govtool/issues/1234) - added `bio` `dRepName` `email` `references` `metadataValid` and `metadataStatus` fields to `drep/list` - added `metadatavalidationmaxconcurrentrequests` field to the backend config - added `metadata/validate` endpoint [Issue 876](https://github.com/IntersectMBO/govtool/issues/876) diff --git a/govtool/backend/app/Main.hs b/govtool/backend/app/Main.hs index a9f3cc65b..cc08afdd3 100644 --- a/govtool/backend/app/Main.hs +++ b/govtool/backend/app/Main.hs @@ -8,6 +8,7 @@ module Main where +import Control.Concurrent (forkIO) import Control.Concurrent.QSem (newQSem) import Control.Exception (Exception, SomeException, @@ -70,6 +71,7 @@ import VVA.API import VVA.API.Types import VVA.CommandLine import VVA.Config +import VVA.Metadata (startFetchProcess) import VVA.Types (AppEnv (..), AppError (CriticalError, NotFoundError, ValidationError, InternalError), CacheEnv (..)) @@ -136,6 +138,13 @@ startApp vvaConfig = do vvaTlsManager <- newManager tlsManagerSettings qsem <- newQSem (metadataValidationMaxConcurrentRequests vvaConfig) let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager, vvaMetadataQSem=qsem} + + _ <- forkIO $ do + result <- runReaderT (runExceptT startFetchProcess) appEnv + case result of + Left e -> throw e + Right _ -> return () + server' <- mkVVAServer appEnv runSettings settings server' diff --git a/govtool/backend/example-config.json b/govtool/backend/example-config.json index d406c8727..04ab9f55c 100644 --- a/govtool/backend/example-config.json +++ b/govtool/backend/example-config.json @@ -13,5 +13,7 @@ "sentryenv": "dev", "metadatavalidationhost": "localhost", "metadatavalidationport": 3001, - "metadatavalidationmaxconcurrentrequests": 10 + "metadatavalidationmaxconcurrentrequests": 10, + "redishost": "localhost", + "redisport": 6379 } diff --git a/govtool/backend/sql/get-voting-anchors.sql b/govtool/backend/sql/get-voting-anchors.sql new file mode 100644 index 000000000..cedbc26de --- /dev/null +++ b/govtool/backend/sql/get-voting-anchors.sql @@ -0,0 +1,3 @@ +select id, url, encode(data_hash, 'hex'), type::text +from voting_anchor +where voting_anchor.id > ? \ No newline at end of file diff --git a/govtool/backend/src/VVA/Config.hs b/govtool/backend/src/VVA/Config.hs index 557e1b319..08dac2bc9 100644 --- a/govtool/backend/src/VVA/Config.hs +++ b/govtool/backend/src/VVA/Config.hs @@ -23,6 +23,8 @@ module VVA.Config , getDbSyncConnectionString , getServerHost , getServerPort + , getRedisHost + , getRedisPort , vvaConfigToText , getMetadataValidationHost , getMetadataValidationPort @@ -88,6 +90,10 @@ data VVAConfigInternal , vVAConfigInternalMetadataValidationPort :: Int -- | Maximum number of concurrent metadata requests , vVAConfigInternalMetadataValidationMaxConcurrentRequests :: Int + -- | Redis host + , vVAConfigInternalRedisHost :: Text + -- | Redis port + , vVAConfigInternalRedisPort :: Int } deriving (FromConfig, Generic, Show) @@ -102,7 +108,9 @@ instance DefaultConfig VVAConfigInternal where vVAConfigInternalSentryEnv = "development", vVAConfigInternalMetadataValidationHost = "localhost", vVAConfigInternalMetadataValidationPort = 3001, - vVAConfigInternalMetadataValidationMaxConcurrentRequests = 10 + vVAConfigInternalMetadataValidationMaxConcurrentRequests = 10, + vVAConfigInternalRedisHost = "localhost", + vVAConfigInternalRedisPort = 6379 } -- | DEX configuration. @@ -126,6 +134,10 @@ data VVAConfig , metadataValidationPort :: Int -- | Maximum number of concurrent metadata requests , metadataValidationMaxConcurrentRequests :: Int + -- | Redis host + , redisHost :: Text + -- | Redis port + , redisPort :: Int } deriving (Generic, Show, ToJSON) @@ -169,7 +181,9 @@ convertConfig VVAConfigInternal {..} = sentryEnv = vVAConfigInternalSentryEnv, metadataValidationHost = vVAConfigInternalMetadataValidationHost, metadataValidationPort = vVAConfigInternalMetadataValidationPort, - metadataValidationMaxConcurrentRequests = vVAConfigInternalMetadataValidationMaxConcurrentRequests + metadataValidationMaxConcurrentRequests = vVAConfigInternalMetadataValidationMaxConcurrentRequests, + redisHost = vVAConfigInternalRedisHost, + redisPort = vVAConfigInternalRedisPort } -- | Load configuration from a file specified on the command line. Load from @@ -208,6 +222,18 @@ getServerHost :: m Text getServerHost = asks (serverHost . getter) +-- | Access redis host +getRedisHost :: + (Has VVAConfig r, MonadReader r m) => + m Text +getRedisHost = asks (redisHost . getter) + +-- | Access redis port +getRedisPort :: + (Has VVAConfig r, MonadReader r m) => + m Int +getRedisPort = asks (redisPort . getter) + -- | Access MetadataValidationService host getMetadataValidationHost :: (Has VVAConfig r, MonadReader r m) => diff --git a/govtool/backend/src/VVA/Metadata.hs b/govtool/backend/src/VVA/Metadata.hs index f3cf05e74..d55e323fa 100644 --- a/govtool/backend/src/VVA/Metadata.hs +++ b/govtool/backend/src/VVA/Metadata.hs @@ -1,10 +1,15 @@ {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TupleSections #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE TypeApplications #-} module VVA.Metadata where -import Prelude hiding (lookup) +import qualified Database.Redis as Redis +import Control.Concurrent (threadDelay) +import Prelude hiding (lookup) import Control.Monad.Except (MonadError, throwError) import Control.Monad.Reader import Control.Exception (try, Exception) @@ -12,16 +17,16 @@ import Control.Exception (try, Exception) import Data.Typeable (Typeable) import Data.Vector (toList) import Data.Aeson.KeyMap (lookup) -import Data.Aeson (Value(..), decode, encode, object, (.=)) +import Data.Aeson (FromJSON, ToJSON, Value(..), decode, encode, object, (.=)) import Data.Maybe (fromJust) -import Data.ByteString (ByteString) +import Data.ByteString (ByteString, fromStrict, toStrict) import Data.FileEmbed (embedFile) import Data.Has (Has, getter) import Data.String (fromString) import Data.Text (Text, unpack, pack) import qualified Data.Text.Encoding as Text import Data.Time.Clock - +import Data.List (partition) import qualified Database.PostgreSQL.Simple as SQL import VVA.Config @@ -30,9 +35,92 @@ import VVA.Types import Network.HTTP.Client import Network.HTTP.Client.TLS import Data.Aeson (encode, object, (.=)) +import Data.Scientific + +sqlFrom :: ByteString -> SQL.Query +sqlFrom bs = fromString $ unpack $ Text.decodeUtf8 bs + +getVotingAnchorsSql :: SQL.Query +getVotingAnchorsSql = sqlFrom $(embedFile "sql/get-voting-anchors.sql") + +getNewVotingAnchors :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) + => Integer + -> m [VotingAnchor] +getNewVotingAnchors lastId = do + anchors <- withPool $ \conn -> do + liftIO $ SQL.query conn getVotingAnchorsSql $ SQL.Only (lastId :: Integer) + return $ map (\(id, url, hash, type') -> VotingAnchor (floor @Scientific id) url hash type') anchors + +startFetchProcess :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) + => m () +startFetchProcess = go 0 + where + go latestKnownId = do + liftIO $ putStrLn "Fetching metadata..." + + anchors <- getNewVotingAnchors latestKnownId + if null anchors + then do + liftIO $ threadDelay (20 * 1000000) + go latestKnownId + else do + (drepMetadata, proposalMetadata) <- processAnchors anchors + storeMetadata drepMetadata + storeMetadata proposalMetadata + + let newId = maximum $ map votingAnchorId anchors + + liftIO $ putStrLn ("Stored " <> show (length anchors) <> " voting anchors") + + liftIO $ threadDelay (20 * 1000000) + go newId + + +processAnchors :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) + => [VotingAnchor] + -> m ( [(Text, MetadataValidationResult DRepMetadata)] + , [(Text, MetadataValidationResult ProposalMetadata)] + ) +processAnchors anchors = do + let (drepAnchors, proposalAnchors) = partition ((== "other") . votingAnchorType) anchors + drepMetadata <- mapM (\(VotingAnchor id url hash _) -> (url<>"#"<>hash, ) <$> getDRepMetadataValidationResult' url hash) drepAnchors + proposalMetadata <- mapM (\(VotingAnchor id url hash _) -> (url<>"#"<>hash, ) <$> getProposalMetadataValidationResult' url hash) proposalAnchors + return (drepMetadata, proposalMetadata) + +storeMetadata :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m, ToJSON a) + => [(Text, MetadataValidationResult a)] + -> m () +storeMetadata metadataResults = do + port <- getRedisPort + host <- getRedisHost + conn <- liftIO $ Redis.checkedConnect $ Redis.defaultConnectInfo {Redis.connectHost = unpack host, Redis.connectPort = Redis.PortNumber $ fromIntegral port} + liftIO $ Redis.runRedis conn $ do + forM metadataResults $ \(reddisId, metadataValidationResult) -> do + _ <- Redis.set (Text.encodeUtf8 reddisId) (toStrict $ encode metadataValidationResult) + return () + return () + +fetchMetadataValidationResult :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m, FromJSON a) + => Text + -> Text + -> m (Maybe (MetadataValidationResult a)) +fetchMetadataValidationResult url hash = do + conn <- liftIO $ Redis.checkedConnect Redis.defaultConnectInfo + result <- liftIO $ Redis.runRedis conn $ Redis.get (Text.encodeUtf8 $ url<>"#"<>hash) + case result of + Left _ -> return Nothing + Right (Just x) -> case decode $ fromStrict x of + Nothing -> return Nothing + Just x -> return $ Just x + Right Nothing -> return Nothing -validateMetadata - :: (Has VVAConfig r, Has Manager r, MonadReader r m, MonadIO m, MonadError AppError m) +validateMetadata :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) => Text -> Text -> Maybe Text @@ -55,12 +143,25 @@ validateMetadata url hash standard = do Nothing -> throwError $ InternalError "Failed to validate metadata" Just x -> return $ Right x + getProposalMetadataValidationResult :: (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) => Text -> Text -> m (MetadataValidationResult ProposalMetadata) getProposalMetadataValidationResult url hash = do + result <- fetchMetadataValidationResult url hash + case result of + Just x -> return x + Nothing -> getProposalMetadataValidationResult' url hash + + +getProposalMetadataValidationResult' :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) => + Text -> + Text -> + m (MetadataValidationResult ProposalMetadata) +getProposalMetadataValidationResult' url hash = do result <- validateMetadata url hash (Just "CIP108") case result of Left e -> return $ MetadataValidationResult False (Just e) Nothing @@ -83,14 +184,24 @@ getProposalMetadataValidationResult url hash = do ProposalMetadata <$> abstract <*> motivation <*> rationale <*> title <*> references return $ MetadataValidationResult valid status proposalMetadata - - getDRepMetadataValidationResult :: (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) => Text -> Text -> m (MetadataValidationResult DRepMetadata) getDRepMetadataValidationResult url hash = do + result <- fetchMetadataValidationResult url hash + case result of + Just x -> return x + Nothing -> getDRepMetadataValidationResult' url hash + + +getDRepMetadataValidationResult' :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) => + Text -> + Text -> + m (MetadataValidationResult DRepMetadata) +getDRepMetadataValidationResult' url hash = do result <- validateMetadata url hash (Just "CIPQQQ") case result of Left e -> return $ MetadataValidationResult False (Just e) Nothing diff --git a/govtool/backend/src/VVA/Types.hs b/govtool/backend/src/VVA/Types.hs index 701bdda87..ef8e97303 100644 --- a/govtool/backend/src/VVA/Types.hs +++ b/govtool/backend/src/VVA/Types.hs @@ -3,16 +3,22 @@ {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE DeriveGeneric #-} module VVA.Types where +import Data.Aeson.TH (deriveJSON) +import VVA.API.Utils (jsonOptions) +import GHC.Generics (Generic) import Control.Exception import Control.Monad.Except (MonadError) import Control.Monad.Fail (MonadFail) import Control.Monad.IO.Class (MonadIO) import Control.Monad.Reader (MonadReader) -import Data.Aeson (Value) +import Data.Aeson (Value, ToJSON, FromJSON) import qualified Data.Cache as Cache import Data.Has import Data.Pool (Pool) @@ -26,38 +32,6 @@ import VVA.Config import Network.HTTP.Client (Manager) import Control.Concurrent.QSem -type App m = (MonadReader AppEnv m, MonadIO m, MonadFail m, MonadError AppError m) - -data AppEnv - = AppEnv - { vvaConfig :: VVAConfig - , vvaCache :: CacheEnv - , vvaConnectionPool :: Pool Connection - , vvaTlsManager :: Manager - , vvaMetadataQSem :: QSem - } - -instance Has VVAConfig AppEnv where - getter AppEnv {vvaConfig} = vvaConfig - modifier f a@AppEnv {vvaConfig} = a {vvaConfig = f vvaConfig} - -instance Has CacheEnv AppEnv where - getter AppEnv {vvaCache} = vvaCache - modifier f a@AppEnv {vvaCache} = a {vvaCache = f vvaCache} - -instance Has (Pool Connection) AppEnv where - getter AppEnv {vvaConnectionPool} = vvaConnectionPool - modifier f a@AppEnv {vvaConnectionPool} = a {vvaConnectionPool = f vvaConnectionPool} - -instance Has Manager AppEnv where - getter AppEnv {vvaTlsManager} = vvaTlsManager - modifier f a@AppEnv {vvaTlsManager} = a {vvaTlsManager = f vvaTlsManager} - -instance Has QSem AppEnv where - getter AppEnv {vvaMetadataQSem} = vvaMetadataQSem - modifier f a@AppEnv {vvaMetadataQSem} = a {vvaMetadataQSem = f vvaMetadataQSem} - - data AppError = ValidationError Text | NotFoundError Text @@ -148,7 +122,10 @@ data ProposalMetadata = , proposalMetadataRationale :: Text , proposalMetadataTitle :: Text , proposalMetadataReferences :: [Text] - } deriving (Show) + } deriving (Show, Generic) + +deriveJSON (jsonOptions "proposalMetadata") ''ProposalMetadata + data DRepMetadata = DRepMetadata @@ -156,7 +133,10 @@ data DRepMetadata = , dRepMetadataDRepName :: Text , dRepMetadataEmail :: Text , dRepMetadataReferences :: [Text] - } deriving (Show) + } deriving (Show, Generic) + +deriveJSON (jsonOptions "dRepMetadata") ''DRepMetadata + data MetadataValidationResult a = MetadataValidationResult @@ -165,6 +145,7 @@ data MetadataValidationResult a = , metadataValidationResultMetadata :: Maybe a } deriving (Show) +deriveJSON (jsonOptions "metadataValidationResult") ''MetadataValidationResult @@ -213,3 +194,42 @@ data MetadataValidationStatus | UrlNotFound +data VotingAnchor + = VotingAnchor + { votingAnchorId :: Integer + , votingAnchorUrl :: Text + , votingAnchorHash :: Text + , votingAnchorType :: Text + } + + +type App m = (MonadReader AppEnv m, MonadIO m, MonadFail m, MonadError AppError m) + +data AppEnv + = AppEnv + { vvaConfig :: VVAConfig + , vvaCache :: CacheEnv + , vvaConnectionPool :: Pool Connection + , vvaTlsManager :: Manager + , vvaMetadataQSem :: QSem + } + +instance Has VVAConfig AppEnv where + getter AppEnv {vvaConfig} = vvaConfig + modifier f a@AppEnv {vvaConfig} = a {vvaConfig = f vvaConfig} + +instance Has CacheEnv AppEnv where + getter AppEnv {vvaCache} = vvaCache + modifier f a@AppEnv {vvaCache} = a {vvaCache = f vvaCache} + +instance Has (Pool Connection) AppEnv where + getter AppEnv {vvaConnectionPool} = vvaConnectionPool + modifier f a@AppEnv {vvaConnectionPool} = a {vvaConnectionPool = f vvaConnectionPool} + +instance Has Manager AppEnv where + getter AppEnv {vvaTlsManager} = vvaTlsManager + modifier f a@AppEnv {vvaTlsManager} = a {vvaTlsManager = f vvaTlsManager} + +instance Has QSem AppEnv where + getter AppEnv {vvaMetadataQSem} = vvaMetadataQSem + modifier f a@AppEnv {vvaMetadataQSem} = a {vvaMetadataQSem = f vvaMetadataQSem} diff --git a/govtool/backend/vva-be.cabal b/govtool/backend/vva-be.cabal index a69b43acb..0a5ef69d6 100644 --- a/govtool/backend/vva-be.cabal +++ b/govtool/backend/vva-be.cabal @@ -33,6 +33,7 @@ extra-source-files: sql/get-transaction-status.sql sql/get-stake-key-voting-power.sql sql/get-network-metrics.sql + sql/get-voting-anchors.sql executable vva-be main-is: Main.hs @@ -103,6 +104,7 @@ library , http-client-tls , vector , async + , hedis exposed-modules: VVA.Config , VVA.CommandLine diff --git a/scripts/govtool/config/templates/backend-config.json.tpl b/scripts/govtool/config/templates/backend-config.json.tpl index 9f57cbbc2..8b544ffe2 100644 --- a/scripts/govtool/config/templates/backend-config.json.tpl +++ b/scripts/govtool/config/templates/backend-config.json.tpl @@ -13,5 +13,7 @@ "sentryenv": "", "metadatavalidationhost": "http://metadata-validation", "metadatavalidationport": "3000", - "metadatavalidationmaxconcurrentrequests": 10 + "metadatavalidationmaxconcurrentrequests": 10, + "redishost": "localhost", + "redisport": 6379 }