From f3bd683586d82a381d2ab78cb9a0ba1489dd8e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sza=C5=82owski?= Date: Mon, 1 Jul 2024 13:44:49 +0200 Subject: [PATCH] feat(#1234): add reddis support for storing metadata validation results --- .github/workflows/build-and-deploy-beta.yml | 1 + .github/workflows/build-and-deploy-dev.yml | 1 + .../workflows/build-and-deploy-staging.yml | 1 + .github/workflows/build-and-deploy-test.yml | 1 + CHANGELOG.md | 2 + govtool/backend/app/Main.hs | 9 ++ govtool/backend/example-config.json | 7 +- govtool/backend/sql/get-voting-anchors.sql | 3 + govtool/backend/src/VVA/Config.hs | 50 ++++++- govtool/backend/src/VVA/Metadata.hs | 132 ++++++++++++++++-- govtool/backend/src/VVA/Types.hs | 90 +++++++----- govtool/backend/vva-be.cabal | 2 + scripts/govtool/config.mk | 1 + .../config/templates/backend-config.json.tpl | 7 +- .../config/templates/docker-compose.yml.tpl | 10 ++ 15 files changed, 270 insertions(+), 47 deletions(-) create mode 100644 govtool/backend/sql/get-voting-anchors.sql diff --git a/.github/workflows/build-and-deploy-beta.yml b/.github/workflows/build-and-deploy-beta.yml index 6af69324b..58c6cc095 100644 --- a/.github/workflows/build-and-deploy-beta.yml +++ b/.github/workflows/build-and-deploy-beta.yml @@ -57,6 +57,7 @@ jobs: TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com" USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }} IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{ inputs.isProposalDiscussionForumEnabled == 'enabled' }} + REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/build-and-deploy-dev.yml b/.github/workflows/build-and-deploy-dev.yml index 7755dc10f..8296d896c 100644 --- a/.github/workflows/build-and-deploy-dev.yml +++ b/.github/workflows/build-and-deploy-dev.yml @@ -57,6 +57,7 @@ jobs: TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com" USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }} IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{ inputs.isProposalDiscussionForumEnabled == 'enabled' }} + REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/build-and-deploy-staging.yml b/.github/workflows/build-and-deploy-staging.yml index 7534040d3..49d730b65 100644 --- a/.github/workflows/build-and-deploy-staging.yml +++ b/.github/workflows/build-and-deploy-staging.yml @@ -60,6 +60,7 @@ jobs: TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com" USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }} IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{github.event_name == 'push' && 'false' || inputs.isProposalDiscussionForumEnabled == 'enabled'}} + REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/build-and-deploy-test.yml b/.github/workflows/build-and-deploy-test.yml index b2c99fd83..a7b146ba7 100644 --- a/.github/workflows/build-and-deploy-test.yml +++ b/.github/workflows/build-and-deploy-test.yml @@ -60,6 +60,7 @@ jobs: TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com" USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }} IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{github.event_name == 'push' && 'false' || inputs.isProposalDiscussionForumEnabled == 'enabled'}} + REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 20195f18c..a34eacced 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ changes. ### Added +- added separate async process that fetches new voting_anchors, validates their metadata using metadata-validation service, and then stores it in Redis database [Issue 1234](https://github.com/IntersectMBO/govtool/issues/1234) - added `bio` `dRepName` `email` `references` `metadataValid` and `metadataStatus` fields to `drep/list` - added `metadatavalidationmaxconcurrentrequests` field to the backend config - added `metadata/validate` endpoint [Issue 876](https://github.com/IntersectMBO/govtool/issues/876) @@ -111,6 +112,7 @@ changes. ### Changed +- `redis` config fields changed [Issue 1234](https://github.com/IntersectMBO/govtool/issues/1234) - `proposal.about` changed to `proposal.abstract` - `drep/info` now returns 4 different tx hashes instead of one latest tx hash [Issue 688](https://github.com/IntersectMBO/govtool/issues/688) - `proposal/list` allows user to search by tx hash [Issue 603](https://github.com/IntersectMBO/govtool/issues/603) diff --git a/govtool/backend/app/Main.hs b/govtool/backend/app/Main.hs index a9f3cc65b..cc08afdd3 100644 --- a/govtool/backend/app/Main.hs +++ b/govtool/backend/app/Main.hs @@ -8,6 +8,7 @@ module Main where +import Control.Concurrent (forkIO) import Control.Concurrent.QSem (newQSem) import Control.Exception (Exception, SomeException, @@ -70,6 +71,7 @@ import VVA.API import VVA.API.Types import VVA.CommandLine import VVA.Config +import VVA.Metadata (startFetchProcess) import VVA.Types (AppEnv (..), AppError (CriticalError, NotFoundError, ValidationError, InternalError), CacheEnv (..)) @@ -136,6 +138,13 @@ startApp vvaConfig = do vvaTlsManager <- newManager tlsManagerSettings qsem <- newQSem (metadataValidationMaxConcurrentRequests vvaConfig) let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager, vvaMetadataQSem=qsem} + + _ <- forkIO $ do + result <- runReaderT (runExceptT startFetchProcess) appEnv + case result of + Left e -> throw e + Right _ -> return () + server' <- mkVVAServer appEnv runSettings settings server' diff --git a/govtool/backend/example-config.json b/govtool/backend/example-config.json index dd99856ae..8456bd144 100644 --- a/govtool/backend/example-config.json +++ b/govtool/backend/example-config.json @@ -14,5 +14,10 @@ "metadatavalidationenabled": true, "metadatavalidationhost": "localhost", "metadatavalidationport": 3001, - "metadatavalidationmaxconcurrentrequests": 10 + "metadatavalidationmaxconcurrentrequests": 10, + "redisconfig" : { + "host" : "localhost", + "port" : 8094, + "password": null + } } diff --git a/govtool/backend/sql/get-voting-anchors.sql b/govtool/backend/sql/get-voting-anchors.sql new file mode 100644 index 000000000..cedbc26de --- /dev/null +++ b/govtool/backend/sql/get-voting-anchors.sql @@ -0,0 +1,3 @@ +select id, url, encode(data_hash, 'hex'), type::text +from voting_anchor +where voting_anchor.id > ? \ No newline at end of file diff --git a/govtool/backend/src/VVA/Config.hs b/govtool/backend/src/VVA/Config.hs index 4d82ae354..44c6e9586 100644 --- a/govtool/backend/src/VVA/Config.hs +++ b/govtool/backend/src/VVA/Config.hs @@ -23,6 +23,9 @@ module VVA.Config , getDbSyncConnectionString , getServerHost , getServerPort + , getRedisHost + , getRedisPort + , getRedisPassword , vvaConfigToText , getMetadataValidationEnabled , getMetadataValidationHost @@ -67,6 +70,14 @@ data DBConfig instance DefaultConfig DBConfig where configDef = DBConfig "localhost" "cexplorer" "postgres" "test" 9903 +data RedisInternalConfig + = RedisInternalConfig + { redisInternalConfigHost :: Text + , redisInternalConfigPort :: Int + , redisInternalConfigPassword :: Maybe Text + } + deriving (FromConfig, Generic, Show) + -- | Internal, backend-dependent representation of configuration for DEX. This -- data type should not be exported from this module. data VVAConfigInternal @@ -91,6 +102,8 @@ data VVAConfigInternal , vVAConfigInternalMetadataValidationPort :: Int -- | Maximum number of concurrent metadata requests , vVAConfigInternalMetadataValidationMaxConcurrentRequests :: Int + -- | Redis config + , vVAConfigInternalRedisConfig :: RedisInternalConfig } deriving (FromConfig, Generic, Show) @@ -106,9 +119,18 @@ instance DefaultConfig VVAConfigInternal where vVAConfigInternalMetadataValidationEnabled = True, vVAConfigInternalMetadataValidationHost = "localhost", vVAConfigInternalMetadataValidationPort = 3001, - vVAConfigInternalMetadataValidationMaxConcurrentRequests = 10 + vVAConfigInternalMetadataValidationMaxConcurrentRequests = 10, + vVAConfigInternalRedisConfig = RedisInternalConfig "localhost" 6379 Nothing } +data RedisConfig + = RedisConfig + { redisHost :: Text + , redisPort :: Int + , redisPassword :: Maybe Text + } + deriving (Generic, Show, ToJSON) + -- | DEX configuration. data VVAConfig = VVAConfig @@ -132,6 +154,8 @@ data VVAConfig , metadataValidationPort :: Int -- | Maximum number of concurrent metadata requests , metadataValidationMaxConcurrentRequests :: Int + -- | Redis config + , redisConfig :: RedisConfig } deriving (Generic, Show, ToJSON) @@ -176,7 +200,12 @@ convertConfig VVAConfigInternal {..} = metadataValidationEnabled = vVAConfigInternalMetadataValidationEnabled, metadataValidationHost = vVAConfigInternalMetadataValidationHost, metadataValidationPort = vVAConfigInternalMetadataValidationPort, - metadataValidationMaxConcurrentRequests = vVAConfigInternalMetadataValidationMaxConcurrentRequests + metadataValidationMaxConcurrentRequests = vVAConfigInternalMetadataValidationMaxConcurrentRequests, + redisConfig = RedisConfig + { redisHost = redisInternalConfigHost $ vVAConfigInternalRedisConfig, + redisPort = redisInternalConfigPort $ vVAConfigInternalRedisConfig, + redisPassword = redisInternalConfigPassword $ vVAConfigInternalRedisConfig + } } -- | Load configuration from a file specified on the command line. Load from @@ -215,6 +244,23 @@ getServerHost :: m Text getServerHost = asks (serverHost . getter) +-- | Access redis host +getRedisHost :: + (Has VVAConfig r, MonadReader r m) => + m Text +getRedisHost = asks (redisHost . redisConfig . getter) + +-- | Access redis port +getRedisPort :: + (Has VVAConfig r, MonadReader r m) => + m Int +getRedisPort = asks (redisPort . redisConfig . getter) + +getRedisPassword :: + (Has VVAConfig r, MonadReader r m) => + m (Maybe Text) +getRedisPassword = asks (redisPassword . redisConfig . getter) + -- | Access MetadataValidationService enabled getMetadataValidationEnabled :: (Has VVAConfig r, MonadReader r m) => diff --git a/govtool/backend/src/VVA/Metadata.hs b/govtool/backend/src/VVA/Metadata.hs index 2aa46d684..8dba8e8a6 100644 --- a/govtool/backend/src/VVA/Metadata.hs +++ b/govtool/backend/src/VVA/Metadata.hs @@ -1,10 +1,15 @@ {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TupleSections #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE TypeApplications #-} module VVA.Metadata where -import Prelude hiding (lookup) +import qualified Database.Redis as Redis +import Control.Concurrent (threadDelay) +import Prelude hiding (lookup) import Control.Monad.Except (MonadError, throwError) import Control.Monad.Reader import Control.Exception (try, Exception) @@ -12,16 +17,16 @@ import Control.Exception (try, Exception) import Data.Typeable (Typeable) import Data.Vector (toList) import Data.Aeson.KeyMap (lookup) -import Data.Aeson (Value(..), decode, encode, object, (.=)) +import Data.Aeson (FromJSON, ToJSON, Value(..), decode, encode, object, (.=)) import Data.Maybe (fromJust) -import Data.ByteString (ByteString) +import Data.ByteString (ByteString, fromStrict, toStrict) import Data.FileEmbed (embedFile) import Data.Has (Has, getter) import Data.String (fromString) import Data.Text (Text, unpack, pack) import qualified Data.Text.Encoding as Text import Data.Time.Clock - +import Data.List (partition) import qualified Database.PostgreSQL.Simple as SQL import VVA.Config @@ -30,9 +35,97 @@ import VVA.Types import Network.HTTP.Client import Network.HTTP.Client.TLS import Data.Aeson (encode, object, (.=)) +import Data.Scientific + +sqlFrom :: ByteString -> SQL.Query +sqlFrom bs = fromString $ unpack $ Text.decodeUtf8 bs + +getVotingAnchorsSql :: SQL.Query +getVotingAnchorsSql = sqlFrom $(embedFile "sql/get-voting-anchors.sql") + +getNewVotingAnchors :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) + => Integer + -> m [VotingAnchor] +getNewVotingAnchors lastId = do + anchors <- withPool $ \conn -> do + liftIO $ SQL.query conn getVotingAnchorsSql $ SQL.Only (lastId :: Integer) + return $ map (\(id, url, hash, type') -> VotingAnchor (floor @Scientific id) url hash type') anchors + +startFetchProcess :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) + => m () +startFetchProcess = go 0 + where + go latestKnownId = do + liftIO $ putStrLn "Fetching metadata..." + + anchors <- getNewVotingAnchors latestKnownId + if null anchors + then do + liftIO $ threadDelay (20 * 1000000) + go latestKnownId + else do + (drepMetadata, proposalMetadata) <- processAnchors anchors + storeMetadata drepMetadata + storeMetadata proposalMetadata + + let newId = maximum $ map votingAnchorId anchors + + liftIO $ putStrLn ("Stored " <> show (length anchors) <> " voting anchors") + + liftIO $ threadDelay (20 * 1000000) + go newId + + +processAnchors :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) + => [VotingAnchor] + -> m ( [(Text, MetadataValidationResult DRepMetadata)] + , [(Text, MetadataValidationResult ProposalMetadata)] + ) +processAnchors anchors = do + let (drepAnchors, proposalAnchors) = partition ((== "other") . votingAnchorType) anchors + drepMetadata <- mapM (\(VotingAnchor id url hash _) -> (url<>"#"<>hash, ) <$> getDRepMetadataValidationResult' url hash) drepAnchors + proposalMetadata <- mapM (\(VotingAnchor id url hash _) -> (url<>"#"<>hash, ) <$> getProposalMetadataValidationResult' url hash) proposalAnchors + return (drepMetadata, proposalMetadata) + +storeMetadata :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m, ToJSON a) + => [(Text, MetadataValidationResult a)] + -> m () +storeMetadata metadataResults = do + port <- getRedisPort + host <- getRedisHost + pass <- fmap Text.encodeUtf8 <$> getRedisPassword + conn <- liftIO $ Redis.checkedConnect $ Redis.defaultConnectInfo + { Redis.connectHost = unpack host + , Redis.connectPort = Redis.PortNumber $ fromIntegral port + , Redis.connectAuth = pass + } + liftIO $ Redis.runRedis conn $ do + forM metadataResults $ \(reddisId, metadataValidationResult) -> do + _ <- Redis.set (Text.encodeUtf8 reddisId) (toStrict $ encode metadataValidationResult) + return () + return () + +fetchMetadataValidationResult :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m, FromJSON a) + => Text + -> Text + -> m (Maybe (MetadataValidationResult a)) +fetchMetadataValidationResult url hash = do + conn <- liftIO $ Redis.checkedConnect Redis.defaultConnectInfo + result <- liftIO $ Redis.runRedis conn $ Redis.get (Text.encodeUtf8 $ url<>"#"<>hash) + case result of + Left _ -> return Nothing + Right (Just x) -> case decode $ fromStrict x of + Nothing -> return Nothing + Just x -> return $ Just x + Right Nothing -> return Nothing -validateMetadata - :: (Has VVAConfig r, Has Manager r, MonadReader r m, MonadIO m, MonadError AppError m) +validateMetadata :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) => Text -> Text -> Maybe Text @@ -59,12 +152,25 @@ validateMetadata url hash standard = do Just x -> return $ Right x False -> return $ Right "" + getProposalMetadataValidationResult :: (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) => Text -> Text -> m (MetadataValidationResult ProposalMetadata) getProposalMetadataValidationResult url hash = do + result <- fetchMetadataValidationResult url hash + case result of + Just x -> return x + Nothing -> getProposalMetadataValidationResult' url hash + + +getProposalMetadataValidationResult' :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) => + Text -> + Text -> + m (MetadataValidationResult ProposalMetadata) +getProposalMetadataValidationResult' url hash = do result <- validateMetadata url hash (Just "CIP108") case result of Left e -> return $ MetadataValidationResult False (Just e) Nothing @@ -88,14 +194,24 @@ getProposalMetadataValidationResult url hash = do ProposalMetadata <$> abstract <*> motivation <*> rationale <*> title <*> references return $ MetadataValidationResult valid status proposalMetadata - - getDRepMetadataValidationResult :: (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) => Text -> Text -> m (MetadataValidationResult DRepMetadata) getDRepMetadataValidationResult url hash = do + result <- fetchMetadataValidationResult url hash + case result of + Just x -> return x + Nothing -> getDRepMetadataValidationResult' url hash + + +getDRepMetadataValidationResult' :: + (Has ConnectionPool r, Has Manager r, Has VVAConfig r, MonadReader r m, MonadIO m, MonadFail m, MonadError AppError m) => + Text -> + Text -> + m (MetadataValidationResult DRepMetadata) +getDRepMetadataValidationResult' url hash = do result <- validateMetadata url hash (Just "CIPQQQ") case result of Left e -> return $ MetadataValidationResult False (Just e) Nothing diff --git a/govtool/backend/src/VVA/Types.hs b/govtool/backend/src/VVA/Types.hs index aa6b1079a..dd1ebbf4f 100644 --- a/govtool/backend/src/VVA/Types.hs +++ b/govtool/backend/src/VVA/Types.hs @@ -3,16 +3,22 @@ {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE DeriveGeneric #-} module VVA.Types where +import Data.Aeson.TH (deriveJSON) +import VVA.API.Utils (jsonOptions) +import GHC.Generics (Generic) import Control.Exception import Control.Monad.Except (MonadError) import Control.Monad.Fail (MonadFail) import Control.Monad.IO.Class (MonadIO) import Control.Monad.Reader (MonadReader) -import Data.Aeson (Value) +import Data.Aeson (Value, ToJSON, FromJSON) import qualified Data.Cache as Cache import Data.Has import Data.Pool (Pool) @@ -26,38 +32,6 @@ import VVA.Config import Network.HTTP.Client (Manager) import Control.Concurrent.QSem -type App m = (MonadReader AppEnv m, MonadIO m, MonadFail m, MonadError AppError m) - -data AppEnv - = AppEnv - { vvaConfig :: VVAConfig - , vvaCache :: CacheEnv - , vvaConnectionPool :: Pool Connection - , vvaTlsManager :: Manager - , vvaMetadataQSem :: QSem - } - -instance Has VVAConfig AppEnv where - getter AppEnv {vvaConfig} = vvaConfig - modifier f a@AppEnv {vvaConfig} = a {vvaConfig = f vvaConfig} - -instance Has CacheEnv AppEnv where - getter AppEnv {vvaCache} = vvaCache - modifier f a@AppEnv {vvaCache} = a {vvaCache = f vvaCache} - -instance Has (Pool Connection) AppEnv where - getter AppEnv {vvaConnectionPool} = vvaConnectionPool - modifier f a@AppEnv {vvaConnectionPool} = a {vvaConnectionPool = f vvaConnectionPool} - -instance Has Manager AppEnv where - getter AppEnv {vvaTlsManager} = vvaTlsManager - modifier f a@AppEnv {vvaTlsManager} = a {vvaTlsManager = f vvaTlsManager} - -instance Has QSem AppEnv where - getter AppEnv {vvaMetadataQSem} = vvaMetadataQSem - modifier f a@AppEnv {vvaMetadataQSem} = a {vvaMetadataQSem = f vvaMetadataQSem} - - data AppError = ValidationError Text | NotFoundError Text @@ -148,7 +122,10 @@ data ProposalMetadata = , proposalMetadataRationale :: Text , proposalMetadataTitle :: Text , proposalMetadataReferences :: [Text] - } deriving (Show) + } deriving (Show, Generic) + +deriveJSON (jsonOptions "proposalMetadata") ''ProposalMetadata + data DRepMetadata = DRepMetadata @@ -156,7 +133,10 @@ data DRepMetadata = , dRepMetadataDRepName :: Text , dRepMetadataEmail :: Text , dRepMetadataReferences :: [Text] - } deriving (Show) + } deriving (Show, Generic) + +deriveJSON (jsonOptions "dRepMetadata") ''DRepMetadata + data MetadataValidationResult a = MetadataValidationResult @@ -165,6 +145,7 @@ data MetadataValidationResult a = , metadataValidationResultMetadata :: Maybe a } deriving (Show) +deriveJSON (jsonOptions "metadataValidationResult") ''MetadataValidationResult @@ -213,3 +194,42 @@ data MetadataValidationStatus | UrlNotFound +data VotingAnchor + = VotingAnchor + { votingAnchorId :: Integer + , votingAnchorUrl :: Text + , votingAnchorHash :: Text + , votingAnchorType :: Text + } + + +type App m = (MonadReader AppEnv m, MonadIO m, MonadFail m, MonadError AppError m) + +data AppEnv + = AppEnv + { vvaConfig :: VVAConfig + , vvaCache :: CacheEnv + , vvaConnectionPool :: Pool Connection + , vvaTlsManager :: Manager + , vvaMetadataQSem :: QSem + } + +instance Has VVAConfig AppEnv where + getter AppEnv {vvaConfig} = vvaConfig + modifier f a@AppEnv {vvaConfig} = a {vvaConfig = f vvaConfig} + +instance Has CacheEnv AppEnv where + getter AppEnv {vvaCache} = vvaCache + modifier f a@AppEnv {vvaCache} = a {vvaCache = f vvaCache} + +instance Has (Pool Connection) AppEnv where + getter AppEnv {vvaConnectionPool} = vvaConnectionPool + modifier f a@AppEnv {vvaConnectionPool} = a {vvaConnectionPool = f vvaConnectionPool} + +instance Has Manager AppEnv where + getter AppEnv {vvaTlsManager} = vvaTlsManager + modifier f a@AppEnv {vvaTlsManager} = a {vvaTlsManager = f vvaTlsManager} + +instance Has QSem AppEnv where + getter AppEnv {vvaMetadataQSem} = vvaMetadataQSem + modifier f a@AppEnv {vvaMetadataQSem} = a {vvaMetadataQSem = f vvaMetadataQSem} diff --git a/govtool/backend/vva-be.cabal b/govtool/backend/vva-be.cabal index a69b43acb..0a5ef69d6 100644 --- a/govtool/backend/vva-be.cabal +++ b/govtool/backend/vva-be.cabal @@ -33,6 +33,7 @@ extra-source-files: sql/get-transaction-status.sql sql/get-stake-key-voting-power.sql sql/get-network-metrics.sql + sql/get-voting-anchors.sql executable vva-be main-is: Main.hs @@ -103,6 +104,7 @@ library , http-client-tls , vector , async + , hedis exposed-modules: VVA.Config , VVA.CommandLine diff --git a/scripts/govtool/config.mk b/scripts/govtool/config.mk index f7481188d..b3efc2a3c 100644 --- a/scripts/govtool/config.mk +++ b/scripts/govtool/config.mk @@ -81,6 +81,7 @@ $(target_config_dir)/backend-config.json: $(config_dir)/templates/backend-config -e "s||$${DBSYNC_POSTGRES_PASSWORD}|" \ -e "s||$${SENTRY_DSN_BACKEND}|" \ -e "s||$(env)|" \ + -e "s||$${REDIS_PASSWORD}|" \ $< > $@ $(target_config_dir)/%.yml: $(template_config_dir)/%.yml $(target_config_dir)/ diff --git a/scripts/govtool/config/templates/backend-config.json.tpl b/scripts/govtool/config/templates/backend-config.json.tpl index 9f57cbbc2..bda65e26d 100644 --- a/scripts/govtool/config/templates/backend-config.json.tpl +++ b/scripts/govtool/config/templates/backend-config.json.tpl @@ -13,5 +13,10 @@ "sentryenv": "", "metadatavalidationhost": "http://metadata-validation", "metadatavalidationport": "3000", - "metadatavalidationmaxconcurrentrequests": 10 + "metadatavalidationmaxconcurrentrequests": 10, + "redisconfig" : { + "host" : "http://redis", + "port" : 8094, + "password": "" + } } diff --git a/scripts/govtool/config/templates/docker-compose.yml.tpl b/scripts/govtool/config/templates/docker-compose.yml.tpl index b73b22e02..25e5b5825 100644 --- a/scripts/govtool/config/templates/docker-compose.yml.tpl +++ b/scripts/govtool/config/templates/docker-compose.yml.tpl @@ -228,6 +228,15 @@ services: - "traefik.http.routers.to-analytics-dashboard.entrypoints=websecure" - "traefik.http.routers.to-analytics-dashboard.tls.certresolver=myresolver" - "traefik.http.services.analytics-dashboard.loadbalancer.server.port=3000" + + redis: + image: docker.io/bitnami/redis:7.2 + ports: + - '8094:8094' + environment: + - REDIS_PASSWORD=${REDIS_PASSWORD} + volumes: + - 'redis_data:/bitnami/redis/data' backend: image: /backend:${BACKEND_TAG} @@ -302,3 +311,4 @@ volumes: node-db: node-ipc: loki-data: + redis_data: \ No newline at end of file