From 3d8016041595d99f6ff35786e27f52c1ccb9905f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sza=C5=82owski?= Date: Mon, 23 Dec 2024 15:22:43 +0100 Subject: [PATCH] chore: optimize the db connection and drep list sql --- govtool/backend/app/Main.hs | 18 +++++-- govtool/backend/sql/list-dreps.sql | 78 +++++++++++++++++------------- 2 files changed, 59 insertions(+), 37 deletions(-) diff --git a/govtool/backend/app/Main.hs b/govtool/backend/app/Main.hs index 02f217e2..9accf102 100644 --- a/govtool/backend/app/Main.hs +++ b/govtool/backend/app/Main.hs @@ -26,7 +26,7 @@ import Data.Monoid (mempty) import Data.OpenApi (OpenApi, Server (Server), _openApiServers, _serverDescription, _serverUrl, _serverVariables, servers) -import Data.Pool (createPool) +import Data.Pool (createPool, Pool) import Data.Proxy import Data.String (fromString) import Data.String.Conversions (cs) @@ -36,7 +36,7 @@ import qualified Data.Text.IO as Text import qualified Data.Text.Lazy as LazyText import qualified Data.Text.Lazy.Encoding as LazyText -import Database.PostgreSQL.Simple (close, connectPostgreSQL) +import Database.PostgreSQL.Simple (close, connectPostgreSQL, Connection) import Network.Wai import Network.Wai.Handler.Warp @@ -65,6 +65,15 @@ import VVA.Types (AppEnv (..), AppError (CriticalError, InternalError, NotFoundError, ValidationError), CacheEnv (..)) +-- Function to create a connection pool with optimized settings +createOptimizedConnectionPool :: BS.ByteString -> IO (Pool Connection) +createOptimizedConnectionPool connectionString = createPool + (connectPostgreSQL connectionString) -- Connection creation function + close -- Connection destruction function + 1 -- Number of stripes (sub-pools) + 60 -- Idle timeout (seconds) + 50 -- Maximum number of connections per stripe + proxyAPI :: Proxy (VVAApi :<|> SwaggerAPI) proxyAPI = Proxy @@ -125,7 +134,10 @@ startApp vvaConfig sentryService = do , dRepListCache , networkMetricsCache } - connectionPool <- createPool (connectPostgreSQL (encodeUtf8 (dbSyncConnectionString $ getter vvaConfig))) close 1 10 60 + + let connectionString = encodeUtf8 (dbSyncConnectionString $ getter vvaConfig) + connectionPool <- createOptimizedConnectionPool connectionString + let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool } server' <- mkVVAServer appEnv runSettings settings server' diff --git a/govtool/backend/sql/list-dreps.sql b/govtool/backend/sql/list-dreps.sql index e6518a7c..f5680c31 100644 --- a/govtool/backend/sql/list-dreps.sql +++ b/govtool/backend/sql/list-dreps.sql @@ -1,6 +1,6 @@ WITH DRepDistr AS ( SELECT - *, + drep_distr.*, ROW_NUMBER() OVER (PARTITION BY drep_hash.id ORDER BY drep_distr.epoch_no DESC) AS rn FROM drep_distr @@ -8,8 +8,8 @@ WITH DRepDistr AS ( ), DRepActivity AS ( SELECT - drep_activity AS drep_activity, - epoch_no AS epoch_no + drep_activity, + epoch_no FROM epoch_param WHERE @@ -26,7 +26,7 @@ SELECT encode(va.data_hash, 'hex'), dr_deposit.deposit, DRepDistr.amount, - (DRepActivity.epoch_no - Max(coalesce(block.epoch_no, block_first_register.epoch_no))) <= DRepActivity.drep_activity AS active, + (DRepActivity.epoch_no - COALESCE(block.epoch_no, block_first_register.epoch_no)) <= DRepActivity.drep_activity AS active, encode(dr_voting_anchor.tx_hash, 'hex') AS tx_hash, newestRegister.time AS last_register_time, COALESCE(latestDeposit.deposit, 0), @@ -50,17 +50,17 @@ FROM FROM drep_registration dr WHERE - dr.deposit IS NOT NULL) AS dr_deposit ON dr_deposit.drep_hash_id = dh.id - AND dr_deposit.rn = 1 - JOIN ( + dr.deposit IS NOT NULL + ) AS dr_deposit ON dr_deposit.drep_hash_id = dh.id AND dr_deposit.rn = 1 + JOIN ( SELECT dr.id, dr.drep_hash_id, dr.deposit, ROW_NUMBER() OVER (PARTITION BY dr.drep_hash_id ORDER BY dr.tx_id DESC) AS rn FROM - drep_registration dr) AS latestDeposit ON latestDeposit.drep_hash_id = dh.id - AND latestDeposit.rn = 1 + drep_registration dr + ) AS latestDeposit ON latestDeposit.drep_hash_id = dh.id AND latestDeposit.rn = 1 LEFT JOIN ( SELECT dr.id, @@ -70,9 +70,11 @@ FROM tx.hash AS tx_hash FROM drep_registration dr - JOIN tx ON tx.id = dr.tx_id) AS dr_voting_anchor ON dr_voting_anchor.drep_hash_id = dh.id - AND dr_voting_anchor.rn = 1 - LEFT JOIN ( + JOIN tx ON tx.id = dr.tx_id + WHERE + dr.deposit IS NOT NULL AND dr.deposit >= 0 + ) AS dr_voting_anchor ON dr_voting_anchor.drep_hash_id = dh.id AND dr_voting_anchor.rn = 1 + LEFT JOIN ( SELECT dr.id, dr.drep_hash_id, @@ -82,9 +84,7 @@ FROM FROM drep_registration dr JOIN tx ON tx.id = dr.tx_id - WHERE dr.deposit is not null - AND dr.deposit >= 0) AS dr_non_deregister_voting_anchor ON dr_non_deregister_voting_anchor.drep_hash_id = dh.id - AND dr_non_deregister_voting_anchor.rn = 1 + ) AS dr_non_deregister_voting_anchor ON dr_non_deregister_voting_anchor.drep_hash_id = dh.id AND dr_non_deregister_voting_anchor.rn = 1 LEFT JOIN ( SELECT dr.id, @@ -92,26 +92,34 @@ FROM dr.voting_anchor_id, ROW_NUMBER() OVER (PARTITION BY dr.drep_hash_id ORDER BY dr.tx_id DESC) AS rn FROM - drep_registration dr) AS second_to_newest_drep_registration ON second_to_newest_drep_registration.drep_hash_id = dh.id - AND second_to_newest_drep_registration.rn = 2 - LEFT JOIN DRepDistr ON DRepDistr.hash_id = dh.id - AND DRepDistr.rn = 1 + drep_registration dr + ) AS second_to_newest_drep_registration ON second_to_newest_drep_registration.drep_hash_id = dh.id AND second_to_newest_drep_registration.rn = 2 + LEFT JOIN DRepDistr ON DRepDistr.hash_id = dh.id AND DRepDistr.rn = 1 LEFT JOIN voting_anchor va ON va.id = dr_voting_anchor.voting_anchor_id LEFT JOIN voting_anchor non_deregister_voting_anchor ON non_deregister_voting_anchor.id = dr_non_deregister_voting_anchor.voting_anchor_id LEFT JOIN ( - SELECT fetch_error as message, voting_anchor_id - FROM off_chain_vote_fetch_error - WHERE fetch_time = ( - SELECT max(fetch_time) - FROM off_chain_vote_fetch_error) - GROUP BY fetch_error, voting_anchor_id + SELECT + fetch_error as message, + voting_anchor_id + FROM + off_chain_vote_fetch_error + WHERE + fetch_time = ( + SELECT + max(fetch_time) + FROM + off_chain_vote_fetch_error + ) + GROUP BY + fetch_error, + voting_anchor_id ) AS fetch_error ON fetch_error.voting_anchor_id = va.id LEFT JOIN off_chain_vote_data ON off_chain_vote_data.voting_anchor_id = va.id - LEFT JOIN off_chain_vote_drep_data on off_chain_vote_drep_data.off_chain_vote_data_id = off_chain_vote_data.id + LEFT JOIN off_chain_vote_drep_data ON off_chain_vote_drep_data.off_chain_vote_data_id = off_chain_vote_data.id CROSS JOIN DRepActivity - LEFT JOIN voting_procedure AS voting_procedure ON voting_procedure.drep_voter = dh.id - LEFT JOIN tx AS tx ON tx.id = voting_procedure.tx_id - LEFT JOIN block AS block ON block.id = tx.block_id + LEFT JOIN voting_procedure ON voting_procedure.drep_voter = dh.id + LEFT JOIN tx ON tx.id = voting_procedure.tx_id + LEFT JOIN block ON block.id = tx.block_id LEFT JOIN ( SELECT block.time, @@ -122,16 +130,16 @@ FROM JOIN tx ON tx.id = dr.tx_id JOIN block ON block.id = tx.block_id WHERE - NOT (dr.deposit < 0)) AS newestRegister ON newestRegister.drep_hash_id = dh.id - AND newestRegister.rn = 1 + NOT (dr.deposit < 0) + ) AS newestRegister ON newestRegister.drep_hash_id = dh.id AND newestRegister.rn = 1 LEFT JOIN ( SELECT dr.tx_id, dr.drep_hash_id, ROW_NUMBER() OVER (PARTITION BY dr.drep_hash_id ORDER BY dr.tx_id ASC) AS rn FROM - drep_registration dr) AS dr_first_register ON dr_first_register.drep_hash_id = dh.id - AND dr_first_register.rn = 1 + drep_registration dr + ) AS dr_first_register ON dr_first_register.drep_hash_id = dh.id AND dr_first_register.rn = 1 LEFT JOIN tx AS tx_first_register ON tx_first_register.id = dr_first_register.tx_id LEFT JOIN block AS block_first_register ON block_first_register.id = tx_first_register.block_id WHERE @@ -142,6 +150,8 @@ WHERE off_chain_vote_drep_data.given_name ILIKE ? ) GROUP BY + block_first_register.epoch_no, + block.epoch_no, dh.raw, second_to_newest_drep_registration.voting_anchor_id, dh.view, @@ -163,4 +173,4 @@ GROUP BY off_chain_vote_drep_data.motivations, off_chain_vote_drep_data.qualifications, off_chain_vote_drep_data.image_url, - off_chain_vote_drep_data.image_hash + off_chain_vote_drep_data.image_hash;