Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: optimize the db connection and drep list sql #2574

Merged
merged 1 commit into from
Dec 24, 2024
+59 −37
Merged

chore: optimize the db connection and drep list sql #2574

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions govtool/backend/app/Main.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import Data.Monoid (mempty)
import Data.OpenApi (OpenApi, Server (Server), _openApiServers,
_serverDescription, _serverUrl, _serverVariables,
servers)
import Data.Pool (createPool)
import Data.Pool (createPool, Pool)
import Data.Proxy
import Data.String (fromString)
import Data.String.Conversions (cs)
Expand All @@ -36,7 +36,7 @@ import qualified Data.Text.IO as Text
import qualified Data.Text.Lazy as LazyText
import qualified Data.Text.Lazy.Encoding as LazyText

import Database.PostgreSQL.Simple (close, connectPostgreSQL)
import Database.PostgreSQL.Simple (close, connectPostgreSQL, Connection)

import Network.Wai
import Network.Wai.Handler.Warp
Expand Down Expand Up @@ -65,6 +65,15 @@ import VVA.Types (AppEnv (..),
AppError (CriticalError, InternalError, NotFoundError, ValidationError),
CacheEnv (..))

-- Function to create a connection pool with optimized settings
createOptimizedConnectionPool :: BS.ByteString -> IO (Pool Connection)
createOptimizedConnectionPool connectionString = createPool
(connectPostgreSQL connectionString) -- Connection creation function
close -- Connection destruction function
1 -- Number of stripes (sub-pools)
60 -- Idle timeout (seconds)
50 -- Maximum number of connections per stripe

proxyAPI :: Proxy (VVAApi :<|> SwaggerAPI)
proxyAPI = Proxy

Expand Down Expand Up @@ -125,7 +134,10 @@ startApp vvaConfig sentryService = do
, dRepListCache
, networkMetricsCache
}
connectionPool <- createPool (connectPostgreSQL (encodeUtf8 (dbSyncConnectionString $ getter vvaConfig))) close 1 10 60

let connectionString = encodeUtf8 (dbSyncConnectionString $ getter vvaConfig)
connectionPool <- createOptimizedConnectionPool connectionString

let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool }
server' <- mkVVAServer appEnv
runSettings settings server'
Expand Down
78 changes: 44 additions & 34 deletions govtool/backend/sql/list-dreps.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
WITH DRepDistr AS (
SELECT
*,
drep_distr.*,
ROW_NUMBER() OVER (PARTITION BY drep_hash.id ORDER BY drep_distr.epoch_no DESC) AS rn
FROM
drep_distr
JOIN drep_hash ON drep_hash.id = drep_distr.hash_id
),
DRepActivity AS (
SELECT
drep_activity AS drep_activity,
epoch_no AS epoch_no
drep_activity,
epoch_no
FROM
epoch_param
WHERE
Expand All @@ -26,7 +26,7 @@ SELECT
encode(va.data_hash, 'hex'),
dr_deposit.deposit,
DRepDistr.amount,
(DRepActivity.epoch_no - Max(coalesce(block.epoch_no, block_first_register.epoch_no))) <= DRepActivity.drep_activity AS active,
(DRepActivity.epoch_no - COALESCE(block.epoch_no, block_first_register.epoch_no)) <= DRepActivity.drep_activity AS active,
encode(dr_voting_anchor.tx_hash, 'hex') AS tx_hash,
newestRegister.time AS last_register_time,
COALESCE(latestDeposit.deposit, 0),
Expand All @@ -50,17 +50,17 @@ FROM
FROM
drep_registration dr
WHERE
dr.deposit IS NOT NULL) AS dr_deposit ON dr_deposit.drep_hash_id = dh.id
AND dr_deposit.rn = 1
JOIN (
dr.deposit IS NOT NULL
) AS dr_deposit ON dr_deposit.drep_hash_id = dh.id AND dr_deposit.rn = 1
JOIN (
SELECT
dr.id,
dr.drep_hash_id,
dr.deposit,
ROW_NUMBER() OVER (PARTITION BY dr.drep_hash_id ORDER BY dr.tx_id DESC) AS rn
FROM
drep_registration dr) AS latestDeposit ON latestDeposit.drep_hash_id = dh.id
AND latestDeposit.rn = 1
drep_registration dr
) AS latestDeposit ON latestDeposit.drep_hash_id = dh.id AND latestDeposit.rn = 1
LEFT JOIN (
SELECT
dr.id,
Expand All @@ -70,9 +70,11 @@ FROM
tx.hash AS tx_hash
FROM
drep_registration dr
JOIN tx ON tx.id = dr.tx_id) AS dr_voting_anchor ON dr_voting_anchor.drep_hash_id = dh.id
AND dr_voting_anchor.rn = 1
LEFT JOIN (
JOIN tx ON tx.id = dr.tx_id
WHERE
dr.deposit IS NOT NULL AND dr.deposit >= 0
) AS dr_voting_anchor ON dr_voting_anchor.drep_hash_id = dh.id AND dr_voting_anchor.rn = 1
LEFT JOIN (
SELECT
dr.id,
dr.drep_hash_id,
Expand All @@ -82,36 +84,42 @@ FROM
FROM
drep_registration dr
JOIN tx ON tx.id = dr.tx_id
WHERE dr.deposit is not null
AND dr.deposit >= 0) AS dr_non_deregister_voting_anchor ON dr_non_deregister_voting_anchor.drep_hash_id = dh.id
AND dr_non_deregister_voting_anchor.rn = 1
) AS dr_non_deregister_voting_anchor ON dr_non_deregister_voting_anchor.drep_hash_id = dh.id AND dr_non_deregister_voting_anchor.rn = 1
LEFT JOIN (
SELECT
dr.id,
dr.drep_hash_id,
dr.voting_anchor_id,
ROW_NUMBER() OVER (PARTITION BY dr.drep_hash_id ORDER BY dr.tx_id DESC) AS rn
FROM
drep_registration dr) AS second_to_newest_drep_registration ON second_to_newest_drep_registration.drep_hash_id = dh.id
AND second_to_newest_drep_registration.rn = 2
LEFT JOIN DRepDistr ON DRepDistr.hash_id = dh.id
AND DRepDistr.rn = 1
drep_registration dr
) AS second_to_newest_drep_registration ON second_to_newest_drep_registration.drep_hash_id = dh.id AND second_to_newest_drep_registration.rn = 2
LEFT JOIN DRepDistr ON DRepDistr.hash_id = dh.id AND DRepDistr.rn = 1
LEFT JOIN voting_anchor va ON va.id = dr_voting_anchor.voting_anchor_id
LEFT JOIN voting_anchor non_deregister_voting_anchor ON non_deregister_voting_anchor.id = dr_non_deregister_voting_anchor.voting_anchor_id
LEFT JOIN (
SELECT fetch_error as message, voting_anchor_id
FROM off_chain_vote_fetch_error
WHERE fetch_time = (
SELECT max(fetch_time)
FROM off_chain_vote_fetch_error)
GROUP BY fetch_error, voting_anchor_id
SELECT
fetch_error as message,
voting_anchor_id
FROM
off_chain_vote_fetch_error
WHERE
fetch_time = (
SELECT
max(fetch_time)
FROM
off_chain_vote_fetch_error
)
GROUP BY
fetch_error,
voting_anchor_id
) AS fetch_error ON fetch_error.voting_anchor_id = va.id
LEFT JOIN off_chain_vote_data ON off_chain_vote_data.voting_anchor_id = va.id
LEFT JOIN off_chain_vote_drep_data on off_chain_vote_drep_data.off_chain_vote_data_id = off_chain_vote_data.id
LEFT JOIN off_chain_vote_drep_data ON off_chain_vote_drep_data.off_chain_vote_data_id = off_chain_vote_data.id
CROSS JOIN DRepActivity
LEFT JOIN voting_procedure AS voting_procedure ON voting_procedure.drep_voter = dh.id
LEFT JOIN tx AS tx ON tx.id = voting_procedure.tx_id
LEFT JOIN block AS block ON block.id = tx.block_id
LEFT JOIN voting_procedure ON voting_procedure.drep_voter = dh.id
LEFT JOIN tx ON tx.id = voting_procedure.tx_id
LEFT JOIN block ON block.id = tx.block_id
LEFT JOIN (
SELECT
block.time,
Expand All @@ -122,16 +130,16 @@ FROM
JOIN tx ON tx.id = dr.tx_id
JOIN block ON block.id = tx.block_id
WHERE
NOT (dr.deposit < 0)) AS newestRegister ON newestRegister.drep_hash_id = dh.id
AND newestRegister.rn = 1
NOT (dr.deposit < 0)
) AS newestRegister ON newestRegister.drep_hash_id = dh.id AND newestRegister.rn = 1
LEFT JOIN (
SELECT
dr.tx_id,
dr.drep_hash_id,
ROW_NUMBER() OVER (PARTITION BY dr.drep_hash_id ORDER BY dr.tx_id ASC) AS rn
FROM
drep_registration dr) AS dr_first_register ON dr_first_register.drep_hash_id = dh.id
AND dr_first_register.rn = 1
drep_registration dr
) AS dr_first_register ON dr_first_register.drep_hash_id = dh.id AND dr_first_register.rn = 1
LEFT JOIN tx AS tx_first_register ON tx_first_register.id = dr_first_register.tx_id
LEFT JOIN block AS block_first_register ON block_first_register.id = tx_first_register.block_id
WHERE
Expand All @@ -142,6 +150,8 @@ WHERE
off_chain_vote_drep_data.given_name ILIKE ?
)
GROUP BY
block_first_register.epoch_no,
block.epoch_no,
dh.raw,
second_to_newest_drep_registration.voting_anchor_id,
dh.view,
Expand All @@ -163,4 +173,4 @@ GROUP BY
off_chain_vote_drep_data.motivations,
off_chain_vote_drep_data.qualifications,
off_chain_vote_drep_data.image_url,
off_chain_vote_drep_data.image_hash
off_chain_vote_drep_data.image_hash;
Loading