Feat: Apply migration to vemnde (#75)

* feat: apply migration to vemnde * fix: change localhost url * fix: deprecate mnde * fix: check if voted validator is eligible * Update scripts/scoring.R Co-authored-by: janlegner <[email protected]> * fix: update reports_scoring --------- Co-authored-by: janlegner <[email protected]>
marinade-finance · Jul 31, 2023 · dbc1532 · dbc1532
1 parent a5e65f0
commit dbc1532
Show file tree

Hide file tree

Showing 23 changed files with 205 additions and 220 deletions.
diff --git a/api.md b/api.md
@@ -43,7 +43,6 @@ curl -sfLS 'localhost:8000/validators?limit=1&offset=0' | jq
       "commission_effective": null,
       "commission_aggregated": null,
       "version": "1.13.5",
-      "mnde_votes": "0",
       "activated_stake": "9488392700088216",
       "marinade_stake": "0",
       "decentralizer_stake": "0",
@@ -59,7 +58,6 @@ curl -sfLS 'localhost:8000/validators?limit=1&offset=0' | jq
           "commission_advertised": 8,
           "commission_effective": null,
           "version": "1.13.5",
-          "mnde_votes": 0,
           "activated_stake": 9488392700088216,
           "marinade_stake": 0,
           "decentralizer_stake": 0,
@@ -356,10 +354,10 @@ curl -sfLS 'localhost:8000/validators/score-breakdown?query_vote_account=DumiCKH
       "DC_CONCENTRATION"
     ],
     "eligible_stake_algo": true,
-    "eligible_stake_mnde": true,
+    "eligible_stake_vemnde": true,
     "eligible_stake_msol": true,
     "target_stake_algo": 0,
-    "target_stake_mnde": 0,
+    "target_stake_vemnde": 0,
     "target_stake_msol": 1214035,
     "scoring_run_id": 4,
     "created_at": "2023-03-03T06:23:00.734614Z",

diff --git a/api/src/handlers/admin_score_upload.rs b/api/src/handlers/admin_score_upload.rs
@@ -126,7 +126,10 @@ pub async fn handler(
     .await;
 
     Ok(match result {
-        Ok(_) => warp::reply::with_status(json(&ResponseAdminScoreUpload { rows_processed }), StatusCode::OK),
+        Ok(_) => warp::reply::with_status(
+            json(&ResponseAdminScoreUpload { rows_processed }),
+            StatusCode::OK,
+        ),
         Err(err) => {
             log::error!("Failed to store the scoring: {}", err);
             response_error_500("Failed to store the scoring!".into())

diff --git a/api/src/handlers/cluster_stats.rs b/api/src/handlers/cluster_stats.rs
@@ -42,9 +42,10 @@ pub async fn handler(
         .get_cluster_stats(query_params.epochs.unwrap_or(DEFAULT_EPOCHS));
 
     Ok(match cluster_stats {
-        Some(cluster_stats) => {
-            warp::reply::with_status(json(&ResponseClusterStats { cluster_stats }), StatusCode::OK)
-        }
+        Some(cluster_stats) => warp::reply::with_status(
+            json(&ResponseClusterStats { cluster_stats }),
+            StatusCode::OK,
+        ),
         _ => {
             error!("No cluster stats found");
             response_error(StatusCode::NOT_FOUND, "Failed to fetch records!".into())

diff --git a/api/src/handlers/list_validators.rs b/api/src/handlers/list_validators.rs
@@ -45,7 +45,6 @@ pub struct QueryParams {
 #[derive(Deserialize, Serialize, Debug)]
 pub enum OrderField {
     Stake,
-    MndeVotes,
     Credits,
     MarinadeScore,
     Apy,
@@ -116,7 +115,6 @@ pub async fn get_validators(
 fn get_field_extractor(order_field: OrderField) -> Box<dyn Fn(&ValidatorRecord) -> Decimal> {
     match order_field {
         OrderField::Stake => Box::new(|a: &ValidatorRecord| a.activated_stake),
-        OrderField::MndeVotes => Box::new(|a: &ValidatorRecord| a.mnde_votes.unwrap_or(0.into())),
         OrderField::Credits => Box::new(|a: &ValidatorRecord| Decimal::from(a.credits)),
         OrderField::MarinadeScore => {
             Box::new(|a: &ValidatorRecord| Decimal::from(to_fixed_for_sort(a.score.unwrap_or(0.0))))

diff --git a/api/src/handlers/reports_scoring.rs b/api/src/handlers/reports_scoring.rs
@@ -46,6 +46,33 @@ fn md_pre_msol_votes(ui_id: &String) -> String {
     ", ui_id, ui_id, ui_id, ui_id, ui_id, ui_id)
 }
 
+fn md_pre_vemnde_votes(ui_id: &String) -> String {
+    format!(
+        "# Report {}\n\
+        - [HTML report](https://validators-api-dev.marinade.finance/reports/scoring/{})\n\
+        - [CSV Scores](https://raw.githubusercontent.com/marinade-finance/delegation-strategy-pipeline/master/scoring/{}/scores.csv)\n\
+        ## Reproduce the results\n\
+        Get the source data:\n\
+        ```bash\n\
+        mkdir -p \"scoring-{}\"\n\
+        cd \"scoring-{}\"\n\
+        wget --base \"https://raw.githubusercontent.com/marinade-finance/delegation-strategy-pipeline/master/scoring/{}/\" \
+            --input-file - --no-clobber <<<$'validators.csv\nmsol-votes.csv\nparams.env\nblacklist.csv'\n\
+        ```\n\
+        Install dependencies for R (assumes you have R installed already):\n\
+        ```bash\n\
+        bash -c \"$(curl -sSfL https://raw.githubusercontent.com/marinade-finance/delegation-strategy-2/master/scripts/scoring-install.bash)\"\n\
+        ```\n\
+        Generate scores:\n\
+        ```bash\n\
+        wget https://raw.githubusercontent.com/marinade-finance/delegation-strategy-2/7c51106/scripts/scoring.R\n\
+        export SCORING_WORKING_DIRECTORY=.\n\
+        export SCORING_R=./scoring.R\n\
+        bash -c \"$(curl -sSfL https://raw.githubusercontent.com/marinade-finance/delegation-strategy-2/7c51106/scripts/scoring-run.bash)\"\n\
+        ```\n\
+    ", ui_id, ui_id, ui_id, ui_id, ui_id, ui_id)
+}
+
 fn md_latest(ui_id: &String) -> String {
     format!(
         "# Report {}\n\
@@ -76,7 +103,13 @@ fn md_latest(ui_id: &String) -> String {
 fn scoring_run_to_report(scoring_run: ScoringRunRecord) -> Report {
     Report {
         created_at: scoring_run.created_at,
-        md: if scoring_run.epoch < 500 { md_pre_msol_votes(&scoring_run.ui_id) } else { md_latest(&scoring_run.ui_id) },
+        md: if scoring_run.epoch < 454 {
+            md_pre_msol_votes(&scoring_run.ui_id)
+        } else if scoring_run.epoch < 481 {
+            md_pre_vemnde_votes(&scoring_run.ui_id)
+        } else {
+            md_latest(&scoring_run.ui_id)
+        },
     }
 }
 

diff --git a/api/src/handlers/reports_staking.rs b/api/src/handlers/reports_staking.rs
@@ -1,7 +1,7 @@
 use crate::{
     cache::CachedSingleRunScores, context::WrappedContext, metrics, utils::response_error,
 };
-use log::{error, info};
+use log::{error, info, warn};
 use serde::Serialize;
 use solana_program::native_token::LAMPORTS_PER_SOL;
 use store::utils::get_last_epoch;
@@ -59,7 +59,7 @@ async fn get_planned_stakes(context: WrappedContext) -> anyhow::Result<Vec<Staki
             .get(vote_account)
             .filter(|v| v.has_last_epoch_stats);
         let should_have = score_record.target_stake_algo
-            + score_record.target_stake_mnde
+            + score_record.target_stake_vemnde
             + score_record.target_stake_msol;
         match validator {
             Some(validator) => {
@@ -86,7 +86,7 @@ async fn get_planned_stakes(context: WrappedContext) -> anyhow::Result<Vec<Staki
                 }
             }
             None => {
-                error!("Couldn't find info for {} in current epoch", vote_account);
+                warn!("Couldn't find info for {} in current epoch", vote_account);
                 continue;
             }
         }

diff --git a/api/src/handlers/unstake_hints.rs b/api/src/handlers/unstake_hints.rs
@@ -38,9 +38,10 @@ pub async fn handler(
     .await;
 
     Ok(match unstake_hints {
-        Ok(unstake_hints) => {
-            warp::reply::with_status(json(&ResponseUnstakeHints { unstake_hints }), StatusCode::OK)
-        }
+        Ok(unstake_hints) => warp::reply::with_status(
+            json(&ResponseUnstakeHints { unstake_hints }),
+            StatusCode::OK,
+        ),
         Err(err) => {
             error!("Failed to load unstake hints: {}", err);
             response_error_500("Failed to load unstake hints!".into())

diff --git a/api/src/handlers/validator_score_breakdown.rs b/api/src/handlers/validator_score_breakdown.rs
@@ -25,16 +25,21 @@ pub struct ScoreBreakdown {
     pub min_score_eligible_algo: Option<f64>,
     pub rank: i32,
     pub ui_hints: Vec<String>,
-    pub mnde_votes: u64,
+    pub vemnde_votes: u64,
+    pub msol_votes: u64,
     pub component_scores: Vec<f64>,
     pub component_ranks: Vec<i32>,
     pub component_values: Vec<Option<String>>,
     pub component_weights: Vec<f64>,
     pub components: Vec<String>,
     pub eligible_stake_algo: bool,
+    pub eligible_stake_vemnde: bool,
+    #[deprecated  = "Use `eligible_stake_vemnde` instead"]
     pub eligible_stake_mnde: bool,
     pub eligible_stake_msol: bool,
     pub target_stake_algo: u64,
+    pub target_stake_vemnde: u64,
+    #[deprecated  = "Use `target_stake_vemnde` instead"]
     pub target_stake_mnde: u64,
     pub target_stake_msol: u64,
     pub scoring_run_id: i64,
@@ -92,15 +97,16 @@ pub async fn handler(
         score,
         rank,
         ui_hints,
-        mnde_votes,
+        vemnde_votes,
+        msol_votes,
         component_scores,
         component_ranks,
         component_values,
         eligible_stake_algo,
-        eligible_stake_mnde,
+        eligible_stake_vemnde,
         eligible_stake_msol,
         target_stake_algo,
-        target_stake_mnde,
+        target_stake_vemnde,
         target_stake_msol,
         scoring_run_id,
         created_at,
@@ -129,17 +135,20 @@ pub async fn handler(
                 min_score_eligible_algo,
                 rank,
                 ui_hints,
-                mnde_votes,
+                vemnde_votes,
+                msol_votes,
                 component_scores,
                 component_ranks,
                 component_values,
                 component_weights,
                 components,
                 eligible_stake_algo,
-                eligible_stake_mnde,
+                eligible_stake_vemnde,
+                eligible_stake_mnde: eligible_stake_vemnde,
                 eligible_stake_msol,
                 target_stake_algo,
-                target_stake_mnde,
+                target_stake_vemnde,
+                target_stake_mnde: target_stake_vemnde,
                 target_stake_msol,
                 scoring_run_id,
                 created_at,

diff --git a/api/src/handlers/validator_score_breakdowns.rs b/api/src/handlers/validator_score_breakdowns.rs
@@ -184,17 +184,20 @@ fn compute_score_breakdowns(
                     .get(&scoring_run.scoring_run_id)
                     .unwrap(),
                 ui_hints: score.ui_hints.clone(),
-                mnde_votes: score.mnde_votes,
+                vemnde_votes: score.vemnde_votes,
+                msol_votes: score.msol_votes,
                 component_scores: score.component_scores.clone(),
                 component_ranks: score.component_ranks.clone(),
                 component_values: score.component_values.clone(),
                 component_weights: scoring_run.component_weights.clone(),
                 components: scoring_run.components.clone(),
                 eligible_stake_algo: score.eligible_stake_algo,
-                eligible_stake_mnde: score.eligible_stake_mnde,
+                eligible_stake_vemnde: score.eligible_stake_vemnde,
+                eligible_stake_mnde: score.eligible_stake_vemnde,
                 eligible_stake_msol: score.eligible_stake_msol,
                 target_stake_algo: score.target_stake_algo,
-                target_stake_mnde: score.target_stake_mnde,
+                target_stake_vemnde: score.target_stake_vemnde,
+                target_stake_mnde: score.target_stake_vemnde,
                 target_stake_msol: score.target_stake_msol,
                 scoring_run_id: score.scoring_run_id,
                 created_at: score.created_at,

diff --git a/collect/src/marinade_service.rs b/collect/src/marinade_service.rs
@@ -1,5 +1,3 @@
-use anchor_lang::prelude::*;
-use log::info;
 use solana_account_decoder::*;
 use solana_client::{
     rpc_client::RpcClient,
@@ -105,59 +103,4 @@ fn get_stake_accounts(
         .iter()
         .map(|(pubkey, account)| (pubkey.clone(), bincode::deserialize(&account.data).unwrap()))
         .collect())
-}
-
-#[derive(Debug, Default, borsh::BorshDeserialize, borsh::BorshSchema)]
-pub struct Gauge {
-    pub gaugemeister: Pubkey,
-    pub total_weight: u64,
-    pub vote_count: u64,
-    pub is_disabled: bool,
-    // snapshots make reading more flexible and make time of reading predicted (no delays because of inet/cpu)
-    pub snapshot_time: i64,
-    pub snapshot_slot: u64,
-    pub snapshot_total_weight: u64,
-    pub info: Vec<u8>,
-}
-
-impl Gauge {
-    pub const LEN: usize = 200;
-}
-
-pub fn get_mnde_votes(
-    rpc_client: &RpcClient,
-    escrow_relocker: Pubkey,
-    gauge_meister: Pubkey,
-) -> anyhow::Result<HashMap<String, u64>> {
-    info!("Getting MNDE votes");
-    let accounts = rpc_client.get_program_accounts_with_config(
-        &escrow_relocker,
-        RpcProgramAccountsConfig {
-            filters: Some(vec![RpcFilterType::Memcmp(Memcmp {
-                offset: 8,
-                bytes: MemcmpEncodedBytes::Binary(gauge_meister.to_string()),
-                encoding: None,
-            })]),
-            account_config: RpcAccountInfoConfig {
-                encoding: Some(UiAccountEncoding::Base64),
-                commitment: Some(rpc_client.commitment()),
-                min_context_slot: None,
-                data_slice: None,
-            },
-            with_context: None,
-        },
-    )?;
-
-    let gauges: Vec<Gauge> = accounts
-        .iter()
-        .flat_map(|(_, account)| Gauge::deserialize(&mut &account.data[8..]))
-        .collect();
-
-    Ok(gauges
-        .iter()
-        .flat_map(|gauge| match Pubkey::try_from_slice(&gauge.info) {
-            Ok(vote_address) => Some((vote_address.to_string(), gauge.total_weight)),
-            _ => None,
-        })
-        .collect())
-}
+}
diff --git a/collect/src/validators.rs b/collect/src/validators.rs
@@ -7,17 +7,10 @@ use crate::whois_service::*;
 use log::info;
 use serde::{Deserialize, Serialize};
 use solana_sdk::clock::Epoch;
-use solana_sdk::pubkey::Pubkey;
 use structopt::StructOpt;
 
 #[derive(Debug, StructOpt)]
 pub struct ValidatorsOptions {
-    #[structopt(long = "gauge-meister", help = "Gauge meister of the vote gauges.")]
-    gauge_meister: Option<Pubkey>,
-
-    #[structopt(long = "escrow-relocker", help = "Escrow relocker program address.")]
-    escrow_relocker: Option<Pubkey>,
-
     #[structopt(long = "whois", help = "Base URL for whois API.")]
     whois: Option<String>,
 
@@ -73,7 +66,6 @@ pub struct ValidatorSnapshot {
     pub info_url: Option<String>,
     pub info_details: Option<String>,
     pub info_keybase: Option<String>,
-    pub mnde_votes: Option<u64>,
     pub data_center: Option<ValidatorDataCenter>,
     pub activated_stake: u64,
     pub marinade_stake: u64,
@@ -132,13 +124,6 @@ pub fn collect_validators_info(
     let decentralizer_stake = get_decentralizer_stakes(&client)?;
 
     let validators_info = get_validators_info(&client)?;
-    let mnde_votes = if let (Some(escrow_relocker), Some(gauge_meister)) =
-        (options.escrow_relocker, options.gauge_meister)
-    {
-        Some(get_mnde_votes(&client, escrow_relocker, gauge_meister)?)
-    } else {
-        None
-    };
     let node_ips = get_cluster_nodes_ips(&client)?;
 
     let data_centers = match options.whois {
@@ -173,9 +158,6 @@ pub fn collect_validators_info(
             vote_account: vote_pubkey.clone(),
             identity: identity.clone(),
             node_ip: data_centers.get(&identity).map(|(ip, _)| ip.clone()),
-            mnde_votes: mnde_votes
-                .clone()
-                .map_or(None, |v| Some(*v.get(&vote_pubkey).unwrap_or(&0))),
             data_center: data_centers
                 .get(&identity)
                 .map_or(None, |(_ip, data_center)| {

diff --git a/migrations/0002-scores.sql b/migrations/0002-scores.sql
@@ -15,10 +15,10 @@ CREATE TABLE "scores" (
     "rank" int NOT NULL,
     "ui_hints" text[] NOT NULL,
     "eligible_stake_algo" boolean,
-    "eligible_stake_mnde" boolean,
+    "eligible_stake_vemnde" boolean,
     "eligible_stake_msol" boolean,
     "target_stake_algo" numeric,
-    "target_stake_mnde" numeric,
+    "target_stake_vemnde" numeric,
     "target_stake_msol" numeric,
     "scoring_run_id" bigint,
     PRIMARY KEY ("score_id"),

diff --git a/migrations/0007-scores-msol-votes.sql b/migrations/0007-scores-msol-votes.sql
@@ -0,0 +1,2 @@
+alter table scores add column "msol_votes" numeric NOT NULL default 0;
+alter table scores alter column "msol_votes" drop default;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		alter table scores add column "msol_votes" numeric NOT NULL default 0;
		alter table scores alter column "msol_votes" drop default;