diff --git a/src/gridcoin/scraper/scraper.cpp b/src/gridcoin/scraper/scraper.cpp index 7bbc99bf69..67a570e0ce 100755 --- a/src/gridcoin/scraper/scraper.cpp +++ b/src/gridcoin/scraper/scraper.cpp @@ -372,9 +372,10 @@ bool MarkScraperFileManifestEntryNonCurrent(ScraperFileManifestEntry& entry); * @param filetype * @param sProject * @param excludefromcsmanifest + * @param all_cpid_total_credit */ void AlignScraperFileManifestEntries(const fs::path& file, const std::string& filetype, const std::string& sProject, - const bool& excludefromcsmanifest); + const bool& excludefromcsmanifest, const uint64_t &all_cpid_total_credit); /** * @brief Constructs the scraper statistics from the current state of the scraper, which is all of the in scope files at the * time the function is called @@ -527,11 +528,12 @@ bool DownloadProjectRacFilesByCPID(const WhitelistSnapshot& projectWhitelist); * @param Consensus * @param GlobalVerifiedBeaconsCopy * @param IncomingVerifiedBeacons + * @param all_cpid_total_credit * @return bool true if successful */ bool ProcessProjectRacFileByCPID(const std::string& project, const fs::path& file, const std::string& etag, BeaconConsensus& Consensus, ScraperVerifiedBeacons& GlobalVerifiedBeaconsCopy, - ScraperVerifiedBeacons& IncomingVerifiedBeacons); + ScraperVerifiedBeacons& IncomingVerifiedBeacons, uint64_t& all_cpid_total_credit); /** * @brief Clears the authentication ETag auth.dat file */ @@ -2111,7 +2113,7 @@ bool DownloadProjectHostFiles(const WhitelistSnapshot& projectWhitelist) } // Save host xml files to file manifest map with exclude from CSManifest flag set to true. - AlignScraperFileManifestEntries(host_file, "host", prjs.m_name, true); + AlignScraperFileManifestEntries(host_file, "host", prjs.m_name, true, 0); } return true; @@ -2285,7 +2287,7 @@ bool DownloadProjectTeamFiles(const WhitelistSnapshot& projectWhitelist) // If in explorer mode and new file downloaded, save team xml files to file manifest map with exclude from CSManifest // flag set to true. If not in explorer mode, this is not necessary, because the team xml file is just temporary and // can be discarded after processing. - if (explorer_mode() && bDownloadFlag) AlignScraperFileManifestEntries(team_file, "team", prjs.m_name, true); + if (explorer_mode() && bDownloadFlag) AlignScraperFileManifestEntries(team_file, "team", prjs.m_name, true, 0); // If require team whitelist is set and bETagChanged is true, then process the file. This also populates/updated the // team whitelist TeamIDs in the TeamIDMap and the ETag entries in the ProjTeamETags map. @@ -2561,13 +2563,15 @@ bool DownloadProjectRacFilesByCPID(const WhitelistSnapshot& projectWhitelist) continue; } - // If in explorer mode, save user (rac) source xml files to file manifest map with exclude from CSManifest flag set - // to true. - if (explorer_mode()) AlignScraperFileManifestEntries(rac_file, "user_source", prjs.m_name, true); + uint64_t all_cpid_total_credit = 0; // Now that the source file is handled, process the file. ProcessProjectRacFileByCPID(prjs.m_name, rac_file, sRacETag, Consensus, - GlobalVerifiedBeaconsCopy, IncomingVerifiedBeacons); + GlobalVerifiedBeaconsCopy, IncomingVerifiedBeacons, all_cpid_total_credit); + + // If in explorer mode, save user (rac) source xml files to file manifest map with exclude from CSManifest flag set + // to true. + if (explorer_mode()) AlignScraperFileManifestEntries(rac_file, "user_source", prjs.m_name, true, all_cpid_total_credit); } // for prjs : projectWhitelist // Get the global verified beacons and copy the incoming verified beacons from the @@ -2615,7 +2619,7 @@ bool DownloadProjectRacFilesByCPID(const WhitelistSnapshot& projectWhitelist) // This version uses a consensus beacon map (and teamid, if team filtering is specified by policy) to filter statistics. bool ProcessProjectRacFileByCPID(const std::string& project, const fs::path& file, const std::string& etag, BeaconConsensus& Consensus, ScraperVerifiedBeacons& GlobalVerifiedBeaconsCopy, - ScraperVerifiedBeacons& IncomingVerifiedBeacons) + ScraperVerifiedBeacons& IncomingVerifiedBeacons, uint64_t& all_cpid_total_credit) { auto explorer_mode = []() { LOCK(cs_ScraperGlobals); return fExplorer; }; auto require_team_whitelist_membership = []() { LOCK(cs_ScraperGlobals); return REQUIRE_TEAM_WHITELIST_MEMBERSHIP; }; @@ -2721,6 +2725,16 @@ bool ProcessProjectRacFileByCPID(const std::string& project, const fs::path& fil } } + // We need to accumulate total credit across ALL project cpids, regardless of their beacon status, to get + // an "all cpid" total credit sum to be used for automatic greylisting purposes. + std::string s_cpid_total_credit = ExtractXML(data, "", ""); + uint64_t cpid_total_credit = 0; + + if (!ParseUInt64(s_cpid_total_credit, &cpid_total_credit)) { + _log(logattribute::ERR, __func__, "Bad team id in user stats file data."); + continue; + } + // We do NOT want to add a just verified CPID to the statistics this iteration, if it was // not already active, because we may be halfway through processing the set of projects. // Instead, add to the incoming verification map (above), which will be handled in the @@ -2764,7 +2778,7 @@ bool ProcessProjectRacFileByCPID(const std::string& project, const fs::path& fil } // User beacon verified. Append its statistics to the CSV output. - out << ExtractXML(data, "", "") << "," + out << s_cpid_total_credit << "," << ExtractXML(data, "", "") << "," << ExtractXML(data, "", "") << "," << cpid @@ -2835,7 +2849,7 @@ bool ProcessProjectRacFileByCPID(const std::string& project, const fs::path& fil // Here, regardless of explorer mode, save processed rac files to file manifest map with exclude from CSManifest flag // set to false. - AlignScraperFileManifestEntries(gzetagfile, "user", project, false); + AlignScraperFileManifestEntries(gzetagfile, "user", project, false, all_cpid_total_credit); _log(logattribute::INFO, "ProcessProjectRacFileByCPID", "Complete Process"); @@ -3221,7 +3235,7 @@ EXCLUSIVE_LOCKS_REQUIRED(cs_StructScraperFileManifest) } void AlignScraperFileManifestEntries(const fs::path& file, const std::string& filetype, - const std::string& sProject, const bool& excludefromcsmanifest) + const std::string& sProject, const bool& excludefromcsmanifest, const uint64_t& all_cpid_total_credit) { ScraperFileManifestEntry NewRecord; @@ -3238,6 +3252,7 @@ void AlignScraperFileManifestEntries(const fs::path& file, const std::string& fi NewRecord.current = true; NewRecord.excludefromcsmanifest = excludefromcsmanifest; NewRecord.filetype = filetype; + NewRecord.all_cpid_total_credit = all_cpid_total_credit; // Code block to lock StructScraperFileManifest during record insertion and delete because we want this atomic. { @@ -3381,6 +3396,27 @@ bool LoadScraperFileManifest(const fs::path& file) LoadEntry.filetype = "user"; } + // This handles startup with legacy manifest file without the all_cpid_total_credit column. + if (vline.size() >= 8) { + // In scraper for superblock v3 and autogreylist, we have to record total credit across all cpids, regardless + // of whether they are active beaconholders to support auto greylisting. + + uint64_t all_cpid_total_credit = 0; + + if (!ParseUInt64(vline[7], &all_cpid_total_credit)) { + // This shouldn't happen given the conditional above, but to be thorough... + _log(logattribute::ERR, __func__, "The \"all_cpid_total_credit\" field not parsed correctly for a manifest " + "entry. Skipping."); + continue; + } + + LoadEntry.all_cpid_total_credit = all_cpid_total_credit; + } else { + // The default if the field is not there is user. (Because scraper ver 1 all files in the manifest are + // user.) + LoadEntry.filetype = "user"; + } + // Lock cs_StructScraperFileManifest before updating // global structure. { @@ -3425,6 +3461,7 @@ bool StoreScraperFileManifest(const fs::path& file) << "Filename," << "ExcludeFromCSManifest," << "Filetype" + << "All_cpid_total_credit" << "\n"; for (auto const& entry : StructScraperFileManifest.mScraperFileManifest) @@ -3437,7 +3474,9 @@ bool StoreScraperFileManifest(const fs::path& file) + entry.second.project + "," + entry.first + "," + ToString(entry.second.excludefromcsmanifest) + "," - + entry.second.filetype + "\n"; + + entry.second.filetype + + ToString(entry.second.all_cpid_total_credit) + + "\n"; stream << sScraperFileManifestEntry; } } diff --git a/src/gridcoin/scraper/scraper.h b/src/gridcoin/scraper/scraper.h index 2340e17b21..3f3c7a438f 100644 --- a/src/gridcoin/scraper/scraper.h +++ b/src/gridcoin/scraper/scraper.h @@ -80,6 +80,7 @@ struct ScraperFileManifestEntry bool current = true; bool excludefromcsmanifest = true; std::string filetype; + uint64_t all_cpid_total_credit = 0; }; /**