Add second source for fiat currency service, first one may be down

src/api/common/include/commonapi.hpp

@@ -6,6 +6,7 @@
 
 #include "cachedresult.hpp"
 #include "cct_flatset.hpp"
+#include "cct_vector.hpp"
 #include "curlhandle.hpp"
 #include "currencycode.hpp"
 #include "exchangebase.hpp"
@@ -41,12 +42,18 @@ class CommonAPI : public ExchangeBase {
   void updateCacheFile() const override;
 
  private:
-  struct FiatsFunc {
+  class FiatsFunc {
+   public:
     FiatsFunc();
 
     Fiats operator()();
 
-    CurlHandle _curlHandle;
+    vector<CurrencyCode> retrieveFiatsSource1();
+    vector<CurrencyCode> retrieveFiatsSource2();
+
+   private:
+    CurlHandle _curlHandle1;
+    CurlHandle _curlHandle2;
   };
 
   CachedResultVault _cachedResultVault;

src/api/common/src/commonapi.cpp

@@ -46,32 +46,96 @@ CommonAPI::CommonAPI(const CoincenterInfo& config, Duration fiatsUpdateFrequency
 }
 
 namespace {
-constexpr std::string_view kFiatsUrl = "https://datahub.io/core/currency-codes/r/codes-all.json";
-}
+constexpr std::string_view kFiatsUrlSource1 = "https://datahub.io/core/currency-codes/r/codes-all.json";
+constexpr std::string_view kFiatsUrlSource2 = "https://www.iban.com/currency-codes";
+}  // namespace
 
 CommonAPI::FiatsFunc::FiatsFunc()
-    : _curlHandle(kFiatsUrl, nullptr, PermanentCurlOptions::Builder().setFollowLocation().build()) {}
+    : _curlHandle1(kFiatsUrlSource1, nullptr,
+                   PermanentCurlOptions::Builder()
+                       .setFollowLocation()
+                       .setTooManyErrorsPolicy(PermanentCurlOptions::TooManyErrorsPolicy::kReturnEmptyResponse)
+                       .build()),
+      _curlHandle2(kFiatsUrlSource2, nullptr,
+                   PermanentCurlOptions::Builder()
+                       .setTooManyErrorsPolicy(PermanentCurlOptions::TooManyErrorsPolicy::kReturnEmptyResponse)
+                       .build()) {}
 
 CommonAPI::Fiats CommonAPI::FiatsFunc::operator()() {
-  json dataCSV = json::parse(_curlHandle.query("", CurlOptions(HttpRequestType::kGet)));
+  vector<CurrencyCode> fiatsVec = retrieveFiatsSource1();
   Fiats fiats;
+  if (!fiats.empty()) {
+    fiats = Fiats(std::move(fiatsVec));
+    log::info("Stored {} fiats from first source", fiats.size());
+  } else {
+    fiats = Fiats(retrieveFiatsSource2());
+    log::info("Stored {} fiats from second source", fiats.size());
+  }
+  return fiats;
+}
+
+vector<CurrencyCode> CommonAPI::FiatsFunc::retrieveFiatsSource1() {
+  vector<CurrencyCode> fiatsVec;
+
+  std::string_view data = _curlHandle1.query("", CurlOptions(HttpRequestType::kGet));
+  if (data.empty()) {
+    log::error("Error parsing currency codes, no fiats found from first source");
+    return fiatsVec;
+  }
+  json dataCSV = json::parse(data);
   for (const json& fiatData : dataCSV) {
     static constexpr std::string_view kCodeKey = "AlphabeticCode";
     static constexpr std::string_view kWithdrawalDateKey = "WithdrawalDate";
     auto codeIt = fiatData.find(kCodeKey);
     auto withdrawalDateIt = fiatData.find(kWithdrawalDateKey);
     if (codeIt != fiatData.end() && !codeIt->is_null() && withdrawalDateIt != fiatData.end() &&
         withdrawalDateIt->is_null()) {
-      fiats.insert(CurrencyCode(codeIt->get<std::string_view>()));
+      fiatsVec.emplace_back(codeIt->get<std::string_view>());
       log::debug("Stored {} fiat", codeIt->get<std::string_view>());
     }
   }
-  if (fiats.empty()) {
-    throw exception("Error parsing currency codes, no fiats found in {}", dataCSV.dump());
+
+  return fiatsVec;
+}
+
+vector<CurrencyCode> CommonAPI::FiatsFunc::retrieveFiatsSource2() {
+  vector<CurrencyCode> fiatsVec;
+  std::string_view data = _curlHandle2.query("", CurlOptions(HttpRequestType::kGet));
+  if (data.empty()) {
+    log::error("Error parsing currency codes, no fiats found from second source");
+    return fiatsVec;
   }
 
-  log::info("Stored {} fiats", fiats.size());
-  return fiats;
+  static constexpr std::string_view kTheadColumnStart = "<th class=\"head\">";
+  static constexpr std::string_view kCodeStrName = "Code";
+  auto pos = data.find("<thead>");
+  int codePos = 0;
+  for (pos = data.find(kTheadColumnStart, pos + 1U); pos != std::string_view::npos;
+       pos = data.find(kTheadColumnStart, pos + 1U), ++codePos) {
+    auto endPos = data.find("</th>", pos + 1);
+    std::string_view colName(data.begin() + pos + kTheadColumnStart.size(), data.begin() + endPos);
+    if (colName == kCodeStrName) {
+      ++codePos;
+      break;
+    }
+    pos = endPos;
+  }
+
+  pos = data.find("<tbody>");
+  for (pos = data.find("<tr>", pos + 1U); pos != std::string_view::npos; pos = data.find("<tr>", pos + 1U)) {
+    static constexpr std::string_view kColStart = "<td>";
+    for (int col = 0; col < codePos; ++col) {
+      pos = data.find(kColStart, pos + 1);
+    }
+    auto endPos = data.find("</td>", pos + 1);
+    std::string_view curStr(data.begin() + pos + kColStart.size(), data.begin() + endPos);
+    if (!curStr.empty()) {
+      // Fiat data is sometimes empty
+      fiatsVec.emplace_back(curStr);
+    }
+  }
+
+  return fiatsVec;
 }
 
 void CommonAPI::updateCacheFile() const {

src/api/common/src/exchangepublicapi.cpp

@@ -393,4 +393,4 @@ MonetaryAmount ExchangePublic::queryWithdrawalFeeOrZero(CurrencyCode currencyCod
   return withdrawFee;
 }
 
-}  // namespace cct::api
+}  // namespace cct::api

src/api/common/test/commonapi_test.cpp

@@ -12,7 +12,7 @@ class CommonAPITest : public ::testing::Test {
  protected:
   settings::RunMode runMode = settings::RunMode::kTestKeys;
   CoincenterInfo config{runMode};
-  CommonAPI commonAPI{config};
+  CommonAPI commonAPI{config, Duration::max(), CommonAPI::AtInit::kNoLoadFromFileCache};
 };
 
 TEST_F(CommonAPITest, IsFiatService) {

src/api/exchanges/include/binancepublicapi.hpp

@@ -147,4 +147,4 @@ class BinancePublic : public ExchangePublic {
 };
 
 }  // namespace api
-}  // namespace cct
+}  // namespace cct

src/api/exchanges/include/bithumbpublicapi.hpp

@@ -111,4 +111,4 @@ class BithumbPublic : public ExchangePublic {
 };
 
 }  // namespace api
-}  // namespace cct
+}  // namespace cct

src/api/exchanges/include/upbitprivateapi.hpp

@@ -101,4 +101,4 @@ class UpbitPrivate : public ExchangePrivate {
   CachedResult<WithdrawFeesFunc, CurrencyCode> _withdrawalFeesCache;
 };
 }  // namespace api
-}  // namespace cct
+}  // namespace cct

src/api/exchanges/include/upbitpublicapi.hpp

@@ -124,4 +124,4 @@ class UpbitPublic : public ExchangePublic {
 };
 
 }  // namespace api
-}  // namespace cct
+}  // namespace cct

src/api/interface/src/exchange.cpp

@@ -47,4 +47,4 @@ void Exchange::updateCacheFile() const {
     _pExchangePrivate->updateCacheFile();
   }
 }
-}  // namespace cct
+}  // namespace cct

src/http-request/include/besturlpicker.hpp

@@ -21,6 +21,8 @@ class BestURLPicker {
   static constexpr int kNbMaxBaseUrl = 4;
 
  public:
+  BestURLPicker() noexcept = default;
+
   /// Builds a BestURLPicker that will work with several base URLs.
   /// Warning: given base URL should come from static storage
   template <unsigned N, std::enable_if_t<(N > 0) && N <= kNbMaxBaseUrl, bool> = true>
@@ -67,7 +69,7 @@ class BestURLPicker {
   using ResponseTimeStatsPerBaseUrl = FixedCapacityVector<ResponseTimeStats, kNbMaxBaseUrl>;
 
   // Non-owning pointer, should come from static storage (default special operations are fine)
-  const std::string_view *_pBaseUrls;
+  const std::string_view *_pBaseUrls{};
   ResponseTimeStatsPerBaseUrl _responseTimeStatsPerBaseUrl;
 };
 }  // namespace cct

src/http-request/include/curlhandle.hpp

@@ -26,6 +26,8 @@ string GetCurlVersionInfo();
 /// CurlHandle for faster similar queries.
 class CurlHandle {
  public:
+  CurlHandle() noexcept = default;
+
   /// Constructs a new CurlHandle.
   /// @param bestURLPicker object managing which URL to pick at each query based on response time stats
   /// @param pMetricGateway if not null, queries will export some metrics
@@ -38,9 +40,8 @@ class CurlHandle {
   CurlHandle(const CurlHandle &) = delete;
   CurlHandle &operator=(const CurlHandle &) = delete;
 
-  // Move operations are deleted but could be implemented if needed. It's just to avoid useless code.
-  CurlHandle(CurlHandle &&) = delete;
-  CurlHandle &operator=(CurlHandle &&) = delete;
+  CurlHandle(CurlHandle &&rhs) noexcept;
+  CurlHandle &operator=(CurlHandle &&rhs) noexcept;
 
   ~CurlHandle();
 
@@ -61,21 +62,25 @@ class CurlHandle {
   /// complexity in a flat key value string.
   void setOverridenQueryResponses(const std::map<string, string> &queryResponsesMap);
 
+  void swap(CurlHandle &rhs) noexcept;
+
   using trivially_relocatable = is_trivially_relocatable<string>::type;
 
  private:
   void setUpProxy(const char *proxyUrl, bool reset);
 
   // void pointer instead of CURL to avoid having to forward declare (we don't know about the underlying definition)
   // and to avoid clients to pull unnecessary curl dependencies by just including the header
-  void *_handle;
-  AbstractMetricGateway *_pMetricGateway;  // non-owning pointer
-  Duration _minDurationBetweenQueries;
+  void *_handle = nullptr;
+  AbstractMetricGateway *_pMetricGateway = nullptr;  // non-owning pointer
+  Duration _minDurationBetweenQueries{};
   TimePoint _lastQueryTime{};
   BestURLPicker _bestUrlPicker;
   string _queryData;
-  log::level::level_enum _requestCallLogLevel;
-  log::level::level_enum _requestAnswerLogLevel;
+  log::level::level_enum _requestCallLogLevel = log::level::level_enum::off;
+  log::level::level_enum _requestAnswerLogLevel = log::level::level_enum::off;
+  int _nbMaxRetries = PermanentCurlOptions::kDefaultNbMaxRetries;
+  PermanentCurlOptions::TooManyErrorsPolicy _tooManyErrorsPolicy = PermanentCurlOptions::TooManyErrorsPolicy::kThrow;
 };
 
 // Simple RAII class managing global init and clean up of Curl library.

src/http-request/include/permanentcurloptions.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstdint>
 #include <string_view>
 
 #include "cct_log.hpp"
@@ -10,18 +11,26 @@ namespace cct {
 
 class PermanentCurlOptions {
  public:
+  static constexpr auto kDefaultNbMaxRetries = 5;
+
+  enum class TooManyErrorsPolicy : int8_t { kThrow, kReturnEmptyResponse };
+
   PermanentCurlOptions() noexcept = default;
 
-  const string &getUserAgent() const { return _userAgent; }
+  const auto &getUserAgent() const { return _userAgent; }
+
+  const auto &getAcceptedEncoding() const { return _acceptedEncoding; }
 
-  const string &getAcceptedEncoding() const { return _acceptedEncoding; }
+  auto minDurationBetweenQueries() const { return _minDurationBetweenQueries; }
 
-  Duration minDurationBetweenQueries() const { return _minDurationBetweenQueries; }
+  auto followLocation() const { return _followLocation; }
 
-  bool followLocation() const { return _followLocation; }
+  auto requestCallLogLevel() const { return _requestCallLogLevel; }
+  auto requestAnswerLogLevel() const { return _requestAnswerLogLevel; }
 
-  log::level::level_enum requestCallLogLevel() const { return _requestCallLogLevel; }
-  log::level::level_enum requestAnswerLogLevel() const { return _requestAnswerLogLevel; }
+  auto tooManyErrorsPolicy() const { return _tooManyErrorsPolicy; }
+
+  auto nbMaxRetries() const { return _nbMaxRetries; }
 
   class Builder {
    public:
@@ -72,14 +81,25 @@ class PermanentCurlOptions {
       return *this;
     }
 
+    Builder &setNbMaxRetries(int nbMaxRetries) {
+      _nbMaxRetries = nbMaxRetries;
+      return *this;
+    }
+
     Builder &setFollowLocation() {
       _followLocation = true;
       return *this;
     }
 
+    Builder &setTooManyErrorsPolicy(TooManyErrorsPolicy tooManyErrorsPolicy) {
+      _tooManyErrorsPolicy = tooManyErrorsPolicy;
+      return *this;
+    }
+
     PermanentCurlOptions build() {
       return {std::move(_userAgent), std::move(_acceptedEncoding), _minDurationBetweenQueries,
-              _requestCallLogLevel,  _requestAnswerLogLevel,       _followLocation};
+              _requestCallLogLevel,  _requestAnswerLogLevel,       _nbMaxRetries,
+              _followLocation,       _tooManyErrorsPolicy};
     }
 
    private:
@@ -88,26 +108,32 @@ class PermanentCurlOptions {
     Duration _minDurationBetweenQueries{};
     log::level::level_enum _requestCallLogLevel = log::level::level_enum::info;
     log::level::level_enum _requestAnswerLogLevel = log::level::level_enum::trace;
+    int _nbMaxRetries = kDefaultNbMaxRetries;
     bool _followLocation = false;
+    TooManyErrorsPolicy _tooManyErrorsPolicy = TooManyErrorsPolicy::kThrow;
   };
 
  private:
   PermanentCurlOptions(string userAgent, string acceptedEncoding, Duration minDurationBetweenQueries,
                        log::level::level_enum requestCallLogLevel, log::level::level_enum requestAnswerLogLevel,
-                       bool followLocation)
+                       int nbMaxRetries, bool followLocation, TooManyErrorsPolicy tooManyErrorsPolicy)
       : _userAgent(std::move(userAgent)),
         _acceptedEncoding(std::move(acceptedEncoding)),
         _minDurationBetweenQueries(minDurationBetweenQueries),
         _requestCallLogLevel(requestCallLogLevel),
         _requestAnswerLogLevel(requestAnswerLogLevel),
-        _followLocation(followLocation) {}
+        _nbMaxRetries(nbMaxRetries),
+        _followLocation(followLocation),
+        _tooManyErrorsPolicy(tooManyErrorsPolicy) {}
 
   string _userAgent;
   string _acceptedEncoding;
   Duration _minDurationBetweenQueries;
   log::level::level_enum _requestCallLogLevel;
   log::level::level_enum _requestAnswerLogLevel;
+  int _nbMaxRetries;
   bool _followLocation;
+  TooManyErrorsPolicy _tooManyErrorsPolicy;
 };
 
 }  // namespace cct