Skip to content

Commit

Permalink
retry tao price shovel request on corrupted CMC data
Browse files Browse the repository at this point in the history
  • Loading branch information
@hey-michal
hey-michal committed Jan 23, 2025
1 parent 472232c commit 5db47dc
Showing 1 changed file with 50 additions and 16 deletions.
66 changes: 50 additions & 16 deletions scraper_service/shovel_tao_price/cmc_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,16 @@ def fetch_cmc_data(params, endpoint):
try:
response = requests.get(url, headers=headers, params=params, timeout=30) # Add timeout
except requests.Timeout:
logging.warning("CMC API request timed out")
raise ShovelProcessingError("CMC API request timed out")
except requests.ConnectionError:
logging.warning("Failed to connect to CMC API")
raise ShovelProcessingError("Failed to connect to CMC API")

# Handle rate limiting explicitly
if response.status_code == 429:
retry_after = int(response.headers.get('Retry-After', 60))
logging.warning(f"CMC API rate limit exceeded, retry after {retry_after} seconds")
raise ShovelProcessingError("CMC API rate limit exceeded")

# Handle other common error codes
Expand All @@ -36,25 +40,55 @@ def fetch_cmc_data(params, endpoint):
elif response.status_code == 403:
raise ShovelProcessingError("CMC API access forbidden")
elif response.status_code >= 500:
logging.warning(f"CMC API server error: {response.status_code}")
raise ShovelProcessingError(f"CMC API server error: {response.status_code}")
elif response.status_code != 200:
logging.warning(f"CMC API request failed with status code: {response.status_code}")
raise ShovelProcessingError(f"CMC API request failed with status code: {response.status_code}")

try:
data = response.json()
except ValueError:
logging.warning("Failed to parse CMC API response as JSON")
raise ShovelProcessingError("Failed to parse CMC API response as JSON")

# Check for API-level errors
if 'status' in data and 'error_code' in data['status'] and data['status']['error_code'] != 0:
error_message = data['status'].get('error_message', 'Unknown API error')
logging.warning(f"CMC API error: {error_message}")
raise ShovelProcessingError(f"CMC API error: {error_message}")

if 'data' not in data:
logging.warning("Invalid CMC API response: missing 'data' field")
raise DatabaseConnectionError("Invalid CMC API response: missing 'data' field")
if 'quotes' not in data['data']:
logging.warning("Invalid CMC API response: missing 'quotes' field")
raise DatabaseConnectionError("Invalid CMC API response: missing 'quotes' field")
if not data['data']['quotes']:
logging.warning(f"No price data available for timestamp {params.get('time_start', 'latest')}")
raise DatabaseConnectionError(f"No price data available for timestamp {params.get('time_start', 'latest')}")

quote = data['data']['quotes'][0] if endpoint == 'historical' else data['data'][str(CMC_TAO_ID)]
if 'quote' not in quote or 'USD' not in quote['quote']:
logging.warning("Invalid CMC API response: missing USD quote data")
raise DatabaseConnectionError("Invalid CMC API response: missing USD quote data")

usd_quote = quote['quote']['USD']
required_fields = ['price', 'market_cap', 'volume_24h']
for field in required_fields:
if field not in usd_quote:
logging.warning(f"Invalid CMC API response: missing {field} field")
raise DatabaseConnectionError(f"Invalid CMC API response: missing {field} field")
if usd_quote[field] is None:
logging.warning(f"Invalid CMC API response: {field} is None")
raise DatabaseConnectionError(f"Invalid CMC API response: {field} is None")

return data, response.status_code
except requests.exceptions.RequestException as e:
raise ShovelProcessingError(f"Failed to make CMC API request: {str(e)}")
except ShovelProcessingError:
raise
except Exception as e:
raise ShovelProcessingError(f"Unexpected error in CMC API request: {str(e)}")
logging.warning(f"Unexpected error in CMC API request: {str(e)}")
raise DatabaseConnectionError(f"Unexpected error in CMC API request: {str(e)}")

def get_price_by_time(timestamp):
if timestamp is None or timestamp <= 0:
Expand Down Expand Up @@ -93,23 +127,23 @@ def get_price_by_time(timestamp):
logging.debug(f"Got response with status code: {status_code}")

if 'data' not in data:
raise ShovelProcessingError(f"Invalid CMC API response: missing 'data' field")
raise DatabaseConnectionError(f"Invalid CMC API response: missing 'data' field")
if 'quotes' not in data['data']:
raise ShovelProcessingError(f"Invalid CMC API response: missing 'quotes' field")
raise DatabaseConnectionError(f"Invalid CMC API response: missing 'quotes' field")
if not data['data']['quotes']:
raise ShovelProcessingError(f"No price data available for timestamp {timestamp}")
raise DatabaseConnectionError(f"No price data available for timestamp {timestamp}")

quote = data['data']['quotes'][0]
if 'quote' not in quote or 'USD' not in quote['quote']:
raise ShovelProcessingError(f"Invalid CMC API response: missing USD quote data")
raise DatabaseConnectionError(f"Invalid CMC API response: missing USD quote data")

usd_quote = quote['quote']['USD']
required_fields = ['price', 'market_cap', 'volume_24h']
for field in required_fields:
if field not in usd_quote:
raise ShovelProcessingError(f"Invalid CMC API response: missing {field} field")
raise DatabaseConnectionError(f"Invalid CMC API response: missing {field} field")
if usd_quote[field] is None:
raise ShovelProcessingError(f"Invalid CMC API response: {field} is None")
raise DatabaseConnectionError(f"Invalid CMC API response: {field} is None")

price = usd_quote['price']
market_cap = usd_quote['market_cap']
Expand All @@ -125,7 +159,7 @@ def get_price_by_time(timestamp):
except ShovelProcessingError:
raise
except Exception as e:
raise ShovelProcessingError(f"Failed to get price data: {str(e)}")
raise DatabaseConnectionError(f"Failed to get price data: {str(e)}")

def get_latest_price():
try:
Expand All @@ -137,23 +171,23 @@ def get_latest_price():
data, status_code = fetch_cmc_data(parameters, 'latest')

if 'data' not in data:
raise ShovelProcessingError(f"Invalid CMC API response: missing 'data' field")
raise DatabaseConnectionError(f"Invalid CMC API response: missing 'data' field")

tao_id_str = str(CMC_TAO_ID)
if tao_id_str not in data['data']:
raise ShovelProcessingError(f"No data available for TAO (ID: {CMC_TAO_ID})")
raise DatabaseConnectionError(f"No data available for TAO (ID: {CMC_TAO_ID})")

tao_data = data['data'][tao_id_str]
if 'quote' not in tao_data or 'USD' not in tao_data['quote']:
raise ShovelProcessingError(f"Invalid CMC API response: missing USD quote data")
raise DatabaseConnectionError(f"Invalid CMC API response: missing USD quote data")

usd_quote = tao_data['quote']['USD']
required_fields = ['price', 'market_cap', 'volume_24h']
for field in required_fields:
if field not in usd_quote:
raise ShovelProcessingError(f"Invalid CMC API response: missing {field} field")
raise DatabaseConnectionError(f"Invalid CMC API response: missing {field} field")
if usd_quote[field] is None:
raise ShovelProcessingError(f"Invalid CMC API response: {field} is None")
raise DatabaseConnectionError(f"Invalid CMC API response: {field} is None")

price = usd_quote['price']
market_cap = usd_quote['market_cap']
Expand All @@ -168,4 +202,4 @@ def get_latest_price():
except ShovelProcessingError:
raise
except Exception as e:
raise ShovelProcessingError(f"Failed to get latest price data: {str(e)}")
raise DatabaseConnectionError(f"Failed to get latest price data: {str(e)}")

0 comments on commit 5db47dc

Please sign in to comment.