Skip to content

Commit

Permalink
Merge pull request #162 from CartoDB/data_observatory_quota_override
Browse files Browse the repository at this point in the history
Use redis or db data observatory quota value
  • Loading branch information
Mario de Frutos committed Apr 25, 2016
2 parents 976697c + 4407ff6 commit 5efb81d
Show file tree
Hide file tree
Showing 17 changed files with 310 additions and 93 deletions.
4 changes: 2 additions & 2 deletions server/extension/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql)
# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
DATA = $(NEW_EXTENSION_ARTIFACT) \
$(OLD_VERSIONS) \
cdb_dataservices_server--0.6.2--0.7.0.sql \
cdb_dataservices_server--0.7.0--0.6.2.sql
cdb_dataservices_server--0.7.0--0.7.1.sql \
cdb_dataservices_server--0.7.1--0.7.0.sql

REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
TEST_DIR = test/
Expand Down
94 changes: 94 additions & 0 deletions server/extension/cdb_dataservices_server--0.7.0--0.7.1.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.1'" to load this file. \quit
DROP FUNCTION IF EXISTS cdb_dataservices_server._get_data_observatory_config(text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_obs_snapshot_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import ObservatorySnapshotConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = obs_snapshot_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;

CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT '2009 - 2013',
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
import json

plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]

quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')

try:
obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [geom, time_span, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
import json

plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]

quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')

try:
obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [geom, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
94 changes: 94 additions & 0 deletions server/extension/cdb_dataservices_server--0.7.1--0.7.0.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.7.0'" to load this file. \quit
DROP FUNCTION IF EXISTS cdb_dataservices_server._get_obs_snapshot_config(text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_data_observatory_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_data_observatory_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import DataObservatoryConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
data_observatory_config = DataObservatoryConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = data_observatory_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;

CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT '2009 - 2013',
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
import json

plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)]

quota_service = QuotaService(user_data_observatory_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')

try:
obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetDemographicSnapshot($1, $2, $3) as snapshot;", ["geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [geom, time_span, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
plpy.notice(traceback.format_exception(type_, value_, traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
username TEXT,
orgname TEXT,
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT '"us.census.tiger".block_group')
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
import json

plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)]

quota_service = QuotaService(user_data_observatory_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')

try:
obs_plan = plpy.prepare("SELECT cdb_observatory.OBS_GetSegmentSnapshot($1, $2) as snapshot;", ["geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [geom, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
2 changes: 1 addition & 1 deletion server/extension/cdb_dataservices_server.control
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.7.0'
default_version = '0.7.1'
requires = 'plpythonu, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server
12 changes: 6 additions & 6 deletions server/extension/sql/110_data_observatory.sql
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@ RETURNS json AS $$

plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]

quota_service = QuotaService(user_data_observatory_config, redis_conn)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')

Expand Down Expand Up @@ -48,10 +48,10 @@ RETURNS json AS $$

plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_data_observatory_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_data_observatory_config = GD["user_data_observatory_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]

quota_service = QuotaService(user_data_observatory_config, redis_conn)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')

Expand Down
10 changes: 5 additions & 5 deletions server/extension/sql/15_config_helper.sql
Original file line number Diff line number Diff line change
Expand Up @@ -54,16 +54,16 @@ RETURNS boolean AS $$
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;

CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_data_observatory_config(username text, orgname text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_obs_snapshot_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_data_observatory_config_{0}".format(username)
cache_key = "user_obs_snapshot_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import DataObservatoryConfig
from cartodb_services.metrics import ObservatorySnapshotConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
data_observatory_config = DataObservatoryConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = data_observatory_config
obs_snapshot_config = ObservatorySnapshotConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = obs_snapshot_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, DataObservatoryConfig
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig
from quota import QuotaService
from user import UserMetricsService
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,31 @@ def username(self):
def organization(self):
return self._orgname

class DataObservatoryConfig(ServiceConfig):

class ObservatorySnapshotConfig(ServiceConfig):

SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit'
QUOTA_KEY = 'obs_snapshot_quota'
PERIOD_END_DATE = 'period_end_date'

def __init__(self, redis_connection, db_conn, username, orgname=None):
super(DataObservatoryConfig, self).__init__(redis_connection, db_conn,
super(ObservatorySnapshotConfig, self).__init__(redis_connection, db_conn,
username, orgname)
self._monthly_quota = self._db_config.data_observatory_monthly_quota
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
if self.SOFT_LIMIT_KEY in self._redis_config and self._redis_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_limit = True
else:
self._soft_limit = False
# Mixed config between db and redis. If we don't update all the users
# in redis, we could use the db value as default
if self.QUOTA_KEY in self._redis_config:
self._monthly_quota = float(self._redis_config[self.QUOTA_KEY])
else:
self._monthly_quota = float(self._db_config.data_observatory_monthly_quota)

@property
def service_type(self):
return 'data_observatory'
return 'obs_snapshot'

@property
def monthly_quota(self):
Expand All @@ -54,6 +66,10 @@ def monthly_quota(self):
def period_end_date(self):
return self._period_end_date

@property
def soft_limit(self):
return self._soft_limit


class RoutingConfig(ServiceConfig):

Expand Down Expand Up @@ -200,28 +216,6 @@ def __init__(self, redis_connection, db_conn, username, orgname=None):
self.__parse_config(filtered_config, self._db_config)
self.__check_config(filtered_config)

def __get_user_config(self, username, orgname):
user_config = self._redis_connection.hgetall(
"rails:users:{0}".format(username))
if not user_config:
raise ConfigException("""There is no user config available. Please check your configuration.'""")
else:
if orgname:
self.__get_organization_config(orgname, user_config)

return user_config

def __get_organization_config(self, orgname, user_config):
org_config = self._redis_connection.hgetall(
"rails:orgs:{0}".format(orgname))
if not org_config:
raise ConfigException("""There is no organization config available. Please check your configuration.'""")
else:
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]

def __check_config(self, filtered_config):
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())) or \
Expand Down Expand Up @@ -438,24 +432,24 @@ class ServicesRedisConfig:
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
QUOTA_KEY = 'geocoding_quota'
ISOLINES_QUOTA_KEY = 'here_isolines_quota'
OBS_SNAPSHOT_QUOTA_KEY = 'obs_snapshot_quota'
PERIOD_END_DATE = 'period_end_date'

def __init__(self, redis_conn):
self._redis_connection = redis_conn

def build(self, username, password):
return self.__get_user_config(username, password)
def build(self, username, orgname):
return self.__get_user_config(username, orgname)

def __get_user_config(self, username, orgname):
user_config = self._redis_connection.hgetall(
"rails:users:{0}".format(username))
if not user_config:
raise ConfigException("""There is no user config available. Please check your configuration.'""")
else:
if orgname:
self.__get_organization_config(orgname, user_config)
elif orgname:
self.__get_organization_config(orgname, user_config)

return user_config
return user_config

def __get_organization_config(self, orgname, user_config):
org_config = self._redis_connection.hgetall(
Expand All @@ -465,6 +459,8 @@ def __get_organization_config(self, orgname, user_config):
else:
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
user_config[self.ISOLINES_QUOTA_KEY] = org_config[self.ISOLINES_QUOTA_KEY]
if self.OBS_SNAPSHOT_QUOTA_KEY in org_config:
user_config[self.OBS_SNAPSHOT_QUOTA_KEY] = org_config[self.OBS_SNAPSHOT_QUOTA_KEY]
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
Loading

0 comments on commit 5efb81d

Please sign in to comment.