Change last requests in last day to lastest failed requests (fix holm…

…es-app#114)

holmes/config/__init__.py

@@ -33,6 +33,7 @@
 Config.define('NUMBER_OF_REVIEWS_TO_KEEP', 4, 'Maximum number of reviews to keep', 'Review')
 
 Config.define('DAYS_TO_KEEP_REQUESTS', 12, 'Number of days to keep requests', 'Requests')
+Config.define('MAX_REQUESTS_FOR_FAILED_RESPONSES', 1000, 'Number of requests for falied responses', 'Requests')
 
 Config.define('MAX_ENQUEUE_BUFFER_LENGTH', 1000,
               'Number of urls to enqueue before submitting to the /pages route', 'Validators')
@@ -117,7 +118,7 @@
     'violation_count_by_category_for_domains': 3 * MINUTE + 11,
     'blacklist_domain_count': 10 * MINUTE + 1,
     'most_common_violations': HOUR + 7,
-    'requests_in_last_day_count': HOUR + 13,
+    'failed_responses_count': HOUR + 13,
 }
 Config.define('MATERIALS_EXPIRATION_IN_SECONDS', materials_expiration_in_seconds, 'Expire times for materials', 'material')
 
@@ -126,7 +127,7 @@
     'violation_count_by_category_for_domains': 2 * materials_expiration_in_seconds['violation_count_by_category_for_domains'],
     'blacklist_domain_count': 2 * materials_expiration_in_seconds['blacklist_domain_count'],
     'most_common_violations': 2 * materials_expiration_in_seconds['most_common_violations'],
-    'requests_in_last_day_count': 2 * materials_expiration_in_seconds['requests_in_last_day_count'],
+    'failed_responses_count': 2 * materials_expiration_in_seconds['failed_responses_count'],
 }
 Config.define('MATERIALS_GRACE_PERIOD_IN_SECONDS', materials_grace_period_in_seconds, 'Grace period times for materials', 'material')
 

holmes/handlers/request.py

@@ -75,10 +75,10 @@ def get(self):
         )
 
 
-class RequestsInLastDayHandler(BaseHandler):
+class FailedResponsesHandler(BaseHandler):
     @coroutine
     def get(self):
-        requests = self.girl.get('requests_in_last_day')
+        requests = self.girl.get('failed_responses_count')
         domain_filter = self.get_argument('domain_filter', None)
 
         if not domain_filter:

holmes/material.py

@@ -48,10 +48,10 @@ def configure_materials(girl, db, config):
     )
 
     girl.add_material(
-        'requests_in_last_day',
-        partial(MaterialConveyor.get_requests_in_last_day, db),
-        config.MATERIALS_EXPIRATION_IN_SECONDS['requests_in_last_day_count'],
-        config.MATERIALS_GRACE_PERIOD_IN_SECONDS['requests_in_last_day_count']
+        'failed_responses_count',
+        partial(MaterialConveyor.failed_responses_count, db, config),
+        config.MATERIALS_EXPIRATION_IN_SECONDS['failed_responses_count'],
+        config.MATERIALS_GRACE_PERIOD_IN_SECONDS['failed_responses_count']
     )
 
 
@@ -67,9 +67,11 @@ def get_blacklist_domain_count(cls, db):
         return [dict(zip(('domain', 'count'), x)) for x in blacklist]
 
     @classmethod
-    def get_requests_in_last_day(cls, db):
-        from_date = datetime.datetime.utcnow() - datetime.timedelta(days=1)
-        return Request.get_requests_count_by_status_in_period_of_days(db, from_date=from_date)
+    def failed_responses_count(cls, db, config):
+        return Request.get_requests_count_by_status(
+            db,
+            config.MAX_REQUESTS_FOR_FAILED_RESPONSES
+        )
 
 
 class MaterialWorker(BaseCLI):

holmes/models/request.py

@@ -102,23 +102,20 @@ def get_last_requests(self, db, current_page=1, page_size=10,
         return query.order_by('id desc')[lower_bound:upper_bound]
 
     @classmethod
-    def get_requests_count_by_status_in_period_of_days(self, db, from_date,
-                                                       to_date=None):
-        if to_date is None:
-            to_date = datetime.utcnow()
-
+    def get_requests_count_by_status(self, db, limit=1000):
         per_domains = {'_all': defaultdict(int)}
 
         from holmes.models.domain import Domain
         for domain in db.query(Domain).all():
             requests = db \
-                .query(Request.status_code,
-                       sa.func.count(Request.id).label('count')) \
-                .filter(Request.completed_date.between(from_date.date(),
-                                                       to_date.date())) \
+                .query(
+                    Request.status_code,
+                    sa.func.count(Request.id).label('count')
+                ) \
                 .filter(Request.domain_name == domain.name) \
                 .group_by(Request.status_code) \
                 .order_by('count DESC') \
+                .limit(limit) \
                 .all()
             per_domains[domain.name] = requests
 

holmes/server.py

@@ -33,7 +33,7 @@
     TaxHandler
 )
 from holmes.handlers.request import (
-    RequestDomainHandler, LastRequestsHandler, RequestsInLastDayHandler,
+    RequestDomainHandler, LastRequestsHandler, FailedResponsesHandler,
     LastRequestsStatusCodeHandler
 )
 from holmes.handlers.limiter import LimiterHandler
@@ -113,7 +113,7 @@ def get_handlers(self):
             ('/next-jobs/?', NextJobHandler),
             ('/last-requests/?', LastRequestsHandler),
             ('/last-requests/status-code/?', LastRequestsStatusCodeHandler),
-            ('/requests-in-last-day/?', RequestsInLastDayHandler),
+            ('/last-requests/failed-responses/?', FailedResponsesHandler),
             ('/version/?', VersionHandler),
         ]
 

tests/unit/handlers/test_requests.py

@@ -79,9 +79,11 @@ def test_get_last_requests_filter_by_staus_code(self):
         expect(response_body['requests'][0]['domain_name']).to_be_like('globo.com')
 
 
-class TestRequestsInLastDayHandler(ApiTestCase):
+class TestFailedResponsesHandler(ApiTestCase):
     @gen_test
-    def test_get_requests_in_last_day(self):
+    def test_can_get_latest_failed_responses(self):
+        self.db.query(Request).delete()
+
         utcnow = datetime.utcnow()
 
         DomainFactory.create(name='globo.com')
@@ -112,31 +114,31 @@ def test_get_requests_in_last_day(self):
 
         self.db.flush()
 
-        response = yield self.http_client.fetch(self.get_url('/requests-in-last-day/'))
+        response = yield self.http_client.fetch(self.get_url('/last-requests/failed-responses/'))
         expect(response.code).to_equal(200)
         expect(loads(response.body)).to_be_like([
-            {u'count': 2, u'statusCodeTitle': u'OK', u'statusCode': 200},
-            {u'count': 4, u'statusCodeTitle': u'Not Found', u'statusCode': 404}
+            {u'count': 3, u'statusCodeTitle': u'OK', u'statusCode': 200},
+            {u'count': 6, u'statusCodeTitle': u'Not Found', u'statusCode': 404}
         ])
 
-        response = yield self.http_client.fetch(self.get_url('/requests-in-last-day/?domain_filter=globo.com'))
+        response = yield self.http_client.fetch(self.get_url('/last-requests/failed-responses/?domain_filter=globo.com'))
         expect(response.code).to_equal(200)
         expect(loads(response.body)).to_be_like([
-            {u'count': 2, u'statusCodeTitle': u'OK', u'statusCode': 200},
-            {u'count': 2, u'statusCodeTitle': u'Not Found', u'statusCode': 404}
+            {u'count': 3, u'statusCodeTitle': u'OK', u'statusCode': 200},
+            {u'count': 3, u'statusCodeTitle': u'Not Found', u'statusCode': 404}
         ])
 
-        response = yield self.http_client.fetch(self.get_url('/requests-in-last-day/?domain_filter=globoesporte.com'))
+        response = yield self.http_client.fetch(self.get_url('/last-requests/failed-responses/?domain_filter=globoesporte.com'))
         expect(response.code).to_equal(200)
         expect(loads(response.body)).to_be_like([
-            {u'count': 2, u'statusCodeTitle': u'Not Found', u'statusCode': 404}
+            {u'count': 3, u'statusCodeTitle': u'Not Found', u'statusCode': 404}
         ])
 
-        response = yield self.http_client.fetch(self.get_url('/requests-in-last-day/?domain_filter=g1.globo.com'))
+        response = yield self.http_client.fetch(self.get_url('/last-requests/failed-responses/?domain_filter=g1.globo.com'))
         expect(response.code).to_equal(200)
         expect(loads(response.body)).to_be_like([])
 
-        response = yield self.http_client.fetch(self.get_url('/requests-in-last-day/?domain_filter=domain3.com'))
+        response = yield self.http_client.fetch(self.get_url('/last-requests/failed-responses/?domain_filter=domain3.com'))
         expect(response.code).to_equal(200)
         expect(loads(response.body)).to_be_like([])
 

tests/unit/models/test_request.py

@@ -102,7 +102,7 @@ def test_can_get_requests_by_status_count(self):
         )
         expect(invalid_code).to_equal([])
 
-    def test_can_get_requests_count_by_status_in_period_of_days(self):
+    def test_can_get_requests_count_by_status(self):
         utcnow = datetime.utcnow()
 
         DomainFactory.create(name='globo.com')
@@ -132,16 +132,13 @@ def test_can_get_requests_count_by_status_in_period_of_days(self):
 
         self.db.flush()
 
-        one_day_ago = utcnow - timedelta(days=1)
-        counts = Request.get_requests_count_by_status_in_period_of_days(
-            self.db, one_day_ago
-        )
-        expect(counts).to_equal(
-            {'_all': [(200L, 2L), (404L, 4L)],
-             u'globo.com': [(200L, 2L), (404L, 2L)],
+        counts = Request.get_requests_count_by_status(self.db)
+        expect(counts).to_equal({
+            '_all': [(200, 3), (404, 6)],
+             u'globo.com': [(200, 3), (404, 3)],
              u'domain3.com': [],
-             u'globoesporte.com': [(404L, 2L)]}
-        )
+             u'globoesporte.com': [(404, 3)]
+        })
 
     def test_can_remove_old_requests(self):
         self.db.query(Request).delete()