From 8cf7c636ff887f0ec8520f9cc3de4dd5a31187bf Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Mon, 24 Sep 2018 22:05:53 +0530 Subject: [PATCH 01/14] 505-dailymotion-bug-solved --- app/scrapers/dailymotion.py | 1 - app/scrapers/generalized.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/app/scrapers/dailymotion.py b/app/scrapers/dailymotion.py index 394ce30b..37bbeaf4 100644 --- a/app/scrapers/dailymotion.py +++ b/app/scrapers/dailymotion.py @@ -21,7 +21,6 @@ def parse_response(soup): [[Tile1,url1], [Title2, url2],..] """ urls = [] - video_list = json.loads(str(soup))['list'] for item in video_list: title = item['title'] diff --git a/app/scrapers/generalized.py b/app/scrapers/generalized.py index 12ade912..810696e2 100644 --- a/app/scrapers/generalized.py +++ b/app/scrapers/generalized.py @@ -42,6 +42,8 @@ def get_page(self, query, startIndex=0, qtype=''): if self.name == 'mojeek' and qtype == 'news': payload['fmt'] = 'news' response = requests.get(url, headers=self.headers, params=payload) + if "dailymotion" in url: + response = requests.get(url=self.refactor_url(response.url), headers=self.headers) print(response.url) return response @@ -57,6 +59,13 @@ def parse_video_response(soup): def next_start(current_start, prev_results): return current_start + len(prev_results) + @staticmethod + def refactor_url(url): + url = str(url) + index_of_question_mark = url.index('?') + url = url[0:index_of_question_mark + 1] + url[index_of_question_mark + 3:] + return url + def search(self, query, num_results, qtype=''): """ Search for the query and return set of urls @@ -98,3 +107,7 @@ def search_without_count(self, query): soup = BeautifulSoup(response.text, 'html.parser') urls = self.parse_response(soup) return urls + + + + From 9bc94a933d43bb21607c258ae3d3a52d4627516a Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Mon, 24 Sep 2018 22:27:07 +0530 Subject: [PATCH 02/14] 505-dailymotion-bug --- app/scrapers/generalized.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/scrapers/generalized.py b/app/scrapers/generalized.py index 810696e2..ad53bb32 100644 --- a/app/scrapers/generalized.py +++ b/app/scrapers/generalized.py @@ -63,7 +63,9 @@ def next_start(current_start, prev_results): def refactor_url(url): url = str(url) index_of_question_mark = url.index('?') - url = url[0:index_of_question_mark + 1] + url[index_of_question_mark + 3:] + first_part = url[0:index_of_question_mark+1] + second_part = url[index_of_question_mark+3:len(url)] + url = first_part + second_part return url def search(self, query, num_results, qtype=''): From 49effbdf29b57299c10fd5d41ab379dce659c8cd Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Mon, 24 Sep 2018 22:36:08 +0530 Subject: [PATCH 03/14] 505-mock-bug-solved --- app/scrapers/generalized.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/scrapers/generalized.py b/app/scrapers/generalized.py index ad53bb32..19c35baf 100644 --- a/app/scrapers/generalized.py +++ b/app/scrapers/generalized.py @@ -43,7 +43,10 @@ def get_page(self, query, startIndex=0, qtype=''): payload['fmt'] = 'news' response = requests.get(url, headers=self.headers, params=payload) if "dailymotion" in url: - response = requests.get(url=self.refactor_url(response.url), headers=self.headers) + response = requests.get( + url=self.refactor_url(response.url), + headers=self.headers + ) print(response.url) return response From 8e5f85220be1c0a0712ee0e3f5b9224766ae3bfe Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Mon, 24 Sep 2018 22:39:43 +0530 Subject: [PATCH 04/14] 505-mock-bug-solved --- app/scrapers/generalized.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/scrapers/generalized.py b/app/scrapers/generalized.py index 19c35baf..c749882c 100644 --- a/app/scrapers/generalized.py +++ b/app/scrapers/generalized.py @@ -112,7 +112,3 @@ def search_without_count(self, query): soup = BeautifulSoup(response.text, 'html.parser') urls = self.parse_response(soup) return urls - - - - From fc245ca207d888d0f0a62a1b555a53a3105f1eb3 Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Mon, 24 Sep 2018 23:01:50 +0530 Subject: [PATCH 05/14] 505-dailymotion-bug-solved --- app/scrapers/dailymotion.py | 1 + app/scrapers/generalized.py | 12 +++--------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/app/scrapers/dailymotion.py b/app/scrapers/dailymotion.py index 37bbeaf4..394ce30b 100644 --- a/app/scrapers/dailymotion.py +++ b/app/scrapers/dailymotion.py @@ -21,6 +21,7 @@ def parse_response(soup): [[Tile1,url1], [Title2, url2],..] """ urls = [] + video_list = json.loads(str(soup))['list'] for item in video_list: title = item['title'] diff --git a/app/scrapers/generalized.py b/app/scrapers/generalized.py index c749882c..bb88fed1 100644 --- a/app/scrapers/generalized.py +++ b/app/scrapers/generalized.py @@ -44,7 +44,7 @@ def get_page(self, query, startIndex=0, qtype=''): response = requests.get(url, headers=self.headers, params=payload) if "dailymotion" in url: response = requests.get( - url=self.refactor_url(response.url), + url=self.refactor_url(str(response.url)), headers=self.headers ) print(response.url) @@ -62,14 +62,8 @@ def parse_video_response(soup): def next_start(current_start, prev_results): return current_start + len(prev_results) - @staticmethod - def refactor_url(url): - url = str(url) - index_of_question_mark = url.index('?') - first_part = url[0:index_of_question_mark+1] - second_part = url[index_of_question_mark+3:len(url)] - url = first_part + second_part - return url + def refactor_url(self, url): + return url[0:url.index('?')+1]+url[url.index('?')+3:len(url)] def search(self, query, num_results, qtype=''): """ From fa10cd2494f4d023b3cb31c0cdb9c17f4defb979 Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Mon, 24 Sep 2018 23:23:51 +0530 Subject: [PATCH 06/14] 505-mock-bug-solved --- app/scrapers/generalized.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/scrapers/generalized.py b/app/scrapers/generalized.py index bb88fed1..93d66c69 100644 --- a/app/scrapers/generalized.py +++ b/app/scrapers/generalized.py @@ -44,7 +44,7 @@ def get_page(self, query, startIndex=0, qtype=''): response = requests.get(url, headers=self.headers, params=payload) if "dailymotion" in url: response = requests.get( - url=self.refactor_url(str(response.url)), + url=url[0:url.index('?') + 1] + url[url.index('?') + 3:len(url)], headers=self.headers ) print(response.url) @@ -62,9 +62,6 @@ def parse_video_response(soup): def next_start(current_start, prev_results): return current_start + len(prev_results) - def refactor_url(self, url): - return url[0:url.index('?')+1]+url[url.index('?')+3:len(url)] - def search(self, query, num_results, qtype=''): """ Search for the query and return set of urls From f7887d544e5185c05c6fd34e47bc11246b298037 Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Mon, 24 Sep 2018 23:27:00 +0530 Subject: [PATCH 07/14] 505-mock-bug-solved --- app/scrapers/generalized.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scrapers/generalized.py b/app/scrapers/generalized.py index 93d66c69..6b0bf8ae 100644 --- a/app/scrapers/generalized.py +++ b/app/scrapers/generalized.py @@ -44,7 +44,7 @@ def get_page(self, query, startIndex=0, qtype=''): response = requests.get(url, headers=self.headers, params=payload) if "dailymotion" in url: response = requests.get( - url=url[0:url.index('?') + 1] + url[url.index('?') + 3:len(url)], + url=url[0:url.index('?')+ 1] + url[url.index('?')+3:len(url)], headers=self.headers ) print(response.url) From 97a30772f563fb51996f2bad43fb9a199a72d13e Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Mon, 24 Sep 2018 23:30:04 +0530 Subject: [PATCH 08/14] 505-mock-bug-solved --- app/scrapers/generalized.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/scrapers/generalized.py b/app/scrapers/generalized.py index 6b0bf8ae..06ed68e2 100644 --- a/app/scrapers/generalized.py +++ b/app/scrapers/generalized.py @@ -44,7 +44,7 @@ def get_page(self, query, startIndex=0, qtype=''): response = requests.get(url, headers=self.headers, params=payload) if "dailymotion" in url: response = requests.get( - url=url[0:url.index('?')+ 1] + url[url.index('?')+3:len(url)], + url=url[0:url.index('?')+1] + url[url.index('?')+3:len(url)], headers=self.headers ) print(response.url) From d3d06096ccbd56040d4e2d530139ec073677bba8 Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Tue, 25 Sep 2018 00:56:05 +0530 Subject: [PATCH 09/14] 505-Test-case-added --- app/scrapers/generalized.py | 7 ++++-- test/test_dailymotion.py | 45 +++++++++++++++++++++++++++++++++++++ test/test_generalized.py | 21 +++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 test/test_dailymotion.py diff --git a/app/scrapers/generalized.py b/app/scrapers/generalized.py index 06ed68e2..29fe7607 100644 --- a/app/scrapers/generalized.py +++ b/app/scrapers/generalized.py @@ -42,9 +42,12 @@ def get_page(self, query, startIndex=0, qtype=''): if self.name == 'mojeek' and qtype == 'news': payload['fmt'] = 'news' response = requests.get(url, headers=self.headers, params=payload) - if "dailymotion" in url: + if "dailymotion" in response.url: + url = response.url + index = url.index('?') + url = url[0:index + 1] + url[index + 3:len(url)] response = requests.get( - url=url[0:url.index('?')+1] + url[url.index('?')+3:len(url)], + url=url, headers=self.headers ) print(response.url) diff --git a/test/test_dailymotion.py b/test/test_dailymotion.py new file mode 100644 index 00000000..f9ba58f5 --- /dev/null +++ b/test/test_dailymotion.py @@ -0,0 +1,45 @@ +import json +from app.scrapers import DailyMotion + + +def test_parse_response(): + json_data = '{"list":[{"id":"x6ilw2u","title":"Job Interviews - Stand Up Comedy by Piyush Sharma","channel":"fun",' \ + '"owner":"x24pst6"},{"id":"x65slib",' \ + '"title":"Beyhadh actor Piyush Sahdev(Samay) will soon enter colors Devanshi opposite Helly shah",' \ + '"channel":"fun","owner":"x21w079"},{"id":"x5fj9kg",' \ + '"title":"Deepika-Piyush Se Mulaqat - Dhhai Kilo Prem",' \ + '"channel":"fun","owner":"x1isdap"},{"id":"x59m108",' \ + '"title":"Sasural Simar Ka TOOT GAYA PIYUSH VAIDAHI KA RISHTA 26 January 2017 News",' \ + '"channel":"fun","owner":"x1wmonz"},' \ + '{"id":"x6aj8su","title":"Beyhadh actor Piyush Sahdev ARRESTED on Rape charges | FilmiBeat",' \ + '"channel":"fun","owner":"x1x1a21"},{"id":"x4eq3kp","title":"Funny Balloon Tricks By Piyush Seth",' \ + '"channel":"fun","owner":"x1swlum"},{"id":"x6cmk7x",' \ + '"title":"Beyhadh Actor Piyush Sahdev Finally REACTS On His Molestation Case",' \ + '"channel":"fun","owner":"x22ch35"},{"id":"x3i6xr3",' \ + '"title":"Revolver Rani Movie || I Am Brutal Video Song || Kangana Ranaut, Piyush Mishra, Vir Das",' \ + '"channel":"fun","owner":"x1ocl9w"},{"id":"xhv1dj",' \ + '"title":"Birthday party of t v actor piyush sachdev","channel":"fun","owner":"xmoljl"},' \ + '{"id":"x6647o3","title":"Piyush Mishra Aarambh Hai Prachand (Live At Hindu College)",' \ + '"channel":"fun","owner":"x21w0ck"}]}' + + video_list = json.loads(json_data)['list'] + expected_resp = [{ + 'link': 'https://www.dailymotion.com/video/x6ilw2u', + 'title': u'Job Interviews - Stand Up Comedy by Piyush Sharma'}, + {'link': 'https://www.dailymotion.com/video/x65slib', + 'title': u'Beyhadh actor Piyush Sahdev(Samay) will soon enter colors Devanshi opposite Helly shah'}, + {'link': 'https://www.dailymotion.com/video/x5fj9kg', 'title': u'Deepika-Piyush Se Mulaqat - Dhhai Kilo Prem'}, + {'link': 'https://www.dailymotion.com/video/x59m108', + 'title': u'Sasural Simar Ka TOOT GAYA PIYUSH VAIDAHI KA RISHTA 26 January 2017 News'}, + {'link': 'https://www.dailymotion.com/video/x6aj8su', + 'title': u'Beyhadh actor Piyush Sahdev ARRESTED on Rape charges | FilmiBeat'}, + {'link': 'https://www.dailymotion.com/video/x4eq3kp', 'title': u'Funny Balloon Tricks By Piyush Seth'}, + {'link': 'https://www.dailymotion.com/video/x6cmk7x', + 'title': u'Beyhadh Actor Piyush Sahdev Finally REACTS On His Molestation Case'}, + {'link': 'https://www.dailymotion.com/video/x3i6xr3', + 'title': u'Revolver Rani Movie || I Am Brutal Video Song || Kangana Ranaut, Piyush Mishra, Vir Das'}, + {'link': 'https://www.dailymotion.com/video/xhv1dj', 'title': u'Birthday party of t v actor piyush sachdev'}, + {'link': 'https://www.dailymotion.com/video/x6647o3', + 'title': u'Piyush Mishra Aarambh Hai Prachand (Live At Hindu College)'}] + resp = DailyMotion().parse_response(json_data) + assert resp == expected_resp diff --git a/test/test_generalized.py b/test/test_generalized.py index d0775d6d..0736ceb5 100644 --- a/test/test_generalized.py +++ b/test/test_generalized.py @@ -22,6 +22,27 @@ def test_get_page(mock_request_get, mock_response): mock_request_get.assert_called_with( '', headers=expected_headers, params=expected_payload) +# +# @patch('') +# @patch('') +# def test_get_page_of_dailymotion(mock_request_get, mock_response): +# mock_request_get.return_value = mock_response +# mock_response.url = "Mock Url" +# import pdb +# pdb.set_trace() +# Scraper().get_page("dummy_query") +# assert response == mock_response +# expected_payload = {'q': 'dummy_query', '': ''} +# expected_headers = { +# 'User-Agent': ( +# 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) ' +# 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 ' +# 'Safari/537.36' +# ) +# } +# mock_request_get.assert_called_with( +# '', headers=expected_headers, params=expected_payload) + def test_parse_response(): with pytest.raises(NotImplementedError): From db8a5083ec994d4d862ca0924f3dd90af49ae361 Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Tue, 25 Sep 2018 01:01:15 +0530 Subject: [PATCH 10/14] 505-change --- app/scrapers/dailymotion.py | 1 + test/test_dailymotion.py | 55 +++++++++++++++++++++++++------------ 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/app/scrapers/dailymotion.py b/app/scrapers/dailymotion.py index 394ce30b..c376a125 100644 --- a/app/scrapers/dailymotion.py +++ b/app/scrapers/dailymotion.py @@ -23,6 +23,7 @@ def parse_response(soup): urls = [] video_list = json.loads(str(soup))['list'] + for item in video_list: title = item['title'] link = 'https://www.dailymotion.com/video/' + str(item['id']) diff --git a/test/test_dailymotion.py b/test/test_dailymotion.py index f9ba58f5..2c6d1874 100644 --- a/test/test_dailymotion.py +++ b/test/test_dailymotion.py @@ -3,23 +3,33 @@ def test_parse_response(): - json_data = '{"list":[{"id":"x6ilw2u","title":"Job Interviews - Stand Up Comedy by Piyush Sharma","channel":"fun",' \ + json_data = '{"list":[{"id":"x6ilw2u",' \ + '"title":"Job Interviews - Stand Up Comedy by Piyush Sharma",' \ + '"channel":"fun",' \ '"owner":"x24pst6"},{"id":"x65slib",' \ - '"title":"Beyhadh actor Piyush Sahdev(Samay) will soon enter colors Devanshi opposite Helly shah",' \ + '"title":"Beyhadh actor Piyush Sahdev(Samay) will soon enter' \ + ' colors Devanshi opposite Helly shah",' \ '"channel":"fun","owner":"x21w079"},{"id":"x5fj9kg",' \ '"title":"Deepika-Piyush Se Mulaqat - Dhhai Kilo Prem",' \ '"channel":"fun","owner":"x1isdap"},{"id":"x59m108",' \ - '"title":"Sasural Simar Ka TOOT GAYA PIYUSH VAIDAHI KA RISHTA 26 January 2017 News",' \ + '"title":"Sasural Simar Ka TOOT GAYA PIYUSH VAIDAHI KA' \ + ' RISHTA 26 January 2017 News",' \ '"channel":"fun","owner":"x1wmonz"},' \ - '{"id":"x6aj8su","title":"Beyhadh actor Piyush Sahdev ARRESTED on Rape charges | FilmiBeat",' \ - '"channel":"fun","owner":"x1x1a21"},{"id":"x4eq3kp","title":"Funny Balloon Tricks By Piyush Seth",' \ + '{"id":"x6aj8su","title":"Beyhadh actor Piyush Sahdev ' \ + 'ARRESTED on Rape charges | FilmiBeat",' \ + '"channel":"fun","owner":"x1x1a21"},{"id":"x4eq3kp",' \ + '"title":"Funny Balloon Tricks By Piyush Seth",' \ '"channel":"fun","owner":"x1swlum"},{"id":"x6cmk7x",' \ - '"title":"Beyhadh Actor Piyush Sahdev Finally REACTS On His Molestation Case",' \ + '"title":"Beyhadh Actor Piyush Sahdev Finally REACTS On' \ + ' His Molestation Case",' \ '"channel":"fun","owner":"x22ch35"},{"id":"x3i6xr3",' \ - '"title":"Revolver Rani Movie || I Am Brutal Video Song || Kangana Ranaut, Piyush Mishra, Vir Das",' \ + '"title":"Revolver Rani Movie || I Am Brutal Video Song ' \ + '|| Kangana Ranaut, Piyush Mishra, Vir Das",' \ '"channel":"fun","owner":"x1ocl9w"},{"id":"xhv1dj",' \ - '"title":"Birthday party of t v actor piyush sachdev","channel":"fun","owner":"xmoljl"},' \ - '{"id":"x6647o3","title":"Piyush Mishra Aarambh Hai Prachand (Live At Hindu College)",' \ + '"title":"Birthday party of t v actor piyush sachdev"' \ + ',"channel":"fun","owner":"xmoljl"},' \ + '{"id":"x6647o3","title":"Piyush Mishra Aarambh Hai' \ + ' Prachand (Live At Hindu College)",' \ '"channel":"fun","owner":"x21w0ck"}]}' video_list = json.loads(json_data)['list'] @@ -27,19 +37,28 @@ def test_parse_response(): 'link': 'https://www.dailymotion.com/video/x6ilw2u', 'title': u'Job Interviews - Stand Up Comedy by Piyush Sharma'}, {'link': 'https://www.dailymotion.com/video/x65slib', - 'title': u'Beyhadh actor Piyush Sahdev(Samay) will soon enter colors Devanshi opposite Helly shah'}, - {'link': 'https://www.dailymotion.com/video/x5fj9kg', 'title': u'Deepika-Piyush Se Mulaqat - Dhhai Kilo Prem'}, + 'title': u'Beyhadh actor Piyush Sahdev(Samay) will soon ' + u'enter colors Devanshi opposite Helly shah'}, + {'link': 'https://www.dailymotion.com/video/x5fj9kg', + 'title': u'Deepika-Piyush Se Mulaqat - Dhhai Kilo Prem'}, {'link': 'https://www.dailymotion.com/video/x59m108', - 'title': u'Sasural Simar Ka TOOT GAYA PIYUSH VAIDAHI KA RISHTA 26 January 2017 News'}, + 'title': u'Sasural Simar Ka TOOT GAYA PIYUSH VAIDAHI KA' + u' RISHTA 26 January 2017 News'}, {'link': 'https://www.dailymotion.com/video/x6aj8su', - 'title': u'Beyhadh actor Piyush Sahdev ARRESTED on Rape charges | FilmiBeat'}, - {'link': 'https://www.dailymotion.com/video/x4eq3kp', 'title': u'Funny Balloon Tricks By Piyush Seth'}, + 'title': u'Beyhadh actor Piyush Sahdev ARRESTED on ' + u'Rape charges | FilmiBeat'}, + {'link': 'https://www.dailymotion.com/video/x4eq3kp', + 'title': u'Funny Balloon Tricks By Piyush Seth'}, {'link': 'https://www.dailymotion.com/video/x6cmk7x', - 'title': u'Beyhadh Actor Piyush Sahdev Finally REACTS On His Molestation Case'}, + 'title': u'Beyhadh Actor Piyush Sahdev Finally REACTS' + u' On His Molestation Case'}, {'link': 'https://www.dailymotion.com/video/x3i6xr3', - 'title': u'Revolver Rani Movie || I Am Brutal Video Song || Kangana Ranaut, Piyush Mishra, Vir Das'}, - {'link': 'https://www.dailymotion.com/video/xhv1dj', 'title': u'Birthday party of t v actor piyush sachdev'}, + 'title': u'Revolver Rani Movie || I Am Brutal Video' + u' Song || Kangana Ranaut, Piyush Mishra, Vir Das'}, + {'link': 'https://www.dailymotion.com/video/xhv1dj', + 'title': u'Birthday party of t v actor piyush sachdev'}, {'link': 'https://www.dailymotion.com/video/x6647o3', - 'title': u'Piyush Mishra Aarambh Hai Prachand (Live At Hindu College)'}] + 'title': u'Piyush Mishra Aarambh Hai Prachand ' + u'(Live At Hindu College)'}] resp = DailyMotion().parse_response(json_data) assert resp == expected_resp From dae1fbc5701f91036aa1b19f1ed925b2c7d1f038 Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Tue, 25 Sep 2018 01:03:47 +0530 Subject: [PATCH 11/14] 505-change-done --- test/test_dailymotion.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_dailymotion.py b/test/test_dailymotion.py index 2c6d1874..daca54fb 100644 --- a/test/test_dailymotion.py +++ b/test/test_dailymotion.py @@ -4,10 +4,12 @@ def test_parse_response(): json_data = '{"list":[{"id":"x6ilw2u",' \ - '"title":"Job Interviews - Stand Up Comedy by Piyush Sharma",' \ + '"title":"Job Interviews - Stand Up Comedy by ' \ + 'Piyush Sharma",' \ '"channel":"fun",' \ '"owner":"x24pst6"},{"id":"x65slib",' \ - '"title":"Beyhadh actor Piyush Sahdev(Samay) will soon enter' \ + '"title":"Beyhadh actor Piyush Sahdev(Samay) will ' \ + 'soon enter' \ ' colors Devanshi opposite Helly shah",' \ '"channel":"fun","owner":"x21w079"},{"id":"x5fj9kg",' \ '"title":"Deepika-Piyush Se Mulaqat - Dhhai Kilo Prem",' \ From 34102131705c3ff11c9f9f213f8b628c529d7b8b Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Tue, 25 Sep 2018 01:06:09 +0530 Subject: [PATCH 12/14] 505-Changes-made-travis --- test/test_dailymotion.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_dailymotion.py b/test/test_dailymotion.py index daca54fb..1729af5d 100644 --- a/test/test_dailymotion.py +++ b/test/test_dailymotion.py @@ -34,7 +34,6 @@ def test_parse_response(): ' Prachand (Live At Hindu College)",' \ '"channel":"fun","owner":"x21w0ck"}]}' - video_list = json.loads(json_data)['list'] expected_resp = [{ 'link': 'https://www.dailymotion.com/video/x6ilw2u', 'title': u'Job Interviews - Stand Up Comedy by Piyush Sharma'}, From d54a386ed3204cfab40d2a54ec22ce182cc42e3e Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Tue, 25 Sep 2018 01:09:07 +0530 Subject: [PATCH 13/14] 505-Changes-made-travis --- test/test_dailymotion.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_dailymotion.py b/test/test_dailymotion.py index 1729af5d..e16eeeb9 100644 --- a/test/test_dailymotion.py +++ b/test/test_dailymotion.py @@ -1,4 +1,3 @@ -import json from app.scrapers import DailyMotion From a39c28f9e0a3e6aafb78670953ae7c5cb325e23e Mon Sep 17 00:00:00 2001 From: nalawadepiyush Date: Tue, 25 Sep 2018 02:11:49 +0530 Subject: [PATCH 14/14] 505 issue solved --- test/test_dailymotion.py | 56 +++------------------------------------- test/test_generalized.py | 21 --------------- 2 files changed, 3 insertions(+), 74 deletions(-) diff --git a/test/test_dailymotion.py b/test/test_dailymotion.py index e16eeeb9..82417f1d 100644 --- a/test/test_dailymotion.py +++ b/test/test_dailymotion.py @@ -4,61 +4,11 @@ def test_parse_response(): json_data = '{"list":[{"id":"x6ilw2u",' \ '"title":"Job Interviews - Stand Up Comedy by ' \ - 'Piyush Sharma",' \ - '"channel":"fun",' \ - '"owner":"x24pst6"},{"id":"x65slib",' \ - '"title":"Beyhadh actor Piyush Sahdev(Samay) will ' \ - 'soon enter' \ - ' colors Devanshi opposite Helly shah",' \ - '"channel":"fun","owner":"x21w079"},{"id":"x5fj9kg",' \ - '"title":"Deepika-Piyush Se Mulaqat - Dhhai Kilo Prem",' \ - '"channel":"fun","owner":"x1isdap"},{"id":"x59m108",' \ - '"title":"Sasural Simar Ka TOOT GAYA PIYUSH VAIDAHI KA' \ - ' RISHTA 26 January 2017 News",' \ - '"channel":"fun","owner":"x1wmonz"},' \ - '{"id":"x6aj8su","title":"Beyhadh actor Piyush Sahdev ' \ - 'ARRESTED on Rape charges | FilmiBeat",' \ - '"channel":"fun","owner":"x1x1a21"},{"id":"x4eq3kp",' \ - '"title":"Funny Balloon Tricks By Piyush Seth",' \ - '"channel":"fun","owner":"x1swlum"},{"id":"x6cmk7x",' \ - '"title":"Beyhadh Actor Piyush Sahdev Finally REACTS On' \ - ' His Molestation Case",' \ - '"channel":"fun","owner":"x22ch35"},{"id":"x3i6xr3",' \ - '"title":"Revolver Rani Movie || I Am Brutal Video Song ' \ - '|| Kangana Ranaut, Piyush Mishra, Vir Das",' \ - '"channel":"fun","owner":"x1ocl9w"},{"id":"xhv1dj",' \ - '"title":"Birthday party of t v actor piyush sachdev"' \ - ',"channel":"fun","owner":"xmoljl"},' \ - '{"id":"x6647o3","title":"Piyush Mishra Aarambh Hai' \ - ' Prachand (Live At Hindu College)",' \ - '"channel":"fun","owner":"x21w0ck"}]}' + 'Piyush Sharma","channel":"fun",' \ + '"owner":"x24pst6"}]}' expected_resp = [{ 'link': 'https://www.dailymotion.com/video/x6ilw2u', - 'title': u'Job Interviews - Stand Up Comedy by Piyush Sharma'}, - {'link': 'https://www.dailymotion.com/video/x65slib', - 'title': u'Beyhadh actor Piyush Sahdev(Samay) will soon ' - u'enter colors Devanshi opposite Helly shah'}, - {'link': 'https://www.dailymotion.com/video/x5fj9kg', - 'title': u'Deepika-Piyush Se Mulaqat - Dhhai Kilo Prem'}, - {'link': 'https://www.dailymotion.com/video/x59m108', - 'title': u'Sasural Simar Ka TOOT GAYA PIYUSH VAIDAHI KA' - u' RISHTA 26 January 2017 News'}, - {'link': 'https://www.dailymotion.com/video/x6aj8su', - 'title': u'Beyhadh actor Piyush Sahdev ARRESTED on ' - u'Rape charges | FilmiBeat'}, - {'link': 'https://www.dailymotion.com/video/x4eq3kp', - 'title': u'Funny Balloon Tricks By Piyush Seth'}, - {'link': 'https://www.dailymotion.com/video/x6cmk7x', - 'title': u'Beyhadh Actor Piyush Sahdev Finally REACTS' - u' On His Molestation Case'}, - {'link': 'https://www.dailymotion.com/video/x3i6xr3', - 'title': u'Revolver Rani Movie || I Am Brutal Video' - u' Song || Kangana Ranaut, Piyush Mishra, Vir Das'}, - {'link': 'https://www.dailymotion.com/video/xhv1dj', - 'title': u'Birthday party of t v actor piyush sachdev'}, - {'link': 'https://www.dailymotion.com/video/x6647o3', - 'title': u'Piyush Mishra Aarambh Hai Prachand ' - u'(Live At Hindu College)'}] + 'title': u'Job Interviews - Stand Up Comedy by Piyush Sharma'}] resp = DailyMotion().parse_response(json_data) assert resp == expected_resp diff --git a/test/test_generalized.py b/test/test_generalized.py index 0736ceb5..d0775d6d 100644 --- a/test/test_generalized.py +++ b/test/test_generalized.py @@ -22,27 +22,6 @@ def test_get_page(mock_request_get, mock_response): mock_request_get.assert_called_with( '', headers=expected_headers, params=expected_payload) -# -# @patch('') -# @patch('') -# def test_get_page_of_dailymotion(mock_request_get, mock_response): -# mock_request_get.return_value = mock_response -# mock_response.url = "Mock Url" -# import pdb -# pdb.set_trace() -# Scraper().get_page("dummy_query") -# assert response == mock_response -# expected_payload = {'q': 'dummy_query', '': ''} -# expected_headers = { -# 'User-Agent': ( -# 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) ' -# 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 ' -# 'Safari/537.36' -# ) -# } -# mock_request_get.assert_called_with( -# '', headers=expected_headers, params=expected_payload) - def test_parse_response(): with pytest.raises(NotImplementedError):