diff --git a/app/scrapers/dailymotion.py b/app/scrapers/dailymotion.py index 394ce30b..c376a125 100644 --- a/app/scrapers/dailymotion.py +++ b/app/scrapers/dailymotion.py @@ -23,6 +23,7 @@ def parse_response(soup): urls = [] video_list = json.loads(str(soup))['list'] + for item in video_list: title = item['title'] link = 'https://www.dailymotion.com/video/' + str(item['id']) diff --git a/app/scrapers/generalized.py b/app/scrapers/generalized.py index 12ade912..29fe7607 100644 --- a/app/scrapers/generalized.py +++ b/app/scrapers/generalized.py @@ -42,6 +42,14 @@ def get_page(self, query, startIndex=0, qtype=''): if self.name == 'mojeek' and qtype == 'news': payload['fmt'] = 'news' response = requests.get(url, headers=self.headers, params=payload) + if "dailymotion" in response.url: + url = response.url + index = url.index('?') + url = url[0:index + 1] + url[index + 3:len(url)] + response = requests.get( + url=url, + headers=self.headers + ) print(response.url) return response diff --git a/test/test_dailymotion.py b/test/test_dailymotion.py new file mode 100644 index 00000000..82417f1d --- /dev/null +++ b/test/test_dailymotion.py @@ -0,0 +1,14 @@ +from app.scrapers import DailyMotion + + +def test_parse_response(): + json_data = '{"list":[{"id":"x6ilw2u",' \ + '"title":"Job Interviews - Stand Up Comedy by ' \ + 'Piyush Sharma","channel":"fun",' \ + '"owner":"x24pst6"}]}' + + expected_resp = [{ + 'link': 'https://www.dailymotion.com/video/x6ilw2u', + 'title': u'Job Interviews - Stand Up Comedy by Piyush Sharma'}] + resp = DailyMotion().parse_response(json_data) + assert resp == expected_resp