From 91fb24eb648b770143990226d1da1b625d247a6f Mon Sep 17 00:00:00 2001 From: Swapniel Date: Sat, 24 Aug 2019 19:17:08 +0530 Subject: [PATCH 01/13] requirements.txt --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 9aee4d6..ee8cb78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ sphinxcontrib-googleanalytics==0.1 sphinxcontrib-newsfeed==0.1.4 +sphinx-rtd-theme==0.4.3 +python-dateutil==2.8.0 \ No newline at end of file From 468b57e592b168d4198ef2a16349f8d8b35f03be Mon Sep 17 00:00:00 2001 From: Swapniel Date: Sat, 24 Aug 2019 21:09:25 +0530 Subject: [PATCH 02/13] Added corporate actions api --- nse.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/nse.py b/nse.py index 2741317..4cfea5d 100644 --- a/nse.py +++ b/nse.py @@ -23,7 +23,6 @@ """ import six -import ast import re import json import zipfile @@ -31,7 +30,7 @@ from dateutil import parser from nsetools.bases import AbstractBaseExchange from nsetools.utils import byte_adaptor -from nsetools.utils import js_adaptor + # import paths differ in python 2 and python 3 if six.PY2: from urllib2 import build_opener, HTTPCookieProcessor, Request @@ -74,6 +73,8 @@ def __init__(self): self.preopen_niftybank_url =\ "https://www.nseindia.com/live_market/dynaContent/live_analysis/pre_open/niftybank.json" self.fno_lot_size_url = "https://www.nseindia.com/content/fo/fo_mktlots.csv" + self.bhavcopyPR_base_url = "https://www.nseindia.com/archives/equities/bhavcopy/pr/PR%s%s%s.zip" + self.corp_act_base_filename = "Bc%s%s%s.csv" def get_fno_lot_sizes(self, cached=True, as_json=False): """ @@ -415,7 +416,7 @@ def render_response(self, data, as_json=False): return data def get_bhavcopy_url(self, d): - """take date and return bhavcopy url""" + """Take date and return bhavcopy url. Starts from 1st November 1994.""" d = parser.parse(d).date() day_of_month = d.strftime("%d") mon = d.strftime("%b").upper() @@ -431,6 +432,23 @@ def get_bhavcopy_filename(self, d): filename = self.bhavcopy_base_filename % (day_of_month, mon, year) return filename + def get_bhavcopyPR_url(self, d): + """Take date and return bhavcopyPR zip url. Starts from 4th January 2010.""" + d = parser.parse(d).date() + day_of_month = d.strftime("%d") + mon = d.strftime("%m") + year = d.strftime("%y") + url = self.bhavcopyPR_base_url % (day_of_month, mon, year) + return url + + def get_corp_act_filename(self, d): + d = parser.parse(d).date() + day_of_month = d.strftime("%d") + mon = d.strftime("%m") + year = d.strftime("%y") + filename = self.corp_act_base_filename % (day_of_month, mon, year) + return filename + def download_bhavcopy(self, d): """returns bhavcopy as csv file.""" # ex_url = "https://www.nseindia.com/content/historical/EQUITIES/2011/NOV/cm08NOV2011bhav.csv.zip" @@ -447,6 +465,22 @@ def download_bhavcopy(self, d): return result + def download_corp_act(self, d): + """returns Corporate Actions file as csv file.""" + # ex_url = "https://www.nseindia.com/archives/equities/bhavcopy/pr/PR230819.zip" + url = self.get_bhavcopyPR_url(d) + filename = self.get_corp_act_filename(d) + # response = requests.get(url, headers=self.headers) + response = self.opener.open(Request(url, None, self.headers)) + zip_file_handle = io.BytesIO(response.read()) + zf = zipfile.ZipFile(zip_file_handle) + try: + result = zf.read(filename) + except KeyError: + result = zf.read(zf.filelist[0].filename) + + return result + def download_index_copy(self, d): """returns index copy file""" pass From 8d8e31b1e5a2d171622e703414f9c86f748631a3 Mon Sep 17 00:00:00 2001 From: Swapniel Date: Sat, 24 Aug 2019 21:11:16 +0530 Subject: [PATCH 03/13] added test for corporate actions. --- tests/nse_tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/nse_tests.py b/tests/nse_tests.py index 0879815..c8121a9 100644 --- a/tests/nse_tests.py +++ b/tests/nse_tests.py @@ -211,6 +211,10 @@ def test_6th_Dec_1994(self): data = self.nse.download_bhavcopy('1994-12-06') self.assertIsInstance(self, data, bytes) + def test_4th_Jan_2010(self): + data = self.nse.download_corp_act('2010-01-04') + self.assertIsInstance(self, data, bytes) + def test_top_fno_gainers_losers(self): fno_gainer = self.nse.get_top_fno_gainers() self.assertIsInstance(fno_gainer, list) From 80692a9c4aae686ac1c17169fcf1244e5394e42d Mon Sep 17 00:00:00 2001 From: Swapniel Date: Sat, 24 Aug 2019 22:12:33 +0530 Subject: [PATCH 04/13] bugfix in test. --- tests/nse_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nse_tests.py b/tests/nse_tests.py index 0879815..e5ad4ea 100644 --- a/tests/nse_tests.py +++ b/tests/nse_tests.py @@ -209,7 +209,7 @@ def test_nse_lot_sizes(self): def test_6th_Dec_1994(self): data = self.nse.download_bhavcopy('1994-12-06') - self.assertIsInstance(self, data, bytes) + self.assertIsInstance(data, bytes) def test_top_fno_gainers_losers(self): fno_gainer = self.nse.get_top_fno_gainers() From 4694685d9a3d064702e736935319157ac13918ad Mon Sep 17 00:00:00 2001 From: Swapniel Date: Sat, 24 Aug 2019 22:17:09 +0530 Subject: [PATCH 05/13] bugfix in test --- tests/nse_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/nse_tests.py b/tests/nse_tests.py index c8121a9..6242e3a 100644 --- a/tests/nse_tests.py +++ b/tests/nse_tests.py @@ -209,11 +209,11 @@ def test_nse_lot_sizes(self): def test_6th_Dec_1994(self): data = self.nse.download_bhavcopy('1994-12-06') - self.assertIsInstance(self, data, bytes) + self.assertIsInstance(data, bytes) def test_4th_Jan_2010(self): data = self.nse.download_corp_act('2010-01-04') - self.assertIsInstance(self, data, bytes) + self.assertIsInstance(data, bytes) def test_top_fno_gainers_losers(self): fno_gainer = self.nse.get_top_fno_gainers() From dc2be89f88fbef55401dd02f2f3484a17f285457 Mon Sep 17 00:00:00 2001 From: Swapniel Date: Sat, 24 Aug 2019 23:27:04 +0530 Subject: [PATCH 06/13] opener test missing assert. --- tests/nse_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/nse_tests.py b/tests/nse_tests.py index e5ad4ea..c67afe2 100644 --- a/tests/nse_tests.py +++ b/tests/nse_tests.py @@ -6,6 +6,7 @@ import json import re import six +from urllib.request import OpenerDirector from nsetools.bases import AbstractBaseExchange from nsetools import Nse from nsetools.utils import js_adaptor, byte_adaptor @@ -34,6 +35,7 @@ def test_nse_headers(self): def test_nse_opener(self): ''' should not raise any exception ''' opener = self.nse.nse_opener() + self.assertIsInstance(opener, OpenerDirector) def test_build_url_for_quote(self): test_code = 'infy' From cc2d663f406f95987f398cffcae8dcccbedb5a4a Mon Sep 17 00:00:00 2001 From: Swapnil Gusani Date: Fri, 29 Jul 2022 23:53:50 +0530 Subject: [PATCH 07/13] update requirements --- requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index db437ac..506e556 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ sphinxcontrib-googleanalytics==0.1 sphinxcontrib-newsfeed==0.1.4 -sphinx-rtd-theme==0.4.3 -python-dateutil==2.8.0 +sphinx-rtd-theme==1.0.0 +python-dateutil==2.8.2 +six From da82bbc2792ff240179927f76dc7b5acf3e9262f Mon Sep 17 00:00:00 2001 From: Swapnil Gusani Date: Sat, 30 Jul 2022 00:25:34 +0530 Subject: [PATCH 08/13] bugfix --- datemgr.py | 2 +- downloader.py | 4 ++-- nse.py | 35 +++++++++++++++++++++++++++++------ 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/datemgr.py b/datemgr.py index 16f7f62..2a35b29 100644 --- a/datemgr.py +++ b/datemgr.py @@ -2,7 +2,7 @@ from dateutil.relativedelta import relativedelta from dateutil.parser import parse from dateutil import rrule -from .errors import DateFormatError +from nsetools.errors import DateFormatError def get_nearest_business_day(d): diff --git a/downloader.py b/downloader.py index 790f142..801050f 100644 --- a/downloader.py +++ b/downloader.py @@ -27,8 +27,8 @@ import zipfile import datetime as dt from urllib.request import Request -from .datemgr import mkdate, get_date_range -from .nse import Nse +from nsetools.datemgr import mkdate, get_date_range +from nsetools import Nse from abc import ABCMeta, abstractmethod diff --git a/nse.py b/nse.py index 7f1ba66..7ca0869 100644 --- a/nse.py +++ b/nse.py @@ -27,10 +27,9 @@ import json import zipfile import io -from dateutil import parser -from .bases import AbstractBaseExchange -from .utils import byte_adaptor -from .datemgr import mkdate +from nsetools.bases import AbstractBaseExchange +from nsetools.utils import byte_adaptor +from nsetools.datemgr import mkdate # import paths differ in python 2 and python 3 if six.PY2: @@ -76,6 +75,8 @@ def __init__(self): self.fno_lot_size_url = "https://www1.nseindia.com/content/fo/fo_mktlots.csv" self.bhavcopyPR_base_url = "https://www1.nseindia.com/archives/equities/bhavcopy/pr/PR%s%s%s.zip" self.corp_act_base_filename = "Bc%s%s%s.csv" + self.daily_volatility_files = "https://www1.nseindia.com/archives/nsccl/volt/FOVOLT_%s%s%s.csv" + self.daily_volatility_filename = "FOVOLT_%s%s%s.csv" def get_fno_lot_sizes(self, cached=True, as_json=False): """ @@ -433,7 +434,7 @@ def get_bhavcopy_filename(self, d): def get_bhavcopyPR_url(self, d): """Take date and return bhavcopyPR zip url. Starts from 4th January 2010.""" - d = parser.parse(d).date() + d = mkdate(d) day_of_month = d.strftime("%d") mon = d.strftime("%m") year = d.strftime("%y") @@ -441,13 +442,29 @@ def get_bhavcopyPR_url(self, d): return url def get_corp_act_filename(self, d): - d = parser.parse(d).date() + d = mkdate(d) day_of_month = d.strftime("%d") mon = d.strftime("%m") year = d.strftime("%y") filename = self.corp_act_base_filename % (day_of_month, mon, year) return filename + def get_daily_volatility_file_url(self, d): + d = mkdate(d) + day_of_month = d.strftime("%d") + mon = d.strftime("%m") + year = d.strftime("%y") + url = self.daily_volatility_files % (day_of_month, mon, year) + return url + + def get_daily_volatility_filename(self, d): + d = mkdate(d) + day_of_month = d.strftime("%d") + mon = d.strftime("%m") + year = d.strftime("%y") + filename = self.daily_volatility_filename % (day_of_month, mon, year) + return filename + def download_bhavcopy(self, d): """returns bhavcopy as csv file.""" # ex_url = "https://www.nseindia.com/content/historical/EQUITIES/2011/NOV/cm08NOV2011bhav.csv.zip" @@ -479,6 +496,12 @@ def download_corp_act(self, d): return result + def download_daily_volatility_file(self, d): + url = self.get_daily_volatility_file_url(d) + filename = self.get_daily_volatility_filename(d) + response = self.opener.open(Request(url, None, self.headers)) + return response + def download_index_copy(self, d): """returns index copy file""" pass From f042ec922853cc67065af26179fd0b1c9571a96d Mon Sep 17 00:00:00 2001 From: Swapnil Gusani Date: Sat, 30 Jul 2022 00:49:46 +0530 Subject: [PATCH 09/13] encode utf 8 --- nse.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/nse.py b/nse.py index 7ca0869..b1dca4e 100644 --- a/nse.py +++ b/nse.py @@ -453,18 +453,10 @@ def get_daily_volatility_file_url(self, d): d = mkdate(d) day_of_month = d.strftime("%d") mon = d.strftime("%m") - year = d.strftime("%y") + year = d.year url = self.daily_volatility_files % (day_of_month, mon, year) return url - def get_daily_volatility_filename(self, d): - d = mkdate(d) - day_of_month = d.strftime("%d") - mon = d.strftime("%m") - year = d.strftime("%y") - filename = self.daily_volatility_filename % (day_of_month, mon, year) - return filename - def download_bhavcopy(self, d): """returns bhavcopy as csv file.""" # ex_url = "https://www.nseindia.com/content/historical/EQUITIES/2011/NOV/cm08NOV2011bhav.csv.zip" @@ -478,7 +470,7 @@ def download_bhavcopy(self, d): result = zf.read(filename) except KeyError: result = zf.read(zf.filelist[0].filename) - return zf.read(filename).decode("utf-8") + return result.decode("utf-8") def download_corp_act(self, d): """returns Corporate Actions file as csv file.""" @@ -493,14 +485,12 @@ def download_corp_act(self, d): result = zf.read(filename) except KeyError: result = zf.read(zf.filelist[0].filename) - - return result + return result.decode('utf-8') def download_daily_volatility_file(self, d): url = self.get_daily_volatility_file_url(d) - filename = self.get_daily_volatility_filename(d) - response = self.opener.open(Request(url, None, self.headers)) - return response + response = self.opener.open(Request(url, None, self.headers)).read() + return response.decode('utf-8') def download_index_copy(self, d): """returns index copy file""" From 2e47846e98a7df9e9a3f0919892927f14a981115 Mon Sep 17 00:00:00 2001 From: Swapnil Gusani Date: Sat, 30 Jul 2022 00:55:29 +0530 Subject: [PATCH 10/13] proper refactor --- __init__.py => nsetools/__init__.py | 0 bases.py => nsetools/bases.py | 0 datemgr.py => nsetools/datemgr.py | 24 ++++++++++++------------ downloader.py => nsetools/downloader.py | 0 errors.py => nsetools/errors.py | 0 nse.py => nsetools/nse.py | 0 ohl.py => nsetools/ohl.py | 0 utils.py => nsetools/utils.py | 0 8 files changed, 12 insertions(+), 12 deletions(-) rename __init__.py => nsetools/__init__.py (100%) rename bases.py => nsetools/bases.py (100%) rename datemgr.py => nsetools/datemgr.py (86%) rename downloader.py => nsetools/downloader.py (100%) rename errors.py => nsetools/errors.py (100%) rename nse.py => nsetools/nse.py (100%) rename ohl.py => nsetools/ohl.py (100%) rename utils.py => nsetools/utils.py (100%) diff --git a/__init__.py b/nsetools/__init__.py similarity index 100% rename from __init__.py rename to nsetools/__init__.py diff --git a/bases.py b/nsetools/bases.py similarity index 100% rename from bases.py rename to nsetools/bases.py diff --git a/datemgr.py b/nsetools/datemgr.py similarity index 86% rename from datemgr.py rename to nsetools/datemgr.py index 2a35b29..e142add 100644 --- a/datemgr.py +++ b/nsetools/datemgr.py @@ -7,28 +7,28 @@ def get_nearest_business_day(d): """ takes datetime object""" - if d.isoweekday() is 7 or d.isoweekday() is 6: + if d.isoweekday() == 7 or d.isoweekday() == 6: d = d - relativedelta(days=1) return get_nearest_business_day(d) # republic day - elif d.month is 1 and d.day is 26: + elif d.month == 1 and d.day == 26: d = d - relativedelta(days=1) return get_nearest_business_day(d) # labour day - elif d.month is 5 and d.day is 1: + elif d.month == 5 and d.day == 1: d = d - relativedelta(days=1) return get_nearest_business_day(d) # independece day - elif d.month is 8 and d.day is 15: + elif d.month == 8 and d.day == 15: d = d - relativedelta(days=1) return get_nearest_business_day(d) # Gandhi Jayanti - elif d.month is 10 and d.day is 2: + elif d.month == 10 and d.day == 2: d = d - relativedelta(days=1) return get_nearest_business_day(d) # chirstmas - elif d.month is 12 and d.day is 25: + elif d.month == 12 and d.day == 25: d = d - relativedelta(days=1) return get_nearest_business_day(d) else: @@ -47,20 +47,20 @@ def is_known_holiday(d): # declare the list of holidays here. # republic day. - if d.month is 1 and d.day is 26: + if d.month == 1 and d.day == 26: return True # labour day - elif d.month is 5 and d.day is 1: + elif d.month == 5 and d.day == 1: d = d - relativedelta(days=1) return get_nearest_business_day(d) # independence day - elif d.month is 8 and d.day is 15: + elif d.month == 8 and d.day == 15: return True # gandhi jayanti - elif d.month is 10 and d.day is 2: + elif d.month == 10 and d.day == 2: return True # christmas - elif d.month is 12 and d.day is 25: + elif d.month == 12 and d.day == 25: return True else: return False @@ -72,7 +72,7 @@ def mkdate(d): """ # check if the it is a string return_date = "" - if type(d) is str: + if type(d) == str: if d == "today": return_date = dt.date.today() elif d == "yesterday": diff --git a/downloader.py b/nsetools/downloader.py similarity index 100% rename from downloader.py rename to nsetools/downloader.py diff --git a/errors.py b/nsetools/errors.py similarity index 100% rename from errors.py rename to nsetools/errors.py diff --git a/nse.py b/nsetools/nse.py similarity index 100% rename from nse.py rename to nsetools/nse.py diff --git a/ohl.py b/nsetools/ohl.py similarity index 100% rename from ohl.py rename to nsetools/ohl.py diff --git a/utils.py b/nsetools/utils.py similarity index 100% rename from utils.py rename to nsetools/utils.py From b07ab6eed84ad2297bf44de284d95357a3472943 Mon Sep 17 00:00:00 2001 From: Swapnil Gusani Date: Sat, 30 Jul 2022 00:58:24 +0530 Subject: [PATCH 11/13] minor change --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 506e556..2407f61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ sphinxcontrib-googleanalytics==0.1 sphinxcontrib-newsfeed==0.1.4 sphinx-rtd-theme==1.0.0 -python-dateutil==2.8.2 +dateutils six From 00ba6a17d5de91d09e8914d857109cd58f371856 Mon Sep 17 00:00:00 2001 From: Swapnil Gusani Date: Sat, 30 Jul 2022 01:02:48 +0530 Subject: [PATCH 12/13] minor change --- nsetools/ohl.py | 2 -- requirements.txt | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/nsetools/ohl.py b/nsetools/ohl.py index 6e16c03..b440d98 100755 --- a/nsetools/ohl.py +++ b/nsetools/ohl.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import requests -import json import ast import re import os @@ -8,7 +7,6 @@ import datetime as dt import pickle from argparse import ArgumentParser -from pprint import pprint INVESTMENT_AMOUNT = 10000 URL = "https://nseindia.com/live_market/dynaContent/live_watch/stock_watch/foSecStockWatch.json" diff --git a/requirements.txt b/requirements.txt index 2407f61..d787905 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ sphinxcontrib-newsfeed==0.1.4 sphinx-rtd-theme==1.0.0 dateutils six +pandas From 5af2e6ad6ea13873f025e08ae4b367dc70c19930 Mon Sep 17 00:00:00 2001 From: Swapnil Gusani Date: Sat, 30 Jul 2022 01:48:31 +0530 Subject: [PATCH 13/13] update http headers --- nsetools/nse.py | 11 +++++++++-- tests/nse_tests.py | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/nsetools/nse.py b/nsetools/nse.py index b1dca4e..c28d2d6 100644 --- a/nsetools/nse.py +++ b/nsetools/nse.py @@ -357,8 +357,15 @@ def nse_headers(self): return {'Accept': '*/*', 'Accept-Language': 'en-US,en;q=0.5', 'Host': 'www1.nseindia.com', - 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0', - 'X-Requested-With': 'XMLHttpRequest' + 'Connection': 'keep-alive', + 'Referer': "https://www1.nseindia.com/products/content/equities/equities/archieve_eq.htm", + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:102.0) Gecko/20100101 Firefox/102.0', + 'X-Requested-With': 'XMLHttpRequest', + 'Sec - Fetch - Dest': 'document', + 'Sec - Fetch - Mode': 'navigate', + 'Sec - Fetch - Site': 'same - origin', + 'Sec - Fetch - User': '?1', + 'Sec - GPC': '1' } def nse_opener(self): diff --git a/tests/nse_tests.py b/tests/nse_tests.py index cda3e57..551f49b 100644 --- a/tests/nse_tests.py +++ b/tests/nse_tests.py @@ -211,11 +211,11 @@ def test_nse_lot_sizes(self): def test_6th_Dec_1994(self): data = self.nse.download_bhavcopy('1994-12-06') - self.assertIsInstance(data, bytes) + self.assertIsInstance(data, str) def test_4th_Jan_2010(self): data = self.nse.download_corp_act('2010-01-04') - self.assertIsInstance(data, bytes) + self.assertIsInstance(data, str) def test_top_fno_gainers_losers(self): fno_gainer = self.nse.get_top_fno_gainers()