From 3345edeb17acc475b6103516e1c3998d0ff527ec Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Sun, 21 Jan 2018 11:17:01 -0500 Subject: [PATCH 01/22] start implementing new studyspaces api --- penn/studyspaces.py | 106 +++++++++----------------------------- tests/studyspaces_test.py | 26 +++------- 2 files changed, 30 insertions(+), 102 deletions(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index d63361a..f689edd 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -1,5 +1,7 @@ -from bs4 import BeautifulSoup import requests +import json + +from bs4 import BeautifulSoup BASE_URL = "http://libcal.library.upenn.edu" @@ -9,85 +11,23 @@ class StudySpaces(object): def __init__(self): pass - @staticmethod - def date_parse(original): - """Parses the date to dashed format. - - :param original: string with date in the format MM/DD/YYYY. - """ - l = original.split("-") - final = [l[1], l[2], l[0]] - return '-'.join(final) - - def get_id_json(self): - """Makes JSON with each element associating URL, ID, and building - name. - """ - group_study_codes = [] - url = BASE_URL + "/booking/vpdlc" - soup = BeautifulSoup(requests.get(url).text, 'html5lib') - l = soup.find('select', {'id': 'lid'}).find_all('option') - for element in l: - if element['value'] != '0': - url2 = "{}/spaces?lid={}".format(BASE_URL, str(element['value'])) - new_dict = {} - new_dict['id'] = int(str(element['value'])) - new_dict['name'] = str(element.contents[0]) - new_dict['url'] = url2 - group_study_codes.append(new_dict) - return group_study_codes - - def get_id_dict(self): - """Extracts the ID's of the room into a dictionary. Used as a - helper for the extract_times method. - """ - group_study_codes = {} - url = BASE_URL + "/booking/vpdlc" - soup = BeautifulSoup(requests.get(url).text, 'html5lib') - options = soup.find('select', {'id': 'lid'}).find_all('option') - for element in options: - if element['value'] != '0': - group_study_codes[int(str(element['value']))] = str(element.contents[0]) - return group_study_codes - - def extract_times(self, id, date, name): - """Scrapes the avaiable rooms with the given ID and date. - - :param id: the ID of the building - :param date: the date to acquire available rooms from - :param name: the name of the building; obtained via get_id_dict - """ - url = BASE_URL + "/rooms_acc.php?gid=%s&d=%s&cap=0" % (int(id), date) - soup = BeautifulSoup(requests.get(url).text, 'html5lib') - - time_slots = soup.find_all('form') - unparsed_rooms = time_slots[1].contents[2:-2] - - roomTimes = [] - - for i in unparsed_rooms: - room = BeautifulSoup(str(i), 'html5lib') - try: - # extract room names - roomName = room.fieldset.legend.h2.contents[0] - except AttributeError: - # in case the contents aren't a list - continue - newRoom = str(roomName)[:-1] - times = [] - - filtered = room.fieldset.find_all('label') - - for t in filtered: - # getting the individual times for each room - dict_item = {} - dict_item['room_name'] = newRoom - time = str(t).split("\t\t\t\t\t")[2][1:-1] - times.append(time) - startAndEnd = time.split(" - ") - dict_item['start_time'] = startAndEnd[0].upper() - dict_item['end_time'] = startAndEnd[1].upper() - roomTimes.append(dict_item) - dict_item['date'] = self.date_parse(date) - dict_item['building'] = name - return roomTimes + def get_buildings(self): + """Returns a dictionary matching building IDs to their names.""" + + soup = BeautifulSoup(requests.get("{}/spaces".format(BASE_URL)).content, "html5lib") + options = soup.find("select", {"id": "lid"}).find_all("option") + return {int(option["value"]): str(option.text) for option in options} + + def get_rooms(self, building, start, end): + """Returns a dictionary matching all rooms given a building id and a date range.""" + + room_endpoint = "{}/process_equip_p_availability.php".format(BASE_URL) + data = { + "lid": building, + "gid": 0, + "start": start.strftime("%Y-%m-%d"), + "end": end.strftime("%Y-%m-%d"), + "bookings": [] + } + resp = requests.post(room_endpoint, data=json.dumps(data), headers={'Referer': "{}/spaces?lid={}".format(BASE_URL, building)}) + return resp.json() diff --git a/tests/studyspaces_test.py b/tests/studyspaces_test.py index 983562e..74ddd41 100644 --- a/tests/studyspaces_test.py +++ b/tests/studyspaces_test.py @@ -8,23 +8,11 @@ class TestStudySpaces(): def setUp(self): self.studyspaces = StudySpaces() - def test_json(self): - json_id = self.studyspaces.get_id_json() - ok_(len(json_id) > 0) - for i in json_id: - ok_(i['id'] > 0) - ok_(i['name'] != '') - ok_(i['url'] != '') + def test_buildings(self): + buildings = self.studyspaces.get_buildings() + ok_(len(buildings) > 0) - def test_extraction(self): - dict_id = self.studyspaces.get_id_dict() - ok_(len(dict_id) > 0) - d = datetime.datetime.now() + datetime.timedelta(days=1) - next_date = d.strftime("%Y-%m-%d") - s = self.studyspaces.extract_times(1799, next_date, "Van Pelt-Dietrich Library Center Group Study Rooms") - for i in s: - ok_("building" in i) - ok_("start_time" in i) - ok_("end_time" in i) - ok_("date" in i) - ok_("room_name" in i) + def test_rooms(self): + now = datetime.datetime.now() + rooms = self.studyspaces.get_rooms(2683, now, now + datetime.timedelta(days=3)) + ok_(len(rooms) > 0) From 486bc16d439876940783b75c6bce122f9e54bf71 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Sun, 21 Jan 2018 11:17:07 -0500 Subject: [PATCH 02/22] style fixes --- penn/calendar3year.py | 6 +++--- penn/dining.py | 2 ++ penn/directory.py | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/penn/calendar3year.py b/penn/calendar3year.py index d1e9b52..c8d7fe2 100644 --- a/penn/calendar3year.py +++ b/penn/calendar3year.py @@ -21,9 +21,9 @@ def pull_3year(self): """ events = [] r = requests.get(BASE_URL).text - l = r.split("\r\n") + lines = r.split("\r\n") d = {} - for line in l: + for line in lines: if line == "BEGIN:VEVENT": d = {} elif line[:7] == "DTSTART": @@ -32,7 +32,7 @@ def pull_3year(self): d['start'] = start_date.strftime('%Y-%m-%d') elif line[:5] == "DTEND": raw_date = line.split(":")[1] - end_date = datetime.datetime.strptime(raw_date,'%Y%m%d').date() + end_date = datetime.datetime.strptime(raw_date, '%Y%m%d').date() d['end'] = end_date.strftime('%Y-%m-%d') elif line[:7] == "SUMMARY": name = line.split(":")[1] diff --git a/penn/dining.py b/penn/dining.py index 6c98285..e918502 100644 --- a/penn/dining.py +++ b/penn/dining.py @@ -81,6 +81,7 @@ def get_meals(v2_response, building_id): meals.append({"tblStation": stations, "txtDayPartDescription": meal["label"]}) return meals + class DiningV2(WrapperBase): """The client for the Registrar. Used to make requests to the API. @@ -141,6 +142,7 @@ def item(self, item_id): response = self._request(V2_ENDPOINTS['ITEMS'] + item_id) return response + class Dining(WrapperBase): """The client for the Registrar. Used to make requests to the API. diff --git a/penn/directory.py b/penn/directory.py index 11bbfc5..bee4ea9 100644 --- a/penn/directory.py +++ b/penn/directory.py @@ -38,7 +38,6 @@ def standardize(res): res['list_affiliation'] = res['list_affiliation'].replace('Faculty - ', '') return res - def search(self, params, standardize=False): """Get a list of person objects for the given search params. From 646b8548e5133250eb13bd391b04b2a16a44acd7 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 19:50:41 -0500 Subject: [PATCH 03/22] format buildings output --- .gitignore | 2 ++ penn/studyspaces.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 43fa68f..7012b83 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,5 @@ MANIFEST docs/_build .pypirc .idea/ +*.swp +*.swo diff --git a/penn/studyspaces.py b/penn/studyspaces.py index f689edd..5bb6e48 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -12,11 +12,11 @@ def __init__(self): pass def get_buildings(self): - """Returns a dictionary matching building IDs to their names.""" + """Returns a list of building IDs, building names, and services.""" soup = BeautifulSoup(requests.get("{}/spaces".format(BASE_URL)).content, "html5lib") options = soup.find("select", {"id": "lid"}).find_all("option") - return {int(option["value"]): str(option.text) for option in options} + return [{"id": int(opt["value"]), "name": str(opt.text), "service": "libcal"} for opt in options] def get_rooms(self, building, start, end): """Returns a dictionary matching all rooms given a building id and a date range.""" From 5e88c3c73ca87316d986c3ab25f777ab488463c4 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 20:12:45 -0500 Subject: [PATCH 04/22] format building room data --- penn/studyspaces.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index 5bb6e48..1dc419f 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -30,4 +30,14 @@ def get_rooms(self, building, start, end): "bookings": [] } resp = requests.post(room_endpoint, data=json.dumps(data), headers={'Referer': "{}/spaces?lid={}".format(BASE_URL, building)}) - return resp.json() + rooms = {} + for row in resp.json(): + room_id = int(row["resourceId"][4:]) + if room_id not in rooms: + rooms[room_id] = [] + rooms[room_id].append({ + "start": row["start"], + "end": row["end"], + "booked": row["status"] != 0 + }) + return [{"room_id": k, "times": v} for k, v in rooms.items()] From 89450c01f1bfee02d4b189f81ba81eefc20e6af9 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 20:23:06 -0500 Subject: [PATCH 05/22] format date with iso time --- penn/studyspaces.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index 1dc419f..7e6b35a 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -1,5 +1,7 @@ import requests +import datetime import json +import pytz from bs4 import BeautifulSoup @@ -18,6 +20,11 @@ def get_buildings(self): options = soup.find("select", {"id": "lid"}).find_all("option") return [{"id": int(opt["value"]), "name": str(opt.text), "service": "libcal"} for opt in options] + @staticmethod + def format_date(date): + date = pytz.timezone("US/Eastern").localize(datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")) + return date.isoformat() + def get_rooms(self, building, start, end): """Returns a dictionary matching all rooms given a building id and a date range.""" @@ -36,8 +43,8 @@ def get_rooms(self, building, start, end): if room_id not in rooms: rooms[room_id] = [] rooms[room_id].append({ - "start": row["start"], - "end": row["end"], + "start": self.format_date(row["start"]), + "end": self.format_date(row["end"]), "booked": row["status"] != 0 }) return [{"room_id": k, "times": v} for k, v in rooms.items()] From b20f35f0fd91497c15c9688c12a752945bf602e6 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 21:18:25 -0500 Subject: [PATCH 06/22] add room and thumbnail info --- penn/studyspaces.py | 23 ++++++++++++++++++++++- requirements.txt | 6 ++++-- tests/studyspaces_test.py | 4 ++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index 7e6b35a..721ddc1 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -2,6 +2,8 @@ import datetime import json import pytz +import re +import six from bs4 import BeautifulSoup @@ -25,9 +27,23 @@ def format_date(date): date = pytz.timezone("US/Eastern").localize(datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")) return date.isoformat() + @staticmethod + def get_room_id_name_mapping(building): + data = requests.get("{}/spaces?lid={}".format(BASE_URL, building)).content.decode("utf8") + # find all of the javascript room definitions + out = {} + for item in re.findall(r"resources.push\(((?s).*?)\);", data, re.MULTILINE): + items = {k: v for k, v in re.findall(r'(\w+?):\s*(.*?),', item)} + out[int(items["eid"])] = { + "name": items["title"][1:-1].encode().decode("unicode_escape" if six.PY3 else "string_escape"), + "thumbnail": items["thumbnail"][1:-1] + } + return out + def get_rooms(self, building, start, end): """Returns a dictionary matching all rooms given a building id and a date range.""" + mapping = self.get_room_id_name_mapping(building) room_endpoint = "{}/process_equip_p_availability.php".format(BASE_URL) data = { "lid": building, @@ -47,4 +63,9 @@ def get_rooms(self, building, start, end): "end": self.format_date(row["end"]), "booked": row["status"] != 0 }) - return [{"room_id": k, "times": v} for k, v in rooms.items()] + return [{ + "room_id": k, + "times": v, + "name": mapping[k]["name"] if k in mapping else None, + "thumbnail": mapping[k]["thumbnail"] if k in mapping else None + } for k, v in rooms.items()] diff --git a/requirements.txt b/requirements.txt index 30f7ee3..7882071 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ -nose==1.3.4 -requests==2.4.3 beautifulsoup4==4.4.1 html5lib==0.999 mock==2.0.0 nameparser==0.4.0 +nose==1.3.4 +pytz==2017.3 +requests==2.4.3 +six==1.11.0 diff --git a/tests/studyspaces_test.py b/tests/studyspaces_test.py index 74ddd41..3ebf60c 100644 --- a/tests/studyspaces_test.py +++ b/tests/studyspaces_test.py @@ -12,6 +12,10 @@ def test_buildings(self): buildings = self.studyspaces.get_buildings() ok_(len(buildings) > 0) + def test_room_name_mapping(self): + mapping = self.studyspaces.get_room_id_name_mapping(2683) + ok_(len(mapping) > 0) + def test_rooms(self): now = datetime.datetime.now() rooms = self.studyspaces.get_rooms(2683, now, now + datetime.timedelta(days=3)) From 1b7a8266822f2ced1ecda4e1ce6f9873bc2e0c2f Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 21:25:10 -0500 Subject: [PATCH 07/22] separate capacity field --- penn/studyspaces.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index 721ddc1..3778cdd 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -34,9 +34,13 @@ def get_room_id_name_mapping(building): out = {} for item in re.findall(r"resources.push\(((?s).*?)\);", data, re.MULTILINE): items = {k: v for k, v in re.findall(r'(\w+?):\s*(.*?),', item)} + title = items["title"][1:-1] + title = title.encode().decode("unicode_escape" if six.PY3 else "string_escape") + title = re.sub(r" \(Capacity [0-9]+\)", r"", title) out[int(items["eid"])] = { - "name": items["title"][1:-1].encode().decode("unicode_escape" if six.PY3 else "string_escape"), - "thumbnail": items["thumbnail"][1:-1] + "name": title, + "thumbnail": items["thumbnail"][1:-1], + "capacity": int(items["capacity"]) } return out @@ -63,9 +67,13 @@ def get_rooms(self, building, start, end): "end": self.format_date(row["end"]), "booked": row["status"] != 0 }) - return [{ - "room_id": k, - "times": v, - "name": mapping[k]["name"] if k in mapping else None, - "thumbnail": mapping[k]["thumbnail"] if k in mapping else None - } for k, v in rooms.items()] + out = [] + for k, v in rooms.items(): + item = { + "room_id": k, + "times": v + } + if k in mapping: + item.update(mapping[k]) + out.append(item) + return out From 402aaf9f33a2648816692be16057615facdc9b09 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 21:29:49 -0500 Subject: [PATCH 08/22] add code style check --- .travis.yml | 4 +++- tests/calendar_test.py | 26 +++++++++++++------------- tests/dining_test.py | 3 +-- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 279d0a9..6e27e4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,9 @@ python: - "2.7" - "3.4" - "3.5" -script: nosetests +script: + - flake8 --max-line-length=150 --exclude=venv,docs,build --ignore=F401,F403,F405 . + - nosetests deploy: provider: pypi user: 'pennappslabs' diff --git a/tests/calendar_test.py b/tests/calendar_test.py index 175d060..267ad88 100644 --- a/tests/calendar_test.py +++ b/tests/calendar_test.py @@ -9,15 +9,15 @@ def setUp(self): self.calendar = Calendar() def test_pull(self): - l = self.calendar.pull_3year() - ok_(len(l) > 0) - for event in l: + cal = self.calendar.pull_3year() + ok_(len(cal) > 0) + for event in cal: ok_(len(event) == 3) def test_date(self): - l = self.calendar.pull_3year() - ok_(len(l) > 0) - for event in l: + cal = self.calendar.pull_3year() + ok_(len(cal) > 0) + for event in cal: ok_(isinstance(event['name'], str)) if event['name'] == "Independence Day Observed (no classes)": independence = event['start'] @@ -25,18 +25,18 @@ def test_date(self): ok_(d.month == 7) def test_name(self): - l = self.calendar.pull_3year() - ok_(len(l) > 0) - for event in l: + cal = self.calendar.pull_3year() + ok_(len(cal) > 0) + for event in cal: ok_(isinstance(event['name'], str)) start = datetime.datetime.strptime(event['start'], '%Y-%m-%d').date() end = datetime.datetime.strptime(event['end'], '%Y-%m-%d').date() ok_((end - start).total_seconds() >= 0) def test_chrono(self): - l = self.calendar.pull_3year() - ok_(len(l) > 0) - for i, event in enumerate(l[:-1]): + cal = self.calendar.pull_3year() + ok_(len(cal) > 0) + for i, event in enumerate(cal[:-1]): start = datetime.datetime.strptime(event['start'], '%Y-%m-%d').date() - nextstart = datetime.datetime.strptime(l[i]['start'], '%Y-%m-%d').date() + nextstart = datetime.datetime.strptime(cal[i]['start'], '%Y-%m-%d').date() ok_((nextstart - start).total_seconds() >= 0) diff --git a/tests/dining_test.py b/tests/dining_test.py index 7754df5..44500bc 100644 --- a/tests/dining_test.py +++ b/tests/dining_test.py @@ -18,8 +18,7 @@ def test_dining(self): self.assertTrue(len(venues) > 0) id = str(venues["id"]) data = self.din.menu_daily(id) - self.assertTrue( - len(data["result_data"]["Document"]["tblMenu"]["tblDayPart"][0]) >= 2) + self.assertTrue(len(data["result_data"]["Document"]["tblMenu"]["tblDayPart"][0]) >= 2) def test_dining_normalization(self): data = self.din.menu_daily("593") From 8855f35abc14427d6ffd996ba0e50a6f34fb0e4d Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 21:33:20 -0500 Subject: [PATCH 09/22] install flake8 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6e27e4b..c2b2044 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ python: - "2.7" - "3.4" - "3.5" +install: + - pip install flake8 script: - flake8 --max-line-length=150 --exclude=venv,docs,build --ignore=F401,F403,F405 . - nosetests From 742d0472f46ec74eb42e2e7886e9d0087ab1fde7 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 21:35:44 -0500 Subject: [PATCH 10/22] install python deps --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c2b2044..9273425 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ python: - "3.4" - "3.5" install: - - pip install flake8 + - pip install flake8 -r requirements.txt script: - flake8 --max-line-length=150 --exclude=venv,docs,build --ignore=F401,F403,F405 . - nosetests From 07f5094f9f1254de3f0cd7ff1ab028f518d23e5d Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 22:40:04 -0500 Subject: [PATCH 11/22] increment pip version --- penn/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/penn/__init__.py b/penn/__init__.py index 5c5f399..0ee73ee 100644 --- a/penn/__init__.py +++ b/penn/__init__.py @@ -1,4 +1,4 @@ -__version__ = '1.6.6' +__version__ = '1.6.7' from .registrar import Registrar from .directory import Directory diff --git a/setup.py b/setup.py index cc9a657..dc3b58b 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ url='https://github.com/pennlabs/penn-sdk-python', author='Penn Labs', author_email='admin@pennlabs.org', - version='1.6.6', + version='1.6.7', packages=['penn'], license='MIT', package_data={ From 69ec3c35a4e8706f16412511d9fb2c595ba330ad Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 23:03:16 -0500 Subject: [PATCH 12/22] change booked -> available --- penn/studyspaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index 3778cdd..965d8bf 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -65,7 +65,7 @@ def get_rooms(self, building, start, end): rooms[room_id].append({ "start": self.format_date(row["start"]), "end": self.format_date(row["end"]), - "booked": row["status"] != 0 + "available": row["status"] == 0 }) out = [] for k, v in rooms.items(): From 48cb4d28e41ef615b4164e2e2bd0ea466748b29e Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 23:24:28 -0500 Subject: [PATCH 13/22] add ability for precise time ranges --- penn/studyspaces.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index 965d8bf..54bcc8a 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -23,8 +23,12 @@ def get_buildings(self): return [{"id": int(opt["value"]), "name": str(opt.text), "service": "libcal"} for opt in options] @staticmethod - def format_date(date): - date = pytz.timezone("US/Eastern").localize(datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S")) + def parse_date(date): + return datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S") + + @staticmethod + def localize_date(date): + date = pytz.timezone("US/Eastern").localize(date) return date.isoformat() @staticmethod @@ -53,7 +57,7 @@ def get_rooms(self, building, start, end): "lid": building, "gid": 0, "start": start.strftime("%Y-%m-%d"), - "end": end.strftime("%Y-%m-%d"), + "end": (end + datetime.timedelta(days=1)).strftime("%Y-%m-%d"), "bookings": [] } resp = requests.post(room_endpoint, data=json.dumps(data), headers={'Referer': "{}/spaces?lid={}".format(BASE_URL, building)}) @@ -62,11 +66,14 @@ def get_rooms(self, building, start, end): room_id = int(row["resourceId"][4:]) if room_id not in rooms: rooms[room_id] = [] - rooms[room_id].append({ - "start": self.format_date(row["start"]), - "end": self.format_date(row["end"]), - "available": row["status"] == 0 - }) + room_start = self.parse_date(row["start"]) + room_end = self.parse_date(row["end"]) + if start <= room_start <= end: + rooms[room_id].append({ + "start": self.localize_date(room_start), + "end": self.localize_date(room_end), + "available": row["status"] == 0 + }) out = [] for k, v in rooms.items(): item = { From 34eb84b440dc5b589d1e199a7ba28a5bef494377 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 23:31:32 -0500 Subject: [PATCH 14/22] compare localized dates --- penn/studyspaces.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index 54bcc8a..956c3bc 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -24,12 +24,8 @@ def get_buildings(self): @staticmethod def parse_date(date): - return datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S") - - @staticmethod - def localize_date(date): - date = pytz.timezone("US/Eastern").localize(date) - return date.isoformat() + date = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S") + return pytz.timezone("US/Eastern").localize(date) @staticmethod def get_room_id_name_mapping(building): @@ -70,8 +66,8 @@ def get_rooms(self, building, start, end): room_end = self.parse_date(row["end"]) if start <= room_start <= end: rooms[room_id].append({ - "start": self.localize_date(room_start), - "end": self.localize_date(room_end), + "start": room_start.isoformat(), + "end": room_end.isoformat(), "available": row["status"] == 0 }) out = [] From 8db20307e9abf9b75c101a7531f43b35fced1f04 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Thu, 25 Jan 2018 23:36:55 -0500 Subject: [PATCH 15/22] fix test --- tests/studyspaces_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/studyspaces_test.py b/tests/studyspaces_test.py index 3ebf60c..b0b35d4 100644 --- a/tests/studyspaces_test.py +++ b/tests/studyspaces_test.py @@ -1,6 +1,8 @@ +import datetime +import pytz + from nose.tools import ok_ from penn import StudySpaces -import datetime class TestStudySpaces(): @@ -17,6 +19,6 @@ def test_room_name_mapping(self): ok_(len(mapping) > 0) def test_rooms(self): - now = datetime.datetime.now() + now = pytz.timezone("US/Eastern").localize(datetime.datetime.now()) rooms = self.studyspaces.get_rooms(2683, now, now + datetime.timedelta(days=3)) ok_(len(rooms) > 0) From 888463a56ff6ac77d6e76420aa930397034a1bad Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Fri, 26 Jan 2018 00:09:55 -0500 Subject: [PATCH 16/22] localize dates if no timezone info --- penn/studyspaces.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index 956c3bc..23b43d6 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -47,6 +47,11 @@ def get_room_id_name_mapping(building): def get_rooms(self, building, start, end): """Returns a dictionary matching all rooms given a building id and a date range.""" + if start.tzinfo is None: + start = pytz.timezone("US/Eastern").localize(self.localize_date(start)) + if end.tzinfo is None: + end = pytz.timezone("US/Eastern").localize(self.localize_date(end)) + mapping = self.get_room_id_name_mapping(building) room_endpoint = "{}/process_equip_p_availability.php".format(BASE_URL) data = { From 71175768c2b6c99dc0cad7dea4934e7963089d06 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Fri, 26 Jan 2018 00:12:49 -0500 Subject: [PATCH 17/22] fix typo --- penn/studyspaces.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index 23b43d6..8dffb0b 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -48,9 +48,9 @@ def get_rooms(self, building, start, end): """Returns a dictionary matching all rooms given a building id and a date range.""" if start.tzinfo is None: - start = pytz.timezone("US/Eastern").localize(self.localize_date(start)) + start = pytz.timezone("US/Eastern").localize(start) if end.tzinfo is None: - end = pytz.timezone("US/Eastern").localize(self.localize_date(end)) + end = pytz.timezone("US/Eastern").localize(end) mapping = self.get_room_id_name_mapping(building) room_endpoint = "{}/process_equip_p_availability.php".format(BASE_URL) From d7bd4e39b5f8c795a7a841dd48d45aea9b73e43b Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Fri, 26 Jan 2018 10:06:59 -0500 Subject: [PATCH 18/22] add http prefix to thumbnail --- penn/studyspaces.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index 8dffb0b..cc0d183 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -37,9 +37,12 @@ def get_room_id_name_mapping(building): title = items["title"][1:-1] title = title.encode().decode("unicode_escape" if six.PY3 else "string_escape") title = re.sub(r" \(Capacity [0-9]+\)", r"", title) + thumbnail = items["thumbnail"][1:-1] + if thumbnail: + thumbnail = "https:" + thumbnail out[int(items["eid"])] = { "name": title, - "thumbnail": items["thumbnail"][1:-1], + "thumbnail": thumbnail, "capacity": int(items["capacity"]) } return out From d81dde6997e2e2fb80fe81ff973a58fc2995ea0f Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 26 Jan 2018 20:05:39 -0500 Subject: [PATCH 19/22] changed thumbnail empty string to null --- penn/studyspaces.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index cc0d183..8c3e5ea 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -42,9 +42,10 @@ def get_room_id_name_mapping(building): thumbnail = "https:" + thumbnail out[int(items["eid"])] = { "name": title, - "thumbnail": thumbnail, "capacity": int(items["capacity"]) } + if thumbnail: + out[int(items["eid"])]["thumbnail"] = thumbnail return out def get_rooms(self, building, start, end): From 127c2e5a51be63752e8a014a1db337ffcae230e4 Mon Sep 17 00:00:00 2001 From: Josh Date: Sat, 27 Jan 2018 11:41:02 -0500 Subject: [PATCH 20/22] changed empyty thumbnail from no key to key with null value --- penn/studyspaces.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index 8c3e5ea..775f397 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -42,10 +42,9 @@ def get_room_id_name_mapping(building): thumbnail = "https:" + thumbnail out[int(items["eid"])] = { "name": title, + "thumbnail": thumbnail or None, "capacity": int(items["capacity"]) } - if thumbnail: - out[int(items["eid"])]["thumbnail"] = thumbnail return out def get_rooms(self, building, start, end): From ee445f3ebff4597fe038d0a959a095238a0dd9a0 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Sat, 27 Jan 2018 15:23:26 -0500 Subject: [PATCH 21/22] bump versions --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7882071..cb7d83d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -beautifulsoup4==4.4.1 +beautifulsoup4==4.6.0 html5lib==0.999 mock==2.0.0 nameparser==0.4.0 -nose==1.3.4 +nose==1.3.7 pytz==2017.3 requests==2.4.3 six==1.11.0 From 8160aec83eb06a82be14cea45532d6473c22c0ea Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Sun, 28 Jan 2018 13:26:09 -0500 Subject: [PATCH 22/22] comment code better --- penn/studyspaces.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/penn/studyspaces.py b/penn/studyspaces.py index 775f397..2475400 100644 --- a/penn/studyspaces.py +++ b/penn/studyspaces.py @@ -8,10 +8,18 @@ from bs4 import BeautifulSoup -BASE_URL = "http://libcal.library.upenn.edu" +BASE_URL = "https://libcal.library.upenn.edu" class StudySpaces(object): + """Used for interacting with the UPenn library GSR booking system. + + Usage:: + + >>> from penn import StudySpaces + >>> s = StudySpaces() + """ + def __init__(self): pass @@ -24,23 +32,34 @@ def get_buildings(self): @staticmethod def parse_date(date): + """Converts library system dates into timezone aware Python datetime objects.""" + date = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S") return pytz.timezone("US/Eastern").localize(date) @staticmethod def get_room_id_name_mapping(building): + """ Returns a dictionary mapping id to name, thumbnail, and capacity. """ + data = requests.get("{}/spaces?lid={}".format(BASE_URL, building)).content.decode("utf8") # find all of the javascript room definitions out = {} for item in re.findall(r"resources.push\(((?s).*?)\);", data, re.MULTILINE): + # parse all of the room attributes items = {k: v for k, v in re.findall(r'(\w+?):\s*(.*?),', item)} + + # room name formatting title = items["title"][1:-1] title = title.encode().decode("unicode_escape" if six.PY3 else "string_escape") title = re.sub(r" \(Capacity [0-9]+\)", r"", title) + + # turn thumbnail into proper url thumbnail = items["thumbnail"][1:-1] if thumbnail: thumbnail = "https:" + thumbnail - out[int(items["eid"])] = { + + room_id = int(items["eid"]) + out[room_id] = { "name": title, "thumbnail": thumbnail or None, "capacity": int(items["capacity"])