From dd493bd5059f8aa62f5908a02168c54684a5b847 Mon Sep 17 00:00:00 2001 From: raymondlin1 Date: Tue, 12 May 2020 23:05:15 -0700 Subject: [PATCH] cleaned master --- .gitignore | 6 +- busyness/__init__.py | 56 + busyness/__pycache__/crawler.cpython-37.pyc | Bin 0 -> 14044 bytes busyness/crawler.py | 558 ++++ busyness/crawler.pyc | Bin 0 -> 14622 bytes busyness/lambda_function.py | 65 + classroomScraper/index.js | 519 ++++ classroomScraper/package-lock.json | 1197 +++++++++ classroomScraper/package.json | 24 + libHours/index.js | 184 ++ libHours/package-lock.json | 825 ++++++ libHours/package.json | 17 + libinfo-scraper.py | 151 -- librarystudyrooms-scraper.js | 109 - librooms-scraper/__MACOSX/._index.js | Bin 0 -> 176 bytes librooms-scraper/__MACOSX/._package.json | Bin 0 -> 176 bytes librooms-scraper/__MACOSX/._test.js | Bin 0 -> 176 bytes librooms-scraper/index.js | 468 ++++ librooms-scraper/package.json | 29 + librooms-scraper/source.html | 2679 +++++++++++++++++++ librooms-scraper/test.js | 351 +++ package-lock.json | 643 ----- reslife-scraper.js | 125 - reslife-scraper/package.json | 30 + reslife-scraper/webscraper.js | 309 +++ scheduler.js | 24 - 26 files changed, 7316 insertions(+), 1053 deletions(-) create mode 100644 busyness/__init__.py create mode 100644 busyness/__pycache__/crawler.cpython-37.pyc create mode 100644 busyness/crawler.py create mode 100644 busyness/crawler.pyc create mode 100644 busyness/lambda_function.py create mode 100644 classroomScraper/index.js create mode 100644 classroomScraper/package-lock.json create mode 100644 classroomScraper/package.json create mode 100644 libHours/index.js create mode 100644 libHours/package-lock.json create mode 100644 libHours/package.json delete mode 100644 libinfo-scraper.py delete mode 100644 librarystudyrooms-scraper.js create mode 100644 librooms-scraper/__MACOSX/._index.js create mode 100644 librooms-scraper/__MACOSX/._package.json create mode 100644 librooms-scraper/__MACOSX/._test.js create mode 100644 librooms-scraper/index.js create mode 100644 librooms-scraper/package.json create mode 100644 librooms-scraper/source.html create mode 100644 librooms-scraper/test.js delete mode 100644 package-lock.json delete mode 100644 reslife-scraper.js create mode 100644 reslife-scraper/package.json create mode 100644 reslife-scraper/webscraper.js delete mode 100644 scheduler.js diff --git a/.gitignore b/.gitignore index f85c6b1..fb6e305 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ -config.py \ No newline at end of file +*/node_modules/ +libinfo-scraper/ +!libinfo-scraper/__pycache__ +!libinfo-scraper/easy_install.py +!libinfo-scraper/libinfo-scraper.py \ No newline at end of file diff --git a/busyness/__init__.py b/busyness/__init__.py new file mode 100644 index 0000000..8b8fd6c --- /dev/null +++ b/busyness/__init__.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from .crawler import run +from .crawler import get_populartimes + +import logging +logging.getLogger(__name__).addHandler(logging.NullHandler()) + +""" + +ENTRY POINT + +""" + + +def get(api_key, types, p1, p2, n_threads=20, radius=180, all_places=False): + """ + :param api_key: str; api key from google places web service + :param types: [str]; placetypes + :param p1: (float, float); lat/lng of the south-west delimiting point + :param p2: (float, float); lat/lng of the north-east delimiting point + :param n_threads: int; number of threads to use + :param radius: int; meters; + :param all_places: bool; include/exclude places without populartimes + :return: see readme + """ + params = { + "API_key": api_key, + "radius": radius, + "type": types, + "n_threads": n_threads, + "all_places": all_places, + "bounds": { + "lower": { + "lat": min(p1[0], p2[0]), + "lng": min(p1[1], p2[1]) + }, + "upper": { + "lat": max(p1[0], p2[0]), + "lng": max(p1[1], p2[1]) + } + } + } + + return run(params) + + +def get_id(api_key, place_id): + """ + retrieves the current popularity for a given place + :param api_key: + :param place_id: + :return: see readme + """ + return get_populartimes(api_key, place_id) diff --git a/busyness/__pycache__/crawler.cpython-37.pyc b/busyness/__pycache__/crawler.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f65139b0e2c62ca7a032516b491943490d730536 GIT binary patch literal 14044 zcmcgzTWlQHd7j(u&MucrilRhHmgS*r%c4y!$t5XXB41>Yk`+soEK-&{v9TTQ8Inux zO=o7s<$AXWLKSg!^r26EXp5jP0j32Spie6L*at%`kbeI^ zyCj#A8zU{c#GL!}KmYk}=jYqq-3bN1M{_^>(7dWB|4Nzm&p_fWeBM`8MG=Zn3yK}a zs;zQfvo(%&Tj$uY4UQvrgyX0kQpGYY-MBlZz4ixm2bNA07mCSn7MeH^igcmQ!6@t{bEuDoiWxT=Vx z=zge(ZujJlZa*hdqUWJvpAx;|2;%2OpE!#6wCESd5WgUfivh%z7!)TE4~bFnoH+GR zwa#c7nhC|(d2;+MpbID`0Q@uGMM@mcY*IE(ld@rpQy_?#FPuOc26uZa=FuZr^` zjrcX8E-Axfj|swy!&(sW3a(oT3_o9Vht(kRUe&E~;#N6d^3lM2kT2!jlD{4FzTZmC z=RMy+Vg0pr->-OcqoYNq;-%Nh<+XzARPtUrS1xkdC~~8ff|GMc>$B1kj`Und=GI3y zymINyLOJL7`Eu#?kDDVOH_xIzU-ghV>u*4sT;ItTyl_fh zOJ2|GZ?}>)gBVEZ?|3Vioe6u65PTZI*|v%j>dSbVsKq z(_`t(%;Z>lcE(zARvnpd^~%w4w0{~BZ{hO>5x5E=I*?oif@?y@5gNk45k`Vzx5|}j z!I5-y-a;+sRtT8&pX?Eu0ESgU1exl)#;XXQ(lzwRQcRQ+@!Vc~bl_ho*i>bu?? z=c&xCRix{A7|uF>K5Tuk45MtV=X2{k7LRO|bGfRNuGkv_Yx0~mm-=J0f>Yw7w#uvf zow#jI$1dqdF>U=aKCc%+Lv1KorJ*4N2(V?95H_*DwjkD;WLOUj&n>LVIJ!w9CqGtX z648@|Q9534)G1Y^Q(9YDulmmDyz6cH<;tjVAJj%G2Y}d_$w_Cc;L3DmJ211^d@1i| zv#()LkEoKOM%DW1XP!X1-FOj)gq|g*=w`D)BAYFiMYTZbWHx)h>J(Zvv20e9bJ?s+ zVO(DKfuNhpOaw3Datrh9de=8(I{D?m7}e71~3k8EeEEMgxG{(VHgH5dc$1 zI*w9vP-!Mm5=Y$CNHh}tN~5dM)vpXF&156lh&H1A$|iwbUwrC}mN18R3IPqii8VJs zRgl&;4LR7*D9z`^`@rew51c}v7xN_qPVLa{f=I656e|-nCbI&~R1KBB`b*Ny`Pp1v z<^Z-_xlqUxPNpvsq`6}_Nw*#6#szoPw-cO{`L%W5?&9=H+4svu+vE%m-!@yL z*7j1{K{O;m@;H`a$8&|U=X!p&wx8SvHv#cOQ5bK7t5Dv%e|S&Fc%s8AAEQ0eDJ!}e zPpV1XR875ClXTPffK2st;C!0@g~XcD)O}@F73w3kq4g_GLuedFP>So(FrHH$sm*9R zC-mo(!B(j-aGo))L0^l+u-08VdldNVA(dei@~CNe#+FQ5%ju$|iE0>%CcU&)*+-$U+b_H85Q<_kfDsHGc3ol3UyBcKnHiQqqQs{za8BU<3z56n~`;K61Xv(E(>8tD2a=+oh`4fdVz{$ zjgw(bEl{5#2*>%1vwVf<2tffr0K9jwWbY-~K)0c%G*e5dC$;0C7AZA}Fsd1pLydkgzoTtvP2*YZ z!hUGY{kUBOH^d-%3u+tvrn0IET^J9|Msyyt5?uynN1Ty1W)mu{R!ccz;30N-%q9uF z7>Qead!<}036D$m5aS@`tc=fDUb*V8pWg!IvRci*R&aa>B&D^FI@)IDtWsH`t?T@< zw&9(@*cCUIU(G|N2unsR$aEE-C5(04I7_%CFYj;jz4&rC>#sZhp$lcLxc-)l`&=k* zxpK4$9F)9rMq2qCjxR^(OFzPFVuMHVd2bT(n{Ieh%1bD^zV#^;?uP&-Ae>ygq_e6Xq= zumt4wM+#Vjb{FRXh--D5kJqxc%5oFLopAz7@PHUtAi)x(8T9TxX(vqJp@B%$0}UN% zg^EFIjJXz(Y-JaRjUaYE%MAk^g9dThT$qphj<*Tnt3)b~k@wx=VLqM=mlU@D4sD7^ zi4xHupqQ$hLV6DuyO1FXl=of-B*5%IP(IX}Y7^vSR|VtKe2vJ2(06sD!1f-gh$Dz2 zV0g?DMGR7u6efs`u|s3TPk@~_BOoURWUUn+aU(LwumsUbphedSFx+Vp6Qa8@-`t3! z9D3)c+D3u}12Axcd?PASV5~8kCxW}-CmXRHbsALhBFzJKEP6I0@^Ad^-IVAh_Kme2 zK@YR`^+pWhK@Fl~T=b#cQIsYcrszld7?^j0LIWdhs-LNnYL16Bo2vY2$-rs<8Ed_Dj3IsI(!3RPXMQ2B1lLI&kX5 zVxU*b9^Zo}Y6A)`bjl!h>DIL@6hHYQjwz%jbaQR901^m(bHCnowgRoiFK@b~*X^E? zTl2G2kmVBGW&=`!42+87d4w_!Xl)P!H>?(X@9{;9Re$|9=fI7r{1$9Jrvvu$BckUP&0q9M@^%rzGcd*e{L z9OpV8dk>Cct+`D=#;cS|9?9Us#hh*_h!x6fYaoGkY|D`)B(UNvufvlh<^udxf4ECd zB5fO4B0vbyKF(J!>-o|)p%G-aoIC_}&&>hQ9@N#qpxJClAh_D{P3j)uac$h*@}c`F zGpPRoK93TLnlud6GLn!4Qn;Q;wO2c?nvfLwkxpo)p3stNT21S{dV)$+?w_Ns^!wF_ z*FB~*PmwO7CeNfIp^pei_awK2-04lDVaSo%e>4mbjejT7MN)n;7NINtNQ3605{Uvy)Z~vu>=V))_Ij3TA{n;ajLLuTjYf=S*-;O* zx-Gi*T7ACXN~HRgLoLsB)^ZN(-BAY)wZwR&zw}^L*Rdkfmtg*D>i4fh+Znm5u*BC3 zh?)BkYnsr1j)=a87>i)KLxl3{-&7$R9_?&d(xCCh8)hTE3RQZzzy1#6#%&htA8-)C{PZG=<<-6?Ii@hywChw$x5{37hhWT<>232Mqy8A^Tl zmttMJ*Z%>?3^FbT2Jx6-UH%#^(xG4lL3`=M_yDFjvEXOH$!sq^st-r`lF3`>5@-^l z)M_4(FTkt^g-E(IQX)v~-(b0hB1yo3t&9$*Y{%(1*jjDJ2}NL2#Wnwknso6JM1C#r zdv*#u@c=C#qn*!U0)rWg7c_L90@hZJKO3$;Ljy03Kr#E&J}ss7LM(s~p!Vx22nI>D z4p&sCk!dZbatEK5f z^np}JrYvk8HXn5G50ZJVpJO8dixHgfY_RrVt~~+FLu*iF!}@T*IcpJW4=tlq_7BVt zjzZRmu#pA1q#D|$*!F&n*1JGY2|;`r0yQUxbvwe-LmZZC4}u8q>d=)4;#``=QS}j4 z5qD5bO@kPw8AhKf>E7jt*3yr3EQ0)Hd~B|1Xfj1F0Ybf~l==lI1z7$WX!xKNVJQrZ z=>};~+C>N{T>m!eb*|T;-$djhG$;r%NJGDZeZbgJA*>kChhQWZ5tPTIgqa+g5Ykck z>o9^VB8D{1cG#@{=l=OI=PmPti&JO=iy1K64EW$2AqbR3vKWRSB`v`sdUWuhLK%&( zUyEahj65vwD3W+XDXl%DC|l|%+`ok##r#48W84>1)6LAn!c+2Fy;_LEizVAQO%64I z<7~-3iHi?|D8K7Cg=~dPrP)%oxB>;K@9DXMWNY3Q1>ouE+6RgKnZe8H`VE@_tSIzu z$8ZUvVFrhWJ71-sNWlXNY6$EE(+?i`Fyam19J1}Bf{p~8A4G;&(ARq$;$#Z`VE!r@nl~Yk?%(!6T!3udDay1hcHL& z8jm0^@4$)=SyA&NyHQBG+5ju9er(tDV;ga(>oiVcql?2Nb)HJlWBlB{=A zkCfdWk$R-=_KF_N)(av88GHALI6~?^6y#p0@joP~7irAE_VkF~w|le^L3=3hkgO>e zgY4Nwwdfb~`;mv#im!hs0(rF`iuqA-j9!dd>ElQzI6WYGk?v}xj|pwZXeK!~$Ti)K zZpgR)CQgWxkSCAr9v9Cwx_1Y}De?Rx?UBZI(3Cjc=-!Nq7byHptt<+QQkv52Y4(aP zye;%#jC!M&LzpCRnNw^&8R}Qov}Si{h3oztrKl4q^#>a%=nLAWhCaJtc`16R*9edM z&tYvxkALj!L34;Rw1Y-Bueh7vH@fgX*1Z}5QFyWb315$v?hJYxs@!D9CE?t$SaoH3 z0Mzqbt93CBNbv`WL0gf>B_um5UIK`Xx5)_bRkJTC1?J($?IPGY|SX zG+7+a42>5jhQ`I@*w|Ehc6xezDwDY|H8Fc(awsFtk7qLJ$;{YvW_EgV=EB76WM*hW z;30cvY>nes=o8(B$g)M0$K^I-eOE z8XsF39~#dTFL1Z1;`q=7H#3wOb1?>2U7-3|44L6!CT3BaA0I>evEl`qj0Vq)p&q5- zWSRWr&=k5|z^LO~KZE9Z1YvG+YzEm3T2B`zsStCy7-|LGXns z_5e^v9#eGWsr|w6W)J1Z0kDn^TyU_*UJIIPd}tEiEP7-Jc;lWkyZZiJ*U8@S?#(#j z-pt0#rG=L#E@4~s8+SJ^+{o9)Zf-2pZqBbwT)%qf!i}Zc*3II=lwGjz*>|t3F5g@H z;O5mUn|J5e@84Xwas_RNrrjAj8v@Dn7mYs!%ov|>$8okmAB0%dR54uP1f}uu5su+# zXLu--WN7gjs>@L21XVJ4D3`ggciJ=9ULGfo2ALTgMEeA(7$2YQ>jtkMTBGsW<%c@1kfx_hDKp}q8Okz zC@4PzN=wUNllT+$^+ky0I&M3}_fU!64AcjQxSL$Xcx!Dj9XiJ70a=*YVRjY1C)sdW z2ej$7rcqCZRp8`Bh=h>k^r|e#bqs6A)?G(HY{WWZxiXl2&96tQ{_6RedUW*FGiP2M zWkv9b+LSr8e@vTvOt07SYm_1m`FNQE;@ErOAiaxPF5&Y?Ut}#X*;chetuK`H;g7Tm z<0RaWS`b)0ynQkwe5<|vs8S#MlH1?2Ak>o|idR2OLxFAJd!mj6YMQm!P%qKeLq+yi zTD>e_b$e^~pSAYi-Cz6gi69El7xF7Xgjt21xL+;%E{v*GPkt3cL9&KT(Ub2{+8|Z~ z{O}5PVrl96@O5yO4p!YEMvJ)(B zd0EU&CSG#=;+x5e}{^F3J7gP(YqZ zC!Yeu3Lf%V=$6;eM&732bqZcU06e}z>1_%+VJhE8+4l%^^c)FoTkBQ1Y|p?b^O-oDf4YhO#;f&r*C`!BZ zHqS0UIDhEs^#0i{K}>EzF!@8Ehw)v+p-92)0y;V@E4VDciXUj-21YdKQY04bBcaIc zyCMr!k7Q&UPOI?UgoV5w`M7C)+zgFTwErN&&ZY7#q~#?90Fag!+yIaY`^!J1mR$!P zpTUC}?xY6ty&jIpN0esh{V}D!N5Qjh2OYy-68!p*P_#jqYaA&ml1n^ zivZ>qG6qAN^YGThX&rAar-&Z5@)6|c01z_OpzQz*Haprk;5y2!P1N4&sEr?}O`!Hc zM{U=k+V6g0Z4!O|qNBE(`gWA0sDx>EPrb@ifQ^4-bOTeC+X%}(JkA^sE-&zR@v1U> z1GS(!mO)we{C3&Bw&h*h@*|^H(z%eDP?DZ!I|vPD>}W`nZ0+W|cG6zBeKUJ|Vd>8G z<)uYCb$j8xI}1z8+4+UVYYX%B)Kzxiwr-KXwrAB}YY7V07S;#H9=QFCFuB7cx`iVp zyJfxv3t?WQ?cSU3LzuvLw?D{Uzjou=a{ap>l&f(4&AG0?Tjw_Hz1DlY>FfM3POPhQ zLK+sn)Yg(KaedtpD+E?s&W5+HOA?H1dz!PCIg1?L=$5;39uM=>Z?wvO#~|L-${HS# z-EeLOb=vN^w)p;~>(}N12Ht$#UIqaVPb1_Ew5_63AfiGW-a9egL5tKt0cZ6duiB${ z(+=&l?7mLD;Z#GeYiT>acyl>><>sBm`T9If!;7`9%`-&F?7B++T-@7*n;VW>lmU6F zacg+sAT%K0L(R{_w>n+Fa;KEXNw94|gg)J&GZ;Fz&kbN4!Ax-K0%M}r`?m3sQ3Q(< zaCTw;VMHu`brOGOA^$|z^(=0n5*>t79Yh_t*KufY2zxn7!Vu-$XYU%5;7^uZzQ#3- z1XEV%1B7eofJ*Hs^l6G?I1xXJT#R$36ZaFlUF7DfDsVDQhQ2cFXnGoQLia|>?-BZ^ zFp&wPp=W#HdU}LCiQc|To+_Y+O`~S}eUKB#mjxW8l>#4G(~yQIO{>l;;RNaWK!uST$TjcoFv~^AP)9I9%^X${`BJNhgZ}r_7nuxf zNv8soa&uMoFof$yK8L$&!4kS&29ncd1TGW3sj*GBVBu-)87>L+6PF+j0pr7aM0KMe>hfUo&=dlccn=E*6RdU#ILX3hp9+`&}$F zFnjXPkPp+9^(_$?K5*M=11ABx+!rZ(mwkAQ9G% z-@Y_|={8HVS5YH>pMviq2;zJHyuuWU9go?yF$Dhz@aCt2P$}J$QdsX`!Ib4H zN%93#CkD&`a5K?kjnKRp{$Iu$v>|pfCh_+0UoX7}fZ+AwNVrZnBfZ_`pgCf`V0M{l Wvn%lx^O!koe$`BwedbYwFaI|R$ipiD literal 0 HcmV?d00001 diff --git a/busyness/crawler.py b/busyness/crawler.py new file mode 100644 index 0000000..850c4f9 --- /dev/null +++ b/busyness/crawler.py @@ -0,0 +1,558 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import calendar +import datetime +import json +import logging +import math +import re +import ssl +import threading +import urllib.request +import urllib.parse +from time import sleep, time +from queue import Queue + +# import requests +from botocore.vendored import requests + +# from geopy import Point +# from geopy.distance import vincenty, VincentyDistance + +# urls for google api web service +BASE_URL = "https://maps.googleapis.com/maps/api/place/" +RADAR_URL = BASE_URL + "radarsearch/json?location={},{}&radius={}&types={}&key={}" +NEARBY_URL = BASE_URL + "nearbysearch/json?location={},{}&radius={}&types={}&key={}" +DETAIL_URL = BASE_URL + "details/json?placeid={}&key={}" + +# user agent for populartimes request +USER_AGENT = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/54.0.2840.98 Safari/537.36"} + + +class PopulartimesException(Exception): + """Exception raised for errors in the input. + + Attributes: + expression -- input expression in which the error occurred + message -- explanation of the error + """ + + def __init__(self, expression, message): + self.expression = expression + self.message = message + + +def rect_circle_collision(rect_left, rect_right, rect_bottom, rect_top, circle_x, circle_y, radius): + # returns true iff circle intersects rectangle + + def clamp(val, min, max): + # limits value to the range min..max + if val < min: + return min + if val > max: + return max + return val + + # Find the closest point to the circle within the rectangle + closest_x = clamp(circle_x, rect_left, rect_right); + closest_y = clamp(circle_y, rect_bottom, rect_top); + + # Calculate the distance between the circle's center and this closest point + dist_x = circle_x - closest_x; + dist_y = circle_y - closest_y; + + # If the distance is less than the circle's radius, an intersection occurs + dist_sq = (dist_x * dist_x) + (dist_y * dist_y); + + return dist_sq < (radius * radius); + +def cover_rect_with_cicles(w, h, r): + """ + fully cover a rectangle of given width and height with + circles of radius r. This algorithm uses a hexagonal + honeycomb pattern to cover the area. + + :param w: width of rectangle + :param h: height of reclangle + :param r: radius of circles + :return: list of circle centers (x,y) + """ + + #initialize result list + res = [] + + # horizontal distance between circle centers + x_dist = math.sqrt(3) * r + # vertical distance between circle centers + y_dist = 1.5 * r + # number of circles per row (different for even/odd rows) + cnt_x_even = math.ceil(w / x_dist) + cnt_x_odd = math.ceil((w - x_dist/2) / x_dist) + 1 + # number of rows + cnt_y = math.ceil((h-r) / y_dist) + 1 + + y_offs = 0.5 * r + for y in range(cnt_y): + if y % 2 == 0: + # shift even rows to the right + x_offs = x_dist/2 + cnt_x = cnt_x_even + else: + x_offs = 0 + cnt_x = cnt_x_odd + + for x in range(cnt_x): + res.append((x_offs + x*x_dist, y_offs + y*y_dist)) + + # top-right circle is not always required + if res and not rect_circle_collision(0, w, 0, h, res[-1][0], res[-1][1], r): + res = res[0:-1] + + return res + +def get_circle_centers(b1, b2, radius): + """ + the function covers the area within the bounds with circles + + :param b1: south-west bounds [lat, lng] + :param b2: north-east bounds [lat, lng] + :param radius: specified radius, adapt for high density areas + :return: list of circle centers that cover the area between lower/upper + """ + + sw = Point(b1) + ne = Point(b2) + + # north/east distances + dist_lat = vincenty(Point(sw[0], sw[1]), Point(ne[0], sw[1])).meters + dist_lng = vincenty(Point(sw[0], sw[1]), Point(sw[0], ne[1])).meters + + circles = cover_rect_with_cicles(dist_lat, dist_lng, radius) + cords = [ + VincentyDistance(meters=c[0]) + .destination( + VincentyDistance(meters=c[1]) + .destination(point=sw, bearing=90), + bearing=0 + )[:2] + for c in circles + ] + + return cords + + +def worker_radar(): + """ + worker that gets coordinates of queue and starts radar search + :return: + """ + while True: + item = q_radar.get() + get_radar(item) + q_radar.task_done() + + +def get_radar(item): + _lat, _lng = item["pos"] + + # places - nearby search + # https://developers.google.com/places/web-service/search?hl=en#PlaceSearchRequests + radar_str = NEARBY_URL.format( + _lat, _lng, params["radius"], "|".join(params["type"]), params["API_key"] + ) + + # is this a next page request? + if item["res"] > 0: + # possibly wait remaining time until next_page_token becomes valid + min_wait = 2 # wait at least 2 seconds before the next page request + sec_passed = time() - item["last_req"] + if sec_passed < min_wait: + sleep(min_wait - sec_passed) + radar_str += "&pagetoken=" + item["next_page_token"] + + resp = json.loads(requests.get(radar_str, auth=('user', 'pass')).text) + check_response_code(resp) + + radar = resp["results"] + + item["res"] += len(radar) + if item["res"] >= 60: + logging.warning("Result limit in search radius reached, some data may get lost") + + bounds = params["bounds"] + + # retrieve google ids for detail search + for place in radar: + + geo = place["geometry"]["location"] + if bounds["lower"]["lat"] <= geo["lat"] <= bounds["upper"]["lat"] \ + and bounds["lower"]["lng"] <= geo["lng"] <= bounds["upper"]["lng"]: + # this isn't thread safe, but we don't really care, + # since in worst case a set entry is simply overwritten + g_places[place["place_id"]] = place + + # if there are more results, schedule next page requests + if "next_page_token" in resp: + item["next_page_token"] = resp["next_page_token"] + item["last_req"] = time() + q_radar.put(item) + + +def worker_detail(): + """ + worker that gets item of queue and starts detailed data retrieval + :return: + """ + while True: + item = q_detail.get() + get_detail(item) + q_detail.task_done() + + +def get_popularity_for_day(popularity,name=""): + """ + Returns popularity for day + :param popularity: + :return: + """ + + # Initialize empty matrix with 0s + pop_json = [[0 for _ in range(24)] for _ in range(7)] + wait_json = [[0 for _ in range(24)] for _ in range(7)] + + for day in popularity: + + day_no, pop_times = day[:2] + + if pop_times: + for hour_info in pop_times: + + hour = hour_info[0] + pop_json[day_no - 1][hour] = hour_info[1] + + # check if the waiting string is available and convert no minutes + if len(hour_info) > 5: + wait_digits = re.findall(r'\d+', hour_info[3]) + + if len(wait_digits) == 0: + wait_json[day_no - 1][hour] = 0 + elif "min" in hour_info[3]: + wait_json[day_no - 1][hour] = int(wait_digits[0]) + elif "hour" in hour_info[3]: + wait_json[day_no - 1][hour] = int(wait_digits[0]) * 60 + else: + wait_json[day_no - 1][hour] = int(wait_digits[0]) * 60 + int(wait_digits[1]) + + # day wrap + if hour_info[0] == 23: + day_no = day_no % 7 + 1 + + ret_popularity = [ + + { + "day": list(calendar.day_name)[d], + "busyness": pop_json[d] + } for d in range(7) + ] + + + # waiting time only if applicable + ret_wait = [ + { + "name": list(calendar.day_name)[d], + "data": wait_json[d] + } for d in range(7) + ] if any(any(day) for day in wait_json) else [] + + # {"name" : "monday", "data": [...]} for each weekday as list + return ret_popularity, ret_wait + + +def index_get(array, *argv): + """ + checks if a index is available in the array and returns it + :param array: the data array + :param argv: index integers + :return: None if not available or the return value + """ + + try: + + for index in argv: + array = array[index] + + return array + + # there is either no info available or no popular times + # TypeError: rating/rating_n/populartimes wrong of not available + except (IndexError, TypeError): + return None + + +def add_optional_parameters(detail_json, detail, rating, rating_n, popularity, current_popularity, time_spent): + """ + check for optional return parameters and add them to the result json + :param detail_json: + :param detail: + :param rating: + :param rating_n: + :param popularity: + :param current_popularity: + :param time_spent: + :return: + """ + # print(popularity) + if rating: + detail_json["rating"] = rating + elif "rating" in detail: + detail_json["rating"] = detail["rating"] + + if rating_n: + detail_json["rating_n"] = rating_n + + if "international_phone_number" in detail: + detail_json["international_phone_number"] = detail["international_phone_number"] + + if current_popularity: + detail_json["current_popularity"] = current_popularity + + if popularity: + popularity, wait_times = get_popularity_for_day(popularity) + + detail_json["populartimes"] = popularity + + if wait_times: + detail_json["time_wait"] = wait_times + + if time_spent: + detail_json["time_spent"] = time_spent + + return detail_json + + +def get_populartimes_from_search(place_identifier): + """ + request information for a place and parse current popularity + :param place_identifier: name and address string + :return: + """ + # print("place identifier={}".format(place_identifier)) + params_url = { + "tbm": "map", + "tch": 1, + "hl": "en", + "q": urllib.parse.quote_plus(place_identifier), + "pb": "!4m12!1m3!1d4005.9771522653964!2d-122.42072974863942!3d37.8077459796541!2m3!1f0!2f0!3f0!3m2!1i1125!2i976" + "!4f13.1!7i20!10b1!12m6!2m3!5m1!6e2!20e3!10b1!16b1!19m3!2m2!1i392!2i106!20m61!2m2!1i203!2i100!3m2!2i4!5b1" + "!6m6!1m2!1i86!2i86!1m2!1i408!2i200!7m46!1m3!1e1!2b0!3e3!1m3!1e2!2b1!3e2!1m3!1e2!2b0!3e3!1m3!1e3!2b0!3e3!" + "1m3!1e4!2b0!3e3!1m3!1e8!2b0!3e3!1m3!1e3!2b1!3e2!1m3!1e9!2b1!3e2!1m3!1e10!2b0!3e3!1m3!1e10!2b1!3e2!1m3!1e" + "10!2b0!3e4!2b1!4b1!9b0!22m6!1sa9fVWea_MsX8adX8j8AE%3A1!2zMWk6Mix0OjExODg3LGU6MSxwOmE5ZlZXZWFfTXNYOGFkWDh" + "qOEFFOjE!7e81!12e3!17sa9fVWea_MsX8adX8j8AE%3A564!18e15!24m15!2b1!5m4!2b1!3b1!5b1!6b1!10m1!8e3!17b1!24b1!" + "25b1!26b1!30m1!2b1!36b1!26m3!2m2!1i80!2i92!30m28!1m6!1m2!1i0!2i0!2m2!1i458!2i976!1m6!1m2!1i1075!2i0!2m2!" + "1i1125!2i976!1m6!1m2!1i0!2i0!2m2!1i1125!2i20!1m6!1m2!1i0!2i956!2m2!1i1125!2i976!37m1!1e81!42b1!47m0!49m1" + "!3b1" + } + + search_url = "https://www.google.de/search?" + "&".join(k + "=" + str(v) for k, v in params_url.items()) + logging.info("searchterm: " + search_url) + + # noinspection PyUnresolvedReferences + gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + + resp = urllib.request.urlopen(urllib.request.Request(url=search_url, data=None, headers=USER_AGENT), + context=gcontext) + data = resp.read().decode('utf-8').split('/*""*/')[0] + + # find eof json + jend = data.rfind("}") + if jend >= 0: + data = data[:jend + 1] + + jdata = json.loads(data)["d"] + jdata = json.loads(jdata[4:]) + + # get info from result array, has to be adapted if backend api changes + info = index_get(jdata, 0, 1, 0, 14) + + rating = index_get(info, 4, 7) + rating_n = index_get(info, 4, 8) + + popular_times = index_get(info, 84, 0) + + # current_popularity is also not available if popular_times isn't + current_popularity = index_get(info, 84, 7, 1) + + time_spent = index_get(info, 117, 0) + + # extract wait times and convert to minutes + if time_spent: + + nums = [float(f) for f in re.findall(r'\d*\.\d+|\d+', time_spent.replace(",", "."))] + contains_min, contains_hour = "min" in time_spent, "hour" in time_spent or "hr" in time_spent + + time_spent = None + + if contains_min and contains_hour: + time_spent = [nums[0], nums[1] * 60] + elif contains_hour: + time_spent = [nums[0] * 60, (nums[0] if len(nums) == 1 else nums[1]) * 60] + elif contains_min: + time_spent = [nums[0], nums[0] if len(nums) == 1 else nums[1]] + + time_spent = [int(t) for t in time_spent] + + return rating, rating_n, popular_times, current_popularity, time_spent + + +def get_detail(place_id): + """ + loads data for a given area + :return: + """ + global results + + # detail_json = get_populartimes(params["API_key"], place_id) + detail_json = get_populartimes_by_detail(params["API_key"], g_places[place_id]) + + if params["all_places"] or "populartimes" in detail_json: + results.append(detail_json) + + +def get_populartimes(api_key, place_id): + """ + sends request to detail to get a search string + and uses standard proto buffer to get additional information + on the current status of popular times + :return: json details + """ + + # places api - detail search + # https://developers.google.com/places/web-service/details?hl=de + detail_str = DETAIL_URL.format(place_id, api_key) + resp = json.loads(requests.get(detail_str, auth=('user', 'pass')).text) + check_response_code(resp) + detail = resp["result"] + print(place_id,detail["name"]) + return get_populartimes_by_detail(api_key, detail) + + +def get_populartimes_by_detail(api_key, detail): + # print(detail) + address = detail["formatted_address"] if "formatted_address" in detail else detail["vicinity"] + + place_identifier = "{} {}".format(detail["name"], address) + + detail_json = { + "id": detail["place_id"], + "name": detail["name"], + "address": address, + "types": detail["types"], + "coordinates": detail["geometry"]["location"] + } + + detail_json = add_optional_parameters(detail_json, detail, *get_populartimes_from_search(place_identifier)) + + return detail_json + + +def get_populartimes_by_detail_no_key(place_identifier): + detail_json = { + "id": detail["place_id"], + "name": detail["name"], + "address": address, + "types": detail["types"], + "coordinates": detail["geometry"]["location"] + } + + detail_json = add_optional_parameters(detail_json, detail, *get_populartimes_from_search(place_identifier)) + + return detail_json + + +def check_response_code(resp): + """ + check if query quota has been surpassed or other errors occured + :param resp: json response + :return: + """ + if resp["status"] == "OK" or resp["status"] == "ZERO_RESULTS": + return + + if resp["status"] == "REQUEST_DENIED": + raise PopulartimesException("Google Places " + resp["status"], + "Request was denied, the API key is invalid.") + + if resp["status"] == "OVER_QUERY_LIMIT": + raise PopulartimesException("Google Places " + resp["status"], + "You exceeded your Query Limit for Google Places API Web Service, " + "check https://developers.google.com/places/web-service/usage " + "to upgrade your quota.") + + if resp["status"] == "INVALID_REQUEST": + raise PopulartimesException("Google Places " + resp["status"], + "The query string is malformed, " + "check if your formatting for lat/lng and radius is correct.") + + if resp["status"] == "INVALID_REQUEST": + raise PopulartimesException("Google Places " + resp["status"], + "The query string is malformed, " + "check if your formatting for lat/lng and radius is correct.") + + if resp["status"] == "NOT_FOUND": + raise PopulartimesException("Google Places " + resp["status"], + "The place ID was not found and either does not exist or was retired.") + + raise PopulartimesException("Google Places " + resp["status"], + "Unidentified error with the Places API, please check the response code") + + +def run(_params): + """ + wrap execution logic in method, for later external call + :return: + """ + global params, g_places, q_radar, q_detail, results + + start = datetime.datetime.now() + + # shared variables + params = _params + q_radar, q_detail = Queue(), Queue() + g_places, results = dict(), list() + + logging.info("Adding places to queue...") + + # threading for radar search + for i in range(params["n_threads"]): + t = threading.Thread(target=worker_radar) + t.daemon = True + t.start() + + # cover search area with circles + bounds = params["bounds"] + for lat, lng in get_circle_centers([bounds["lower"]["lat"], bounds["lower"]["lng"]], # southwest + [bounds["upper"]["lat"], bounds["upper"]["lng"]], # northeast + params["radius"]): + q_radar.put(dict(pos=(lat, lng), res=0)) + + q_radar.join() + logging.info("Finished in: {}".format(str(datetime.datetime.now() - start))) + + logging.info("{} places to process...".format(len(g_places))) + + # threading for detail search and popular times + for i in range(params["n_threads"]): + t = threading.Thread(target=worker_detail) + t.daemon = True + t.start() + + for g_place_id in g_places: + q_detail.put(g_place_id) + + q_detail.join() + logging.info("Finished in: {}".format(str(datetime.datetime.now() - start))) + + return results diff --git a/busyness/crawler.pyc b/busyness/crawler.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c73160bef8849142e280419033f86e4ac42cdb4d GIT binary patch literal 14622 zcmb_jTWlOzT0Yh7du+#X?AV^nZpz6_*2yGocl++m?2IRNve6{=Y}pxSyvf#-yUOjd zySv&|)v-NeBSZ#)wA@)hyRZ@(9`<1+1P{y8Y9E#d1VSLR3=;6d6B1&D5J(6KB>2An zRNHMQp4~+xuH$pR{{Q!1&T;zRJKDbX{tq7KRq`*5-*4iJJ~otc@nfi>Qmcn?L#-OM z{FGWv)#7Qjny$qg)M`U5-l$d^Yw;$v+Ej}-tJUUOyhW|H*5YlocuEz|s8YLHZC9l; zYW0jVIGeO8cBoRPTJ6+n%nl3GVrhx$mVXG*=ddQrKZOsu}F z+_Mt-jB>jq@>%7clgK6Ic1y%m?smzCQqkt@o*Ady#;+b5A%mD?|otI8da z$mf*%l0=4-dr>0Sl>4$ouJdS?hCYL{^8W|_7KT!Y#*I9?qUTlO1}6QIH-v-K`CqDf z)w<+6fnSbws}@vw>X-9gIo^xAka#yq%=uC5AU}EynHzCjiDrg}OHL)q6oQ~o^qh(x zW%5Bu@`jNbt`wcTHyll)e(1PP7Z-x%JEg7O>1An(L}P=5WxgBu?nTt&HGjgYt+ z?^QgBYA+$M#Hte|5NkNqPgqEj$e%Qz%nV{jdoO=OshOXedpJ7Q;ZcJ`V%Td5Si6>sfrKOP<*oy?3*m_u)Warwdh z8)nhp^31zlek-_U-rfjv+jg`(kg1h;144&yg6IIj>o-7 zz7tfcMJMEeMf1CPoFDe-kSlqFZ{dsj5O|lL^!uoA*xn2sKk{62 zJqS%N41+K-{jwQvc=%PR#+kM@6aQx8IP}-5u@}uqnt9%CCG?^Qb1-k-)V1sL(96z- zpWl$aq-Qh8=c{4pxrbd~A(2z?*dDzVow6Lg8LS^QlDc_94nNZGZG6!M1p9{CSIR~t zB@tQhlF)KO>DCR7>3CnGd9 zHKp?GpaUOAlI1&y=yN@2R{QvA;0|Czo~6}6i{v!QwxlJeRVuWo{)0BPkAhaj&ZvDP z+EmXzVrNuO-$A?D$4?V~dbSjRgS)+OZ10zl6TN~!kA_-d7l&P9I;FN6RA@57bnPTi zTj}lQoSurP_@KgP(52U5Q%}gkCH-*6)Uv~9sm)5F;c4M01_6Q!C0dd3Y&mbdQst^=~xz|W^aE||dDbStI8KOpI`cM@Ea0+E&5 z70Q@wW^#8@dT7@@TvG=fYCo+uLC~mycmqfi1WRIVM07pHipKVzs+H}ZQ!A(%zF~k+ z&?RV&Atnqpqgf3gF;I&(Ac~gFsvXst2`!^p4Pw{IpxiWwl^Yt3U^4}A)~m(ho|zAx zdZFo!n_>t)p3qgo# zC9?`TLcbf{u2TrgPEi_e1Z8gzf^yBQIC1QSfZ-sS0`b@hJtu+VnTiuSC39ycnFfcd zO;Nf^Ds0TuW~XZaVX=t%8sSWBlBl6+Fw*6r7gxjb48&Fxb1k~ir2O|FB!+fx>|K+Z zKG_%)$bbL&hELFdf^?Ym4{!J^5d~z3>>;RIa^ejQzuRFfk-X;@h2cX?(i28a>uPX8O7riY9jvcM z(yPwc)frTlqzftf6s)2|5_++PpGgjO-UqTocTggrsXCg{=TmK|^Twsr#Z;Sd-snQu znrbqdkoQ`89BE;2&ijzT{{;!~BB+~eD{M}nN4$YM!XnVKp=mv(2A-uP4_tRF57}U6 zBEjixl70j?*iX_1PyrqX&x5;=2@%uG0|)=kJ$ak*1_uZh2MTb=XUf>bD2G{`l4Joq zI8))(XDY=A*I|GW&99`?_SY2v1+1R%;*APk$l+NxR@bhD!2Eay81^R_n2yoQUEe62#n?NKaAYcJW zj5Q+GNTAa4cmV88z3}VGZ4tC^%2N(7ouD3OX2 zMKTP;ceNNt{w*|(UPoX_1{9ByAJZ7s98}X&0Q&h2&%FUCFL|cx#Ew~V_DswGh?t0f z4ec`1f`@8xsO`6!b3sr?@KZK{FhxcbEY&%{Dl&*T1#I6HQ^;E6wu1AHZ{wMt%`W4OzAr6bgWKiSIaJnQ>0PV9Np`>n$Yoc~|2ho|#3Z zLGgv zk~N3=zk@HzAy7tFLz8j6p$qEJIiS)xqc=5Zv_W0EV02MYN_V6>jS*uc-J5PVI*kq^ zFRc*#e*pa>^dqSJ9Ly72Ux@G8eFSkYLNmHg0 z;FG+0B*ljzM|A4u0jLf^UAO!*WIT`RtVXSyL={!AQ&8L>2u1UP`t~ZZJ;{Ppq!HK7 zEC4%IGE>41qW@3S?vb>crxe)zvH|p&d$MFeU!^VBpiGJshzM~tgga_?j0iUOpg+5oUR#+WrQAR5shr0L)-%lOyGy7?MVms^l98Q(fq!W6QAf4J3jP<1<>S)my z5eQ>gVpc|GMLRR84%7AMI(wRLv}Jo#s{M_}*vW53XFm1LBdH`7r%w=%2#k;2*TgKf zD31iHHiBv>BF3kGRI=!-lL+}YPkbhjuzAFfLK2kw@ne$vY4SKyFE3WSi@=c2H$Vykx|gqkRQ?0P>|SGHsDZ9GFlqDQgcYkz~?W zAfka)#s4fR?kPN?>w$&s(Z+%_cm0ANYg^ilJ5hw`+jT>?Bw4I{>i)zusELb54Q)`0 zjoEyHk|={fwM-dPsg4w7Mh8sk3+Zm-LaN*7g~Ygo?@B70?lN9Zr_-NrT}{go^Qef2 zckxAUA=ta3cKdm_b5FX6b2xIa$^fi`?Kb$Apf0VH`&9#0h%Ey_Z?&oLb!EV8g^}=x zf%d9@7kvo{i?D_lecg1Ry|~`4N!QO)+%!0A@E|3|suPCLTIfH*=D^W{25~(HNcoJ| zxguU9eY`^9=}c_^_+$#O_H?=KaK>DKiNI-;gSc*dFdGeOUCD&Js%nSOg7(ppO73yi z^Slg@V_t?rs8ggj_NuFr+hkfohfY{{D#BmZeEPAH$fSym!|f#5(>zfv=~Rc&nQCb` zZ{*3}q!FIgZG6%95#Uh5^9*q$*!5dwT&V%w3{inc7(r50U>2f2JcG9d!BIaYSrh=e z{`*oNAj{6ux<2&(h*-<42Au`*^6kW#0%&{;xza%22Tc@IYE-uX)u@4x=_Yk+``hAA zglPDlcuSDmth=Jx%{4{;h2V%z1pMHLhHJwq)(%B5P|SBB5XW_SWF@)-_0dsbHp#N} z1fjJ6;oSr@HJ0l9V_6~YS;~bMWZC6onMbU5EL(1I;5j>`m;hnJ8B&guRtZ7MWCc<5 z_yRsUYM0}eurI(gxIN_($SK+tIz;VqwX_D4fg1PoWZR4kpCVr#ZgvAWEdrI{o&w$w3 zgntjA=u*(9=1eZu(j!wz*3MeMmcP0dNQFX9_GM_fSk5k4sP*#BnNBNb&!J#e=G|y&|b^L z9lx%IUF%Tc_eFoYpf>xI`x)kJ_A3{+{+H-Y#n0T6ZgtS9;sLdfjWR@0{;qL=-7{Gm zYNb1vL@NJgxR;Z7>HyLSyY`A)kaVf=YHI(SglGvj#6EseX}qF(3c#rjRsKM+LLtQx zRA3313@PrL)`^#Dg=HYva~({peT?(FwK^vo|4OaksBto~7#UcLEVWz37uCL;I`z(P zr8qZGpT5!&H4#{C5tJiW z50>-C2FKBC0=;IXd`eXjw@*lgDyrfnWJ=lIR-pd z9~*QmjUkSuS--x&=@aQJHtbjf$*6B}Sc5}l2gmSDqeYIJmyMk1^>^RJH1Vp$(NSlOC*;O~wsZqDy6&J{-Q-+egoU}<+}u{1xvT3mg!x^icI`O(6A zi+AsAt;}s~FV5e&gSvy0-W1Pwrxo*>`RNz69Oy&02pm@JJ9j!h#E>lLo5tVMjgRC`mjv$K;t zDayEbJ|vHZ-VnaKnx{2GW~GKI*4s5iAOxRW617eI*HjUe>Vw>ZxB>|FX`)h z#`ACv!uJ-2bH>V=1Cd1b+U%o!-96@e(%kKOB! zGcbww@yQBqNSG1pSley_V%k7n7zCL!jgyB<^Oik(cYa}6%nrDNU2Ua$;uhs>qDm3|O2$H3%+^;Bv+zzxS--~M0|p-= zNCwv4N>*Pl22Lziw*}f!L)5$Et=mZ%A~O+(iykmHoj;U0k{$Kr0E8h$EJ(-4*nM8!3vLV zQdLQew%6&x=ED$CS9v=G&38$z(8Pl*@*5|JJ76)HwX&d}Yw)i4dHHU=%^8rY;8m&l z%6|*VG&?Bi;_n0{;F_I^H@1q(umQ~-s74hX+FY7(dFg_i zIldHtKNY}-t0DU2I70M21hOX)XcE_zn$u`y>h&>()tgT32BuC;qT@myvGDCDoKCJ; z2?Nw$tFEu}k~C@Qx_)BX)@gAvCX_kavNhEW-NaS7Moq}y6mmRvWMc6qbJMO4YXM6* zx|Y$lrQ9BApA0FD7f$k$j`tSgTayEyn_r&2cb^wG)|c7I5k|bvB)8vk7_2e)D+Kzk zmY9S$ta35NW0ubJe(}mgbNg?zI=&Yky#UHds`@>W0tr#IbsM<$z{STQIgliT9DW%Q zoOZ$v7Z@SqQ2*BK0^$I>Y~q}ePM$o0WYV-3nFW_Kt;D31b|dKrMj~_IImaha+|0sQ z9DuWWf~12Xh3m+nLRY^ao)Ba%4bcE!h|eL^T6*lcHtb%i)nea2_3>sFUS5%9A0C(= z9$1u1mh2q~9`C4RPm?xWtUUF#mRL1*rpjsY<@|xffway_2ti+uMM8QQLnTKg@R>2I$ z*3{}WiW`4{jEv1sa{I9B;KZN7rSMU2EmCv$WKCG!Cu_Dz&5uvkJabyjKmRdm>g$I8 zI9apfsLvBCfcYx(5)`JRJ>tm<%Zd+_zDHbXhI;_=rL4~gmAvb5YZeL-xm3}I&Ur7*8I}L`^!rUqIp>J zUwSydv~17KFWj4-lLwY}<(0bm4j-k%6nqV9NR$q92Q$W_9X#o|A!vmgaT7PaE|SUtMa582wbfca4+lW zvE>LdazB4>;oaH$_vWxMvKP@RCc6wQ)@P+bo=Yq_Me;Nke|VmriXf+ueYQR4^QVn5^jLw zV2QUYPY^??3iI5@6Y`8GCJ)O#jz`RTSHE7@ufHf+lJmr_-hjgZuecy=c#U3jp$a8o zmacwABd@NtgOCc_spu{8;7^DcS5ZwhnMOmZ3)j)@I&6@5x-P_}wO|N!LIRy2`9Grr za1)Rkvi%uu%mA^FE0FELR}@K@avGE)H+jqgUf_btX;BVbR^osI0#U~ALN9_d3P28p z2^n)wa1Dr_^h*|K19ZI^R15G_JFRSS%)#4+?u_mC)CyZ_9bq@5Ck?Qk2#SGP6pv}) z+F^=9g3oG`s4id+5E*cu%{|G8y(O2s)KGe8Nbm&@1W*&NPasp^e#G-XPJR0m43Apy z`(ZN4sxa>hD(*YBKafv+5u|@=6+5M3KUJHfvLCq{aQmIQ8wF!!_agh7dvac!h;WC( z)hLgJjO}k4E7073;>q$sqo5;|6+x?=(5Zl`y?j-kG~)G!p9i3ra1U<;;5K3dn2NBt zD-Q9ZiOangJS>O?(B&+YEO4Jj0%~61QsAS7GNQXGo;cPoc5(j|@ahm94qNS#&6I85u_ zB5|jLUMxA6_gS5Jc(+s8lXs!gt0Xgg)5*W|(oaQ2QzNRqHM=w~FQ9lx*6iG@C2BRr ziuD5q-$fuNc=%UGg0}LoTR%2^lhx!XDIerqtgkXA2K%9wCsvd7H!PsyrgigwMeOJl zBzE%eNI1(|`pSNN%DIjz78RXuQzT+TVJk^-}AZ)(pPs S*51}@t@m0xTf6c9)&B*!ms^Vf literal 0 HcmV?d00001 diff --git a/busyness/lambda_function.py b/busyness/lambda_function.py new file mode 100644 index 0000000..cf6bd32 --- /dev/null +++ b/busyness/lambda_function.py @@ -0,0 +1,65 @@ +import crawler + +import boto3 +import json + + + + +# ChIJq6qqIYq8woARFNfDahGA4Js + +def lambda_handler(event, context): + # Get the service resource. + dynamodb = boto3.resource('dynamodb') + + # Instantiate a table resource object without actually + # creating a DynamoDB table. Note that the attributes of this table + # are lazy-loaded: a request is not made nor are the attribute + # values populated until the attributes + # on the table resource are accessed or its load() method is called. + table = dynamodb.Table('lib_hours') + + # google does not have data for the commented out libraries + libraries={ + 'Powell Library':'Powell Library', + 'Charles E. Young Research Library': 'Research Library (Charles E. Young)', + 'The Study at Hedrick':'The Study at Hedrick', + 'UCLA Music Library':'Music Library', + 'Rosenfeld Library':'Management Library (Eugene and Maxine Rosenfeld)', + 'UCLA Science and Engineering Library':'Science and Engineering Library' + # 'Hugh & Hazel Darling Law Library', + # 'UCLA Louise M. Darling Biomedical Library', + # 'Rudolph East Asian Library', + # 'Arts Library', + # 'Southern Regional Library Facility' + } + print(libraries) + for Library in libraries.keys(): + + print(Library) + rating, rating_n, popular_times, current_popularity, time_spent= crawler.get_populartimes_from_search(Library) + weekly_busyness=crawler.get_popularity_for_day(popular_times)[0] + + if current_popularity ==None: + current_popularity=0 + + + # popularity={ + # "name":libraries[Library], + # "weekly_busyness" :weekly_busyness, + # "current_busyness":current_popularity + # } + # table.update_item( + # Item=popularity + # ) + response = table.update_item( + Key={"name":libraries[Library]}, + UpdateExpression="set weekly_busyness = :wb, current_busyness=:cb", + ExpressionAttributeValues={ + ':wb': weekly_busyness, + ':cb': current_popularity, + }, + ReturnValues="UPDATED_NEW" + ) + print(response); +lambda_handler(0,0) \ No newline at end of file diff --git a/classroomScraper/index.js b/classroomScraper/index.js new file mode 100644 index 0000000..d7eb445 --- /dev/null +++ b/classroomScraper/index.js @@ -0,0 +1,519 @@ +var AWS = require("aws-sdk"); +AWS.config.update({ region: "us-east-1" }); +//var docClient = new AWS.DynamoDB.DocumentClient(); + +//RDS +// var pg = require('pg'); +// var con = mysql.createConnection({ +// host: 'studysmart-db.chpjzhfmtelr.us-west-1.rds.amazonaws.com', +// user: 'studyroot', +// password: 'studysmart-db', +// port: 5432, +// database: 'mydb' +// }); + +// con.connect(function(err) { +// if (err) throw err; +// console.log("Connected!"); +// }); + +var pg = require('pg'); +var conString = "postgres://studyroot:studysmart-db@studysmart-db.chpjzhfmtelr.us-west-1.rds.amazonaws.com:5432/mydb"; + +var client = new pg.Client(conString); +client.connect(err => { + if (err) { + console.error('Postgres connection error', err.stack); + } else { + console.log('connected to Postgres database'); + } +}); + +const requestPromise = require("request-promise"); +const moment = require("moment"); +const jsdom = require("jsdom"); +const { JSDOM } = jsdom; + +var res = []; +var queryFinished = false; + +exports.handler = async function (event) { + "use strict"; + + // Constants + const mainURL = + "https://www.registrar.ucla.edu/desktopmodules/ClassRoomSearch/api/webapi/GetClassroomItems"; + const classroomURL = + "https://www.registrar.ucla.edu/desktopmodules/ClassRoomSearch/api/webapi/GetCalendarEvents"; + const academicCalendarURL = + "https://www.registrar.ucla.edu/Calendars/Annual-Academic-Calendar"; + + // set the current term based on today's date + function parseDate(date, year) + { + let ret = year.concat("-"); + if(typeof(date) === 'string') + { + let tokens = date.split(", "); + if(tokens.length != 2) + { + console.log("could not parse date, datestring was not in correct format"); + process.exit(1); + } + else + { + let tokens2 = tokens[1].split(" "); + if(tokens2.length != 2) + { + console.log("could not parse date, datestring was not in correct format"); + process.exit(1); + } + else + { + switch(tokens2[0]) + { + case "January": + ret += "01-"; + break; + case "February": + ret += "02-"; + break; + case "March": + ret += "03-"; + break; + case "April": + ret += "04-"; + break; + case "May": + ret += "05-"; + break; + case "June": + ret += "06-"; + break; + case "July": + ret += "07-"; + break; + case "August": + ret += "08-"; + break; + case "September": + ret += "09-"; + break; + case "October": + ret += "10-"; + break; + case "Novemeber": + ret += "11-"; + break; + case "December": + ret += "12-"; + break; + default: + console.log("could not parse date, month is invalid"); + process.exit(1); + } + + if(tokens2[1].length == 1) + ret += "0"; + ret = ret.concat(tokens2[1]); + return ret; + } + } + } + else + { + console.log("could not parse date, parameter was not a string"); + process.exit(1); + } + } + + async function getCurrentTerm() + { + console.log("getting current term"); + console.log("trying to send request to", academicCalendarURL); + async function handleRequest(options, delay) + { + let a = new Promise(function(resolve, reject) { + setTimeout(() => resolve("done"), delay); + }); + + await a; + + return requestPromise(options); + } + + let options = { + method: "GET", + url: academicCalendarURL, + json: true + }; + + var results = await handleRequest(options, 1000); + const dom = new JSDOM(results); + + let currDate = new Date(); + let year = currDate.getFullYear() - 2000; + // we get all the elements where id ends with (for example) 2019-20 and 2020-21 + let academicYears = dom.window.document.querySelectorAll(`[id$='20${year - 1}-${year}'][id^='lt'], [id$='20${year}-${year + 1}'][id^='lt']`); + + // then we loop through the two academic years and we find the dates that "quarter begins" corresponds to + // first maintain a counter to determine what year each date corresponds to + // each time, it only outputs 8 values, because we are searching through two academic years + // 2 academic years * (3 dates for each regular quarter + 1 date for summer session) = 8 dates + let count = 0; + let dates = []; + for(let i = 0; i < academicYears.length; i++) + { + let rows = academicYears[i].querySelectorAll("td"); + for(let j = 0; j < rows.length; j++) + { + let text = rows[j].textContent; + if(text.match(/Quarter begins*/) || text.match(/Summer session begins*/)) + { + switch(count) + { + case 0: + // we'll discard the date from the previous year + break; + case 1: + case 2: + case 3: + case 4: + // we only push the dates that pertain to the current year + dates.push(new Date(parseDate(rows[j + 1].textContent, "20" + year))); + break; + case 5: + case 6: + case 7: + // we'll also discared the dates in the next year + break; + default: + // this is just to check if we've outputted the right number of dates. + // if there were too many, then we know there was an error + console.log("error: too many \"quarter begins\" or \"summer session begins\" dates were outputted"); + } + + count++; + } + } + } + + // our dates list should contain 4 dates in the same calendar year - the start of each quarter: Winter, Spring, Summer Sessions, Fall + let num = -1; + for(let i = 0; i < dates.length; i++) + { + if(currDate.getTime() < dates[i].getTime()) + { + num = i; + break; + } + } + + if(num == -1) + num = 4; + + switch(num) + { + case 0: + return (year - 1) + "F"; + case 1: + return year + "W"; + case 2: + return year + "S"; + case 3: + return year + "1"; + case 4: + return year + "F"; + } + } + + let currentTerm = await getCurrentTerm(); + console.log("the current term is", currentTerm); + + // Helper functions + function processClassroomJson(buildingClassroom) { + return { + building: buildingClassroom.value.slice(0, 8), + room: buildingClassroom.value.substr(9), + buildingName: buildingClassroom.label.slice(0, 8).trim(), + roomName: buildingClassroom.label.substr(9).trim() + }; + } + + function processClassTimeObject(buildingArray, classNameTime) { + if (classNameTime.enroll_total === 0) return buildingArray; + + let processedObject = { + day: "", + start: "", + end: "" + }; + + if (classNameTime.start.substr(0, 9) != "2014-11-1") { + processedObject.day = "Varies"; + processedObject.start = classNameTime.start.substr(0, 5); + processedObject.end = classNameTime.end.substr(0, 5); + } else { + let startTime = moment(classNameTime.start); + let endTime = moment(classNameTime.end); + + processedObject.day = startTime.format("d"); + processedObject.start = startTime.format("HH:mm"); + processedObject.end = endTime.format("HH:mm"); + } + buildingArray.push(processedObject); + return buildingArray; + } + + async function getClassroomTimes(buildingRoomObject, callback) { + let options = { + method: "GET", + url: classroomURL, + qs: { + term: currentTerm, + building: buildingRoomObject.building, + room: buildingRoomObject.room + }, + json: true + }; + + var finishedRequest = false; + while(!finishedRequest) + { + try + { + var body = await handleRequest(options, 100); + finishedRequest = true; + } + catch(e) + { + console.log("RequestError for getting ", buildingRoomObject.building, buildingRoomObject.room); + console.log(e.message); + console.log("trying again...") + } + } + + body = body.reduce(processClassTimeObject, []); + + let finalClassroomObject = { + building: buildingRoomObject.buildingName, + room: buildingRoomObject.roomName, + classTimes: body + }; + let ClassroomAvailabilities = getAvailableTimes(finalClassroomObject); + callback(ClassroomAvailabilities); + } + + function tomins(timeString) { + let timeObj = stoiParser(timeString); + return timeObj['hour'] * 60 + timeObj['minutes']; + } + + function stoiParser(timeString) { + return { + 'hour': parseInt(timeString.substring(0, 2)), + 'minutes': parseInt(timeString.substring(3)) + }; + } + + function getAvailableTimes(classroomTimeObject) { + let inverseObject = {}; + let inverseArray = []; + + inverseObject.building = classroomTimeObject.building; + inverseObject.room = classroomTimeObject.room; + inverseObject.classTimes = []; + inverseObject.class_key = ""; + let timesByDay = { '1': [], '2': [], '3': [], '4': [], '5': [], 'Varies': [] }; + let inverseTimesByDay = { '1': [], '2': [], '3': [], '4': [], '5': [], 'Varies': [] } + for (let timeSlot of classroomTimeObject.classTimes) { + timesByDay[timeSlot.day].push(timeSlot); + } + for (let d in timesByDay) { + let s = '08:00'; + for (let slot of timesByDay[d]) { + if (tomins(slot['start']) - tomins(s) > 15) { + inverseTimesByDay[d].push({ 'day': d, 'start': s, 'end': slot['start'] }); + } + s = slot['end']; + } + inverseTimesByDay[d].push({ 'day': d, 'start': s, 'end': '22:00' }); + } + + for (let d in inverseTimesByDay) { + for (let slot of inverseTimesByDay[d]) { + let NewTimeSlot = {} + NewTimeSlot.day = d; + NewTimeSlot.start = tomins(slot['start']); + NewTimeSlot.end = tomins(slot['end']); + NewTimeSlot.room = inverseObject.room; + NewTimeSlot.building = inverseObject.building; + inverseArray.push(NewTimeSlot); + } + } + + return inverseArray; + } + + // Wrapper function for sending requests, with delay parameter. + async function handleRequest(options, delay) + { + let a = new Promise(function(resolve, reject) { + setTimeout(() => resolve("done"), delay); + }); + + await a; + + return requestPromise(options); + } + + async function queryWrapper(sql) + { + await client.query(sql); + return true; + } + + async function continueExec() + { + // Here is the trick, wait until var callbackCount is set number of callback functions. + if (done_count > 0) + { + return false; + } + + // Finally, do what you need + console.log("got each classroom's schedule"); + let st = "" + res.forEach(function (resp) { + if (resp.day != "Varies" && resp.start < resp.end) { + + let day = resp.day.toString(); + let start = resp.start.toString(); + let end = resp.end.toString(); + let room = resp.room; + let building = resp.building; + st += "(" + day + "," + start + "," + end + "," + "'" + room + "'" + "," + "'" + building + "'" + "),\n"; + + } + }) + + // Run the query in the database. + let deleteSQL = "DELETE FROM classroom_availabilities;"; + let insertSQL = "INSERT INTO classroom_availabilities (day, start_time, end_time, room, building) VALUES" + st.substring(0, st.length - 2); + try + { + console.log("deleting existing data from database..."); + await queryWrapper(deleteSQL); + let delay = new Promise(function(resolve, reject) { + setTimeout(() => resolve("done"), 1000); + }); + await delay; + console.log("inserting into the database..."); + queryFinished = await queryWrapper(insertSQL); + } + catch(err) + { + console.log(err.stack); + queryFinished = true; + } + + return true; + } + + function isUnusedRoom(crAvailList) + { + if(crAvailList.length == 6 + && crAvailList[0].day == '1' + && crAvailList[1].day == '2' + && crAvailList[2].day == '3' + && crAvailList[3].day == '4' + && crAvailList[4].day == '5' + && crAvailList[5].day == 'Varies') + return true; + + return false; + } + + // try to get classrooms for the current quarter + let options = { + method: "GET", + url: mainURL, + json: true + }; + + // Send a request to get all classrooms for the current quarter. + var resolved = false; + var delay = 2000; + while(!resolved) + { + console.log("trying to send request to ", options.url); + try + { + var body = await handleRequest(options, delay); + resolved = true; + } + catch(e) + { + console.log("error sending request to ", options.url, ": ", e.message); + console.log("trying to send it again..."); + } + } + + console.log("got classrooms"); + + // For each classroom, send a request to get its availability. + console.log("trying to send request to server for each classroom's availability..."); + let classrooms = body.map(processClassroomJson); + let done_count = classrooms.length; + + classrooms.forEach(async function (classroom) { + await getClassroomTimes(classroom, function (response) { + // Here you have access to your variable + if(!isUnusedRoom(response)) + res = res.concat(response) + done_count -= 1; + }) + }); + + // We must wait until all the requests for the classrooms have returned. + // The while loop will continue to call "continueExec()" until the variable + // done_count reaches 0. The "continueExec()" function includes the query + // that is run in the database. + while(1) + { + let a = new Promise(function(resolve, reject) { + setTimeout(() => resolve("done"), 2000); + }); + + await a; + + console.log("waiting for all classroom requests to return..."); + if(await continueExec()) + break; + } + + // Wait for the query to finish running and then break. + while(1) + { + if(queryFinished) + { + console.log("finished."); + break; + } + + let a = new Promise(function(resolve, reject) { + setTimeout(() => resolve("done"), 2000); + }); + + await a; + console.log("waiting for query to finish..."); + } +}; + +//uncomment out this part if you want to run it locally +/*try { + exports.handler({}); +} catch (e) { + console.log("error"); + // Deal with the fact the chain failed +}*/ diff --git a/classroomScraper/package-lock.json b/classroomScraper/package-lock.json new file mode 100644 index 0000000..cb6618f --- /dev/null +++ b/classroomScraper/package-lock.json @@ -0,0 +1,1197 @@ +{ + "name": "classroommetadata", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@types/concat-stream": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@types/concat-stream/-/concat-stream-1.6.0.tgz", + "integrity": "sha1-OU2+C7X+5Gs42JZzXoto7yOQ0A0=", + "requires": { + "@types/node": "*" + } + }, + "@types/form-data": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-0.0.33.tgz", + "integrity": "sha1-yayFsqX9GENbjIXZ7LUObWyJP/g=", + "requires": { + "@types/node": "*" + } + }, + "@types/node": { + "version": "10.14.21", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.21.tgz", + "integrity": "sha512-nuFlRdBiqbF+PJIEVxm2jLFcQWN7q7iWEJGsBV4n7v1dbI9qXB8im2pMMKMCUZe092sQb5SQft2DHfuQGK5hqQ==" + }, + "@types/qs": { + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.5.3.tgz", + "integrity": "sha512-Jugo5V/1bS0fRhy2z8+cUAHEyWOATaz4rbyLVvcFs7+dXp5HfwpEwzF1Q11bB10ApUqHf+yTauxI0UXQDwGrbA==" + }, + "abab": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.3.tgz", + "integrity": "sha512-tsFzPpcttalNjFBCFMqsKYQcWxxen1pgJR56by//QwvJc4/OUS3kPOOttx2tSIfjsylB0pYu7f5D3K1RCxUnUg==" + }, + "acorn": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.1.1.tgz", + "integrity": "sha512-add7dgA5ppRPxCFJoAGfMDi7PIBXq1RtGo7BhbLaxwrXPOmw8gq48Y9ozT01hUKy9byMjlR20EJhu5zlkErEkg==" + }, + "acorn-globals": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-6.0.0.tgz", + "integrity": "sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg==", + "requires": { + "acorn": "^7.1.1", + "acorn-walk": "^7.1.1" + } + }, + "acorn-walk": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.1.1.tgz", + "integrity": "sha512-wdlPY2tm/9XBr7QkKlq0WQVgiuGTX6YWPyRyBviSoScBuLfTVQhvwg6wJ369GJ/1nPfTLMfnrFIfjqVg6d+jQQ==" + }, + "ajv": { + "version": "6.10.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz", + "integrity": "sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==", + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=" + }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sdk": { + "version": "2.544.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.544.0.tgz", + "integrity": "sha512-wwiBJgAUGKXY/xoCSrUXVZnNtefoH3YcPwGxQrQXOIDnLMQ32yh/SWc52qmwdxA7WJzpTcIj8y+5keH3P1LYaw==", + "requires": { + "buffer": "4.9.1", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + }, + "aws4": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" + }, + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "requires": { + "tweetnacl": "^0.14.3" + } + }, + "bignumber.js": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", + "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==" + }, + "bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, + "browser-process-hrtime": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", + "integrity": "sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow==" + }, + "buffer": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + }, + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "requires": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "cssom": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.4.4.tgz", + "integrity": "sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw==" + }, + "cssstyle": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.2.0.tgz", + "integrity": "sha512-sEb3XFPx3jNnCAMtqrXPDeSgQr+jojtCeNf8cvMNMh1cG970+lljssvQDzPq6lmmJu2Vhqood/gtEomBiHOGnA==", + "requires": { + "cssom": "~0.3.6" + }, + "dependencies": { + "cssom": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" + } + } + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "data-urls": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", + "integrity": "sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ==", + "requires": { + "abab": "^2.0.3", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.0.0" + } + }, + "decimal.js": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.2.0.tgz", + "integrity": "sha512-vDPw+rDgn3bZe1+F/pyEwb1oMG2XTlRVgAa6B4KccTEpYgF8w6eQllVbQcfIJnZyvzFtFpxnpGtx8dd7DJp/Rw==" + }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "domexception": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-2.0.1.tgz", + "integrity": "sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg==", + "requires": { + "webidl-conversions": "^5.0.0" + }, + "dependencies": { + "webidl-conversions": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", + "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==" + } + } + }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "escodegen": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.1.tgz", + "integrity": "sha512-Bmt7NcRySdIfNPfU2ZoXDrrXsG9ZjvDxcAlMfDUgRBjLOWTuIACXPBFJH7Z+cLb40JeQco5toikyc9t9P8E9SQ==", + "requires": { + "esprima": "^4.0.1", + "estraverse": "^4.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1", + "source-map": "~0.6.1" + } + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" + }, + "estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==" + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==" + }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "get-port": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz", + "integrity": "sha1-3Xzn3hh8Bsi/NTeWrHHgmfCYDrw=" + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "html-encoding-sniffer": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", + "integrity": "sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ==", + "requires": { + "whatwg-encoding": "^1.0.5" + } + }, + "http-basic": { + "version": "8.1.3", + "resolved": "https://registry.npmjs.org/http-basic/-/http-basic-8.1.3.tgz", + "integrity": "sha512-/EcDMwJZh3mABI2NhGfHOGOeOZITqfkEO4p/xK+l3NpyncIHUQBoMvCSF/b5GqvKtySC2srL/GGG3+EtlqlmCw==", + "requires": { + "caseless": "^0.12.0", + "concat-stream": "^1.6.2", + "http-response-object": "^3.0.1", + "parse-cache-control": "^1.0.1" + } + }, + "http-response-object": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/http-response-object/-/http-response-object-3.0.2.tgz", + "integrity": "sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==", + "requires": { + "@types/node": "^10.0.3" + } + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "index.js": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/index.js/-/index.js-0.0.3.tgz", + "integrity": "sha1-JzOx9IbciQ7QJpiQJPWC2DXFI6w=" + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "ip-regex": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", + "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=" + }, + "is-potential-custom-element-name": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.0.tgz", + "integrity": "sha1-DFLlS8yjkbssSUsh6GJtczbG45c=" + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "jsdom": { + "version": "16.2.2", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.2.2.tgz", + "integrity": "sha512-pDFQbcYtKBHxRaP55zGXCJWgFHkDAYbKcsXEK/3Icu9nKYZkutUXfLBwbD+09XDutkYSHcgfQLZ0qvpAAm9mvg==", + "requires": { + "abab": "^2.0.3", + "acorn": "^7.1.1", + "acorn-globals": "^6.0.0", + "cssom": "^0.4.4", + "cssstyle": "^2.2.0", + "data-urls": "^2.0.0", + "decimal.js": "^10.2.0", + "domexception": "^2.0.1", + "escodegen": "^1.14.1", + "html-encoding-sniffer": "^2.0.1", + "is-potential-custom-element-name": "^1.0.0", + "nwsapi": "^2.2.0", + "parse5": "5.1.1", + "request": "^2.88.2", + "request-promise-native": "^1.0.8", + "saxes": "^5.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^3.0.1", + "w3c-hr-time": "^1.0.2", + "w3c-xmlserializer": "^2.0.0", + "webidl-conversions": "^6.0.0", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^8.0.0", + "ws": "^7.2.3", + "xml-name-validator": "^3.0.0" + }, + "dependencies": { + "request": { + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.3", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.5.0", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + }, + "dependencies": { + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + } + } + }, + "tough-cookie": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz", + "integrity": "sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==", + "requires": { + "ip-regex": "^2.1.0", + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + } + } + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "requires": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + } + }, + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + }, + "lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=" + }, + "mime-db": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", + "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==" + }, + "mime-types": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", + "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", + "requires": { + "mime-db": "1.40.0" + } + }, + "moment": { + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", + "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" + }, + "mysql": { + "version": "2.17.1", + "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.17.1.tgz", + "integrity": "sha512-7vMqHQ673SAk5C8fOzTG2LpPcf3bNt0oL3sFpxPEEFp1mdlDcrLK0On7z8ZYKaaHrHwNcQ/MTUz7/oobZ2OyyA==", + "requires": { + "bignumber.js": "7.2.1", + "readable-stream": "2.3.6", + "safe-buffer": "5.1.2", + "sqlstring": "2.3.1" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, + "nwsapi": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz", + "integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ==" + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, + "optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "requires": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + } + }, + "package": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package/-/package-1.0.1.tgz", + "integrity": "sha1-0lofmeJQbcsn1nBLg9yooxLk7cw=" + }, + "package-lock.json": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/package-lock.json/-/package-lock.json-1.0.0.tgz", + "integrity": "sha512-+yEXtNdlCs5N0Zy/9uvkifgf/RqnGu0WqP4j9Wu1Us4YReFe1YNBh2Krmf8B1xGxjpYnta63K55QP8bkafnOzA==" + }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, + "parse-cache-control": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parse-cache-control/-/parse-cache-control-1.0.1.tgz", + "integrity": "sha1-juqz5U+laSD+Fro493+iGqzC104=" + }, + "parse5": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.1.tgz", + "integrity": "sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==" + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, + "pg": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-7.12.1.tgz", + "integrity": "sha512-l1UuyfEvoswYfcUe6k+JaxiN+5vkOgYcVSbSuw3FvdLqDbaoa2RJo1zfJKfPsSYPFVERd4GHvX3s2PjG1asSDA==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "0.1.3", + "pg-pool": "^2.0.4", + "pg-types": "^2.1.0", + "pgpass": "1.x", + "semver": "4.3.2" + } + }, + "pg-connection-string": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", + "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-pool": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.7.tgz", + "integrity": "sha512-UiJyO5B9zZpu32GSlP0tXy8J2NsJ9EFGFfz5v6PSbdz/1hBLX1rNiiy5+mAm5iJJYwfCv4A0EBcQLGWwjbpzZw==" + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", + "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", + "requires": { + "split": "^1.0.0" + } + }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" + }, + "postgres-date": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.4.tgz", + "integrity": "sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "promise": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/promise/-/promise-8.0.3.tgz", + "integrity": "sha512-HeRDUL1RJiLhyA0/grn+PTShlBAcLuh/1BJGtrvjwbvRDCTLLMEz9rOGCV+R3vHY4MixIuoMEd9Yq/XvsTPcjw==", + "requires": { + "asap": "~2.0.6" + } + }, + "psl": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.4.0.tgz", + "integrity": "sha512-HZzqCGPecFLyoRj5HLfuDSKYTJkAfB5thKBIkRHtGjWwY7p1dAyveIbXIq4tO0KYfDF2tHqPUgY9SDnGm00uFw==" + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "request-promise": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.5.tgz", + "integrity": "sha512-ZgnepCykFdmpq86fKGwqntyTiUrHycALuGggpyCZwMvGaZWgxW6yagT0FHkgo5LzYvOaCNvxYwWYIjevSH1EDg==", + "requires": { + "bluebird": "^3.5.0", + "request-promise-core": "1.1.3", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + } + }, + "request-promise-core": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.3.tgz", + "integrity": "sha512-QIs2+ArIGQVp5ZYbWD5ZLCY29D5CfWizP8eWnm8FoGD1TX61veauETVQbrV60662V0oFBkrDOuaBI8XgtuyYAQ==", + "requires": { + "lodash": "^4.17.15" + } + }, + "request-promise-native": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.8.tgz", + "integrity": "sha512-dapwLGqkHtwL5AEbfenuzjTYg35Jd6KPytsC2/TLkVMz8rm+tNt72MGUWT1RP/aYawMpN6HqbNGBQaRcBtjQMQ==", + "requires": { + "request-promise-core": "1.1.3", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + } + }, + "safe-buffer": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", + "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "requires": { + "xmlchars": "^2.2.0" + } + }, + "semver": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", + "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true + }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "requires": { + "through": "2" + } + }, + "sqlstring": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", + "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=" + }, + "sshpk": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, + "stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, + "symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" + }, + "sync-request": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/sync-request/-/sync-request-6.1.0.tgz", + "integrity": "sha512-8fjNkrNlNCrVc/av+Jn+xxqfCjYaBoHqCsDz6mt030UMxJGr+GSfCV1dQt2gRtlL63+VPidwDVLr7V2OcTSdRw==", + "requires": { + "http-response-object": "^3.0.1", + "sync-rpc": "^1.2.1", + "then-request": "^6.0.0" + } + }, + "sync-rpc": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/sync-rpc/-/sync-rpc-1.3.6.tgz", + "integrity": "sha512-J8jTXuZzRlvU7HemDgHi3pGnh/rkoqR/OZSjhTyyZrEkkYQbk7Z33AXp37mkPfPpfdOuj7Ex3H/TJM1z48uPQw==", + "requires": { + "get-port": "^3.1.0" + } + }, + "then-request": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/then-request/-/then-request-6.0.2.tgz", + "integrity": "sha512-3ZBiG7JvP3wbDzA9iNY5zJQcHL4jn/0BWtXIkagfz7QgOL/LqjCEOBQuJNZfu0XYnv5JhKh+cDxCPM4ILrqruA==", + "requires": { + "@types/concat-stream": "^1.6.0", + "@types/form-data": "0.0.33", + "@types/node": "^8.0.0", + "@types/qs": "^6.2.31", + "caseless": "~0.12.0", + "concat-stream": "^1.6.0", + "form-data": "^2.2.0", + "http-basic": "^8.1.1", + "http-response-object": "^3.0.1", + "promise": "^8.0.0", + "qs": "^6.4.0" + }, + "dependencies": { + "@types/node": { + "version": "8.10.54", + "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.54.tgz", + "integrity": "sha512-kaYyLYf6ICn6/isAyD4K1MyWWd5Q3JgH6bnMN089LUx88+s4W8GvK9Q6JMBVu5vsFFp7pMdSxdKmlBXwH/VFRg==" + } + } + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + } + } + }, + "tr46": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.0.2.tgz", + "integrity": "sha512-3n1qG+/5kg+jrbTzwAykB5yRYtQCTqOGKq5U5PE3b0a1/mzo6snDhjGS0zJVJunO0NrT3Dg1MLy5TjWP/UJppg==", + "requires": { + "punycode": "^2.1.1" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, + "type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "requires": { + "prelude-ls": "~1.1.2" + } + }, + "typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" + }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "requires": { + "punycode": "^2.1.0" + } + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + } + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "uuid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", + "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "w3c-hr-time": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", + "integrity": "sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ==", + "requires": { + "browser-process-hrtime": "^1.0.0" + } + }, + "w3c-xmlserializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz", + "integrity": "sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA==", + "requires": { + "xml-name-validator": "^3.0.0" + } + }, + "webidl-conversions": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-6.0.0.tgz", + "integrity": "sha512-jTZAeJnc6D+yAOjygbJOs33kVQIk5H6fj9SFDOhIKjsf9HiAzL/c+tAJsc8ASWafvhNkH+wJZms47pmajkhatA==" + }, + "whatwg-encoding": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "requires": { + "iconv-lite": "0.4.24" + } + }, + "whatwg-mimetype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + }, + "whatwg-url": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.0.0.tgz", + "integrity": "sha512-41ou2Dugpij8/LPO5Pq64K5q++MnRCBpEHvQr26/mArEKTkCV5aoXIqyhuYtE0pkqScXwhf2JP57rkRTYM29lQ==", + "requires": { + "lodash.sortby": "^4.7.0", + "tr46": "^2.0.0", + "webidl-conversions": "^5.0.0" + }, + "dependencies": { + "webidl-conversions": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", + "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==" + } + } + }, + "word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==" + }, + "ws": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.3.tgz", + "integrity": "sha512-HTDl9G9hbkNDk98naoR/cHDws7+EyYMOdL1BmjsZXRUjf7d+MficC4B7HLUPlSiho0vg+CWKrGIt/VJBd1xunQ==" + }, + "xml-name-validator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", + "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + }, + "xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + } + } +} diff --git a/classroomScraper/package.json b/classroomScraper/package.json new file mode 100644 index 0000000..14feb59 --- /dev/null +++ b/classroomScraper/package.json @@ -0,0 +1,24 @@ +{ + "name": "classroommetadata", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "aws-sdk": "^2.544.0", + "index.js": "0.0.3", + "jsdom": "^16.2.2", + "moment": "^2.24.0", + "mysql": "^2.17.1", + "package": "^1.0.1", + "package-lock.json": "^1.0.0", + "pg": "^7.12.1", + "request": "^2.88.0", + "request-promise": "^4.2.5", + "sync-request": "^6.1.0" + } +} diff --git a/libHours/index.js b/libHours/index.js new file mode 100644 index 0000000..97c206b --- /dev/null +++ b/libHours/index.js @@ -0,0 +1,184 @@ +const jsdom = require("jsdom"); +const { JSDOM } = jsdom; +var rp = require("request-promise"); + +var AWS = require("aws-sdk"); +var Promise = require("bluebird"); +AWS.config.update({ region: "us-east-1" }); +var docClient = new AWS.DynamoDB.DocumentClient(); + +/* +// +{ + "name":"Arts Library", + "location":" 1400 Public Affairs Building, Los Angeles, CA 90095-1392", + "phone":"(310) 206-5425", + "image":"https://www.library.ucla.edu/sites/default/files/styles/thumbnail_large/public/location_map_images/map_arts.png?itok=0LT3XP_5", + "start_date":"Feb 17 ", + "department": + [ + { + "department_name":"Arts Library", + "time": + [ + {"dp_open_time":"1pm - 5pm","date":"Su 17"}, + {"dp_open_time":"9am - 5pm","date":"M 18"}, + {"dp_open_time":"8am - 9pm","date":"Tu 19"}, + {"dp_open_time":"8am - 9pm","date":"W 20"}, + {"dp_open_time":"8am - 9pm","date":"Th 21"}, + {"dp_open_time":"8am - 5pm","date":"F 22"}, + {"dp_open_time":"1pm - 5pm","date":"Sa 23"} + ] + }, + { + "department_name":"CLICC Laptop & iPad Lending (Arts Library)", + "time": + [ + {"dp_open_time":"1pm - 4:30pm","date":"Su 17"}, + {"dp_open_time":"9am - 4:30pm","date":"M 18"}, + {"dp_open_time":"8am - 8:30pm","date":"Tu 19"}, + {"dp_open_time":"8am - 8:30pm","date":"W 20"}, + {"dp_open_time":"8am - 8:30pm","date":"Th 21"}, + {"dp_open_time":"8am - 4:30pm","date":"F 22"}, + {"dp_open_time":"1pm - 4:30pm","date":"Sa 23"} + ] + }, + { + "department_name":"Reference Desk", + "time": + [ + {"dp_open_time":"Closed","date":"Su 17"}, + {"dp_open_time":"Closed","date":"M 18"}, + {"dp_open_time":"11am - 4pm","date":"Tu 19"}, + {"dp_open_time":"11am - 4pm","date":"W 20"}, + {"dp_open_time":"11am - 4pm","date":"Th 21"}, + {"dp_open_time":"11am - 4pm","date":"F 22"}, + {"dp_open_time":"Closed","date":"Sa 23"}] + } + ] +}*/ +exports.handler = async function (event) { + try { + // console.log("here"); + console.log("here"); + //Url to visit + let options = { + method: "GET", + url: "https://www.library.ucla.edu/hours", + }; + + var body = await rp(options); + const dom = new JSDOM(body); + + await new Promise((resolve) => setTimeout(resolve, 1500)); + //find names of the 11 libraries + const names = dom.window.document.querySelectorAll('div.pane-node-title h2') + //find address of the 11 libraries + const addresses = dom.window.document.querySelectorAll('div.location__address') + //find opening times of the 11 libraries + const time_of_dp = dom.window.document.querySelectorAll('table.opening-hours-week') + //find phone number + const phones = dom.window.document.querySelectorAll('span.location__phone-number') + //find image + const imgs = dom.window.document.querySelectorAll('img') + console.log(imgs) + const num_rooms = names.length; + const data = []; + for (let n = 0; n < num_rooms; n++) { + var obj = new Object(); + //add name to lib obj + const lib_name = names[n].textContent.trim() + obj["name"] = lib_name + + //add location to lib obj + const lib_loc = addresses[n].textContent.trim() + obj["location"] = lib_loc + //add phone-number to lib obj + const lib_phone = phones[n].textContent.trim() + obj["phone"] = lib_phone + //add image to lib obj + const img = imgs[n + 1].src + obj["image"] = img + //add department to lib obj + const department = [] + lib_dp = time_of_dp[n].textContent.trim() + list_dp = lib_dp.split("\n") + //first line : date + //after : openning time for each department + const date = list_dp[0].split('\t') + obj["start_date"] = date[0].split("-")[0] + const num_dp = list_dp.length - 1 + for (let i = 0; i < num_dp; i++) { + var dp_obj = new Object(); + dp_info = list_dp[i + 1].split('\t') + dp_obj["department_name"] = dp_info[0] + + //console.log(list_dp[i+1].split('\t')); + //dp_open=dp_info[1].split(" ") + //dp_obj["dp_info"]=dp_info[2] + dp_open_time = [] + const open_day = dp_info.length - 1 + for (let j = 0; j < open_day; j++) { + var dp_time_obj = new Object(); + dp_time_obj["dp_open_time"] = dp_info[j + 1] + dp_time_obj["date"] = date[j + 1] + dp_open_time.push(dp_time_obj); + } + dp_obj["time"] = dp_open_time + department.push(dp_obj); + } + obj["department"] = department + data.push(obj); + } + + + console.log(data); + + var numDone = 0; + + data.forEach(obj => { + const params = { + TableName: "lib_hours", + Key: { + "name": obj["name"] + }, + + UpdateExpression: "set #location = :lo, #phone = :ph, #image = :im, #start_date = :st, #department = :dp", + + ExpressionAttributeNames: { + "#location": "location", + "#phone": "phone" , + "#image": "image", + "#start_date": "start_date", + "#department": "department", + }, + + ExpressionAttributeValues: { + ":lo": obj["location"], + ":ph": obj["phone"], + ":im": obj["image"], + ":st": obj["start_date"], + ":dp": obj["department"] + } + } + + docClient.update(params, function(err, data) { + if (err) console.log(err); + + numDone += 1; + + if(numDone == data.length){ + return; + } + }) + }) + } + + //If there is an error, write to console and exit + catch (err) { + console.error(err); + return; + } + +} +// test(); diff --git a/libHours/package-lock.json b/libHours/package-lock.json new file mode 100644 index 0000000..a8ffd11 --- /dev/null +++ b/libHours/package-lock.json @@ -0,0 +1,825 @@ +{ + "name": "libhoursscraper", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "abab": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.0.tgz", + "integrity": "sha512-sY5AXXVZv4Y1VACTtR11UJCPHHudgY5i26Qj5TypE6DKlIApbwb5uqhXcJ5UUGbvZNRh7EeIoW+LrJumBsKp7w==" + }, + "acorn": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz", + "integrity": "sha512-jPTiwtOxaHNaAPg/dmrJ/beuzLRnXtB0kQPQ8JpotKJgTB6rX6c8mlf315941pyjBSaPg8NHXS9fhP4u17DpGA==" + }, + "acorn-globals": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-4.3.2.tgz", + "integrity": "sha512-BbzvZhVtZP+Bs1J1HcwrQe8ycfO0wStkSGxuul3He3GkHOIZ6eTqOkPuw9IP1X3+IkOo4wiJmwkobzXYz4wewQ==", + "requires": { + "acorn": "^6.0.1", + "acorn-walk": "^6.0.1" + } + }, + "acorn-walk": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.1.1.tgz", + "integrity": "sha512-OtUw6JUTgxA2QoqqmrmQ7F2NYqiBPi/L2jqHyFtllhOUvXYQXf0Z1CYUinIfyT4bTCGmrA7gX9FvHA81uzCoVw==" + }, + "ajv": { + "version": "6.10.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", + "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "array-equal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=" + }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, + "async-limiter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", + "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" + }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sdk": { + "version": "2.464.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.464.0.tgz", + "integrity": "sha512-DsJ/V/Eaazio5klO49IXIgnDpBcObgOunFs0KsUbdPz5yxvznZQiez9LqTcaj0SaCS7zsrT9K5p+Jtbt60z1SA==", + "requires": { + "buffer": "4.9.1", + "events": "1.1.1", + "ieee754": "1.1.8", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + } + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + }, + "aws4": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" + }, + "base64-js": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", + "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==" + }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "requires": { + "tweetnacl": "^0.14.3" + } + }, + "bluebird": { + "version": "3.5.5", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.5.tgz", + "integrity": "sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w==" + }, + "browser-process-hrtime": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-0.1.3.tgz", + "integrity": "sha512-bRFnI4NnjO6cnyLmOV/7PVoDEMJChlcfN0z4s1YMBY989/SvlfMI1lgCnkFUs53e9gQF+w7qu7XdllSTiSl8Aw==" + }, + "buffer": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "cssom": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.6.tgz", + "integrity": "sha512-DtUeseGk9/GBW0hl0vVPpU22iHL6YB5BUX7ml1hB+GMpo0NX5G4voX3kdWiMSEguFtcW3Vh3djqNF4aIe6ne0A==" + }, + "cssstyle": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-1.2.2.tgz", + "integrity": "sha512-43wY3kl1CVQSvL7wUY1qXkxVGkStjpkDmVjiIKX8R97uhajy8Bybay78uOtqvh7Q5GK75dNPfW0geWjE6qQQow==", + "requires": { + "cssom": "0.3.x" + } + }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "data-urls": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-1.1.0.tgz", + "integrity": "sha512-YTWYI9se1P55u58gL5GkQHW4P6VJBJ5iBT+B5a7i2Tjadhv52paJG0qHX4A0OR6/t52odI64KP2YvFpkDOi3eQ==", + "requires": { + "abab": "^2.0.0", + "whatwg-mimetype": "^2.2.0", + "whatwg-url": "^7.0.0" + } + }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" + }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "domexception": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz", + "integrity": "sha512-raigMkn7CJNNo6Ihro1fzG7wr3fHuYVytzquZKX5n0yizGsTcYgzdIUwj1X9pK0VvjeihV+XiclP+DjwbsSKug==", + "requires": { + "webidl-conversions": "^4.0.2" + } + }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "escodegen": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.11.1.tgz", + "integrity": "sha512-JwiqFD9KdGVVpeuRa68yU3zZnBEOcPs0nKW7wZzXky8Z7tffdYUHbe11bPCV5jYlK6DVdKLWLm0f5I/QlL0Kmw==", + "requires": { + "esprima": "^3.1.3", + "estraverse": "^4.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1", + "source-map": "~0.6.1" + } + }, + "esprima": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz", + "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=" + }, + "estraverse": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz", + "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=" + }, + "esutils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", + "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=" + }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + }, + "fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "^1.0.0" + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "html-encoding-sniffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz", + "integrity": "sha512-71lZziiDnsuabfdYiUeWdCVyKuqwWi23L8YeIgV9jSSZHCtb6wB1BKWooH7L3tn4/FuZJMVWyNaIDr4RGmaSYw==", + "requires": { + "whatwg-encoding": "^1.0.1" + } + }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "ieee754": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", + "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" + }, + "ip-regex": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", + "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=" + }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "jsdom": { + "version": "15.1.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-15.1.1.tgz", + "integrity": "sha512-cQZRBB33arrDAeCrAEWn1U3SvrvC8XysBua9Oqg1yWrsY/gYcusloJC3RZJXuY5eehSCmws8f2YeliCqGSkrtQ==", + "requires": { + "abab": "^2.0.0", + "acorn": "^6.1.1", + "acorn-globals": "^4.3.2", + "array-equal": "^1.0.0", + "cssom": "^0.3.6", + "cssstyle": "^1.2.2", + "data-urls": "^1.1.0", + "domexception": "^1.0.1", + "escodegen": "^1.11.1", + "html-encoding-sniffer": "^1.0.2", + "nwsapi": "^2.1.4", + "parse5": "5.1.0", + "pn": "^1.1.0", + "request": "^2.88.0", + "request-promise-native": "^1.0.7", + "saxes": "^3.1.9", + "symbol-tree": "^3.2.2", + "tough-cookie": "^3.0.1", + "w3c-hr-time": "^1.0.1", + "w3c-xmlserializer": "^1.1.2", + "webidl-conversions": "^4.0.2", + "whatwg-encoding": "^1.0.5", + "whatwg-mimetype": "^2.3.0", + "whatwg-url": "^7.0.0", + "ws": "^7.0.0", + "xml-name-validator": "^3.0.0" + } + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "requires": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + } + }, + "lodash": { + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + }, + "lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=" + }, + "mime-db": { + "version": "1.40.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", + "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==" + }, + "mime-types": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", + "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", + "requires": { + "mime-db": "1.40.0" + } + }, + "nwsapi": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.1.4.tgz", + "integrity": "sha512-iGfd9Y6SFdTNldEy2L0GUhcarIutFmk+MPWIn9dmj8NMIup03G08uUF2KGbbmv/Ux4RT0VZJoP/sVbWA6d/VIw==" + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, + "optionator": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", + "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", + "requires": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.4", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "wordwrap": "~1.0.0" + } + }, + "parse5": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-5.1.0.tgz", + "integrity": "sha512-fxNG2sQjHvlVAYmzBZS9YlDp6PTSSDwa98vkD4QgVDDCAo84z5X1t5XyJQ62ImdLXx5NdIIfihey6xpum9/gRQ==" + }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, + "pn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", + "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==" + }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" + }, + "psl": { + "version": "1.1.32", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.32.tgz", + "integrity": "sha512-MHACAkHpihU/REGGPLj4sEfc/XKW2bheigvHO1dUqjaKigMp1C8+WLQYRGgeKFMsw5PMfegZcaN8IDXK/cD0+g==" + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + } + } + } + }, + "request-promise": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.4.tgz", + "integrity": "sha512-8wgMrvE546PzbR5WbYxUQogUnUDfM0S7QIFZMID+J73vdFARkFy+HElj4T+MWYhpXwlLp0EQ8Zoj8xUA0he4Vg==", + "requires": { + "bluebird": "^3.5.0", + "request-promise-core": "1.1.2", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + }, + "dependencies": { + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + } + } + }, + "request-promise-core": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.2.tgz", + "integrity": "sha512-UHYyq1MO8GsefGEt7EprS8UrXsm1TxEvFUX1IMTuSLU2Rh7fTIdFtl8xD7JiEYiWU2dl+NYAjCTksTehQUxPag==", + "requires": { + "lodash": "^4.17.11" + } + }, + "request-promise-native": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.7.tgz", + "integrity": "sha512-rIMnbBdgNViL37nZ1b3L/VfPOpSi0TqVDQPAvO6U14lMzOLrt5nilxCQqtDKhZeDiW0/hkCXGoQjhgJd/tCh6w==", + "requires": { + "request-promise-core": "1.1.2", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + }, + "dependencies": { + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + } + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "saxes": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-3.1.9.tgz", + "integrity": "sha512-FZeKhJglhJHk7eWG5YM0z46VHmI3KJpMBAQm3xa9meDvd+wevB5GuBB0wc0exPInZiBBHqi00DbS8AcvCGCFMw==", + "requires": { + "xmlchars": "^1.3.1" + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true + }, + "sshpk": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, + "stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" + }, + "symbol-tree": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.2.tgz", + "integrity": "sha1-rifbOPZgp64uHDt9G8KQgZuFGeY=" + }, + "tough-cookie": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz", + "integrity": "sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==", + "requires": { + "ip-regex": "^2.1.0", + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + }, + "tr46": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-1.0.1.tgz", + "integrity": "sha1-qLE/1r/SSJUZZ0zN5VujaTtwbQk=", + "requires": { + "punycode": "^2.1.0" + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, + "type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "requires": { + "prelude-ls": "~1.1.2" + } + }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "requires": { + "punycode": "^2.1.0" + } + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + } + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "w3c-hr-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.1.tgz", + "integrity": "sha1-gqwr/2PZUOqeMYmlimViX+3xkEU=", + "requires": { + "browser-process-hrtime": "^0.1.2" + } + }, + "w3c-xmlserializer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-1.1.2.tgz", + "integrity": "sha512-p10l/ayESzrBMYWRID6xbuCKh2Fp77+sA0doRuGn4tTIMrrZVeqfpKjXHY+oDh3K4nLdPgNwMTVP6Vp4pvqbNg==", + "requires": { + "domexception": "^1.0.1", + "webidl-conversions": "^4.0.2", + "xml-name-validator": "^3.0.0" + } + }, + "webidl-conversions": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz", + "integrity": "sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==" + }, + "whatwg-encoding": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", + "integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==", + "requires": { + "iconv-lite": "0.4.24" + } + }, + "whatwg-mimetype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", + "integrity": "sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==" + }, + "whatwg-url": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-7.0.0.tgz", + "integrity": "sha512-37GeVSIJ3kn1JgKyjiYNmSLP1yzbpb29jdmwBSgkD9h40/hyrR/OifpVUndji3tmwGgD8qpw7iQu3RSbCrBpsQ==", + "requires": { + "lodash.sortby": "^4.7.0", + "tr46": "^1.0.1", + "webidl-conversions": "^4.0.2" + } + }, + "wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=" + }, + "ws": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.0.0.tgz", + "integrity": "sha512-cknCal4k0EAOrh1SHHPPWWh4qm93g1IuGGGwBjWkXmCG7LsDtL8w9w+YVfaF+KSVwiHQKDIMsSLBVftKf9d1pg==", + "requires": { + "async-limiter": "^1.0.0" + } + }, + "xml-name-validator": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", + "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + }, + "xmlchars": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-1.3.1.tgz", + "integrity": "sha512-tGkGJkN8XqCod7OT+EvGYK5Z4SfDQGD30zAa58OcnAa0RRWgzUEK72tkXhsX1FZd+rgnhRxFtmO+ihkp8LHSkw==" + } + } +} diff --git a/libHours/package.json b/libHours/package.json new file mode 100644 index 0000000..f2e6d73 --- /dev/null +++ b/libHours/package.json @@ -0,0 +1,17 @@ +{ + "name": "libhoursscraper", + "version": "1.0.0", + "description": "", + "main": "libraryinfo-scraper.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "aws-sdk": "^2.464.0", + "bluebird": "^3.5.5", + "jsdom": "^15.1.1", + "request-promise": "^4.2.4" + } +} diff --git a/libinfo-scraper.py b/libinfo-scraper.py deleted file mode 100644 index dde6043..0000000 --- a/libinfo-scraper.py +++ /dev/null @@ -1,151 +0,0 @@ -import requests -import psycopg2 -import config -from bs4 import BeautifulSoup - -""" -This module scrapes the UCLA Library Hours from the web. -URL: https://www.library.ucla.edu/hours. - -Output format: -{ - "name":"Arts Library", - "location":" 1400 Public Affairs Building, Los Angeles, CA 90095-1392", - "phone":"(310) 206-5425", - "image":"https://www.library.ucla.edu/sites/default/files/styles/thumbnail_large/public/location_map_images/map_arts.png?itok=0LT3XP_5", - "department": - [ - { - "department_name":"Arts Library", - "time": - [ - {"dp_open_time":"1pm - 5pm","date":"Su 17"}, - {"dp_open_time":"9am - 5pm","date":"M 18"}, - {"dp_open_time":"8am - 9pm","date":"Tu 19"}, - {"dp_open_time":"8am - 9pm","date":"W 20"}, - {"dp_open_time":"8am - 9pm","date":"Th 21"}, - {"dp_open_time":"8am - 5pm","date":"F 22"}, - {"dp_open_time":"1pm - 5pm","date":"Sa 23"} - ] - }, - { - "department_name":"CLICC Laptop & iPad Lending (Arts Library)", - "time": - [ - {"dp_open_time":"1pm - 4:30pm","date":"Su 17"}, - {"dp_open_time":"9am - 4:30pm","date":"M 18"}, - {"dp_open_time":"8am - 8:30pm","date":"Tu 19"}, - {"dp_open_time":"8am - 8:30pm","date":"W 20"}, - {"dp_open_time":"8am - 8:30pm","date":"Th 21"}, - {"dp_open_time":"8am - 4:30pm","date":"F 22"}, - {"dp_open_time":"1pm - 4:30pm","date":"Sa 23"} - ] - }, - { - "department_name":"Reference Desk", - "time": - [ - {"dp_open_time":"Closed","date":"Su 17"}, - {"dp_open_time":"Closed","date":"M 18"}, - {"dp_open_time":"11am - 4pm","date":"Tu 19"}, - {"dp_open_time":"11am - 4pm","date":"W 20"}, - {"dp_open_time":"11am - 4pm","date":"Th 21"}, - {"dp_open_time":"11am - 4pm","date":"F 22"}, - {"dp_open_time":"Closed","date":"Sa 23"}] - } - ] -} -""" - - -""" -Database schema: - -CREATE TABLE libraries ( - name text PRIMARY KEY, - location text, - phone text); - -CREATE TABLE library_hours ( - library_name text, - dep_name text, - date text, - times text, - FOREIGN KEY (library_name) REFERENCES libraries(name), - PRIMARY KEY (library_name, dep_name)); -""" - -def get_lib_info(): - - # get HTML data - url = 'https://www.library.ucla.edu/hours' - response = requests.get(url) - if response.status_code != 200: - print("ERROR: Could not connect to https://www.library.ucla.edu/hours") - exit() - page = BeautifulSoup(response.content) # Need to specify parser - - # loop through each library - libraries = [] - library_hours = [] - for library in page.find_all('div', {'class': 'views-row'}): - - # extract library name, location, phone, & dates - name = library.find('div', {'class': 'pane-node-title'}).find('a').text - location = library.find('span', {'class': 'location__address-text'}).text.replace('\n', ' ').replace('\r', ' ').replace(' ', ' ').strip(' ') - phone = library.find('span', {'class':'location__phone-number'}).text - libraries.append((name, location, phone)) - - # each library has multiple departments, loop through each department - for department in library.find('tbody').find_all('tr'): - - # extract department name and their hours - table = department.find_all('td') - dep_name = table[0].text.replace('\n', ' ').replace('\r', ' ').replace(' ', ' ').strip(' ') - for day in table[1:]: - date = day.get('data-label') - hours = day.text - library_hours.append((name, dep_name, date, hours)) - - return libraries, library_hours - -def main(event=None, lambda_context=None): - # Print trigger event - print("Triggered by:", event) - - # Get updated library info - libraries, library_hours = get_lib_info() - - # Connect to database and insert data - try: - conn = psycopg2.connect(host=config.host, - port=config.port, - database=config.database, - user=config.user, - password=config.password) - curr = conn.cursor() - print("Connected to database.") - - # Remove old library data - curr.execute("TRUNCATE libraries, library_hours CASCADE;") - - # Insert new library data - sql1 = "INSERT INTO libraries(name, location, phone) VALUES (%s, %s, %s);" - sql2 = "INSERT INTO library_hours(library_name, dep_name, date, times) VALUES (%s, %s, %s, %s);" - curr.executemany(sql1, libraries) - curr.executemany(sql2, library_hours) - - # Commit & close database - conn.commit() - print("Updated data in database.") - curr.close() - except (Exception, psycopg2.DatabaseError) as error: - print("ERROR: ", error) - finally: - if conn is not None: - conn.close() - print('Database connection closed.') - - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/librarystudyrooms-scraper.js b/librarystudyrooms-scraper.js deleted file mode 100644 index fa2221c..0000000 --- a/librarystudyrooms-scraper.js +++ /dev/null @@ -1,109 +0,0 @@ -const puppeteer = require('puppeteer'); -const request = require('request'); - -/* -JSON Format -{ - "Building Name": "Powell Library" - "Room Number": "Group Study Room F" - "Capacity": 6 - "Date": "February 5, 2019" - "Day": "Tuesday" - "Start Time": "1:00PM" -} -*/ - -(async function main() { - - try { - - const browser = await puppeteer.launch(/*{headless:false}*/); - const [page] = await browser.pages(); - //Url to visit - await page.goto('http://calendar.library.ucla.edu/allspaces'); - - const result = await page.evaluate(async () => { - //Wait for webpage to load (may need to change this 2000 ms) - await new Promise((resolve) => setTimeout(resolve, 1500)); - const rooms = document.querySelectorAll('.s-lc-eq-avail'); - //Save length to prevent function call overhead in for loop - const num_rooms = rooms.length; - //Empty array to append JSON objects to - const data = []; - - for (let n = 0; n < num_rooms; n++) { - var obj = new Object(); - //Get title attribute from Anchor html elemet - const title = rooms[n].getAttribute('title'); - //Split by commas - var splitByCommas = title.split(","); - //Split first element of splitByCommas array by spaces - var timeAndDay = splitByCommas[0].split(" "); - //Split third element of splitByCommas array by dashes - var yearAndRoom = splitByCommas[2].split("-"); - var room = yearAndRoom[1].split(" "); - room = room[room.length-1]; - //data.push(room); - - //Specific study room numbers for Powell - if (room == 'A' || room == 'B' || room == 'C' || - room == 'D'|| room == 'E' || room == 'F') - { - obj["Building Name"] = "Powell Library"; - obj["Room"] = yearAndRoom[1].trim(); - obj["Capacity"] = 8; - obj["Date"] = splitByCommas[1].trim() + yearAndRoom[0].trimRight(); - obj["Day"] = timeAndDay[1].trim(); - obj["Start Time"] = timeAndDay[0].trim(); - } - - //Specific room numbers for YRL study rooms - else if(room == 'G01'|| room == 'G02' || room == 'G03' || - room == 'G04'|| room == 'G05' || room == 'G06' || - room == 'G07' || room == 'G08' || room == 'G09'|| - room == 'G10' || room == 'G11' || room == 'G12' || - room == 'G13' || room == 'G14'|| room == 'G15') - { - obj["Building Name"] = "Young Research Library"; - obj["Room"] = yearAndRoom[1].trim(); - obj["Capacity"] = 8; - obj["Date"] = splitByCommas[1].trim() + yearAndRoom[0].trimRight(); - obj["Day"] = timeAndDay[1].trim(); - obj["Start Time"] = timeAndDay[0].trim(); - } - - //Everything else must be a study pod in YRL - else { - obj["Building Name"] = "Young Research Library"; - obj["Room"] = yearAndRoom[1].trim(); - obj["Capacity"] = 6; - obj["Date"] = splitByCommas[1].trim() + yearAndRoom[0].trimRight(); - obj["Day"] = timeAndDay[1].trim(); - obj["Start Time"] = timeAndDay[0].trim(); - } - - //Add to data array*/ - data.push(obj); - } - - return data; - }); - - // for (let i = 0; i < result.length; i++) { - request.post({ url: "http://studysmart-env-2.dqiv29pdi2.us-east-1.elasticbeanstalk.com/librooms", headers: { 'content-type': 'application/json' }, body: JSON.stringify(result) }, function (err, response, body) { - console.log(response.body) - new Promise((resolve) => setTimeout(resolve, 10000)); - }) - // } - - console.log(result); - await browser.close(); - } - - //If there is an error, write to console and exit - catch (err) { - console.error(err); - return; - } - -})(); \ No newline at end of file diff --git a/librooms-scraper/__MACOSX/._index.js b/librooms-scraper/__MACOSX/._index.js new file mode 100644 index 0000000000000000000000000000000000000000..58ea31b2e88f05a2541a23c0e609d9d0bb1762f9 GIT binary patch literal 176 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDI}aUBqY_#1$j2;dkJ5(HHS(y;)D1)zKw z#Rz090AirHRC0c$Vqox1Ojhs@R)|o50+1L3ClDI}aUBqY_#1$j2;dkJ5(HHS(y;)D1)zKw z#Rz090AirHRC0cFrTOC9L09w-( A8~^|S literal 0 HcmV?d00001 diff --git a/librooms-scraper/__MACOSX/._test.js b/librooms-scraper/__MACOSX/._test.js new file mode 100644 index 0000000000000000000000000000000000000000..efe84a4d351bf8da9433d3a600ea586e8f5460a2 GIT binary patch literal 176 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDI}aUBqY_#1$j2;dkJ5(HHS(y;)D1)zKw z#Rz090AirHRC0cXwVAbNS5gcOJd09oA? AH~;_u literal 0 HcmV?d00001 diff --git a/librooms-scraper/index.js b/librooms-scraper/index.js new file mode 100644 index 0000000..ae2d2f4 --- /dev/null +++ b/librooms-scraper/index.js @@ -0,0 +1,468 @@ +var request = require("request"); +const moment = require("moment-timezone"); +var AWS = require("aws-sdk"); +var Promise = require("bluebird"); +var rp = require("request-promise"); +AWS.config.update({ region: "us-east-1" }); +var docClient = new AWS.DynamoDB.DocumentClient(); + +exports.handler = async (event) => { + + var now = moment().tz("America/Los_Angeles"); + const formatString = "YYYY-MM-DD"; + var todayString = now.format(formatString); + now.add(5, "days"); + var fiveDaysLaterString = now.format(formatString); + var yrlOptions = { + method: "POST", + url: "http://calendar.library.ucla.edu/spaces/availability/grid", //TODO: change to https + headers: { + "cache-control": "no-cache", + Referer: "http://calendar.library.ucla.edu/allspaces" + }, + form: { + lid: "5567", + gid: "0", + eid: "-1", + start: todayString, + end: fiveDaysLaterString + }, + json: true + }; + + var powellOptions = { + method: "POST", + url: "http://calendar.library.ucla.edu/spaces/availability/grid", //TODO: change to https + headers: { + "cache-control": "no-cache", + Referer: "http://calendar.library.ucla.edu/allspaces" + }, + form: { + lid: "4361", + gid: "0", + eid: "-1", + start: todayString, + end: fiveDaysLaterString + }, + json: true + }; + + var powellInformation = await rp(powellOptions); + var yrlInformation = await rp(yrlOptions); + var powellData = []; + var yrlData = []; + for(var i = 0; i < powellInformation.length; i++) + { + if(powellInformation[i]["className"] == "s-lc-eq-checkout") + { + continue; + } + else + { + var duration = 30; + var obj = new Object(); + var dateTime = powellInformation[i]["start"].split(" "); + var roomName; + var roomId = powellInformation[i]["itemId"]; + var randomId = false; + switch (roomId) { + case 29694: + roomName = "Powell Group Study Room A" + break; + case 29695: + roomName = "Powell Group Study Room B" + break; + case 29696: + roomName = "Powell Group Study Room C" + break; + case 29697: + roomName = "Powell Group Study Room D" + break; + case 29698: + roomName = "Powell Group Study Room E" + break; + case 29699: + roomName = "Powell Group Study Room F" + break; + default: + randomId = true; + break + } + if (randomId) { continue; } // there is a random id that falsely becomes Study Room F available + + obj["Building Name"] = "Powell Library"; + obj["Room"] = roomName; + obj["Capacity"] = 8; + obj["Date"] = dateTime[0]; + obj["Start Time"] = dateTime[1]; + if(i+1 != powellInformation.length) + { + var dateTimeFuture = powellInformation[i]["start"].split(" "); + while(powellInformation[i+1]["className"] != "s-lc-eq-checkout" && dateTimeFuture[0] == dateTime[0]) + { + const newDateTimeFuture = powellInformation[i+1]["start"].split(" ") + if (newDateTimeFuture[0] == dateTimeFuture[0]) { + duration += 30; + i++; + dateTimeFuture[0] = newDateTimeFuture[0]; + dateTime[0] = newDateTimeFuture[0]; + + } else if (newDateTimeFuture[0] != dateTimeFuture[0] && newDateTimeFuture[1] == '00:00:00') { + duration += 30; + i++; + dateTimeFuture[0] = newDateTimeFuture[0]; + dateTime[0] = newDateTimeFuture[0]; + + } else { + break; + } + + if(i+1 == powellInformation.length) + break; + } + } + obj["Duration"] = duration.toString(); + var item = { + combined: obj.Room + "," + obj["Building Name"]+ "," + obj.Capacity + ","+ obj.Date+ "," + obj.Day+ "," + obj["Start Time"]+ "," + "30", + room: obj.Room, + building: obj["Building Name"], + capacity: obj.Capacity, + date: obj.Date, + start: obj["Start Time"], + duration: obj.Duration + } + powellData.push(item); + } + } + + + var paramsPowell = { + TableName: "lib_rooms", + KeyConditionExpression: "#bd = :buildingQuery", + ExpressionAttributeNames: { + "#bd": "building", + }, + ExpressionAttributeValues: { + ":buildingQuery": "Powell Library", + }, + IndexName: "building-index", + ReturnConsumedCapacity: "TOTAL" + }; + + compareDatabaseData(paramsPowell, powellData); + + for(var i = 0; i < yrlInformation.length; i++) + { + if(yrlInformation[i]["className"] == "s-lc-eq-checkout") + { + continue; + } + else + { + var duration = 30; + var obj = new Object(); + var dateTime = yrlInformation[i]["start"].split(" "); + var roomName; + var roomId = yrlInformation[i]["itemId"]; + var capacity = 0; + var randomId = false; + if(roomId >= 29703 && roomId <= 29718) + { + capacity = 8 + } + else if(roomId <= 29738 && roomId >= 29719) + { + capacity = 6; + } + switch (roomId) { + case 29703: + roomName = "YRL Group Study Room G01" + break; + case 29704: + roomName = "YRL Group Study Room G02" + break; + case 29705: + roomName = "YRL Group Study Room G03" + break; + case 29706: + roomName = "YRL Group Study Room G04" + break; + case 29707: + roomName = "YRL Group Study Room G05" + break; + case 29708: + roomName = "YRL Group Study Room G06" + break; + case 29709: + roomName = "YRL Group Study Room G07" + break; + case 29710: + roomName = "YRL Group Study Room G08" + break; + case 29712: + roomName = "YRL Group Study Room G09" + break; + case 29713: + roomName = "YRL Group Study Room G10" + break; + case 29714: + roomName = "YRL Group Study Room G11" + break; + case 29715: + roomName = "YRL Group Study Room G12" + break; + case 29716: + roomName = "YRL Group Study Room G13" + break; + case 29717: + roomName = "YRL Group Study Room G14" + break; + case 29718: + roomName = "YRL Group Study Room G15" + break; + case 29719: + roomName = "YRL Collaboration Pod R01" + break; + case 29720: + roomName = "YRL Collaboration Pod R02" + break; + case 29721: + roomName = "YRL Collaboration Pod R03" + break; + case 29722: + roomName = "YRL Collaboration Pod R04" + break; + case 29723: + roomName = "YRL Collaboration Pod R05" + break; + case 29724: + roomName = "YRL Collaboration Pod R06" + break; + case 29725: + roomName = "YRL Collaboration Pod R07" + break; + case 29726: + roomName = "YRL Collaboration Pod R08" + break; + case 29727: + roomName = "YRL Collaboration Pod R09" + break; + case 29728: + roomName = "YRL Collaboration Pod R10" + break; + case 29729: + roomName = "YRL Collaboration Pod R11" + break; + case 29730: + roomName = "YRL Collaboration Pod R12" + break; + case 29731: + roomName = "YRL Collaboration Pod R13" + break; + case 29732: + roomName = "YRL Collaboration Pod R14" + break; + case 29733: + roomName = "YRL Collaboration Pod R15" + break; + case 29734: + roomName = "YRL Collaboration Pod R16" + break; + case 29735: + roomName = "YRL Collaboration Pod R17" + break; + case 29736: + roomName = "YRL Collaboration Pod R18" + break; + case 29737: + roomName = "YRL Collaboration Pod R19" + break; + case 29738: + roomName = "YRL Collaboration Pod R20" + break; + default: + randomId = true; + break; + + } + + if (randomId) { + continue; + } + + obj["Building Name"] = "Young Research Library"; + obj["Room"] = roomName; + obj["Capacity"] = capacity; + obj["Date"] = dateTime[0]; + obj["Start Time"] = dateTime[1]; + if(i+1 != yrlInformation.length) + { + var dateTimeFuture = yrlInformation[i]["start"].split(" "); + while(yrlInformation[i+1]["className"] != "s-lc-eq-checkout" && dateTimeFuture[0] == dateTime[0]) + { + const newDateTimeFuture = yrlInformation[i+1]["start"].split(" ") + if (newDateTimeFuture[0] == dateTimeFuture[0]) { + duration += 30; + i++; + dateTimeFuture[0] = newDateTimeFuture[0]; + dateTime[0] = newDateTimeFuture[0]; + } else if (newDateTimeFuture[0] != dateTimeFuture[0] && newDateTimeFuture[1] == '00:00:00') { + duration += 30; + i++; + dateTimeFuture[0] = newDateTimeFuture[0] + dateTime[0] = newDateTimeFuture[0]; + } else { + break; + } + + if(i+1 == yrlInformation.length) + break; + } + } + obj["Duration"] = duration.toString(); + + var item = { + combined: obj.Room + "," + obj["Building Name"]+ "," + obj.Capacity + ","+ obj.Date+ "," + obj.Day+ "," + obj["Start Time"]+ "," + "30", + room: obj.Room, + building: obj["Building Name"], + capacity: obj.Capacity, + date: obj.Date, + start: obj["Start Time"], + duration: obj.Duration + } + yrlData.push(item); + } + + + } + + var paramsYrl = { + TableName: "lib_rooms", + KeyConditionExpression: "#bd = :buildingQuery", + ExpressionAttributeNames: { + "#bd": "building", + }, + ExpressionAttributeValues: { + ":buildingQuery": "Young Research Library", + }, + IndexName: "building-index", + ReturnConsumedCapacity: "TOTAL" + }; + + compareDatabaseData(paramsYrl, yrlData) + +}; + +function finishRequest(toAdd, toDelete) { + + toAdd.forEach(info => { + var entry = { + TableName: "lib_rooms", + Item: info + }; + + setTimeout( + e => { + docClient.put(e, function(err, data) { + if (err) { + console.log(err); + } + }); + }, + 0, + entry + ); + }); + + toDelete.forEach(info => { + var entry = { + TableName: "lib_rooms", + Key: { combined: info.combined} + }; + + setTimeout( + e => { + docClient.delete(e, function(err, data) { + if (err) { + console.log(err); + } + console.log("successfully deleted item"); + console.log(entry); + }); + }, + 0, + entry + ); + + }); +} + +function sortFunction(a, b) { + if (a.combined < b.combined) return -1; + else return 1; +} + +function compareDatabaseData(params, scrapedData) { + + docClient.query(params, function(err, queryData) { + if (err) { + console.error("Unable to query. Error:", JSON.stringify(err, null, 2)); + } else { + var queryItems = queryData.Items; + + queryItems.sort(sortFunction); + scrapedData.sort(sortFunction); + + console.log("queryItems"); + console.log(queryItems); + console.log("scrapedItems"); + console.log(scrapedData); + + let databaseIndex = 0; + let scrapeIndex = 0; + let databaseLength = queryItems.length; + let scrapeLength = scrapedData.length; + + let toAdd = []; + let toBeDeleted = []; + + while (databaseIndex != databaseLength || scrapeIndex != scrapeLength) { + if (databaseIndex == databaseLength) { + toAdd.push(scrapedData[scrapeIndex]); + scrapeIndex++; + } else if (scrapeIndex == scrapeLength) { + toBeDeleted.push(queryItems[databaseIndex]); + databaseIndex++; + } else if ( + queryItems[databaseIndex].combined == scrapedData[scrapeIndex].combined + ) { + const queryDuration = parseInt(queryItems[databaseIndex].duration, 10); + const scrapedDuration = parseInt(scrapedData[scrapeIndex].duration, 10); + console.log("same:", queryItems[databaseIndex].combined, queryDuration, scrapedDuration); + if (queryDuration == scrapedDuration) { + scrapeIndex++; + databaseIndex++; + } else { + console.log("Remove duplicate, wrong duration") + toBeDeleted.push(queryItems[databaseIndex]); + databaseIndex++; + toAdd.push(scrapedData[scrapeIndex]); + scrapeIndex++; + } + } else if ( + queryItems[databaseIndex].combined > scrapedData[scrapeIndex].combined + ) { + toAdd.push(scrapedData[scrapeIndex]); + scrapeIndex++; + } else if ( + queryItems[databaseIndex].combined < scrapedData[scrapeIndex].combined + ) { + toBeDeleted.push(queryItems[databaseIndex]); + databaseIndex++; + } + } + finishRequest(toAdd, toBeDeleted); + } + }); +} + diff --git a/librooms-scraper/package.json b/librooms-scraper/package.json new file mode 100644 index 0000000..44540a5 --- /dev/null +++ b/librooms-scraper/package.json @@ -0,0 +1,29 @@ +{ + "name": "backend-v2", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node scheduler.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/aseem191/backend-v2.git" + }, + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/aseem191/backend-v2/issues" + }, + "homepage": "https://github.com/aseem191/backend-v2#readme", + "dependencies": { + "aws-sdk": "^2.441.0", + "bluebird": "^3.5.4", + "body-parser": "^1.18.3", + "moment": "^2.24.0", + "moment-timezone": "^0.5.23", + "request": "^2.88.0", + "request-promise": "^4.2.4" + } +} diff --git a/librooms-scraper/source.html b/librooms-scraper/source.html new file mode 100644 index 0000000..13f6588 --- /dev/null +++ b/librooms-scraper/source.html @@ -0,0 +1,2679 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + Space Availability - Calendar - UCLA Library + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Skip to Main Content + + +
+
+
+
    +
  1. UCLA Library
    2. +
  2. Calendar
    3. +
  3. Space Availability
    4. +
+
+
+ + + +
+
+
+
+

Space Availability

+ + +
+ +
+
+ Please note, due to the amount of data being loaded, this page + can be slow to load. For a faster view, limit your search to a + specific Location. +
+ + + +
+ + + + + + + + +
+
+
+ + +
+ + + + + + + + + + + + diff --git a/librooms-scraper/test.js b/librooms-scraper/test.js new file mode 100644 index 0000000..e1c3409 --- /dev/null +++ b/librooms-scraper/test.js @@ -0,0 +1,351 @@ +var request = require("request"); +const moment = require("moment-timezone"); +var AWS = require("aws-sdk"); +var Promise = require("bluebird"); +var rp = require("request-promise"); +AWS.config.update({ region: "us-east-1" }); +var docClient = new AWS.DynamoDB.DocumentClient(); + +(async function main () { + + var now = moment().tz("America/Los_Angeles"); + const formatString = "YYYY-MM-DD"; + var todayString = now.format(formatString); + now.add(5, "days"); + var fiveDaysLaterString = now.format(formatString); + var yrlOptions = { + method: "POST", + url: "http://calendar.library.ucla.edu/spaces/availability/grid", //TODO: change to https + headers: { + "cache-control": "no-cache", + Referer: "http://calendar.library.ucla.edu/allspaces" + }, + form: { + lid: "5567", + gid: "0", + eid: "-1", + start: todayString, + end: fiveDaysLaterString + }, + json: true + }; + + var powellOptions = { + method: "POST", + url: "http://calendar.library.ucla.edu/spaces/availability/grid", //TODO: change to https + headers: { + "cache-control": "no-cache", + Referer: "http://calendar.library.ucla.edu/allspaces" + }, + form: { + lid: "4361", + gid: "0", + eid: "-1", + start: todayString, + end: fiveDaysLaterString + }, + json: true + }; + + var powellInformation = await rp(powellOptions); + var yrlInformation = await rp(yrlOptions); + var powellData = []; + var yrlData = []; + for(var i = 0; i < powellInformation.length; i++) + { + if(powellInformation[i]["className"] == "s-lc-eq-checkout") + { + continue; + } + else + { + var duration = 30; + var obj = new Object(); + var dateTime = powellInformation[i]["start"].split(" "); + var roomName; + var roomId = powellInformation[i]["itemId"]; + var randomId = false; + switch (roomId) { + case 29694: + roomName = "Powell Group Study Room A" + break; + case 29695: + roomName = "Powell Group Study Room B" + break; + case 29696: + roomName = "Powell Group Study Room C" + break; + case 29697: + roomName = "Powell Group Study Room D" + break; + case 29698: + roomName = "Powell Group Study Room E" + break; + case 29699: + roomName = "Powell Group Study Room F" + break; + default: + randomId = true; + } + if (randomId) { + continue; + } + obj["Building Name"] = "Powell Library"; + obj["Room"] = roomName; + obj["Capacity"] = 8; + obj["Date"] = dateTime[0]; + obj["Start Time"] = dateTime[1]; + if(i+1 != powellInformation.length) + { + var dateTimeFuture = powellInformation[i]["start"].split(" "); + while(powellInformation[i+1]["className"] != "s-lc-eq-checkout" && dateTimeFuture[0] == dateTime[0]) + { + const newDateTimeFuture = powellInformation[i+1]["start"].split(" ") + if (newDateTimeFuture[0] == dateTimeFuture[0]) { + duration += 30; + i++; + dateTimeFuture[0] = newDateTimeFuture[0]; + dateTime[0] = newDateTimeFuture[0]; + + } else if (newDateTimeFuture[0] != dateTimeFuture[0] && newDateTimeFuture[1] == '00:00:00') { + duration += 30; + i++; + dateTimeFuture[0] = newDateTimeFuture[0]; + dateTime[0] = newDateTimeFuture[0]; + + } else { + break; + } + + if(i+1 == powellInformation.length) + break; + } + } + obj["Duration"] = duration.toString(); + var item = { + combined: obj.Room + "," + obj["Building Name"]+ "," + obj.Capacity + ","+ obj.Date+ "," + obj.Day+ "," + obj["Start Time"]+ "," + obj.Duration, + room: obj.Room, + building: obj["Building Name"], + capacity: obj.Capacity, + date: obj.Date, + start: obj["Start Time"], + duration: obj.Duration + } + powellData.push(item); + } + } + + + var paramsPowell = { + TableName: "lib_rooms", + KeyConditionExpression: "#bd = :buildingQuery", + ExpressionAttributeNames: { + "#bd": "building", + }, + ExpressionAttributeValues: { + ":buildingQuery": "Powell Library", + }, + IndexName: "building-index", + ReturnConsumedCapacity: "TOTAL" + }; + +// compareDatabaseData(paramsPowell, powellData); + + for(var i = 0; i < yrlInformation.length; i++) + { + if(yrlInformation[i]["className"] == "s-lc-eq-checkout") + { + continue; + } + else + { + var duration = 30; + var obj = new Object(); + var dateTime = yrlInformation[i]["start"].split(" "); + var roomName; + var roomId = yrlInformation[i]["itemId"]; + var capacity = 0; + var randomId = false + if(roomId >= 29703 && roomId <= 29718) + { + capacity = 8 + } + else if(roomId <= 29738 && roomId >= 29719) + { + capacity = 6; + } + switch (roomId) { + case 29703: + roomName = "YRL Group Study Room G01" + break; + case 29704: + roomName = "YRL Group Study Room G02" + break; + case 29705: + roomName = "YRL Group Study Room G03" + break; + case 29706: + roomName = "YRL Group Study Room G04" + break; + case 29707: + roomName = "YRL Group Study Room G05" + break; + case 29708: + roomName = "YRL Group Study Room G06" + break; + case 29709: + roomName = "YRL Group Study Room G07" + break; + case 29710: + roomName = "YRL Group Study Room G08" + break; + case 29712: + roomName = "YRL Group Study Room G09" + break; + case 29713: + roomName = "YRL Group Study Room G10" + break; + case 29714: + roomName = "YRL Group Study Room G11" + break; + case 29715: + roomName = "YRL Group Study Room G12" + break; + case 29716: + roomName = "YRL Group Study Room G13" + break; + case 29717: + roomName = "YRL Group Study Room G14" + break; + case 29718: + roomName = "YRL Group Study Room G15" + break; + case 29719: + roomName = "YRL Collaboration Pod R01" + break; + case 29720: + roomName = "YRL Collaboration Pod R02" + break; + case 29721: + roomName = "YRL Collaboration Pod R03" + break; + case 29722: + roomName = "YRL Collaboration Pod R04" + break; + case 29723: + roomName = "YRL Collaboration Pod R05" + break; + case 29724: + roomName = "YRL Collaboration Pod R06" + break; + case 29725: + roomName = "YRL Collaboration Pod R07" + break; + case 29726: + roomName = "YRL Collaboration Pod R08" + break; + case 29727: + roomName = "YRL Collaboration Pod R09" + break; + case 29728: + roomName = "YRL Collaboration Pod R10" + break; + case 29729: + roomName = "YRL Collaboration Pod R11" + break; + case 29730: + roomName = "YRL Collaboration Pod R12" + break; + case 29731: + roomName = "YRL Collaboration Pod R13" + break; + case 29732: + roomName = "YRL Collaboration Pod R14" + break; + case 29733: + roomName = "YRL Collaboration Pod R15" + break; + case 29734: + roomName = "YRL Collaboration Pod R16" + break; + case 29735: + roomName = "YRL Collaboration Pod R17" + break; + case 29736: + roomName = "YRL Collaboration Pod R18" + break; + case 29737: + roomName = "YRL Collaboration Pod R19" + break; + case 29738: + roomName = "YRL Collaboration Pod R20" + break; + default: + randomId = true; + break; + + } + if (randomId) { + continue; + } + obj["Building Name"] = "Young Research Library"; + obj["Room"] = roomName; + obj["Capacity"] = capacity; + obj["Date"] = dateTime[0]; + obj["Start Time"] = dateTime[1]; + if(i+1 != yrlInformation.length) + { + var dateTimeFuture = yrlInformation[i]["start"].split(" "); + while(yrlInformation[i+1]["className"] != "s-lc-eq-checkout" && dateTimeFuture[0] == dateTime[0]) + { + const newDateTimeFuture = yrlInformation[i+1]["start"].split(" ") + if (newDateTimeFuture[0] == dateTimeFuture[0]) { + duration += 30; + i++; + dateTimeFuture[0] = newDateTimeFuture[0]; + dateTime[0] = newDateTimeFuture[0]; + } else if (newDateTimeFuture[0] != dateTimeFuture[0] && newDateTimeFuture[1] == '00:00:00') { + duration += 30; + i++; + dateTimeFuture[0] = newDateTimeFuture[0] + dateTime[0] = newDateTimeFuture[0]; + } else { + break; + } + + if(i+1 == yrlInformation.length) + break; + } + } + obj["Duration"] = duration.toString(); + + var item = { + combined: obj.Room + "," + obj["Building Name"]+ "," + obj.Capacity + ","+ obj.Date+ "," + obj.Day+ "," + obj["Start Time"]+ "," + obj.Duration, + room: obj.Room, + building: obj["Building Name"], + capacity: obj.Capacity, + date: obj.Date, + start: obj["Start Time"], + duration: obj.Duration + } + yrlData.push(item); + } + + } + + var paramsYrl = { + TableName: "lib_rooms", + KeyConditionExpression: "#bd = :buildingQuery", + ExpressionAttributeNames: { + "#bd": "building", + }, + ExpressionAttributeValues: { + ":buildingQuery": "Young Research Library", + }, + IndexName: "building-index", + ReturnConsumedCapacity: "TOTAL" + }; + + //compareDatabaseData(paramsYrl, yrlData) + +})(); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 0bf3f66..0000000 --- a/package-lock.json +++ /dev/null @@ -1,643 +0,0 @@ -{ - "requires": true, - "lockfileVersion": 1, - "dependencies": { - "agent-base": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz", - "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==", - "requires": { - "es6-promisify": "^5.0.0" - } - }, - "ajv": { - "version": "6.10.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", - "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", - "requires": { - "fast-deep-equal": "^2.0.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "requires": { - "safer-buffer": "~2.1.0" - } - }, - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - }, - "async-limiter": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", - "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" - }, - "asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" - }, - "aws-sign2": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" - }, - "aws4": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", - "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" - }, - "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, - "bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "requires": { - "tweetnacl": "^0.14.3" - } - }, - "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" - }, - "caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" - }, - "combined-stream": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", - "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", - "requires": { - "delayed-stream": "~1.0.0" - } - }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "requires": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - } - }, - "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, - "dashdash": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "requires": { - "assert-plus": "^1.0.0" - } - }, - "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "requires": { - "ms": "^2.1.1" - } - }, - "delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" - }, - "ecc-jsbn": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", - "requires": { - "jsbn": "~0.1.0", - "safer-buffer": "^2.1.0" - } - }, - "es6-promise": { - "version": "4.2.6", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.6.tgz", - "integrity": "sha512-aRVgGdnmW2OiySVPUC9e6m+plolMAJKjZnQlCwNSuK5yQ0JN61DZSO1X1Ufd1foqWRAlig0rhduTCHe7sVtK5Q==" - }, - "es6-promisify": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", - "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", - "requires": { - "es6-promise": "^4.0.3" - } - }, - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "extract-zip": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz", - "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=", - "requires": { - "concat-stream": "1.6.2", - "debug": "2.6.9", - "mkdirp": "0.5.1", - "yauzl": "2.4.1" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } - } - }, - "extsprintf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" - }, - "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" - }, - "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" - }, - "fd-slicer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", - "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=", - "requires": { - "pend": "~1.2.0" - } - }, - "forever-agent": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" - }, - "form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - }, - "fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" - }, - "getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "requires": { - "assert-plus": "^1.0.0" - } - }, - "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "har-schema": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" - }, - "har-validator": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", - "requires": { - "ajv": "^6.5.5", - "har-schema": "^2.0.0" - } - }, - "http-signature": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "requires": { - "assert-plus": "^1.0.0", - "jsprim": "^1.2.2", - "sshpk": "^1.7.0" - } - }, - "https-proxy-agent": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", - "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==", - "requires": { - "agent-base": "^4.1.0", - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", - "requires": { - "ms": "^2.1.1" - } - } - } - }, - "inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, - "is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" - }, - "jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" - }, - "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" - }, - "json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" - }, - "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "requires": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.2.3", - "verror": "1.10.0" - } - }, - "mime": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.2.tgz", - "integrity": "sha512-zJBfZDkwRu+j3Pdd2aHsR5GfH2jIWhmL1ZzBoc+X+3JEti2hbArWcyJ+1laC1D2/U/W1a/+Cegj0/OnEU2ybjg==" - }, - "mime-db": { - "version": "1.38.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz", - "integrity": "sha512-bqVioMFFzc2awcdJZIzR3HjZFX20QhilVS7hytkKrv7xFAn8bM1gzc/FOX2awLISvWe0PV8ptFKcon+wZ5qYkg==" - }, - "mime-types": { - "version": "2.1.22", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.22.tgz", - "integrity": "sha512-aGl6TZGnhm/li6F7yx82bJiBZwgiEa4Hf6CNr8YO+r5UHr53tSTYZb102zyU50DOWWKeOv0uQLRL0/9EiKWCog==", - "requires": { - "mime-db": "~1.38.0" - } - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" - }, - "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "requires": { - "minimist": "0.0.8" - } - }, - "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" - }, - "oauth-sign": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" - }, - "once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "requires": { - "wrappy": "1" - } - }, - "path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" - }, - "pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" - }, - "performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" - }, - "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" - }, - "progress": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" - }, - "proxy-from-env": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", - "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=" - }, - "psl": { - "version": "1.1.31", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz", - "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==" - }, - "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" - }, - "puppeteer": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.14.0.tgz", - "integrity": "sha512-SayS2wUX/8LF8Yo2Rkpc5nkAu4Jg3qu+OLTDSOZtisVQMB2Z5vjlY2TdPi/5CgZKiZroYIiyUN3sRX63El9iaw==", - "requires": { - "debug": "^4.1.0", - "extract-zip": "^1.6.6", - "https-proxy-agent": "^2.2.1", - "mime": "^2.0.3", - "progress": "^2.0.1", - "proxy-from-env": "^1.0.0", - "rimraf": "^2.6.1", - "ws": "^6.1.0" - } - }, - "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" - }, - "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", - "requires": { - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "har-validator": "~5.1.0", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", - "tunnel-agent": "^0.6.0", - "uuid": "^3.3.2" - } - }, - "rimraf": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", - "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", - "requires": { - "glob": "^7.1.3" - } - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "sshpk": { - "version": "1.16.1", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", - "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", - "requires": { - "asn1": "~0.2.3", - "assert-plus": "^1.0.0", - "bcrypt-pbkdf": "^1.0.0", - "dashdash": "^1.12.0", - "ecc-jsbn": "~0.1.1", - "getpass": "^0.1.1", - "jsbn": "~0.1.0", - "safer-buffer": "^2.0.2", - "tweetnacl": "~0.14.0" - } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, - "tough-cookie": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", - "requires": { - "psl": "^1.1.24", - "punycode": "^1.4.1" - }, - "dependencies": { - "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" - } - } - }, - "tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "requires": { - "safe-buffer": "^5.0.1" - } - }, - "tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, - "typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" - }, - "uri-js": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", - "requires": { - "punycode": "^2.1.0" - } - }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, - "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" - }, - "verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "requires": { - "assert-plus": "^1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "^1.2.0" - } - }, - "wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" - }, - "ws": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", - "integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==", - "requires": { - "async-limiter": "~1.0.0" - } - }, - "yauzl": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz", - "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=", - "requires": { - "fd-slicer": "~1.0.1" - } - } - } -} diff --git a/reslife-scraper.js b/reslife-scraper.js deleted file mode 100644 index d7f91a5..0000000 --- a/reslife-scraper.js +++ /dev/null @@ -1,125 +0,0 @@ -'use strict'; - -const request = require('request'); -const jsdom = require('jsdom'); -const { JSDOM } = jsdom; -var docClient = new AWS.DynamoDB.DocumentClient(); -/* -JSON Format { - "Room Details": "Sproul Study Room 110E (max 4 people)", - "Time": "11:00pm-midnight on Sun, Mar 03", - "Link": "https://www.orl.ucla.edu/reserve?type=sproulstudy&duration=60&date=2019-03-03&roomid=3584&start=1551682800&stop=1551686400" -} -*/ -async function postRequest(obj) { - request.post({ url: "http://studysmart-env-2.dqiv29pdi2.us-east-1.elasticbeanstalk.com/studyinfo", headers: { 'content-type': 'application/json' }, body: JSON.stringify(obj) }, function (err, response, body) { - console.log(response.body) - new Promise((resolve) => setTimeout(resolve, 10000)); - //console.log(JSON.stringify(response.body)) - }) -} - -module.exports = { - scrape: async function () { - try { - const browser = await puppeteer.launch({headless: false}); - const [page] = await browser.pages(); - - await page.goto('https://reslife.ucla.edu/reserve'); - - const result = await page.evaluate(async () => { - const list_rooms = document.querySelectorAll('.reserve-grid .col-md-4 input').length; - const data1 = []; - for (let n = 0; n < list_rooms; n++) { - const room = document.querySelectorAll('.reserve-grid .col-md-4 input').item(n); - room.click(); - await new Promise((resolve) => setTimeout(resolve, 2000)); - const hour1 = document.querySelectorAll('.reserve-grid .col-md-6 input').item(0) - hour1.click(); - await new Promise((resolve) => setTimeout(resolve, 2000)); - const length = document.querySelectorAll('.calendar-available').length; - for (let j = 0; j < 7 && j < length; j++) { - const data = []; - const element = document.querySelectorAll('.calendar-available').item(j); - element.click(); - await new Promise((resolve) => setTimeout(resolve, 2000)); - let columns = document.querySelectorAll('.col-md-6'); - //Get all links so that you can redirect user to registration page directly - let links = document.getElementsByTagName('a'); - let filteredLinks = []; - //Need to filter all links so that we only save the ones with the class mentioned below - for (k = 0; k < links.length; k++) { - if (links[k].getAttribute("class") == "btn btn-sm btn-block btn-default btn-select") { - filteredLinks.push(links[k]); - } - } - //Counter for filtered links - var k = 0; - //Start at 2 because first value is not relevant - //Increment by 2 because every pair of 2 elements gives us "room details" and "time" - for (i = 2; i < columns.length; i += 2) { - //Create JS object and push required elements - var obj = new Object(); - obj["Room Details"] = columns[i].innerText; - obj["Time"] = columns[i + 1].innerText; - obj["Link"] = filteredLinks[k].getAttribute("href"); - k++; - //Push - data.push(obj); - } - data1.push(data) - } - await new Promise((resolve) => setTimeout(resolve, 2000)); - const hour2 = document.querySelectorAll('.reserve-grid .col-md-6 input').item(1) - hour2.click(); - await new Promise((resolve) => setTimeout(resolve, 2000)); - const length2 = document.querySelectorAll('.calendar-available').length; - for (let j = 0; j < 7 && j < length2; j++) { - const data = []; - const element = document.querySelectorAll('.calendar-available').item(j); - element.click(); - await new Promise((resolve) => setTimeout(resolve, 2000)); - let columns = document.querySelectorAll('.col-md-6'); - //Get all links so that you can redirect user to registration page directly - let links = document.getElementsByTagName('a'); - let filteredLinks = []; - //Need to filter all links so that we only save the ones with the class mentioned below - for (k = 0; k < links.length; k++) { - if (links[k].getAttribute("class") == "btn btn-sm btn-block btn-default btn-select") { - filteredLinks.push(links[k]); - } - } - //Counter for filtered links - var k = 0; - //Start at 2 because first value is not relevant - //Increment by 2 because every pair of 2 elements gives us "room details" and "time" - for (i = 2; i < columns.length; i += 2) { - //Create JS object and push required elements - var obj = new Object(); - obj["Room Details"] = columns[i].innerText; - obj["Time"] = columns[i + 1].innerText; - obj["Link"] = filteredLinks[k].getAttribute("href"); - k++; - //Push - data.push(obj); - } - data1.push(data) - } - } - return data1; - }); - - for (let i = 0; i < result.length; i++) { - request.post({ url: "http://studysmart-env-2.dqiv29pdi2.us-east-1.elasticbeanstalk.com/studyinfo", headers: { 'content-type': 'application/json' }, body: JSON.stringify(result[i]) }, function (err, response, body) { - console.log(response.body) - new Promise((resolve) => setTimeout(resolve, 10000)); - }) - } - console.log(result); - await browser.close(); - } - catch (err) { - console.error(err); - } -} -} \ No newline at end of file diff --git a/reslife-scraper/package.json b/reslife-scraper/package.json new file mode 100644 index 0000000..7662f0b --- /dev/null +++ b/reslife-scraper/package.json @@ -0,0 +1,30 @@ +{ + "name": "backend-v2", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node scheduler.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/aseem191/backend-v2.git" + }, + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/aseem191/backend-v2/issues" + }, + "homepage": "https://github.com/aseem191/backend-v2#readme", + "dependencies": { + "aws-sdk": "^2.441.0", + "bluebird": "^3.5.4", + "body-parser": "^1.18.3", + "jsdom": "^14.0.0", + "moment": "^2.24.0", + "moment-timezone": "^0.5.23", + "request": "^2.88.0", + "request-promise": "^4.2.4" + } +} diff --git a/reslife-scraper/webscraper.js b/reslife-scraper/webscraper.js new file mode 100644 index 0000000..148f96d --- /dev/null +++ b/reslife-scraper/webscraper.js @@ -0,0 +1,309 @@ +"use strict"; + +const moment = require("moment-timezone"); +const jsdom = require("jsdom"); +const { JSDOM } = jsdom; +var AWS = require("aws-sdk"); +var Promise = require("bluebird"); +var rp = require("request-promise"); + +AWS.config.update({ region: "us-east-1" }); +var docClient = new AWS.DynamoDB.DocumentClient(); +/* +JSON Format { + "Room Details": "Sproul Study Room 110E (max 4 people)", + "Time": "11:00pm-midnight on Sun, Mar 03", + "Link": "https://www.orl.ucla.edu/reserve?type=sproulstudy&duration=60&date=2019-03-03&roomid=3584&start=1551682800&stop=1551686400" +} +*/ + +const meetingRooms = [ + "sproulstudy", + "hedrick", + "hedrickstudy", + "hedrickmusic", + "movement", + "music", + "rieber", + "sproulmusic", + "deneve" +]; + +const durations = ["60", "120"]; + +var now = moment().tz("America/Los_Angeles"); +const formatString = "YYYY-MM-DD"; + +let next21DaysStrings = []; + +exports.handler = async event => { + let numSaturdays = 0; + next21DaysStrings = []; + now = moment().tz("America/Los_Angeles"); + for (var i = 0; numSaturdays < 3; i++) { + next21DaysStrings[i] = now.format(formatString); + + if (now.day() == 6) numSaturdays++; + now.add(1, "days"); + } + for (var dayIndex = 0; dayIndex < 7; dayIndex++) { + for ( + var meetingRoomIndex = 0; + meetingRoomIndex < meetingRooms.length; + meetingRoomIndex++ + ) { + for ( + var durationIndex = 0; + durationIndex < durations.length; + durationIndex++ + ) { + var notResolved = true; + var delay = 2000; + while (notResolved) { + try { + var body = await sendRequest( + meetingRoomIndex, + durationIndex, + dayIndex, + delay + ); + scrapeData(body, meetingRoomIndex, durationIndex, dayIndex); //TODO make async so last scrape is not dropped + notResolved = false; + } catch (e) { + console.log(e); + delay *= 2; + } + } + } + } + } +}; + +function finishRequest(toAdd, toDelete) { + console.log("to add: "); + console.log(toAdd); + console.log("to delete: "); + console.log(toDelete); + + toAdd.forEach(itemValue => { + let params = { + TableName: "study_info", + Item: itemValue + }; + + docClient.put(params, function(err, data) { + if (err) { + //throw err; + console.log("error"); + } + console.log("successfully put item"); + console.log(params); + }); + }); + + toDelete.forEach(itemValue => { + let params = { + TableName: "study_info", + Key: { link: itemValue.link } + }; + + docClient.delete(params, function(err, data) { + if (err) { + //throw err; + console.log("error"); + } + console.log("successfully deleted item"); + console.log(params); + }); + + itemValue.deletedTime = moment() + .tz("America/Los_Angeles") + .format(); + + params = { + TableName: "deleted_study_info", + Item: itemValue + }; + + docClient.put(params, function(err, data) { + if (err) { + //throw err; + console.log("error"); + } + console.log(params); + }); + }); +} + +function sortFunction(a, b) { + if (a.link < b.link) return -1; + else return 1; +} + +function compareDatabaseData(params, scrapedData) { + docClient.query(params, function(err, queryData) { + if (err) { + console.error("Unable to query. Error:", JSON.stringify(err, null, 2)); + } else { + var queryItems = queryData.Items; + + queryItems.sort(sortFunction); + scrapedData.sort(sortFunction); + + console.log("queryItems"); + console.log(queryItems); + + let databaseIndex = 0; + let scrapeIndex = 0; + let databaseLength = queryItems.length; + let scrapeLength = scrapedData.length; + + let toAdd = []; + let toBeDeleted = []; + + while (databaseIndex != databaseLength || scrapeIndex != scrapeLength) { + if (databaseIndex == databaseLength) { + toAdd.push(scrapedData[scrapeIndex]); + scrapeIndex++; + } else if (scrapeIndex == scrapeLength) { + toBeDeleted.push(queryItems[databaseIndex]); + databaseIndex++; + } else if ( + queryItems[databaseIndex].link == scrapedData[scrapeIndex].link + ) { + scrapeIndex++; + databaseIndex++; + } else if ( + queryItems[databaseIndex].link > scrapedData[scrapeIndex].link + ) { + toAdd.push(scrapedData[scrapeIndex]); + scrapeIndex++; + } else if ( + queryItems[databaseIndex].link < scrapedData[scrapeIndex].link + ) { + toBeDeleted.push(queryItems[databaseIndex]); + databaseIndex++; + } + } + + finishRequest(toAdd, toBeDeleted); + } + }); +} + +async function sendRequest(meetingRoomIndex, durationIndex, dayIndex, delay) { + let options = { + method: "GET", + url: "http://reslife.ucla.edu/reserve", + qs: { + type: meetingRooms[meetingRoomIndex], + duration: durations[durationIndex], + date: next21DaysStrings[dayIndex], + partial: "" + } + //timeout: 20000, + }; + console.log(delay); + let a = new Promise(function(resolve, reject) { + setTimeout(() => resolve("done"), delay); + }); + await a; + console.log("sending request"); + console.log({ + type: meetingRooms[meetingRoomIndex], + duration: durations[durationIndex], + date: next21DaysStrings[dayIndex] + }); + return rp(options); +} + +function scrapeData(body, meetingRoomIndex, durationIndex, dayIndex) { + const dom = new JSDOM(body); + + let data = []; + let columns = dom.window.document.querySelectorAll(".col-md-6"); + //Get all links so that you can redirect user to registration page directly + let links = dom.window.document.getElementsByTagName("a"); + let filteredLinks = []; + //Need to filter all links so that we only save the ones with the class mentioned below + for (k = 0; k < links.length; k++) { + if ( + links[k].getAttribute("class") == + "btn btn-sm btn-block btn-default btn-select" + ) { + filteredLinks.push(links[k]); + } + } + //Counter for filtered links + var k = 0; + //Start at 2 because first value is not relevant + //Increment by 2 because every pair of 2 elements gives us "room details" and "time" + + for (var i = 2; i < columns.length; i += 2) { + //Create JS object and push required elements + var obj = new Object(); + obj["Room Details"] = columns[i].textContent.trim(); + obj["Time"] = columns[i + 1].textContent.trim(); + obj["Link"] = filteredLinks[k].getAttribute("href"); + k++; + + var splitLink = obj["Link"] + .split("?") + .join("=") + .split("&") + .join("=") + .split("="); + var infoName, infoDate, infoDuration, infoStart; + + for (var linkI = 0; linkI < splitLink.length; linkI++) { + if (splitLink[linkI] == "type") { + infoName = splitLink[linkI + 1]; + } else if (splitLink[linkI] == "date") { + infoDate = splitLink[linkI + 1]; + } else if (splitLink[linkI] == "duration") { + infoDuration = splitLink[linkI + 1]; + } else if (splitLink[linkI] == "start") { + infoStart = splitLink[linkI + 1]; + } + } + + var entry = { + link: obj["Link"], + name: infoName, + date: infoDate, + duration: infoDuration, + start: infoStart, + details: obj["Room Details"], + time: obj["Time"] + }; + + data.push(entry); + } + + var params = { + TableName: "study_info", + KeyConditionExpression: "#dt = :dateQuery AND #nm = :nameQuery", + FilterExpression: "#dur = :durationFilter", + ExpressionAttributeNames: { + "#dt": "date", + "#nm": "name", + "#dur": "duration" + }, + ExpressionAttributeValues: { + ":dateQuery": next21DaysStrings[dayIndex], + ":nameQuery": meetingRooms[meetingRoomIndex], + ":durationFilter": durations[durationIndex] + }, + IndexName: "date-name-index", + ReturnConsumedCapacity: "TOTAL" + }; + console.log("successfully scraped"); + console.log({ + type: meetingRooms[meetingRoomIndex], + duration: durations[durationIndex], + date: next21DaysStrings[dayIndex] + }); + console.log(data); + + compareDatabaseData(params, data); +} diff --git a/scheduler.js b/scheduler.js deleted file mode 100644 index 5553dce..0000000 --- a/scheduler.js +++ /dev/null @@ -1,24 +0,0 @@ -var lib_info_scraper = require("./libraryinfo-scraper"); -var reslife_scraper = require("./reslife-scraper"); - -var min_to_ms = 60000; -var reslife_scraper_interval = 5 * min_to_ms; -var lib_info_scraper_interval = 10080 * min_to_ms -// console.log("I am doing my 5 minutes check"); - -setInterval(function () { - console.log("I am doing my 1 week check"); - lib_info_scraper.scrape(); - //do your stuff here -}, lib_info_scraper_interval); - -setInterval(function () { - console.log("I am doing my 5 minutes check"); - reslife_scraper.scrape(); - // do your stuff here -}, reslife_scraper_interval); - -//setInterval(function () { -// console.log("I am doing my 10 minutes check"); -// // do your stuff here -// }, the_interval * 2); \ No newline at end of file