-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use JSON API and capture new bike_list data (#3)
* Add basic test cases * Add separate request function and associated tests * Remove each-both from mkplace * Replace params with .proc.params * Replace wget with curl * Remove webpage flag * Remove .xml.p wrapper * Use JSON API and update tests to match * Remove unused columns * Fix JSON tests * Add additional comments * Update README references from xml to json * Remove xml test file * Remove unused kdblog flag * Remove xml with maximum prejudice * Add correct error logging function * Update logging * Add logging test * Add bikes namespace * Include notes on log replaying * TorQify those filepaths * Changed to .Q.hg. Added bike_list data. Fixed writedown function. * Removed unnecessary comments. * Fixed errors in README * Fixed typo. Minor appearance changes. * Added stop bikes script. Modified writedown function so it would not override data already in hdb. * Added stop_bikes.sh to readme. * Fixed typo. Changed method of passing in rdb port. * Added error handling to the stop script. Cleaned up writedown method. Updated README. * Removed bug in stop script that was being used for testing. Co-authored-by: Thomas Smyth <[email protected]>
- Loading branch information
1 parent
d47349a
commit e304337
Showing
15 changed files
with
233 additions
and
139 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| *.q linguist-language=q |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1,4 @@ | ||
| .proc.loadprocesscode:1b | ||
| .proc.loadprocesscode:1b; | ||
|
|
||
| hdbdir:hsym`$getenv`KDBHDB; / location of hdb | ||
| webpage:"https://nextbike.net/maps/nextbike-live.json"; / URL for nextbike API |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,54 +1,119 @@ | ||
| / Record station info from nextbike API | ||
|
|
||
| params:.Q.opt[.z.x] | ||
|
|
||
| getbikedata:{ | ||
| /Retrieve data from website and save to xml file | ||
| system[raze"wget -q -O bikes.xml ",params[`webpage],"?city=",params[`cityno]]; | ||
| /Remove any spaces between quotations in preparation for parsing | ||
| l:raze read0`:bikes.xml; pos:o where any (o:where l = " ") within/: 2 cut where "\""=/:l; | ||
| l[pos]:"^";l} | ||
|
|
||
| logbikedata:{[t;f] | ||
| /Open connection to file using current time on request | ||
| hdat:hopen hsym`$raze[params[`xmllog]],"/","xmllog_",ssr[string[.z.D];".";""],"_",raze params[`cityno],".txt"; | ||
| /Write data on single line possibly with time appending each time | ||
| hdat string[t]," -- ", f,"\n"; | ||
| /Close connection to file. | ||
| hclose[hdat]; | ||
| } | ||
|
|
||
| parsedata:{[x] | ||
| /Use xml.q p function to parse | ||
| parsed:.xml.p[x]; parsed} | ||
|
|
||
| mkplace:{[parsed] | ||
| iplace: update "F"$'lat, "F"$'lng, "I"$'uid, "I"$'number, "I"$'bikes, "I"$'bike_racks, "I"$'free_racks, 0^"I"$'"," vs'bike_numbers, "I"$'place_type, "I"$'rack_locks from delete spot, bike_types, bike from `time xcols select from update time:.z.P from update name:{[x]ssr[x;"^";" "]}'[name] from uj/[enlist each parsed[0;2;0;2;0;2;;1]]; | ||
| `place insert iplace;} | ||
|
|
||
| \d .bikes | ||
|
|
||
| hdbdir:@[value;`hdbdir;hsym`$getenv`KDBHDB]; | ||
| webpage:@[value;`webpage;"https://nextbike.net/maps/nextbike-live.json"]; | ||
|
|
||
| // Request data from nextbike API | ||
| request:{ | ||
| .lg.o[`bikes;"Requesting data from nextbike for city ",c:raze .proc.params`cityno]; | ||
| /Retrieve data from website | ||
| req:.Q.hg hsym `$webpage,"?city=",c; | ||
| .lg.o[`bikes;"Returning data for city ",c]; | ||
| :req; | ||
| }; | ||
|
|
||
| // Get JSON log file name for date d | ||
| getjsonlog:{[d] | ||
| :hsym`$raze[.proc.params`jsonlog],"/jsonlog_",(string[d]except"."),"_",raze .proc.params[`cityno],".txt"; | ||
| }; | ||
|
|
||
| // Log output of API request to file | ||
| logbikedata:{[t;j] | ||
| fn:getjsonlog`date$t; | ||
| .lg.o[`bikes;"Writing to JSON log: ",f:.os.pth fn]; | ||
| /Open connection to file using current time on request | ||
| hdat:hopen fn; | ||
| /Write data on single line with corresponding time | ||
| hdat string[t]," -- ",j,"\n"; | ||
| /Close connection to file. | ||
| hclose hdat; | ||
| .lg.o[`bikes;"Finished writing to JSON log: ",f]; | ||
| }; | ||
|
|
||
| // Replay a JSON log into memory | ||
| replayjsonlog:{[d] | ||
| if[()~key fn:getjsonlog d; | ||
| .lg.o[`bikes;"Could not find log file, exiting early: ",.os.pth fn]; | ||
| :(); | ||
| ]; | ||
| .lg.o[`bikes;"Found log file, beginning replay: ",f:.os.pth fn]; | ||
| /Replay each line of log file in turn | ||
| {mkplace . readlogline x}'[read0 fn]; | ||
| .lg.o[`bikes;"Finished replaying log file: ",f]; | ||
| }; | ||
|
|
||
| // Parse line from log file | ||
| readlogline:{@[;1;.j.k]@[0 29 33 cut x;0;"P"$]0 2}; | ||
|
|
||
| // Parse json into in memory table place | ||
| mkplace:{[t;parsed] | ||
| .lg.o[`bikes;"Starting to parse JSON..."] | ||
| /Extract tables from JSON | ||
| tab:first[first[parsed`countries]`cities]`places; | ||
| bike_tab:update name: (exec raze (count each bike_list) #' enlist each name from tab) from exec raze bike_list from tab; | ||
| /Refactor data and extract relevant data | ||
| tab:`address`bike_list`spot`bike_types`bike _`time xcols update time:.z.P^t,name:trim name from tab; | ||
| bike_tab:`pedelec_battery`battery_pack _ `time`name xcols update time:.z.P^t,name:trim name,number:"I"$number,lock_types:raze lock_types from bike_tab; | ||
| /Convert floats to ints where appropriate | ||
| tab:@[tab;`uid`number`bikes`bike_racks`free_racks;`int$]; | ||
| tab:@[tab;`place_type`bike_numbers;"I"$]; | ||
| bike_tab:@[bike_tab;`bike_type`boardcomputer;`int$]; | ||
| .lg.o[`bikes;"Finished parsing JSON, adding to in memory tables"]; | ||
| /Insert data into table in memory | ||
| `place insert tab; | ||
| .lg.o[`bikes;"Added data to in memory table: place"]; | ||
| `bike_list insert bike_tab; | ||
| .lg.o[`bikes;"Added data to in memory table: bike_list"]; | ||
| }; | ||
|
|
||
| // Make request to nextbike API, log to disk and parse into in memory table | ||
| fullbikedata:{ | ||
| /Write messages to out logs as requests are processed | ||
| .lg.o[1;"Starting to make requests"]; | ||
| l:getbikedata[]; | ||
| .lg.o[1;"Finished request"]; | ||
| logbikedata[.z.P;l]; | ||
| .lg.o[1;"Finished logging"]; | ||
| parsed:parsedata[l]; | ||
| .lg.o[1;"Finished parsing"]; | ||
| mkplace[parsed]; | ||
| .lg.o[1;"Requests complete!"]; | ||
| } | ||
| .lg.o[`bikes;"Request started"]; | ||
| /Record time of request | ||
| rt:.z.P; | ||
| /Request data from nextbike API | ||
| l:request[]; | ||
| /Write messages to out logs as requests are processed | ||
| logbikedata[rt;l]; | ||
| /Parse JSON into a table and add to in memory table | ||
| mkplace[rt;.j.k l]; | ||
| .lg.o[`bikes;"Request complete"]; | ||
| }; | ||
|
|
||
| fullbikedataprotected:{[] @[fullbikedata;`;{[x] show "Error running fullbikedata",x}]}; | ||
| fullbikedataprotected:{[]@[fullbikedata;`;{[x].lg.e[`bikes]"Error running fullbikedata: ",x}]}; | ||
|
|
||
| // Write data to disk for date d | ||
| writedown:{[d] | ||
| dir:` sv .Q.par[hdbdir;d;`place],`; | ||
| bikesdir:` sv .Q.par[hdbdir;d;`bike_list],`; | ||
| .lg.o[`bikes;"Writing place data to: ",.os.pth dir]; | ||
| .lg.o[`bikes;"Writing bike_list data to: ",.os.pth bikesdir]; | ||
| /Checks if directory already exists. Appends data if it does. | ||
| checkdir:hsym `$"hdb/",string .z.d; | ||
| cmd:$[`place in key checkdir;(insert);set]; | ||
| cmd[dir; select from `..place where time.date=(d)]; | ||
| cmd:$[`bike_list in key checkdir;insert;set]; | ||
| cmd[bikesdir;select from `..bike_list where time.date=(d)]; | ||
| }; | ||
|
|
||
| //Repeat for 14 days - every 30 seconds | ||
| .timer.repeat[.proc.cp[];.proc.cp[]+14D00:00;0D00:00:30;(fullbikedataprotected;`);"belfastbikes"] | ||
| // Clear data for date d | ||
| cleardate:{[d] | ||
| delete from `..place where time.date=d; | ||
| delete from `..bike_list where time.date=d; | ||
| }; | ||
|
|
||
| //At 6am each day, write down yesterdays data to hdb, and delete the data in memory from 2 days before | ||
| writedown:{(hsym `$raze"hdb/",string (.z.d-1),`$"/place/") set select from place where time.date=(.z.d-1); | ||
| delete from `place where time.date=(.z.d-2)} | ||
| // Write yesterdays data to disk | ||
| eodwritedown:{ | ||
| writedown .z.d-1; | ||
| cleardate .z.d-2; | ||
| }; | ||
|
|
||
| @[writedown;;"writedown failed"] | ||
| \d . | ||
|
|
||
| // Repeat for 14 days - every 30 seconds | ||
| .timer.repeat[.proc.cp[];.proc.cp[]+14D00:00;0D00:00:30;(.bikes.fullbikedataprotected;`);"belfastbikes"]; | ||
|
|
||
| .timer.repeat[(.z.D+1)+06:00:00.000000;.z.d+14;0D01:00:00;(writedown;`);"dailyWritedownBikes"] | ||
| // At 6am each day, write down yesterdays data to hdb, and delete the data in memory from 2 days before | ||
| .timer.repeat[(.z.D+1)+06:00:00.000000;.z.d+14;0D01:00:00;(.bikes.eodwritedown;`);"dailyWritedownBikes"]; |
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| // Gets the rdb port past in from the command line. | ||
| conn:.Q.def[(enlist `conn)!enlist 0Nj;.Q.opt .z.x][`conn]; | ||
| // Opens a handle to rbd and calls the writedown function. | ||
| bikerdb:@[hopen;conn;{-2 "Cannot perform writedown. Unable to open connection, error: ",x;exit 1;}]; | ||
| bikerdb".bikes.writedown[.z.d]"; | ||
| exit 0; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.