-
Notifications
You must be signed in to change notification settings - Fork 0
/
app.js
115 lines (105 loc) · 3.62 KB
/
app.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
var util = require('util')
var debug = require('debug')('app')
var async = require('async')
var MongoClient = require('mongodb').MongoClient
var $ = require('cheerio')
var request = require('request')
var xlsx = require('node-xlsx')
var moment = require('moment')
Array.prototype.mergeAll = function() {
var results = []
this.
forEach(function(subArray) {
results.push.apply(results, subArray)
})
return results
}
Array.prototype.flatMap = function(projectionFunction) {
return this.
map(function(item) {
return projectionFunction(item)
}).
mergeAll()
}
Array.zip = function(left, right, combinerFunction) {
var counter,
results = []
for(counter = 0; counter < Math.min(left.length, right.length); counter++) {
results.push(combinerFunction(left[counter], right[counter]))
}
return results
}
//var stockSheet = xlsx.parse(__dirname + '/tickers.xlsx')
var stockSheet = xlsx.parse(__dirname + '/Bank\ Tickers\ only.xlsx')
var stockList = stockSheet[0].data.
map(function(item) {
return item[0]
})
var url = 'mongodb://localhost:27017/mydb'
MongoClient.connect(url, function(err, db) {
if(err) {
debug(err)
} else {
var collection = db.collection('bank_data_2015');
var missingStocks = []
async.eachLimit(stockList, 20, function(stockName, callback) {
if(stockName && stockName.trim().toLowerCase() !== 'ticker') {
debug('Stock: ' + stockName)
debug('http://finance.yahoo.com/q/ks?s=' + stockName + '+Key+Statistics')
request({
uri: 'http://finance.yahoo.com/q/ks?s=' + stockName + '+Key+Statistics'
}, function(error, response, body) {
if(error) {
return callback(error)
} else {
$ = $.load(body);
var newEntry = {}
newEntry.date = moment().format("YYYY-MM-DD")
newEntry.stock = stockName
newEntry.price = $('.time_rtq_ticker').text()
var titlesObj = $('.yfnc_tablehead1')
var valuesObj = $('.yfnc_tabledata1')
var titles = []
var values = []
titlesObj.each(function(i, elem) {
if(elem.children && elem.children[0].data) {
titles.push(elem.children[0].data)
}
})
valuesObj.each(function(i, elem) {
if(elem.children) {
if(elem.children[0].data) {
values.push(elem.children[0].data)
} else {
values.push(elem.children[0].children[0].data)
}
}
})
var combined = Array.zip(titles, values, function(title, value) {
newEntry[title] = value
return
})
collection.insert(newEntry, function(err, item) {
if(err) {
debug(err)
return callback(err)
} else {
return callback()
}
})
}
})
} else {
return callback()
}
}, function(err) {
if(err) {
debug(err)
} else {
debug('Completed')
debug('Missing Stocks: ' + missingStocks)
}
db.close()
})
}
})