From ea4c8d8ae5700399f7b7e8b4b76588e4efae125a Mon Sep 17 00:00:00 2001
From: yozu <yasuo@ozu.email>
Date: Mon, 20 Mar 2017 20:37:16 +0900
Subject: [PATCH] implemented class command

---
 README.md     |   8 +++-
 app.js        |  46 +++++++++++++++++++-
 convert.js    | 118 --------------------------------------------------
 get_gakubu.js | 102 -------------------------------------------
 lib/class.js  | 101 ++++++++++++++++++++++++++++++++++++++++++
 lib/gakubu.js |  10 +----
 package.json  |   1 -
 7 files changed, 154 insertions(+), 232 deletions(-)
 delete mode 100644 convert.js
 delete mode 100644 get_gakubu.js
 create mode 100644 lib/class.js

diff --git a/README.md b/README.md
index 22aca1e..857f3bb 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,13 @@ node app searchGakubu 基幹理工
 node app listGakubu
 
 # 2. Get Page ID(pid)s to get clas details
-node app fetchGakubu ???? > dest/g_????.txt
+node app fetchGakubu 111973 > dest/g_111973.txt
+
+# 3. Preview JSON output
+node app fetchClass 2600001002012017260000100226
+
+# 4. dump all data
+cat dest/g_111973.txt | node app fetchClass - > dest/o_111973.txt
 
 ```
 
diff --git a/app.js b/app.js
index 69f9e2a..d115e66 100644
--- a/app.js
+++ b/app.js
@@ -1,4 +1,5 @@
 let gakubu = require('./lib/gakubu');
+let classes = require('./lib/class');
 
 if (process.argv[2] == "searchGakubu" && process.argv[3]) {
 	let list = gakubu.search(process.argv[3]);
@@ -21,7 +22,7 @@ if (process.argv[2] == "searchGakubu" && process.argv[3]) {
 	} else {
 		gidList = [process.argv[3]];
 	}
-	function loop() {
+	(function loop() {
 		let gid = gidList.shift();
 		process.stderr.write("fetching gakubu " + gid + "...\n");
 		if (!gakubu.getPageIDList(gid, function(data){
@@ -36,11 +37,52 @@ if (process.argv[2] == "searchGakubu" && process.argv[3]) {
 		})) {
 			process.stderr.write("error: " + gid + " not found\n");
 		}
+	})();
+} else if (process.argv[2] == "fetchClass" && process.argv[3]) {
+	let pidList = [];
+	let res = [];
+	let loop = function loop() {
+		let pid = pidList.shift();
+		process.stderr.write("fetching page " + pid + "...\n");
+		classes.fetch(pid, function(data){
+			if (data) {
+				res.push(data);
+				if (pidList.length) {
+					loop();
+				} else {
+					process.stdout.write(JSON.stringify(res));
+				}
+			} else {
+				process.stderr.write("error: some error occurs when fetching or parsing " + pid + "\n");
+			}
+		});
+	};
+	if (process.argv[3] == "-") {
+		process.stdin.resume();
+		let str = "", ind;
+		process.stdin.on('data', function(chunk) {
+			str += chunk;
+			if (ind = str.indexOf("\n") > -1) {
+				let line = str.splice(0, ind);
+				str.splice(0, 1);
+				pidList.push(line);
+			}
+		});
+		process.stdin.on('end', function() {
+			if (str) {
+				pidList.push(str);
+			}
+			loop();
+		});
+	} else {
+		pidList = [process.argv[3]];
+		loop();
 	}
-	loop();
 } else {
 	console.log("node app searchGakubu <keyword>");
 	console.log("node app listGakubu");
 	console.log("node app fetchGakubu <gid>");
 	console.log("node app fetchGakubu '*'");
+	console.log("node app fetchClass <pid>");
+	console.log("node app fetchClass -");
 }
diff --git a/convert.js b/convert.js
deleted file mode 100644
index 1467261..0000000
--- a/convert.js
+++ /dev/null
@@ -1,118 +0,0 @@
-var fs = require('fs');
-//require('./ext.js');
-
-var gakubuID = "262006";
-
-function findData(m0, mp, me, env){
-	var s, t;
-	env.p = env.text.indexOf(m0, env.p + 1) + m0.length - 1;
-	env.p = env.text.indexOf(mp, env.p + 1);
-	s = env.p + mp.length;
-	env.p = env.text.indexOf(me, env.p + 1);
-	t = env.p;
-	return env.text.substring(s, t);
-}
-
-var youbiList = ['日', '月', '火', '水', '木', '金', '土'];
-var pNumList = ['０', '１', '２', '３', '４', '５', '６'];
-
-function getPeriodList(pStr){
-	var pList0 = pStr.split('／');
-	var pList1 = new Array();
-	var d, p, p1, p2;
-	for(var i = 0; i < pList0.length; i++){
-		if(pList0[i].indexOf(":") != -1){
-			pList0[i] = pList0[i].substring(3);
-		}
-		d = youbiList.indexOf(pList0[i][0]);
-		if(d != -1){
-			if(pList0[i].indexOf('-') == -1){
-				p = pNumList.indexOf(pList0[i][1]);
-				if(p != -1){
-					pList1.push([d, p]);
-				}
-			} else{
-				p1 = pNumList.indexOf(pList0[i][1]);
-				p2 = pNumList.indexOf(pList0[i][3]);
-				for(p = p1; p <= p2; p++){
-					pList1.push([d, p]);
-				}
-			}
-		}
-	}
-	return pList1;
-}
-
-fs.readFile('2016_pages_' + gakubuID + ".json", 'utf8', function (err, text) {
-	if(err){
-		console.log("File open error.");
-		return;
-	}
-	var env = new Object();
-	var tList = new Object();
-	var tMinList = new Object();
-	var uqList = new Array();
-	var tIndexTable = [[], [], [], [], [], [], []];
-	env.p = -1;
-	env.text = text;
-	for(;;){
-		var t = new Object();
-		var tmp;
-    	env.p = text.indexOf("授業情報", env.p + 1);
-		if(env.p == -1){
-			break;
-		}
-		t["開講年度"]		= findData("<td", ">", "<", env);
-		t["開講箇所"]		= findData("<td", ">", "<", env);
-		t["科目名"]			= findData("<td", "div>", "<", env);
-		t["担当教員"]		= findData("<td", ">", "<", env);
-		t["学期曜日時限"]	= findData("<td", ">", "<", env);
-		tmp = t["学期曜日時限"].split("&nbsp;&nbsp;");
-		t["学期"] = tmp[0];
-		t["曜日時限"] = tmp[1];
-		t["pList"] = getPeriodList(t["曜日時限"]);
-		t["科目区分"]		= findData("<td", ">", "<", env);
-		//uqList.pushUnique(t["科目区分"]);
-		t["配当年次"]		= pNumList.indexOf(findData("<td", ">", "<", env)[0]);
-		t["単位数"]			= parseInt(findData("<td", ">", "<", env));
-		t["使用教室"]			= findData("<td", ">", "<", env);
-		t["キャンパス"]			= findData("<td", ">", "<", env);
-		t["科目キー"]			= findData("<td", ">", "<", env);
-		t["科目クラスコード"]	= findData("<td", ">", "<", env);
-		t["授業で使用する言語"]	= findData("<td", ">", "<", env);
-		t["コース・コード"]	= findData("<td", ">", "<", env);
-		t["大分野名称"]	= findData("<td", ">", "<", env);
-		t["中分野名称"]	= findData("<td", ">", "<", env);
-		t["小分野名称"]	= findData("<td", ">", "<", env);
-		t["レベル"]	= findData("<td", ">", "<", env);
-		t["授業形態"]	= findData("<td", ">", "<", env);
-		t["pKey"]	= findData('<input type="hidden" name="pKey"', '"', '"', env);
-		t["URL"] = "https://www.wsl.waseda.jp/syllabus/JAA104.php?pKey=" + t["pKey"] + "&pLng=jp"
-		tList[t["pKey"]] = t;
-		//
-		tMinList[t["pKey"]] = [
-			t["pList"],
-			t["配当年次"],
-			t["単位数"],
-			t["学期"],
-			t["科目名"],
-			t["科目区分"],
-			t["URL"],
-			t["担当教員"],
-			t["使用教室"],
-		];
-
-		for(var i = 0; i < t["pList"].length; i++){
-			if(!tIndexTable[t["pList"][i][0]][t["pList"][i][1]]){
-				tIndexTable[t["pList"][i][0]][t["pList"][i][1]] = [];
-			}
-			tIndexTable[t["pList"][i][0]][t["pList"][i][1]].push(t["pKey"]); 
-		}
-	}
-	//console.log(JSON.stringify(tList, "", "    "));
-	//console.log(JSON.stringify(tMinList, "", "    "));
-	fs.writeFile('2016_converted_' + gakubuID + '.json', JSON.stringify(tMinList));
-	console.log(JSON.stringify(tIndexTable, "", "    "));
-	fs.writeFile('2016_converted_' + gakubuID + '_index.json', JSON.stringify(tIndexTable));
-	//console.log(uqList);
-});
diff --git a/get_gakubu.js b/get_gakubu.js
deleted file mode 100644
index b61227f..0000000
--- a/get_gakubu.js
+++ /dev/null
@@ -1,102 +0,0 @@
-var request = require('request');
-var fs = require('fs');
-
-function findData(m0, mp, me, env){
-	var s, t;
-	env.p = env.text.indexOf(m0, env.p + 1) + m0.length - 1;
-	env.p = env.text.indexOf(mp, env.p + 1);
-	s = env.p + mp.length;
-	env.p = env.text.indexOf(me, env.p + 1);
-	t = env.p;
-	return env.text.substring(s, t);
-}
-
-var URIBase = "https://www.wsl.waseda.jp/syllabus/JAA104.php?pLng=jp&pKey=";
-
-var opt = {
-	uri: '',
-};
-
-var gakubuID = "262006";
-
-/*
-772 <option label="政経" value="111973">政経</option>
-773 <option label="法学" value="121973">法学</option>
-774 <option label="一文" value="132002">一文</option>
-775 <option label="二文" value="142002">二文</option>
-776 <option label="教育" value="151949">教育</option>
-777 <option label="商学" value="161973">商学</option>
-778 <option label="理工" value="171968">理工</option>
-779 <option label="社学" value="181966">社学</option>
-780 <option label="人科" value="192000">人科</option>
-781 <option label="スポーツ" value="202003">スポーツ</option>
-782 <option label="国際教養" value="212004">国際教養</option>
-783 <option label="文構" value="232006">文構</option>
-784 <option label="文" value="242006">文</option>
-785 <option label="人通" value="252003">人通</option>
-786 <option label="基幹" value="262006">基幹</option>
-787 <option label="創造" value="272006">創造</option>
-788 <option label="先進" value="282006">先進</option>
-789 <option label="政研" value="311951">政研</option>
-790 <option label="経研" value="321951">経研</option>
-791 <option label="法研" value="331951">法研</option>
-792 <option label="文研" value="342002">文研</option>
-793 <option label="商研" value="351951">商研</option>
-794 <option label="教研" value="371990">教研</option>
-795 <option label="人研" value="381991">人研</option>
-796 <option label="社学研" value="391994">社学研</option>
-797 <option label="アジア研" value="402003">アジア研</option>
-798 <option label="国情研" value="422000">国情研</option>
-799 <option label="日研" value="432001">日研</option>
-800 <option label="情シス研" value="442003">情シス研</option>
-801 <option label="公共研" value="452003">公共研</option>
-802 <option label="ファイナンス研" value="462004">ファイナンス研</option>
-803 <option label="法務研" value="472004">法務研</option>
-804 <option label="会計研" value="482005">会計研</option>
-805 <option label="スポーツ研" value="502005">スポーツ研</option>
-806 <option label="基幹研" value="512006">基幹研</option>
-807 <option label="創造研" value="522006">創造研</option>
-808 <option label="先進研" value="532006">先進研</option>
-809 <option label="環エネ研" value="542006">環エネ研</option>
-810 <option label="教職研" value="552007">教職研</option>
-811 <option label="国際コミ研" value="562012">国際コミ研</option>
-812 <option label="経管研" value="572015">経管研</option>
-813 <option label="芸術" value="712001">芸術</option>
-814 <option label="日本語" value="922006">日本語</option>
-815 <option label="留学" value="982007">留学</option>
-816 <option label="グローバル" value="9S2013">グローバル</option>
-*/
-
-
-
-var idList = [];
-var htmlStr = "";
-
-function getClassPage(index){
-	opt.uri = URIBase + idList[index];
-	if(index >= idList.length){
-		console.log("sum: " + idList.length);
-		fs.writeFile('2016_pages_' + gakubuID + ".json", htmlStr);
-		return;
-	}
-	console.log(index);
-	request.get(opt, function(error, res, body){
-		if (!error && res.statusCode == 200) {
-			console.log("200 OK");
-			htmlStr += body;
-			getClassPage(index + 1);
-		} else {
-			console.log('error: '+ res.statusCode);
-			return;
-		}
-	});
-}
-
-fs.readFile('./2016_id_' + gakubuID + ".json", 'utf8', function (err, text) {
-	if(err){
-		console.log("File open error." + err);
-		return;
-	}
-	idList = JSON.parse(text);
-	getClassPage(0);
-});
diff --git a/lib/class.js b/lib/class.js
new file mode 100644
index 0000000..ce696b7
--- /dev/null
+++ b/lib/class.js
@@ -0,0 +1,101 @@
+;(function(exports, require) {
+	"use strict";
+
+	let request = require('request');
+
+	function findData(m0, mp, me, env){
+		var s, t;
+		env.p = env.text.indexOf(m0, env.p + 1) + m0.length - 1;
+		env.p = env.text.indexOf(mp, env.p + 1);
+		s = env.p + mp.length;
+		env.p = env.text.indexOf(me, env.p + 1);
+		t = env.p;
+		return env.text.substring(s, t);
+	}
+
+	var youbiList = ['日', '月', '火', '水', '木', '金', '土'];
+	var pNumList = ['０', '１', '２', '３', '４', '５', '６'];
+
+	function getPeriodList(pStr){
+		var pList0 = pStr.split('／');
+		var pList1 = new Array();
+		var d, p, p1, p2;
+		for(var i = 0; i < pList0.length; i++){
+			if(pList0[i].indexOf(":") != -1){
+				pList0[i] = pList0[i].substring(3);
+			}
+			d = youbiList.indexOf(pList0[i][0]);
+			if(d != -1){
+				if(pList0[i].indexOf('-') == -1){
+					p = pNumList.indexOf(pList0[i][1]);
+					if(p != -1){
+						pList1.push([d, p]);
+					}
+				} else{
+					p1 = pNumList.indexOf(pList0[i][1]);
+					p2 = pNumList.indexOf(pList0[i][3]);
+					for(p = p1; p <= p2; p++){
+						pList1.push([d, p]);
+					}
+				}
+			}
+		}
+		return pList1;
+	}
+
+	function parsePageHtml(text) {
+		let t = {};
+		let env = { 
+			p: text.indexOf("授業情報", 0),
+			text: text
+		};
+		t["開講年度"]		= findData("<td", ">", "<", env);
+		t["開講箇所"]		= findData("<td", ">", "<", env);
+		t["科目名"]			= findData("<td", "div>", "<", env);
+		t["担当教員"]		= findData("<td", ">", "<", env);
+		t["学期曜日時限"]	= findData("<td", ">", "<", env);
+		let tmp = t["学期曜日時限"].split("&nbsp;&nbsp;");
+		t["学期"] = tmp[0];
+		t["曜日時限"] = tmp[1];
+		t["pList"] = getPeriodList(t["曜日時限"]);
+		t["科目区分"]		= findData("<td", ">", "<", env);
+		t["配当年次"]		= pNumList.indexOf(findData("<td", ">", "<", env)[0]);
+		t["単位数"]			= parseInt(findData("<td", ">", "<", env));
+		t["使用教室"]			= findData("<td", ">", "<", env);
+		t["キャンパス"]			= findData("<td", ">", "<", env);
+		t["科目キー"]			= findData("<td", ">", "<", env);
+		t["科目クラスコード"]	= findData("<td", ">", "<", env);
+		t["授業で使用する言語"]	= findData("<td", ">", "<", env);
+		t["コース・コード"]	= findData("<td", ">", "<", env);
+		t["大分野名称"]	= findData("<td", ">", "<", env);
+		t["中分野名称"]	= findData("<td", ">", "<", env);
+		t["小分野名称"]	= findData("<td", ">", "<", env);
+		t["レベル"]	= findData("<td", ">", "<", env);
+		t["授業形態"]	= findData("<td", ">", "<", env);
+		t["pKey"]	= findData('<input type="hidden" name="pKey"', '"', '"', env);
+		t["URL"] = "https://www.wsl.waseda.jp/syllabus/JAA104.php?pKey=" + t["pKey"] + "&pLng=jp"
+		return t;
+	}
+
+	var URIBase = "https://www.wsl.waseda.jp/syllabus/JAA104.php?pLng=jp&pKey=";
+	function getClassPage(pid, callback){
+		let opt = {
+			uri: URIBase + pid
+		};
+		process.stderr.write("Fetching " + pid + " ...");
+		request.get(opt, function(error, res, body){
+			if (!error && res.statusCode == 200) {
+				process.stderr.write("  200 OK\n");
+				callback(parsePageHtml(body));
+			} else {
+				process.stderr.write("  " + res.statusCode + " ERROR\n");
+				callback(null);
+			}
+		});
+	}
+
+	exports.fetch = function(pid, callback) {
+		getClassPage(pid, callback);
+	};
+
+})(exports, require);
diff --git a/lib/gakubu.js b/lib/gakubu.js
index 40508f7..bec654b 100644
--- a/lib/gakubu.js
+++ b/lib/gakubu.js
@@ -2,7 +2,6 @@
 	"use strict";
 
 	var request = require('request');
-	var fs = require('fs');
 
 	function findData(m0, mp, me, env){
 		var s, t;
@@ -77,8 +76,8 @@
 			},
 			json: true
 		};
+		process.stderr.write("Fetching page " + page + "...");
 		request.post(opt, function(error, res, body){
-			process.stderr.write("Fetching page " + page + "...");
 			if (!error && res.statusCode == 200) {
 				process.stderr.write("  200 OK.\n");
 				var env = { text: body, p: 0 };
@@ -104,7 +103,6 @@
 			} else {
 				process.stderr.write("  " + res.statusCode + " ERROR\n");
 				callback(pidList);
-				return;
 			}
 		});
 	}
@@ -133,9 +131,5 @@
 		if (isAvailableId) getPage(1, [], id, callback);
 		return isAvailableId;
 	};
-
-	exports.createPageUrl = function(pid) {
-		return "https://www.wsl.waseda.jp/syllabus/JAA104.php?pKey=" + pid + "&pLng=jp";
-
-	}
 })(exports, require);
+
diff --git a/package.json b/package.json
index 713cdde..ad19212 100644
--- a/package.json
+++ b/package.json
@@ -17,7 +17,6 @@
   },
   "homepage": "https://github.com/yasuo-ozu/coursecompass-fetch#readme",
   "dependencies": {
-    "fs": "0.0.1-security",
     "request": "^2.81.0"
   }
 }