发布1.1.2版本

xtuhcy · May 15, 2016 · dbd2f21 · dbd2f21
1 parent 20b956c
commit dbd2f21
Showing 1 changed file with 8 additions and 1 deletion.
diff --git a/src/main/java/com/geccocrawler/gecco/scheduler/UniqueSpiderScheduler.java b/src/main/java/com/geccocrawler/gecco/scheduler/UniqueSpiderScheduler.java
@@ -41,7 +41,11 @@ public int compare(SortHttpRequest o1, SortHttpRequest o2) {
 
 	@Override
 	public HttpRequest out() {
-		HttpRequest request = set.pollFirst().getHttpRequest();
+		SortHttpRequest sortHttpRequest = set.pollFirst();
+		if(sortHttpRequest == null) {
+			return null;
+		}
+		HttpRequest request = sortHttpRequest.getHttpRequest();
 		if(request != null && log.isDebugEnabled()) {
 			log.debug("OUT:"+request.getUrl()+"(Referer:"+request.getHeaders().get("Referer")+")");
 		}
@@ -54,6 +58,9 @@ public void into(HttpRequest request) {
 		if(success && log.isDebugEnabled()) {
 			log.debug("INTO:"+request.getUrl()+"(Referer:"+request.getHeaders().get("Referer")+")");
 		}
+		if(!success) {
+			log.error("not unique request : " + request.getUrl());
+		}
 	}
 
 	private class SortHttpRequest {