From a33060f2061119bae2d91500fb069214c91ed397 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com>
Date: Mon, 28 Oct 2019 11:32:36 +0800
Subject: [PATCH 1/7] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=89=88=E6=9C=AC?=
 =?UTF-8?q?=E5=8F=B7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 frontend/package.json                           | 2 +-
 frontend/src/views/layout/components/Navbar.vue | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/frontend/package.json b/frontend/package.json
index 60ac5cc8a..5b835a600 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
 {
   "name": "crawlab",
-  "version": "0.3.4",
+  "version": "0.3.5",
   "private": true,
   "scripts": {
     "serve": "vue-cli-service serve --ip=0.0.0.0 --mode=development",
diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue
index e294ad0c7..0bd4c6d15 100644
--- a/frontend/src/views/layout/components/Navbar.vue
+++ b/frontend/src/views/layout/components/Navbar.vue
@@ -9,7 +9,7 @@
       </span>
       <el-dropdown-menu slot="dropdown" class="user-dropdown">
         <el-dropdown-item>
-          <span style="display:block;">v0.3.4</span>
+          <span style="display:block;">v0.3.5</span>
         </el-dropdown-item>
         <el-dropdown-item>
           <span style="display:block;" @click="logout">{{$t('Logout')}}</span>

From 5525c82835b8d5b1c3c2ba16ec40d8fbb4de1f2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com>
Date: Mon, 28 Oct 2019 11:38:39 +0800
Subject: [PATCH 2/7] =?UTF-8?q?=E8=AF=BB=E5=8F=96=E6=97=A5=E5=BF=97?=
 =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=BA2M?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/model/log.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/model/log.go b/backend/model/log.go
index abb77ed97..77e5094f3 100644
--- a/backend/model/log.go
+++ b/backend/model/log.go
@@ -24,7 +24,7 @@ func GetLocalLog(logPath string) (fileBytes []byte, err error) {
 	}
 	defer utils.Close(f)
 
-	const bufLen = 1 * 1024 * 1024
+	const bufLen = 2 * 1024 * 1024
 	logBuf := make([]byte, bufLen)
 
 	off := int64(0)

From 76ab8f1fcd43928c986f7f5ef0cd7b46fabf6526 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com>
Date: Mon, 28 Oct 2019 12:03:02 +0800
Subject: [PATCH 3/7] =?UTF-8?q?fix=20=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/model/schedule.go | 3 ++-
 backend/model/task.go     | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/backend/model/schedule.go b/backend/model/schedule.go
index 36799ac3c..ef758fb64 100644
--- a/backend/model/schedule.go
+++ b/backend/model/schedule.go
@@ -5,6 +5,7 @@ import (
 	"crawlab/database"
 	"crawlab/lib/cron"
 	"github.com/apex/log"
+	"github.com/globalsign/mgo"
 	"github.com/globalsign/mgo/bson"
 	"runtime/debug"
 	"time"
@@ -93,7 +94,7 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
 
 		// 获取爬虫名称
 		spider, err := GetSpider(schedule.SpiderId)
-		if err != nil {
+		if err != nil && err == mgo.ErrNotFound {
 			log.Errorf("get spider by id: %s, error: %s", schedule.SpiderId.Hex(), err.Error())
 			debug.PrintStack()
 			_ = schedule.Delete()
diff --git a/backend/model/task.go b/backend/model/task.go
index df046eccf..64f06cd71 100644
--- a/backend/model/task.go
+++ b/backend/model/task.go
@@ -117,7 +117,7 @@ func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Tas
 	for i, task := range tasks {
 		// 获取爬虫名称
 		spider, err := task.GetSpider()
-		if spider.Id.Hex() == "" || err != nil {
+		if err != nil || spider.Id.Hex() == "" {
 			_ = spider.Delete()
 		} else {
 			tasks[i].SpiderName = spider.DisplayName

From 383d5e61353a1b17b959c90810406e9c01670494 Mon Sep 17 00:00:00 2001
From: Marvin Zhang <tikazyq@163.com>
Date: Mon, 28 Oct 2019 12:18:09 +0800
Subject: [PATCH 4/7] Update CHANGELOG.md

---
 CHANGELOG.md | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 04139677c..e38cd5d6d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,15 +1,17 @@
 # 0.3.5 (2019-10-28)
 ### Features / Enhancement
-- **Graceful Showdown**. https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385
-- **Node Info Optimization**. https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7
-- **Append System Environment Variables to Tasks**. https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3
-- **Auto Refresh Task Log**. https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3
-- **Enable HTTPS Deployment**. https://github.com/crawlab-team/crawlab/commit/5d8f6f0c56768a6e58f5e46cbf5adff8c7819228
+- **[Graceful Showdown](https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385)**
+- **[Node Info Optimization](https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7)**
+- **[Append System Environment Variables to Tasks](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)**
+- **[Auto Refresh Task Log](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)**
+- **[Enable HTTPS Deployment](https://github.com/crawlab-team/crawlab/commit/5d8f6f0c56768a6e58f5e46cbf5adff8c7819228)**
 
 ### Bug Fixes
-- **Unable to fetch spider list info in schedule jobs**. https://github.com/crawlab-team/crawlab/commit/311f72da19094e3fa05ab4af49812f58843d8d93
-- **Unable to fetch node info from worker nodes**. https://github.com/crawlab-team/crawlab/commit/6af06efc17685a9e232e8c2b5fd819ec7d2d1674
-- **Unable to select node when trying to run spider tasks**. https://github.com/crawlab-team/crawlab/commit/31f8e03234426e97aed9b0bce6a50562f957edad
+- **Unable to fetch spider list info in schedule jobs**. [detail](https://github.com/crawlab-team/crawlab/commit/311f72da19094e3fa05ab4af49812f58843d8d93)
+- **Unable to fetch node info from worker nodes**. [detail](https://github.com/crawlab-team/crawlab/commit/6af06efc17685a9e232e8c2b5fd819ec7d2d1674)
+- **Unable to select node when trying to run spider tasks**. [detail](https://github.com/crawlab-team/crawlab/commit/31f8e03234426e97aed9b0bce6a50562f957edad)
+- **Unable to fetch result count when result volume is large**. [#260](https://github.com/crawlab-team/crawlab/issues/260)
+- **Node issue in schedule tasks**. [#244](https://github.com/crawlab-team/crawlab/issues/244)
 
 
 # 0.3.1 (2019-08-25)

From ba85c5312cf40ea7f4d3afdaf9a23ae6356548d6 Mon Sep 17 00:00:00 2001
From: Marvin Zhang <tikazyq@163.com>
Date: Mon, 28 Oct 2019 12:19:26 +0800
Subject: [PATCH 5/7] Update CHANGELOG.md

---
 CHANGELOG.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e38cd5d6d..6e85f8ce1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,10 +1,10 @@
 # 0.3.5 (2019-10-28)
 ### Features / Enhancement
-- **[Graceful Showdown](https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385)**
-- **[Node Info Optimization](https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7)**
-- **[Append System Environment Variables to Tasks](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)**
-- **[Auto Refresh Task Log](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)**
-- **[Enable HTTPS Deployment](https://github.com/crawlab-team/crawlab/commit/5d8f6f0c56768a6e58f5e46cbf5adff8c7819228)**
+- **Graceful Showdown**. [detail](https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385)**
+- **Node Info Optimization**. [detail](https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7)
+- **Append System Environment Variables to Tasks**. [detail](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)
+- **Auto Refresh Task Log**. [detail](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)
+- **Enable HTTPS Deployment**. [detail](https://github.com/crawlab-team/crawlab/commit/5d8f6f0c56768a6e58f5e46cbf5adff8c7819228)
 
 ### Bug Fixes
 - **Unable to fetch spider list info in schedule jobs**. [detail](https://github.com/crawlab-team/crawlab/commit/311f72da19094e3fa05ab4af49812f58843d8d93)

From 7b01e650296a91c36ad5e00969aa13b14813cf4d Mon Sep 17 00:00:00 2001
From: Marvin Zhang <tikazyq@163.com>
Date: Mon, 28 Oct 2019 12:19:45 +0800
Subject: [PATCH 6/7] Update CHANGELOG.md

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6e85f8ce1..95ef9cd76 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,6 @@
 # 0.3.5 (2019-10-28)
 ### Features / Enhancement
-- **Graceful Showdown**. [detail](https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385)**
+- **Graceful Showdown**. [detail](https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385)
 - **Node Info Optimization**. [detail](https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7)
 - **Append System Environment Variables to Tasks**. [detail](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)
 - **Auto Refresh Task Log**. [detail](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)

From 965553b285fd0ca28d537ced66f0de8bf5f21545 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com>
Date: Mon, 28 Oct 2019 12:36:00 +0800
Subject: [PATCH 7/7] =?UTF-8?q?fix=20=E4=BB=A3=E7=A0=81=E9=81=97=E6=BC=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/utils/helpers.go | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/backend/utils/helpers.go b/backend/utils/helpers.go
index 541d90027..1c8c96e30 100644
--- a/backend/utils/helpers.go
+++ b/backend/utils/helpers.go
@@ -1,6 +1,8 @@
 package utils
 
 import (
+	"context"
+	"crawlab/database"
 	"crawlab/entity"
 	"encoding/json"
 	"github.com/apex/log"
@@ -40,3 +42,22 @@ func Close(c io.Closer) {
 		log.WithError(err).Error("关闭资源文件失败。")
 	}
 }
+
+func Pub(channel string, msg entity.NodeMessage) error {
+	if _, err := database.RedisClient.Publish(channel, GetJson(msg)); err != nil {
+		log.Errorf("publish redis error: %s", err.Error())
+		debug.PrintStack()
+		return err
+	}
+	return nil
+}
+
+func Sub(channel string, consume database.ConsumeFunc) error {
+	ctx := context.Background()
+	if err := database.RedisClient.Subscribe(ctx, consume, channel); err != nil {
+		log.Errorf("subscribe redis error: %s", err.Error())
+		debug.PrintStack()
+		return err
+	}
+	return nil
+}