From a33060f2061119bae2d91500fb069214c91ed397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 28 Oct 2019 11:32:36 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/package.json | 2 +- frontend/src/views/layout/components/Navbar.vue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 60ac5cc8a..5b835a600 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "crawlab", - "version": "0.3.4", + "version": "0.3.5", "private": true, "scripts": { "serve": "vue-cli-service serve --ip=0.0.0.0 --mode=development", diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index e294ad0c7..0bd4c6d15 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -9,7 +9,7 @@ </span> <el-dropdown-menu slot="dropdown" class="user-dropdown"> <el-dropdown-item> - <span style="display:block;">v0.3.4</span> + <span style="display:block;">v0.3.5</span> </el-dropdown-item> <el-dropdown-item> <span style="display:block;" @click="logout">{{$t('Logout')}}</span> From 5525c82835b8d5b1c3c2ba16ec40d8fbb4de1f2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 28 Oct 2019 11:38:39 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E8=AF=BB=E5=8F=96=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=BA2M?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/log.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/model/log.go b/backend/model/log.go index abb77ed97..77e5094f3 100644 --- a/backend/model/log.go +++ b/backend/model/log.go @@ -24,7 +24,7 @@ func GetLocalLog(logPath string) (fileBytes []byte, err error) { } defer utils.Close(f) - const bufLen = 1 * 1024 * 1024 + const bufLen = 2 * 1024 * 1024 logBuf := make([]byte, bufLen) off := int64(0) From 76ab8f1fcd43928c986f7f5ef0cd7b46fabf6526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 28 Oct 2019 12:03:02 +0800 Subject: [PATCH 3/7] =?UTF-8?q?fix=20=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/schedule.go | 3 ++- backend/model/task.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/model/schedule.go b/backend/model/schedule.go index 36799ac3c..ef758fb64 100644 --- a/backend/model/schedule.go +++ b/backend/model/schedule.go @@ -5,6 +5,7 @@ import ( "crawlab/database" "crawlab/lib/cron" "github.com/apex/log" + "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" "runtime/debug" "time" @@ -93,7 +94,7 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) { // 获取爬虫名称 spider, err := GetSpider(schedule.SpiderId) - if err != nil { + if err != nil && err == mgo.ErrNotFound { log.Errorf("get spider by id: %s, error: %s", schedule.SpiderId.Hex(), err.Error()) debug.PrintStack() _ = schedule.Delete() diff --git a/backend/model/task.go b/backend/model/task.go index df046eccf..64f06cd71 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -117,7 +117,7 @@ func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Tas for i, task := range tasks { // 获取爬虫名称 spider, err := task.GetSpider() - if spider.Id.Hex() == "" || err != nil { + if err != nil || spider.Id.Hex() == "" { _ = spider.Delete() } else { tasks[i].SpiderName = spider.DisplayName From 383d5e61353a1b17b959c90810406e9c01670494 Mon Sep 17 00:00:00 2001 From: Marvin Zhang <tikazyq@163.com> Date: Mon, 28 Oct 2019 12:18:09 +0800 Subject: [PATCH 4/7] Update CHANGELOG.md --- CHANGELOG.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04139677c..e38cd5d6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,17 @@ # 0.3.5 (2019-10-28) ### Features / Enhancement -- **Graceful Showdown**. https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385 -- **Node Info Optimization**. https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7 -- **Append System Environment Variables to Tasks**. https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3 -- **Auto Refresh Task Log**. https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3 -- **Enable HTTPS Deployment**. https://github.com/crawlab-team/crawlab/commit/5d8f6f0c56768a6e58f5e46cbf5adff8c7819228 +- **[Graceful Showdown](https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385)** +- **[Node Info Optimization](https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7)** +- **[Append System Environment Variables to Tasks](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)** +- **[Auto Refresh Task Log](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)** +- **[Enable HTTPS Deployment](https://github.com/crawlab-team/crawlab/commit/5d8f6f0c56768a6e58f5e46cbf5adff8c7819228)** ### Bug Fixes -- **Unable to fetch spider list info in schedule jobs**. https://github.com/crawlab-team/crawlab/commit/311f72da19094e3fa05ab4af49812f58843d8d93 -- **Unable to fetch node info from worker nodes**. https://github.com/crawlab-team/crawlab/commit/6af06efc17685a9e232e8c2b5fd819ec7d2d1674 -- **Unable to select node when trying to run spider tasks**. https://github.com/crawlab-team/crawlab/commit/31f8e03234426e97aed9b0bce6a50562f957edad +- **Unable to fetch spider list info in schedule jobs**. [detail](https://github.com/crawlab-team/crawlab/commit/311f72da19094e3fa05ab4af49812f58843d8d93) +- **Unable to fetch node info from worker nodes**. [detail](https://github.com/crawlab-team/crawlab/commit/6af06efc17685a9e232e8c2b5fd819ec7d2d1674) +- **Unable to select node when trying to run spider tasks**. [detail](https://github.com/crawlab-team/crawlab/commit/31f8e03234426e97aed9b0bce6a50562f957edad) +- **Unable to fetch result count when result volume is large**. [#260](https://github.com/crawlab-team/crawlab/issues/260) +- **Node issue in schedule tasks**. [#244](https://github.com/crawlab-team/crawlab/issues/244) # 0.3.1 (2019-08-25) From ba85c5312cf40ea7f4d3afdaf9a23ae6356548d6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang <tikazyq@163.com> Date: Mon, 28 Oct 2019 12:19:26 +0800 Subject: [PATCH 5/7] Update CHANGELOG.md --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e38cd5d6d..6e85f8ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,10 @@ # 0.3.5 (2019-10-28) ### Features / Enhancement -- **[Graceful Showdown](https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385)** -- **[Node Info Optimization](https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7)** -- **[Append System Environment Variables to Tasks](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)** -- **[Auto Refresh Task Log](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3)** -- **[Enable HTTPS Deployment](https://github.com/crawlab-team/crawlab/commit/5d8f6f0c56768a6e58f5e46cbf5adff8c7819228)** +- **Graceful Showdown**. [detail](https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385)** +- **Node Info Optimization**. [detail](https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7) +- **Append System Environment Variables to Tasks**. [detail](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3) +- **Auto Refresh Task Log**. [detail](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3) +- **Enable HTTPS Deployment**. [detail](https://github.com/crawlab-team/crawlab/commit/5d8f6f0c56768a6e58f5e46cbf5adff8c7819228) ### Bug Fixes - **Unable to fetch spider list info in schedule jobs**. [detail](https://github.com/crawlab-team/crawlab/commit/311f72da19094e3fa05ab4af49812f58843d8d93) From 7b01e650296a91c36ad5e00969aa13b14813cf4d Mon Sep 17 00:00:00 2001 From: Marvin Zhang <tikazyq@163.com> Date: Mon, 28 Oct 2019 12:19:45 +0800 Subject: [PATCH 6/7] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e85f8ce1..95ef9cd76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # 0.3.5 (2019-10-28) ### Features / Enhancement -- **Graceful Showdown**. [detail](https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385)** +- **Graceful Showdown**. [detail](https://github.com/crawlab-team/crawlab/commit/63fab3917b5a29fd9770f9f51f1572b9f0420385) - **Node Info Optimization**. [detail](https://github.com/crawlab-team/crawlab/commit/973251a0fbe7a2184ac0da09e0404a17c736aee7) - **Append System Environment Variables to Tasks**. [detail](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3) - **Auto Refresh Task Log**. [detail](https://github.com/crawlab-team/crawlab/commit/4ab4892471965d6342d30385578ca60dc51f8ad3) From 965553b285fd0ca28d537ced66f0de8bf5f21545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 28 Oct 2019 12:36:00 +0800 Subject: [PATCH 7/7] =?UTF-8?q?fix=20=E4=BB=A3=E7=A0=81=E9=81=97=E6=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/utils/helpers.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/backend/utils/helpers.go b/backend/utils/helpers.go index 541d90027..1c8c96e30 100644 --- a/backend/utils/helpers.go +++ b/backend/utils/helpers.go @@ -1,6 +1,8 @@ package utils import ( + "context" + "crawlab/database" "crawlab/entity" "encoding/json" "github.com/apex/log" @@ -40,3 +42,22 @@ func Close(c io.Closer) { log.WithError(err).Error("关闭资源文件失败。") } } + +func Pub(channel string, msg entity.NodeMessage) error { + if _, err := database.RedisClient.Publish(channel, GetJson(msg)); err != nil { + log.Errorf("publish redis error: %s", err.Error()) + debug.PrintStack() + return err + } + return nil +} + +func Sub(channel string, consume database.ConsumeFunc) error { + ctx := context.Background() + if err := database.RedisClient.Subscribe(ctx, consume, channel); err != nil { + log.Errorf("subscribe redis error: %s", err.Error()) + debug.PrintStack() + return err + } + return nil +}