Release v2022.05.16.1 (#1257)

* fix: missing fields in slowquery
* fix dev build exit unexpectedly (#1207)
* fix wrong time unit for slow query (#1212)
* invalidate assets cache (#1208)
* enable query when conprof is disabled (#1217)
* feat: improve statement (#1234)
* misc: bump ech (#1238)
* feat: use ech for overview charts (#1241)
* feat: show more metrics in overview (#1242)
* feat: show info about data delay for Top SQL (#1244)
* feat: multi-line logs by default (#1245)
* feat: enlarge log display area (#1248)
* feat: select all profiling types and instances by default (#1249)
* fix: prevent enter overview page before auth (#1251)
* ui: polish the layout of profiling (#1253)
* feat: do not autoload data when network slow in slow query (#1254)
* ui: small polish (#1260)

Author: shhdgit

Diff for: .github/workflows/manual-create-pd-pr.yaml

@@ -12,8 +12,9 @@ jobs:
     name: Create PD PR
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        branch: [master, release-5.3, release-5.4]
+        branch: [master, release-5.4, release-6.1]
     steps:
       - name: Check out PD code base
         uses: actions/checkout@master
@@ -27,6 +28,9 @@ jobs:
         run: |
           go get -d "github.com/pingcap/tidb-dashboard@${{ github.event.inputs.release_version }}"
           go mod tidy
+          cd tests/client
+          go mod tidy
+          cd ../..
           make pd-server
           go mod tidy
       - name: Commit PD code base change
@@ -55,6 +59,8 @@ jobs:
           body: |
             ### What problem does this PR solve?
 
+            Issue Number: ref #4257
+
             Update TiDB Dashboard to ${{ github.event.inputs.release_version }}.
 
             Upstream commit: https://github.com/${{ github.repository }}/commit/${{ github.sha }} .

Diff for: .github/workflows/release.yaml

@@ -88,8 +88,9 @@ jobs:
     runs-on: ubuntu-latest
     needs: release
     strategy:
+      fail-fast: false
       matrix:
-        branch: [master, release-6.0]
+        branch: [master, release-5.4, release-6.1]
     steps:
       - name: Check out PD code base
         uses: actions/checkout@master
@@ -103,6 +104,9 @@ jobs:
         run: |
           go get -d "github.com/pingcap/tidb-dashboard@${{ needs.release.outputs.release_version }}"
           go mod tidy
+          cd tests/client
+          go mod tidy
+          cd ../..
           make pd-server
           go mod tidy
       - name: Commit PD code base change
@@ -131,6 +135,8 @@ jobs:
           body: |
             ### What problem does this PR solve?
 
+            Issue Number: ref #4257
+
             This is an automatic updating PR for TiDB Dashboard. See #4257 for details.
 
             This PR updates TiDB Dashboard to ${{ needs.release.outputs.release_version }} for upstream commit: https://github.com/${{ github.repository }}/commit/${{ github.sha }} .

Diff for: .github/workflows/test.yaml

@@ -70,15 +70,24 @@ jobs:
   e2e_test:
     name: E2E Test
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 30
     strategy:
+      fail-fast: false
       matrix:
         # test latest features and compatibility of lower version
         include:
           - feature_version: 6.0.0
             tidb_version: nightly
+            without_ngm: false
+          - feature_version: 6.0.0
+            tidb_version: nightly
+            without_ngm: true
+          - feature_version: 5.4.0
+            tidb_version: v5.4.0
+            without_ngm: false
           - feature_version: 5.0.0
             tidb_version: v5.0.0
+            without_ngm: false
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -114,7 +123,7 @@ jobs:
       - name: Install and run TiUP in the background
         run: |
           chmod u+x scripts/start_tiup.sh
-          scripts/start_tiup.sh ${{ matrix.tidb_version }}
+          scripts/start_tiup.sh ${{ matrix.tidb_version }} ${{ matrix.without_ngm }}
       - name: Build UI
         run: |
           make ui
@@ -147,18 +156,14 @@ jobs:
           UI: 1
           FEATURE_VERSION: ${{ matrix.feature_version }}
       - name: Run E2E Features Test
-        run: make test_e2e
+        run: make e2e_test
         env:
           SERVER_URL: http://127.0.0.1:12333/dashboard/
           CI: true
           FEATURE_VERSION: ${{ matrix.feature_version }}
           TIDB_VERSION: ${{ matrix.tidb_version }}
-      - name: Archive Test Results
-        if: always()
-        run: |
-          cat ui/start_tiup.log
-          echo "==============="
-          cat ui/wait_tiup.log
+          CYPRESS_ALLOW_SCREENSHOT: true
+          WITHOUT_NGM: ${{ matrix.without_ngm }}
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
         with:

Diff for: .github/workflows/upload-e2e-snapshots.yaml

@@ -7,18 +7,24 @@ on:
       ref:
         description: "The branch, tag or SHA to create snapshots"
         required: true
+      spec:
+        description: "Specify the spec files to run, example: `topsql/topsql.spec.ts`"
+        required: true
 
 jobs:
   e2e_test_snapshots:
     name: Take E2E Test Snapshots
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 30
     strategy:
+      fail-fast: false
       matrix:
         # test latest features and compatibility of lower version
         include:
           - feature_version: 6.0.0
             tidb_version: nightly
+          - feature_version: 5.4.0
+            tidb_version: v5.4.0
           - feature_version: 5.0.0
             tidb_version: v5.0.0
     steps:
@@ -58,7 +64,7 @@ jobs:
       - name: Install and run TiUP in the background
         run: |
           chmod u+x scripts/start_tiup.sh
-          scripts/start_tiup.sh ${{ matrix.tidb_version }}
+          scripts/start_tiup.sh ${{ matrix.tidb_version }} false
       - name: Build UI
         run: |
           make ui
@@ -89,22 +95,26 @@ jobs:
         env:
           UI: 1
           FEATURE_VERSION: ${{ matrix.feature_version }}
+      - name: Delete Previous Snapshots
+        run: rm -rf ${{ github.workspace }}/ui/cypress/snapshots
       - name: Run E2E Features Test
-        run: make e2e_test
+        run: make e2e_test_specify
         env:
           SERVER_URL: http://127.0.0.1:12333/dashboard/
           CI: true
           FEATURE_VERSION: ${{ matrix.feature_version }}
           TIDB_VERSION: ${{ matrix.tidb_version }}
           CYPRESS_ALLOW_SCREENSHOT: true
-      - name: Archive Test Results
+          E2E_SPEC: cypress/integration/${{ github.event.inputs.spec }}
+      - name: Archive Test Video
         if: always()
-        run: |
-          cat ui/start_tiup.log
-          echo "==============="
-          cat ui/wait_tiup.log
+        uses: actions/upload-artifact@v2
+        with:
+          name: e2e-video-${{ matrix.feature_version }}
+          path: ${{ github.workspace }}/ui/cypress/videos/**/*
       - name: Upload snapshots artifact
         uses: actions/upload-artifact@v2
+        if: always()
         with:
           name: e2e-snapshots-${{ matrix.feature_version }}
           path: ${{ github.workspace }}/ui/cypress/snapshots/**/*

Diff for: CONTRIBUTING.md

@@ -2,9 +2,7 @@
 
 Thanks for your interest in contributing to TiDB Dashboard! This document outlines some of the conventions on building, running, and testing TiDB Dashboard, the development workflow, commit message formatting, contact points and other resources.
 
-If you need any help (for example, mentoring getting started or understanding the codebase), feel free to join the discussion of [Diagnosis SIG] (Special Interest Group):
-
-- Slack: [#sig-diagnosis](https://slack.tidb.io/invite?team=tidb-community&channel=sig-diagnosis&ref=github_dashboard_repo)
+If you need any help (for example, mentoring getting started or understanding the codebase), feel free to reach out on the [TiDB Internals forum](https://internals.tidb.io/).
 
 ## Setting up a development workspace
 

Diff for: Makefile

@@ -6,6 +6,10 @@ LDFLAGS ?=
 
 FEATURE_VERSION ?= 6.0.0
 
+WITHOUT_NGM ?= false
+
+E2E_SPEC ?=
+
 ifeq ($(UI),1)
 	BUILD_TAGS += ui_server
 endif
@@ -45,6 +49,39 @@ integration_test:
 	@mkdir -p ./coverage
 	@TIDB_VERSION=${TIDB_VERSION} tests/run.sh
 
+.PHONY: e2e_test
+e2e_test:
+	@if $(WITHOUT_NGM); then\
+		make e2e_without_ngm_test;\
+	else\
+		make e2e_compat_features_test;\
+		make e2e_common_features_test;\
+	fi
+
+.PHONY: e2e_compat_features_test
+e2e_compat_features_test:
+	cd ui &&\
+	yarn &&\
+	yarn run:e2e-test:compat-features --env FEATURE_VERSION=$(FEATURE_VERSION) TIDB_VERSION=$(TIDB_VERSION)
+
+.PHONY: e2e_common_features_test
+e2e_common_features_test:
+	cd ui &&\
+	yarn &&\
+	yarn run:e2e-test:common-features --env TIDB_VERSION=$(TIDB_VERSION)
+
+.PHONY: e2e_without_ngm_test
+e2e_without_ngm_test:
+	cd ui &&\
+	yarn &&\
+	yarn run:e2e-test:without-ngm --env TIDB_VERSION=$(TIDB_VERSION) WITHOUT_NGM=$(WITHOUT_NGM)
+
+.PHONY: e2e_test_specify
+e2e_test_specify:
+	cd ui &&\
+	yarn &&\
+	yarn run:e2e-test:specify --env TIDB_VERSION=$(TIDB_VERSION) -- --spec $(E2E_SPEC)
+
 .PHONY: dev
 dev: lint default
 
@@ -74,13 +111,3 @@ endif
 .PHONY: run
 run:
 	bin/tidb-dashboard --debug --experimental --feature-version "$(FEATURE_VERSION)" --host 0.0.0.0
-
-test_e2e_compat_features:
-	cd ui &&\
-	yarn run:e2e-test:compat-features --env FEATURE_VERSION=$(FEATURE_VERSION) TIDB_VERSION=$(TIDB_VERSION)
-
-test_e2e_common_features:
-	cd ui &&\
-	yarn run:e2e-test:common-features TIDB_VERSION=$(TIDB_VERSION)
-
-test_e2e: test_e2e_compat_features test_e2e_common_features

Diff for: README.md

@@ -16,7 +16,7 @@ TiDB Dashboard is a Web UI for monitoring, diagnosing and managing the TiDB clus
 Feel free to [open GitHub issues](https://github.com/pingcap/tidb-dashboard/issues/new/choose)
 for questions, support and suggestions.
 
-You may also consider join our community chat in the Slack channel [#sig-diagnosis].
+You may also consider to reach out on the [TiDB Internals forum](https://internals.tidb.io/) if you encounter any problems about TiDB development.
 
 For Chinese users, you can visit the PingCAP official user forum [AskTUG.com] to make life easier.
 
@@ -37,7 +37,7 @@ for a list of recommended tasks, in which we have also marked the difficulty lev
 See [CONTRIBUTING.md](./CONTRIBUTING.md) for a detailed step-by-step contributing guide, or steps to
 build TiDB Dashboard from source.
 
-If you need any help, feel free to community chat in the Slack channel [#sig-diagnosis].
+If you need any help, feel free to reach out on the [TiDB Internals forum](https://internals.tidb.io/).
 
 Thank you to all the people who already contributed to TiDB Dashboard!
 
@@ -86,5 +86,4 @@ TiDB Dashboard can also be integrated into PD, as follows:
 Copyright 2020 PingCAP, Inc.
 
 [pd]: https://github.com/pingcap/pd
-[#sig-diagnosis]: https://slack.tidb.io/invite?team=tidb-community&channel=sig-diagnosis&ref=github_dashboard_repo
 [asktug.com]: https://asktug.com/

Diff for: pkg/apiserver/statement/models.go

@@ -12,13 +12,9 @@ import (
 	"github.com/pingcap/tidb-dashboard/util/reflectutil"
 )
 
-// TimeRange represents a range of time.
-type TimeRange struct {
-	BeginTime int64 `json:"begin_time"`
-	EndTime   int64 `json:"end_time"`
-}
-
 type Model struct {
+	AggBeginTime             int    `json:"summary_begin_time" agg:"FLOOR(UNIX_TIMESTAMP(MIN(summary_begin_time)))"`
+	AggEndTime               int    `json:"summary_end_time" agg:"FLOOR(UNIX_TIMESTAMP(MAX(summary_end_time)))"`
 	AggDigestText            string `json:"digest_text" agg:"ANY_VALUE(digest_text)"`
 	AggDigest                string `json:"digest" agg:"ANY_VALUE(digest)"`
 	AggExecCount             int    `json:"exec_count" agg:"SUM(exec_count)"`

Diff for: pkg/apiserver/statement/queries.go

@@ -14,19 +14,6 @@ const (
 	statementsTable = "INFORMATION_SCHEMA.CLUSTER_STATEMENTS_SUMMARY_HISTORY"
 )
 
-func queryTimeRanges(db *gorm.DB) (result []*TimeRange, err error) {
-	err = db.
-		Select(`
-			DISTINCT
-			FLOOR(UNIX_TIMESTAMP(summary_begin_time)) AS begin_time,
-			FLOOR(UNIX_TIMESTAMP(summary_end_time)) AS end_time
-		`).
-		Table(statementsTable).
-		Order("begin_time DESC, end_time DESC").
-		Find(&result).Error
-	return
-}
-
 func queryStmtTypes(db *gorm.DB) (result []string, err error) {
 	// why should put DISTINCT inside the `Pluck()` method, see here:
 	// https://github.com/jinzhu/gorm/issues/496
@@ -64,7 +51,8 @@ func (s *Service) queryStatements(
 	query := db.
 		Select(selectStmt).
 		Table(statementsTable).
-		Where("summary_begin_time >= FROM_UNIXTIME(?) AND summary_end_time <= FROM_UNIXTIME(?)", beginTime, endTime).
+		// https://stackoverflow.com/questions/3269434/whats-the-most-efficient-way-to-test-if-two-ranges-overlap
+		Where("summary_begin_time <= FROM_UNIXTIME(?) AND summary_end_time >= FROM_UNIXTIME(?)", endTime, beginTime).
 		Group("schema_name, digest").
 		Order("agg_sum_latency DESC")
 
@@ -130,7 +118,7 @@ func (s *Service) queryPlans(
 	query := db.
 		Select(selectStmt).
 		Table(statementsTable).
-		Where("summary_begin_time >= FROM_UNIXTIME(?) AND summary_end_time <= FROM_UNIXTIME(?)", beginTime, endTime).
+		Where("summary_begin_time <= FROM_UNIXTIME(?) AND summary_end_time >= FROM_UNIXTIME(?)", endTime, beginTime).
 		Group("plan_digest")
 
 	if digest == "" {
@@ -167,7 +155,7 @@ func (s *Service) queryPlanDetail(
 	query := db.
 		Select(selectStmt).
 		Table(statementsTable).
-		Where("summary_begin_time >= FROM_UNIXTIME(?) AND summary_end_time <= FROM_UNIXTIME(?)", beginTime, endTime)
+		Where("summary_begin_time <= FROM_UNIXTIME(?) AND summary_end_time >= FROM_UNIXTIME(?)", endTime, beginTime)
 
 	if digest == "" {
 		// the evicted record's digest will be NULL

Diff for: pkg/apiserver/statement/service.go

@@ -48,7 +48,6 @@ func registerRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
 		{
 			endpoint.GET("/config", s.configHandler)
 			endpoint.POST("/config", auth.MWRequireWritePriv(), s.modifyConfigHandler)
-			endpoint.GET("/time_ranges", s.timeRangesHandler)
 			endpoint.GET("/stmt_types", s.stmtTypesHandler)
 			endpoint.GET("/list", s.listHandler)
 			endpoint.GET("/plans", s.plansHandler)
@@ -114,21 +113,6 @@ func (s *Service) modifyConfigHandler(c *gin.Context) {
 	c.Status(http.StatusNoContent)
 }
 
-// @Summary Get available statement time ranges
-// @Success 200 {array} statement.TimeRange
-// @Router /statements/time_ranges [get]
-// @Security JwtAuth
-// @Failure 401 {object} rest.ErrorResponse
-func (s *Service) timeRangesHandler(c *gin.Context) {
-	db := utils.GetTiDBConnection(c)
-	timeRanges, err := queryTimeRanges(db)
-	if err != nil {
-		rest.Error(c, err)
-		return
-	}
-	c.JSON(http.StatusOK, timeRanges)
-}
-
 // @Summary Get all statement types
 // @Success 200 {array} string
 // @Router /statements/stmt_types [get]

Diff for: pkg/apiserver/statement/statement_gen.go

@@ -19,8 +19,11 @@ func (s *Service) genSelectStmt(tableColumns []string, reqJSONColumns []string)
 
 	// use required fields filter when not all fields are requested
 	if reqJSONColumns[0] != "*" {
-		// "schema_name", "digest" for group, "sum_latency" for order
-		requiredFields := funk.UniqString(append(reqJSONColumns, "schema_name", "digest", "sum_latency"))
+		requiredFields := funk.UniqString(append(reqJSONColumns,
+			"schema_name", "digest", // required by group by
+			"sum_latency", // required by order
+			"summary_begin_time", "summary_end_time",
+		))
 		fields = funk.Filter(fields, func(f Field) bool {
 			return funk.Contains(requiredFields, f.JSONName)
 		}).([]Field)