Support encoded plan in plan meta (#156)

close pingcap/tidb#35964
pingcap · Jul 7, 2022 · 1869d7b · 1869d7b
1 parent d6ec5f4
commit 1869d7b
Show file tree

Hide file tree

Showing 40 changed files with 2,047 additions and 140 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.16'
+          go-version: '1.18'
       - name: Format
         run: make fmt
       - name: Lint

diff --git a/component/conprof/contprof.go b/component/conprof/contprof.go
@@ -1,10 +1,11 @@
 package conprof
 
 import (
-	"github.com/genjidb/genji"
 	"github.com/pingcap/ng-monitoring/component/conprof/scrape"
 	"github.com/pingcap/ng-monitoring/component/conprof/store"
 	"github.com/pingcap/ng-monitoring/component/topology"
+
+	"github.com/genjidb/genji"
 )
 
 var (

diff --git a/component/conprof/http/api.go b/component/conprof/http/api.go
@@ -9,12 +9,13 @@ import (
 	"strings"
 	"time"
 
-	"github.com/gin-gonic/gin"
-	"github.com/pingcap/log"
 	"github.com/pingcap/ng-monitoring/component/conprof"
 	"github.com/pingcap/ng-monitoring/component/conprof/meta"
 	"github.com/pingcap/ng-monitoring/component/topology"
 	"github.com/pingcap/ng-monitoring/config"
+
+	"github.com/gin-gonic/gin"
+	"github.com/pingcap/log"
 	"go.uber.org/zap"
 )
 

diff --git a/component/conprof/http/api_test.go b/component/conprof/http/api_test.go
@@ -15,13 +15,14 @@ import (
 	"testing"
 	"time"
 
-	"github.com/genjidb/genji"
-	"github.com/gin-gonic/gin"
 	"github.com/pingcap/ng-monitoring/component/conprof"
 	"github.com/pingcap/ng-monitoring/component/conprof/meta"
 	"github.com/pingcap/ng-monitoring/component/topology"
 	"github.com/pingcap/ng-monitoring/config"
 	"github.com/pingcap/ng-monitoring/utils/testutil"
+
+	"github.com/genjidb/genji"
+	"github.com/gin-gonic/gin"
 	"github.com/stretchr/testify/require"
 )
 

diff --git a/component/conprof/scrape/manager.go b/component/conprof/scrape/manager.go
@@ -8,12 +8,13 @@ import (
 	"sync"
 	"time"
 
-	"github.com/pingcap/log"
 	"github.com/pingcap/ng-monitoring/component/conprof/meta"
 	"github.com/pingcap/ng-monitoring/component/conprof/store"
 	"github.com/pingcap/ng-monitoring/component/topology"
 	"github.com/pingcap/ng-monitoring/config"
 	"github.com/pingcap/ng-monitoring/utils"
+
+	"github.com/pingcap/log"
 	commonconfig "github.com/prometheus/common/config"
 	"go.uber.org/zap"
 )

diff --git a/component/conprof/scrape/manager_test.go b/component/conprof/scrape/manager_test.go
@@ -7,12 +7,13 @@ import (
 	"testing"
 	"time"
 
-	"github.com/pingcap/log"
 	"github.com/pingcap/ng-monitoring/component/conprof/meta"
 	"github.com/pingcap/ng-monitoring/component/conprof/store"
 	"github.com/pingcap/ng-monitoring/component/topology"
 	"github.com/pingcap/ng-monitoring/config"
 	"github.com/pingcap/ng-monitoring/utils/testutil"
+
+	"github.com/pingcap/log"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 	"go.uber.org/zap"

diff --git a/component/conprof/scrape/scrape.go b/component/conprof/scrape/scrape.go
@@ -11,10 +11,11 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/pingcap/log"
 	"github.com/pingcap/ng-monitoring/component/conprof/meta"
 	"github.com/pingcap/ng-monitoring/component/conprof/store"
 	"github.com/pingcap/ng-monitoring/config"
+
+	"github.com/pingcap/log"
 	"github.com/pkg/errors"
 	"go.uber.org/atomic"
 	"go.uber.org/zap"

diff --git a/component/conprof/store/gc.go b/component/conprof/store/gc.go
@@ -4,11 +4,12 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/pingcap/ng-monitoring/component/conprof/meta"
+	"github.com/pingcap/ng-monitoring/config"
+
 	"github.com/genjidb/genji/document"
 	"github.com/genjidb/genji/types"
 	"github.com/pingcap/log"
-	"github.com/pingcap/ng-monitoring/component/conprof/meta"
-	"github.com/pingcap/ng-monitoring/config"
 	"go.uber.org/zap"
 )
 

diff --git a/component/conprof/store/store.go b/component/conprof/store/store.go
@@ -8,12 +8,13 @@ import (
 	"sync"
 	"time"
 
+	"github.com/pingcap/ng-monitoring/component/conprof/meta"
+	"github.com/pingcap/ng-monitoring/utils"
+
 	"github.com/genjidb/genji"
 	"github.com/genjidb/genji/document"
 	"github.com/genjidb/genji/types"
 	"github.com/pingcap/log"
-	"github.com/pingcap/ng-monitoring/component/conprof/meta"
-	"github.com/pingcap/ng-monitoring/utils"
 	"github.com/valyala/gozstd"
 	"go.uber.org/atomic"
 	"go.uber.org/zap"

diff --git a/component/conprof/store/store_test.go b/component/conprof/store/store_test.go
@@ -10,14 +10,15 @@ import (
 	"testing"
 	"time"
 
+	"github.com/pingcap/ng-monitoring/component/conprof/meta"
+	"github.com/pingcap/ng-monitoring/config"
+	"github.com/pingcap/ng-monitoring/utils/testutil"
+
 	"github.com/dgraph-io/badger/v3"
 	"github.com/genjidb/genji"
 	"github.com/genjidb/genji/document"
 	"github.com/genjidb/genji/engine/badgerengine"
 	"github.com/genjidb/genji/types"
-	"github.com/pingcap/ng-monitoring/component/conprof/meta"
-	"github.com/pingcap/ng-monitoring/config"
-	"github.com/pingcap/ng-monitoring/utils/testutil"
 	"github.com/stretchr/testify/require"
 )
 

diff --git a/component/domain/client.go b/component/domain/client.go
@@ -7,8 +7,9 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/pingcap/log"
 	"github.com/pingcap/ng-monitoring/config"
+
+	"github.com/pingcap/log"
 	"github.com/pingcap/tidb-dashboard/util/client/httpclient"
 	"github.com/pingcap/tidb-dashboard/util/client/pdclient"
 	clientv3 "go.etcd.io/etcd/client/v3"

diff --git a/component/domain/domain.go b/component/domain/domain.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/pingcap/ng-monitoring/config"
 	"github.com/pingcap/ng-monitoring/utils"
+
 	"github.com/pingcap/tidb-dashboard/util/client/pdclient"
 	clientv3 "go.etcd.io/etcd/client/v3"
 )

diff --git a/component/domain/domain_test.go b/component/domain/domain_test.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/pingcap/ng-monitoring/config"
 	"github.com/pingcap/ng-monitoring/utils/testutil"
+
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 )

diff --git a/component/topology/discovery.go b/component/topology/discovery.go
@@ -6,9 +6,10 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/pingcap/log"
 	"github.com/pingcap/ng-monitoring/component/domain"
 	"github.com/pingcap/ng-monitoring/utils"
+
+	"github.com/pingcap/log"
 	"github.com/pingcap/tidb-dashboard/util/topo"
 	"github.com/pingcap/tidb-dashboard/util/topo/pdtopo"
 	"go.uber.org/zap"

diff --git a/component/topology/syncer.go b/component/topology/syncer.go
@@ -8,11 +8,12 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/pingcap/log"
 	"github.com/pingcap/ng-monitoring/component/domain"
 	"github.com/pingcap/ng-monitoring/config"
 	"github.com/pingcap/ng-monitoring/utils"
 	"github.com/pingcap/ng-monitoring/utils/printer"
+
+	"github.com/pingcap/log"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/client/v3/concurrency"
 	"go.uber.org/zap"

diff --git a/component/topology/topology_test.go b/component/topology/topology_test.go
@@ -11,6 +11,7 @@ import (
 	"github.com/pingcap/ng-monitoring/config"
 	"github.com/pingcap/ng-monitoring/utils/printer"
 	"github.com/pingcap/ng-monitoring/utils/testutil"
+
 	"github.com/stretchr/testify/require"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.etcd.io/etcd/tests/v3/integration"
@@ -39,7 +40,7 @@ func TestTopology(t *testing.T) {
 		t.Skip("integration.NewClusterV3 will create file contains a colon which is not allowed on Windows")
 	}
 
-	integration.BeforeTest(t)
+	integration.BeforeTestExternal(t)
 	cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
 	defer cluster.Terminate(t)
 	mockPD := testutil.MockPDHTTPServer{}

diff --git a/component/topsql/codec/plan/plan.go b/component/topsql/codec/plan/plan.go
@@ -0,0 +1,30 @@
+package plan
+
+import (
+	"encoding/base64"
+
+	"github.com/golang/snappy"
+	"github.com/pingcap/tidb/util/plancodec"
+)
+
+func Decode(planString string) (string, error) {
+	binaryPlan, err := decompress(planString)
+	if err != nil {
+		return "", err
+	}
+
+	return plancodec.DecodeNormalizedPlan(binaryPlan)
+}
+
+func decompress(str string) (string, error) {
+	decodeBytes, err := base64.StdEncoding.DecodeString(str)
+	if err != nil {
+		return "", err
+	}
+
+	bs, err := snappy.Decode(nil, decodeBytes)
+	if err != nil {
+		return "", err
+	}
+	return string(bs), nil
+}
diff --git a/component/topsql/codec/plan/plan_test.go b/component/topsql/codec/plan/plan_test.go
@@ -0,0 +1,35 @@
+package plan_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/pingcap/ng-monitoring/component/topsql/codec/plan"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestBigPlan(t *testing.T) {
+	t.Parallel()
+
+	bigEncodedPlan, err := ioutil.ReadFile("testdata/big_encoded_plan.txt")
+	require.NoError(t, err)
+
+	planText, err := plan.Decode(string(bigEncodedPlan))
+	require.NoError(t, err)
+
+	bigDecodedPlan, err := ioutil.ReadFile("testdata/big_decoded_plan.txt")
+	require.NoError(t, err)
+	require.Equal(t, planText, string(bigDecodedPlan))
+}
+
+func TestSmallPlan(t *testing.T) {
+	t.Parallel()
+
+	encodedPlan := "WrAwCTM4CTAJdGFibGU6R0xPQkFMX1ZBUklBQkxFUywgaW5kZXg6UFJJTUFSWSgRGZBfTkFNRSksIGtlZXAgb3JkZXI6ZmFsc2UsIGRlc2M6ZmFsc2UK"
+	expectedPlanText := "\tBatch_Point_Get\troot\ttable:GLOBAL_VARIABLES, index:PRIMARY(VARIABLE_NAME), keep order:false, desc:false"
+
+	planText, err := plan.Decode(encodedPlan)
+	require.NoError(t, err)
+	require.Equal(t, planText, expectedPlanText)
+}