Skip to content

Commit

Permalink
statstics: fix cannot load non-index column with concurrently init stats
Browse files Browse the repository at this point in the history 
Signed-off-by: Weizhen Wang <[email protected]>
  • Loading branch information
@hawkingrei
hawkingrei committed Nov 20, 2024
1 parent 719f68e commit 2e50733
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 80 deletions.
7 changes: 2 additions & 5 deletions pkg/statistics/handle/bootstrap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import (
"go.uber.org/zap"
)

const initStatsStep = int64(500)
const initStatsStep = int64(400)

var maxTidRecord MaxTidRecord

Expand Down Expand Up @@ -340,10 +340,7 @@ func (h *Handle) initStatsHistogramsByPaging(is infoschema.InfoSchema, cache uti
}()

sctx := se.(sessionctx.Context)
// Why do we need to add `is_index=1` in the SQL?
// because it is aligned to the `initStatsTopN` function, which only loads the topn of the index too.
// the other will be loaded by sync load.
sql := "select HIGH_PRIORITY table_id, is_index, hist_id, distinct_count, version, null_count, cm_sketch, tot_col_size, stats_ver, correlation, flag, last_analyze_pos from mysql.stats_histograms where table_id >= %? and table_id < %? and is_index=1"
sql := "select HIGH_PRIORITY table_id, is_index, hist_id, distinct_count, version, null_count, cm_sketch, tot_col_size, stats_ver, correlation, flag, last_analyze_pos from mysql.stats_histograms where table_id >= %? and table_id < %?"
rc, err := util.Exec(sctx, sql, task.StartTid, task.EndTid)
if err != nil {
return errors.Trace(err)
Expand Down
97 changes: 22 additions & 75 deletions pkg/statistics/handle/handletest/statstest/stats_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,49 +239,6 @@ func testInitStatsMemTraceFunc(t *testing.T, liteInitStats bool) {
}

func TestInitStats(t *testing.T) {
originValue := config.GetGlobalConfig().Performance.LiteInitStats
defer func() {
config.GetGlobalConfig().Performance.LiteInitStats = originValue
config.GetGlobalConfig().Performance.ConcurrentlyInitStats = false
}()
config.GetGlobalConfig().Performance.LiteInitStats = false
config.GetGlobalConfig().Performance.ConcurrentlyInitStats = true
store, dom := testkit.CreateMockStoreAndDomain(t)
testKit := testkit.NewTestKit(t, store)
testKit.MustExec("use test")
testKit.MustExec("set @@session.tidb_analyze_version = 1")
testKit.MustExec("create table t(a int, b int, c int, primary key(a), key idx(b))")
testKit.MustExec("insert into t values (1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,7,8)")
testKit.MustExec("analyze table t")
h := dom.StatsHandle()
is := dom.InfoSchema()
tbl, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)
// `Update` will not use load by need strategy when `Lease` is 0, and `InitStats` is only called when
// `Lease` is not 0, so here we just change it.
h.SetLease(time.Millisecond)

h.Clear()
require.NoError(t, h.InitStats(is))
table0 := h.GetTableStats(tbl.Meta())
idx := table0.Indices
require.Equal(t, uint8(0x3), idx[1].LastAnalyzePos.GetBytes()[0])
h.Clear()
require.NoError(t, h.Update(is))
// Index and pk are loaded.
needed := fmt.Sprintf(`Table:%v RealtimeCount:6
index:1 ndv:6
num: 1 lower_bound: 1 upper_bound: 1 repeats: 1 ndv: 0
num: 1 lower_bound: 2 upper_bound: 2 repeats: 1 ndv: 0
num: 1 lower_bound: 3 upper_bound: 3 repeats: 1 ndv: 0
num: 1 lower_bound: 4 upper_bound: 4 repeats: 1 ndv: 0
num: 1 lower_bound: 5 upper_bound: 5 repeats: 1 ndv: 0
num: 1 lower_bound: 7 upper_bound: 7 repeats: 1 ndv: 0`, tbl.Meta().ID)
require.Equal(t, needed, table0.String())
h.SetLease(0)
}

func TestInitStats2(t *testing.T) {
originValue := config.GetGlobalConfig().Performance.LiteInitStats
defer func() {
config.GetGlobalConfig().Performance.LiteInitStats = originValue
Expand Down Expand Up @@ -354,30 +311,26 @@ func TestInitStats51358(t *testing.T) {
}

func TestInitStatsVer2(t *testing.T) {
originValue := config.GetGlobalConfig().Performance.LiteInitStats
concurrentlyInitStatsValue := config.GetGlobalConfig().Performance.ConcurrentlyInitStats
defer func() {
config.GetGlobalConfig().Performance.LiteInitStats = originValue
config.GetGlobalConfig().Performance.ConcurrentlyInitStats = concurrentlyInitStatsValue
}()
config.GetGlobalConfig().Performance.LiteInitStats = false
config.GetGlobalConfig().Performance.ConcurrentlyInitStats = false
initStatsVer2(t, false)
restore := config.RestoreFunc()
defer restore()
config.UpdateGlobal(func(conf *config.Config) {
config.GetGlobalConfig().Performance.LiteInitStats = false
config.GetGlobalConfig().Performance.ConcurrentlyInitStats = false
})
initStatsVer2(t)
}

func TestInitStatsVer2Concurrency(t *testing.T) {
originValue := config.GetGlobalConfig().Performance.LiteInitStats
concurrentlyInitStatsValue := config.GetGlobalConfig().Performance.ConcurrentlyInitStats
defer func() {
config.GetGlobalConfig().Performance.LiteInitStats = originValue
config.GetGlobalConfig().Performance.ConcurrentlyInitStats = concurrentlyInitStatsValue
}()
config.GetGlobalConfig().Performance.LiteInitStats = false
config.GetGlobalConfig().Performance.ConcurrentlyInitStats = true
initStatsVer2(t, true)
restore := config.RestoreFunc()
defer restore()
config.UpdateGlobal(func(conf *config.Config) {
config.GetGlobalConfig().Performance.LiteInitStats = false
config.GetGlobalConfig().Performance.ConcurrentlyInitStats = true
})
initStatsVer2(t)
}

func initStatsVer2(t *testing.T, isConcurrency bool) {
func initStatsVer2(t *testing.T) {
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
Expand All @@ -396,19 +349,13 @@ func initStatsVer2(t *testing.T, isConcurrency bool) {
h.Clear()
require.NoError(t, h.InitStats(is))
table0 := h.GetTableStats(tbl.Meta())
if isConcurrency {
idx := table0.Indices
require.Equal(t, uint8(0x3), idx[1].LastAnalyzePos.GetBytes()[0])
require.Equal(t, uint8(0x3), idx[2].LastAnalyzePos.GetBytes()[0])
} else {
cols := table0.Columns
require.Equal(t, uint8(0x33), cols[1].LastAnalyzePos.GetBytes()[0])
require.Equal(t, uint8(0x33), cols[2].LastAnalyzePos.GetBytes()[0])
require.Equal(t, uint8(0x33), cols[3].LastAnalyzePos.GetBytes()[0])
idx := table0.Indices
require.Equal(t, uint8(0x3), idx[1].LastAnalyzePos.GetBytes()[0])
require.Equal(t, uint8(0x3), idx[2].LastAnalyzePos.GetBytes()[0])
}
cols := table0.Columns
require.Equal(t, uint8(0x33), cols[1].LastAnalyzePos.GetBytes()[0])
require.Equal(t, uint8(0x33), cols[2].LastAnalyzePos.GetBytes()[0])
require.Equal(t, uint8(0x33), cols[3].LastAnalyzePos.GetBytes()[0])
idx := table0.Indices
require.Equal(t, uint8(0x3), idx[1].LastAnalyzePos.GetBytes()[0])
require.Equal(t, uint8(0x3), idx[2].LastAnalyzePos.GetBytes()[0])
h.Clear()
require.NoError(t, h.InitStats(is))
table1 := h.GetTableStats(tbl.Meta())
Expand Down

0 comments on commit 2e50733

Please sign in to comment.