Merge branch 'master' into gregwebs/error-code-tracing

.gitignore

@@ -6,4 +6,4 @@ default.pd
 *.swp
 .DS_Store
 tags
-/_tools/
+/.retools/

CHANGELOG.md

@@ -1,5 +1,20 @@
 # PD Change Log
 
+## v2.1.0-rc2
+### Features
+* Support the `GetAllStores` interface
+* Add the statistics of scheduling estimation in Simulator
+### Improvements
+* Optimize the handling process of down stores to make up replicas as soon as possible
+* Optimize the start of Coordinator to reduce the unnecessary scheduling caused by restarting PD
+* Optimize the memory usage to reduce the overhead caused by heartbeats
+* Optimize error handling and improve the log information
+* Support querying the Region information of a specific store in pd-ctl
+* Support querying the topN Region information based on version
+* Support more accurate TSO decoding in pd-ctl
+### Bug fix
+* Fix the issue that pd-ctl uses the `hot store` command to exit wrongly
+
 ## v2.1.0-rc1
 ### Features
 * Introduce the version control mechanism and support rolling update of the cluster with compatibility

Makefile

@@ -7,9 +7,10 @@ BASIC_TEST_PKGS := $(filter-out github.com/pingcap/pd/pkg/integration_test,$(TES
 PACKAGES := go list ./...
 PACKAGE_DIRECTORIES := $(PACKAGES) | sed 's|github.com/pingcap/pd/||'
 GOCHECKER := awk '{ print } END { if (NR > 0) { exit 1 } }'
+RETOOL:= ./hack/retool
 
-GOFAIL_ENABLE  := $$(find $$PWD/ -type d | grep -vE "(\.git|vendor)" | xargs retool do gofail enable)
-GOFAIL_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|vendor)" | xargs retool do gofail disable)
+GOFAIL_ENABLE  := $$(find $$PWD/ -type d | grep -vE "(\.git|vendor)" | xargs ./hack/retool do gofail enable)
+GOFAIL_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|vendor)" | xargs ./hack/retool do gofail disable)
 
 LDFLAGS += -X "$(PD_PKG)/server.PDReleaseVersion=$(shell git describe --tags --dirty)"
 LDFLAGS += -X "$(PD_PKG)/server.PDBuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')"
@@ -35,9 +36,9 @@ ifeq ("$(WITH_RACE)", "1")
 else
 	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o bin/pd-server cmd/pd-server/main.go
 endif
-	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o bin/pd-ctl cmd/pd-ctl/main.go
-	CGO_ENABLED=0 go build -o bin/pd-tso-bench cmd/pd-tso-bench/main.go
-	CGO_ENABLED=0 go build -o bin/pd-recover cmd/pd-recover/main.go
+	CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o bin/pd-ctl tools/pd-ctl/main.go
+	CGO_ENABLED=0 go build -o bin/pd-tso-bench tools/pd-tso-bench/main.go
+	CGO_ENABLED=0 go build -o bin/pd-recover tools/pd-recover/main.go
 
 test: retool-setup
 	# testing..
@@ -52,34 +53,34 @@ basic_test:
 
 # These need to be fixed before they can be ran regularly
 check-fail:
-	CGO_ENABLED=0 retool do gometalinter.v2 --disable-all \
+	CGO_ENABLED=0 ./hack/retool do gometalinter.v2 --disable-all \
 	  --enable errcheck \
 	  $$($(PACKAGE_DIRECTORIES))
-	CGO_ENABLED=0 retool do gosec $$($(PACKAGE_DIRECTORIES))
+	CGO_ENABLED=0 ./hack/retool do gosec $$($(PACKAGE_DIRECTORIES))
 
 check-all: static lint
 	@echo "checking"
 
 retool-setup:
 	@which retool >/dev/null 2>&1 || go get github.com/twitchtv/retool
-	@retool sync
+	@./hack/retool sync
 
 check: retool-setup check-all
 
 static:
 	@ # Not running vet and fmt through metalinter becauase it ends up looking at vendor
 	gofmt -s -l $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(GOCHECKER)
-	retool do govet --shadow $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(GOCHECKER)
+	./hack/retool do govet --shadow $$($(PACKAGE_DIRECTORIES)) 2>&1 | $(GOCHECKER)
 
-	CGO_ENABLED=0 retool do gometalinter.v2 --disable-all --deadline 120s \
+	CGO_ENABLED=0 ./hack/retool do gometalinter.v2 --disable-all --deadline 120s \
 	  --enable misspell \
 	  --enable megacheck \
 	  --enable ineffassign \
 	  $$($(PACKAGE_DIRECTORIES))
 
 lint:
 	@echo "linting"
-	CGO_ENABLED=0 retool do revive -formatter friendly -config revive.toml $$($(PACKAGES))
+	CGO_ENABLED=0 ./hack/retool do revive -formatter friendly -config revive.toml $$($(PACKAGES))
 
 travis_coverage:
 ifeq ("$(TRAVIS_COVERAGE)", "1")
@@ -100,8 +101,7 @@ endif
 	bash ./hack/clean_vendor.sh
 
 simulator:
-	CGO_ENABLED=0 go build -o bin/simulator cmd/simulator/main.go
-	bin/simulator
+	CGO_ENABLED=0 go build -o bin/pd-simulator tools/pd-simulator/main.go
 
 gofail-enable:
 	# Converting gofail failpoints...

conf/simconfig.toml

@@ -0,0 +1,17 @@
+# PD Simulator Configuration
+
+[tick]
+# the tick interval when starting PD inside (default: "100ms")
+sim-tick-interval = "100ms"
+# the tick interval when connecting with an external PD (default: "1s")
+norm-tick-interval = "1s"
+
+[store]
+# the capacity size of a new store in GB (default: 1024)
+store-capacity = 1024
+# the available size of a new store in GB (default: 1024)
+store-available = 1024
+# the io rate of a new store in MB/s (default: 40)
+store-io-per-second = 40
+# the version of a new store (default: "2.1.0")
+store-version = "2.1.0"

hack/retool

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+#
+# Add standard retool options
+set -euo pipefail
+
+
+cd $(dirname "$0")/..
+exec retool -tool-dir "$PWD/.retools" "$@"

hack/retool-install.sh

@@ -3,21 +3,23 @@ set -euo pipefail
 
 # This script generates tools.json
 # It helps record what releases/branches are being used
+
+cd $(dirname "$0")/..
 which retool >/dev/null || go get github.com/twitchtv/retool
 
 # tool environment
 # check runner
-retool add gopkg.in/alecthomas/gometalinter.v2 v2.0.5
+./hack/retool add gopkg.in/alecthomas/gometalinter.v2 v2.0.5
 # check spelling
-retool add github.com/client9/misspell/cmd/misspell v0.3.4
+./hack/retool add github.com/client9/misspell/cmd/misspell v0.3.4
 # checks correctness
-retool add github.com/gordonklaus/ineffassign 7bae11eba15a3285c75e388f77eb6357a2d73ee2
-retool add honnef.co/go/tools/cmd/megacheck master
-retool add github.com/dnephin/govet 4a96d43e39d340b63daa8bc5576985aa599885f6
+./hack/retool add github.com/gordonklaus/ineffassign 7bae11eba15a3285c75e388f77eb6357a2d73ee2
+./hack/retool add honnef.co/go/tools/cmd/megacheck master
+./hack/retool add github.com/dnephin/govet 4a96d43e39d340b63daa8bc5576985aa599885f6
 # slow checks
-retool add github.com/kisielk/errcheck v1.1.0
+./hack/retool add github.com/kisielk/errcheck v1.1.0
 # linter
-retool add github.com/mgechev/revive 7773f47324c2bf1c8f7a5500aff2b6c01d3ed73b
-retool add github.com/securego/gosec/cmd/gosec 1.0.0
+./hack/retool add github.com/mgechev/revive 7773f47324c2bf1c8f7a5500aff2b6c01d3ed73b
+./hack/retool add github.com/securego/gosec/cmd/gosec 1.0.0
 # go fail
-retool add github.com/etcd-io/gofail master
+./hack/retool add github.com/etcd-io/gofail master