-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
rewrite: first commit for protobuf v1.20 rewrite of Gorums
- Loading branch information
Showing
86 changed files
with
6,587 additions
and
9,174 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,186 +1,68 @@ | ||
| CMD_PKG := cmd | ||
| PLUGINS_PKG := plugins | ||
| BUNDLE_PKG := bundle | ||
| DEV_PKG := dev | ||
| PLUGIN_PKG := gorums | ||
| PLUGIN_PATH := cmd/protoc-gen-gorums | ||
| PLUGIN_DEV_PATH := $(PLUGIN_PATH)/dev | ||
| PLUGIN_INTERNAL_PATH := $(PLUGIN_PATH)/internalgorums | ||
| PLUGIN_TESTS_PATH := $(PLUGIN_PATH)/tests | ||
| PLUGIN_STATIC_FILE := $(PLUGIN_INTERNAL_PATH)/template_static.go | ||
|
|
||
| GORUMS_PKGS := $(shell go list ./... | grep -v /vendor/) | ||
| GORUMS_FILES := $(shell find . -name '*.go' -not -path "*vendor*") | ||
| GORUMS_DIRS := $(shell find . -type d -not -path "*vendor*" -not -path "./.git*" -not -path "*testdata*") | ||
| GORUMS_PKG_PATH := github.com/relab/gorums | ||
| GORUMS_PROTO_NAME := gorums.proto | ||
| GORUMS_DEV_PKG_PATH := $(GORUMS_PKG_PATH)/$(DEV_PKG) | ||
| GORUMS_ENV_GENDEV := GORUMSGENDEV=1 | ||
| # TODO clean up the make targets to include relevant dependent src files | ||
|
|
||
| GORUMS_STATIC_GO := $(PLUGINS_PKG)/$(PLUGIN_PKG)/static.go | ||
| BUNDLE_MAIN_GO := $(CMD_PKG)/$(BUNDLE_PKG)/main.go | ||
| GENTEMPLATES_MAIN_GO := $(CMD_PKG)/gentemplates/main.go | ||
| # TODO evaluate if these should be PHONY (dev and devsingle shouldn't, but need to make dependency handling) | ||
| .PHONY: clean download install-tools dev devsingle | ||
|
|
||
| PROTOC_PLUGIN_PKG := protoc-gen-gorums | ||
| PROTOC_PLUGIN_PKG_PATH := $(GORUMS_PKG_PATH)/$(CMD_PKG)/$(PROTOC_PLUGIN_PKG) | ||
| PROTOC_PLUGIN_NAME := gorums_out | ||
| PROTOC_I_FLAG := ../../../:. | ||
|
|
||
| TMP_DEVGEN_DIR := tmpdevgen | ||
|
|
||
| REG_PROTO_NAME := storage.proto | ||
| REG_PBGO_NAME := storage.pb.go | ||
| REG_PROTO_DEV_RPATH := $(DEV_PKG)/$(REG_PROTO_NAME) | ||
| REG_PBGO_DEV_RPATH := $(DEV_PKG)/$(REG_PBGO_NAME) | ||
|
|
||
| GOGOPROTO_ALIAS := google/protobuf/descriptor.proto=github.com/gogo/protobuf/protoc-gen-gogo/descriptor | ||
|
|
||
| BENCHMARK := BenchmarkRead1KQ2N3Local | ||
|
|
||
| CHECKTOOLS := golang.org/x/tools/cmd/goimports \ | ||
| golang.org/x/lint \ | ||
| github.com/jgautheron/goconst/cmd/goconst \ | ||
| github.com/kisielk/errcheck \ | ||
| github.com/gordonklaus/ineffassign \ | ||
| github.com/mdempsky/unconvert \ | ||
| honnef.co/go/tools/staticcheck \ | ||
| mvdan.cc/interfacer \ | ||
| github.com/client9/misspell/cmd/misspell | ||
|
|
||
| .PHONY: all | ||
| all: build test | ||
|
|
||
| .PHONY: build | ||
| build: | ||
| @echo go build: | ||
| @go build -v -i $(GORUMS_PKGS) | ||
|
|
||
| .PHONY: test | ||
| test: reinstallprotoc | ||
| @echo go test: | ||
| @go test -v $(GORUMS_PKGS) | ||
|
|
||
| .PHONY: testrace | ||
| testrace: reinstallprotoc | ||
| @echo go test -race: | ||
| @go test -v -race -cpu=1,2,4 $(GORUMS_PKGS) | ||
|
|
||
| .PHONY: stresstestdev | ||
| stresstestdev: | ||
| go get -u golang.org/x/tools/cmd/stress | ||
| cd dev; go test -c | ||
| cd dev; stress -p=1 ./dev.test | ||
|
|
||
| .PHONY: benchlocal | ||
| benchlocal: | ||
| go test -v $(GORUMS_DEV_PKG_PATH) -run=^$$ -bench=Local$$ -benchtime=5s | ||
|
|
||
| .PHONY: benchremotewithlocalhost | ||
| benchremotewithlocalhost: | ||
| cd cmd/storageserver; go build | ||
| cmd/storageserver/storageserver -port=8080 & | ||
| cmd/storageserver/storageserver -port=8081 & | ||
| cmd/storageserver/storageserver -port=8082 & | ||
| @echo starting storage servers in background... press enter to start benchmark. | ||
| @read | ||
| go test -v $(GORUMS_DEV_PKG_PATH) -run=^$$ -bench=Remote$$ -benchtime=5s -remotehosts=localhost:8080,localhost:8081,localhost:8082 | ||
| @echo done running benchmark... killing storage servers | ||
| killall storageserver | ||
|
|
||
| .PHONY: clean | ||
| clean: | ||
| go clean -i $(GORUMS_PKG_PATH)/... | ||
| find . -name '*.test' -type f -exec rm -f {} \; | ||
| find . -name '*.prof' -type f -exec rm -f {} \; | ||
|
|
||
| .PHONY: reinstallprotoc | ||
| reinstallprotoc: | ||
| @echo installing protoc-gen-gorums with gorums plugin linked | ||
| @go install $(PROTOC_PLUGIN_PKG_PATH) | ||
|
|
||
| .PHONY: devproto | ||
| devproto: reinstallprotoc | ||
| @echo generating gorumsdev storage proto | ||
| protoc -I=$(PROTOC_I_FLAG) --$(PROTOC_PLUGIN_NAME)=plugins=grpc:. $(REG_PROTO_DEV_RPATH) | ||
|
|
||
| .PHONY: gorumsprotoopts | ||
| gorumsprotoopts: | ||
| @echo generating gorums proto options | ||
| protoc --$(PROTOC_PLUGIN_NAME)=M$(GOGOPROTO_ALIAS):../../../ $(GORUMS_PROTO_NAME) | ||
|
|
||
| .PHONY: static | ||
| static: | ||
| @echo creating static gorums plugin code bundle | ||
| go run $(BUNDLE_MAIN_GO) $(GORUMS_DEV_PKG_PATH) > $(GORUMS_STATIC_GO) | ||
|
|
||
| .PHONY: templates | ||
| templates: | ||
| @echo creating templates for gorums plugin code bundle | ||
| @go run $(GENTEMPLATES_MAIN_GO) | ||
|
|
||
| .PHONY: dev | ||
| dev: static templates reinstallprotoc | ||
| @echo generating _gen.go files for dev | ||
| mkdir -p $(TMP_DEVGEN_DIR) | ||
| $(GORUMS_ENV_GENDEV) protoc -I=$(PROTOC_I_FLAG) --$(PROTOC_PLUGIN_NAME)=plugins=grpc+gorums:$(TMP_DEVGEN_DIR) $(REG_PROTO_DEV_RPATH) | ||
| rm -r $(TMP_DEVGEN_DIR) | ||
|
|
||
| .PHONY: profcpu | ||
| profcpu: | ||
| go test $(GORUMS_DEV_PKG_PATH) -run=NONE -bench=$(BENCHMARK) -cpuprofile cpu.prof | ||
| go tool pprof $(DEV_PKG).test cpu.prof | ||
|
|
||
| .PHONY: profmem | ||
| profmem: | ||
| go test $(GORUMS_DEV_PKG_PATH) -run=NONE -bench=$(BENCHMARK) -memprofile allocmem.prof | ||
| go tool pprof -alloc_space $(DEV_PKG).test allocmem.prof | ||
|
|
||
| .PHONY: profobj | ||
| profobj: | ||
| go test $(GORUMS_DEV_PKG_PATH) -run=NONE -bench=$(BENCHMARK) -memprofile allocobj.prof | ||
| go tool pprof -alloc_objects $(DEV_PKG).test allocobj.prof | ||
|
|
||
| .PHONY: getchecktools | ||
| getchecktools: | ||
| go get -u $(CHECKTOOLS) | ||
|
|
||
| .PHONY: getdevtools | ||
| getdevtools: getchecktools | ||
|
|
||
| .PHONY: check | ||
| check: getchecktools | ||
| @echo static analysis tools: | ||
| @echo "gofmt (simplify)" | ||
| @! gofmt -s -l $(GORUMS_FILES) | grep -vF 'No Exceptions' | ||
| @echo "goimports" | ||
| @! goimports -l $(GORUMS_FILES) | grep -vF 'No Exceptions' | ||
| @echo "vet" | ||
| @! go tool vet $(GORUMS_DIRS) 2>&1 | \ | ||
| grep -vF 'vendor/' | \ | ||
| grep -vE '^dev/config_qc_test.go:.+: constant [0-9]+ not a string in call to Errorf' | ||
| @echo "vet --shadow" | ||
| @! go tool vet --shadow $(GORUMS_DIRS) 2>&1 | grep -vF 'vendor/' | ||
| @echo "golint" | ||
| @for pkg in $(GORUMS_PKGS); do \ | ||
| ! golint $$pkg | \ | ||
| grep -vE '(\.pb\.go)' | \ | ||
| grep -vE 'gorums/plugins/gorums/templates.go' ; \ | ||
| done | ||
| @echo "goconst" | ||
| @for dir in $(GORUMS_DIRS); do \ | ||
| ! goconst $$dir | \ | ||
| grep -vE '("_"|.pb.go)' ; \ | ||
| done | ||
| @echo "errcheck" | ||
| @errcheck -ignore 'fmt:Fprint*,bytes:WriteString,encoding/binary:Write,io:WriteString,os:Close|Remove*,net:Close,github.com/relab/gorums/dev:Close' $(GORUMS_PKGS) | ||
| @echo "ineffassign" | ||
| @for dir in $(GORUMS_DIRS); do \ | ||
| ineffassign -n $$dir ; \ | ||
| done | ||
| @echo "unconvert" | ||
| @! unconvert $(GORUMS_PKGS) | grep -vF '.pb.go' | ||
| @echo "megacheck" | ||
| @megacheck $(GORUMS_PKGS) | ||
| @echo "interfacer" | ||
| @interfacer $(GORUMS_PKGS) | ||
| @echo "missspell" | ||
| @! misspell ./**/* | grep -vF 'vendor' | ||
|
|
||
| .PHONY: updatedeps | ||
| updatedeps: | ||
| go mod tidy | ||
| rm -f $(PLUGIN_STATIC_FILE).bak | ||
|
|
||
| # TODO(meling) remove this when v2 released; for now, it is necessary for compiling gorums.proto | ||
| .PHONY: getv1 | ||
| getv1: | ||
| @echo getting v1 pre-release proto package based on v2 impl | ||
| @go mod tidy | ||
| @go get -u github.com/golang/protobuf/proto@api-v1 | ||
|
|
||
| download: | ||
| @echo Download go.mod dependencies | ||
| @go mod download | ||
|
|
||
| install-tools: download | ||
| @echo Installing tools from tools.go | ||
| @cat tools.go | grep _ | awk -F'"' '{print $$2}' | xargs -tI % go install % | ||
|
|
||
| gorumsproto: install-tools | ||
| @echo Generating gorums proto options | ||
| @protoc --go_out=paths=source_relative:. gorums.proto | ||
|
|
||
| installgorums: gorumsproto | ||
| @echo Installing protoc-gen-gorums compiler plugin for protoc | ||
| @go install github.com/relab/gorums/cmd/protoc-gen-gorums | ||
|
|
||
| bundle: installgorums | ||
| cp $(PLUGIN_STATIC_FILE) $(PLUGIN_STATIC_FILE).bak | ||
| protoc -I$(PLUGIN_DEV_PATH):. --gorums_out=bundle=$(PLUGIN_STATIC_FILE):. $(PLUGIN_DEV_PATH)/zorums.proto | ||
|
|
||
| dev: installgorums | ||
| @echo Generating Gorums code for zorums.proto as a multiple _gorums.pb.go files in dev folder | ||
| rm -f $(PLUGIN_DEV_PATH)/*.pb.go | ||
| protoc -I$(PLUGIN_DEV_PATH):. \ | ||
| --go_out=:. \ | ||
| --go-grpc_out=:. \ | ||
| --gorums_out=dev=true,trace=true:. \ | ||
| $(PLUGIN_DEV_PATH)/zorums.proto | ||
|
|
||
| devsingle: installgorums | ||
| @echo Generating Gorums code for zorums.proto as a single _gorums.pb.go file in dev folder | ||
| rm -f $(PLUGIN_DEV_PATH)/*.pb.go | ||
| protoc -I$(PLUGIN_DEV_PATH):. \ | ||
| --go_out=:. \ | ||
| --go-grpc_out=:. \ | ||
| --gorums_out=trace=true:. \ | ||
| $(PLUGIN_DEV_PATH)/zorums.proto | ||
|
|
||
| # TODO(meling) make test case comparing generated code with old code | ||
|
|
||
| .PHONY: qc | ||
| qc: installgorums | ||
| protoc -I. \ | ||
| --go_out=paths=source_relative:. \ | ||
| --go-grpc_out=paths=source_relative:. \ | ||
| --gorums_out=paths=source_relative,trace=true:. \ | ||
| $(PLUGIN_TESTS_PATH)/quorumcall/quorumcall.proto |
Oops, something went wrong.