Merge m3ctl into monorepo

.buildkite/pipeline.yml

@@ -35,8 +35,15 @@ steps:
         run: app
         workdir: /go/src/github.com/m3db/m3
     <<: *common
-  - name: "Integration (:docker:), Services, Tools, Metalint"
-    command: make install-vendor metalint docker-integration-test tools services
+  - name: "Services, Tools, Metalint"
+    command: make clean install-vendor services tools metalint
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+        workdir: /go/src/github.com/m3db/m3
+    <<: *common
+  - name: "Integration (:docker:)"
+    command: make clean install-vendor docker-integration-test
     env:
       CGO_ENABLED: 0
       GIMME_GO_VERSION: 1.10.x

.gitignore

@@ -34,13 +34,21 @@ test-data
 
 # Coverage imports
 coverage_imports.go
+coverage/
 
 # Release directory
 dist/
 
 # Docs directory
 site/
 
+# Node
+node_modules/
+build/
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
 # Used to serve m3db.io site
 !m3db.io/**/*.html
 # glide does not manage this

Makefile

@@ -12,11 +12,11 @@ auto_gen             := scripts/auto-gen.sh
 process_coverfile    := scripts/process-cover.sh
 gopath_prefix        := $(GOPATH)/src
 gopath_bin_path      := $(GOPATH)/bin
-m3db_package         := github.com/m3db/m3
-m3db_package_path    := $(gopath_prefix)/$(m3db_package)
+m3_package           := github.com/m3db/m3
+m3_package_path      := $(gopath_prefix)/$(m3_package)
 mockgen_package      := github.com/golang/mock/mockgen
-retool_bin_path      := $(m3db_package_path)/_tools/bin
-retool_src_prefix    := $(m3db_package_path)/_tools/src
+retool_bin_path      := $(m3_package_path)/_tools/bin
+retool_src_prefix    := $(m3_package_path)/_tools/src
 retool_package       := github.com/twitchtv/retool
 metalint_check       := .ci/metalint.sh
 metalint_config      := .metalinter.json
@@ -33,8 +33,8 @@ vendor_prefix        := vendor
 cache_policy         ?= recently_read
 
 BUILD                     := $(abspath ./bin)
-VENDOR                    := $(m3db_package_path)/$(vendor_prefix)
-GO_BUILD_LDFLAGS_CMD      := $(abspath ./.ci/go-build-ldflags.sh) $(m3db_package)
+VENDOR                    := $(m3_package_path)/$(vendor_prefix)
+GO_BUILD_LDFLAGS_CMD      := $(abspath ./.ci/go-build-ldflags.sh) $(m3_package)
 GO_BUILD_LDFLAGS          := $(shell $(GO_BUILD_LDFLAGS_CMD))
 GO_BUILD_COMMON_ENV       := CGO_ENABLED=0
 LINUX_AMD64_ENV           := GOOS=linux GOARCH=amd64 $(GO_BUILD_COMMON_ENV)
@@ -50,6 +50,7 @@ SERVICES :=     \
 	m3aggregator  \
 	m3query       \
 	m3collector   \
+	m3ctl         \
 	m3em_agent    \
 	m3nsch_server \
 	m3nsch_client \
@@ -67,6 +68,7 @@ SUBDIRS :=    \
 	m3nsch      \
 	m3ninx      \
 	aggregator  \
+	ctl         \
 
 TOOLS :=               \
 	read_ids             \
@@ -87,6 +89,10 @@ define SERVICE_RULES
 
 .PHONY: $(SERVICE)
 $(SERVICE): setup
+ifeq ($(SERVICE),m3ctl)
+	@echo "Building $(SERVICE) dependencies"
+	make build-ui-ctl-statik-gen
+endif
 	@echo Building $(SERVICE)
 	[ -d $(VENDOR) ] || make install-vendor
 	$(GO_BUILD_COMMON_ENV) go build -ldflags '$(GO_BUILD_LDFLAGS)' -o $(BUILD)/$(SERVICE) ./src/cmd/services/$(SERVICE)/main/.
@@ -237,25 +243,25 @@ define SUBDIR_RULES
 mock-gen-$(SUBDIR): install-tools
 	@echo "--- Generating mocks $(SUBDIR)"
 	@[ ! -d src/$(SUBDIR)/$(mocks_rules_dir) ] || \
-		PATH=$(retool_bin_path):$(PATH) PACKAGE=$(m3db_package) $(auto_gen) src/$(SUBDIR)/$(mocks_output_dir) src/$(SUBDIR)/$(mocks_rules_dir)
+		PATH=$(retool_bin_path):$(PATH) PACKAGE=$(m3_package) $(auto_gen) src/$(SUBDIR)/$(mocks_output_dir) src/$(SUBDIR)/$(mocks_rules_dir)
 
 .PHONY: thrift-gen-$(SUBDIR)
 thrift-gen-$(SUBDIR): install-tools
 	@echo "--- Generating thrift files $(SUBDIR)"
 	@[ ! -d src/$(SUBDIR)/$(thrift_rules_dir) ] || \
-		PATH=$(retool_bin_path):$(PATH) PACKAGE=$(m3db_package) $(auto_gen) src/$(SUBDIR)/$(thrift_output_dir) src/$(SUBDIR)/$(thrift_rules_dir)
+		PATH=$(retool_bin_path):$(PATH) PACKAGE=$(m3_package) $(auto_gen) src/$(SUBDIR)/$(thrift_output_dir) src/$(SUBDIR)/$(thrift_rules_dir)
 
 .PHONY: proto-gen-$(SUBDIR)
 proto-gen-$(SUBDIR): install-tools
 	@echo "--- Generating protobuf files $(SUBDIR)"
 	@[ ! -d src/$(SUBDIR)/$(proto_rules_dir) ] || \
-		PATH=$(retool_bin_path):$(PATH) PACKAGE=$(m3db_package) $(auto_gen) src/$(SUBDIR)/$(proto_output_dir) src/$(SUBDIR)/$(proto_rules_dir)
+		PATH=$(retool_bin_path):$(PATH) PACKAGE=$(m3_package) $(auto_gen) src/$(SUBDIR)/$(proto_output_dir) src/$(SUBDIR)/$(proto_rules_dir)
 
 .PHONY: asset-gen-$(SUBDIR)
 asset-gen-$(SUBDIR): install-tools
 	@echo "--- Generating asset files $(SUBDIR)"
 	@[ ! -d src/$(SUBDIR)/$(assets_rules_dir) ] || \
-		PATH=$(retool_bin_path):$(PATH) PACKAGE=$(m3db_package) $(auto_gen) src/$(SUBDIR)/$(assets_output_dir) src/$(SUBDIR)/$(assets_rules_dir)
+		PATH=$(retool_bin_path):$(PATH) PACKAGE=$(m3_package) $(auto_gen) src/$(SUBDIR)/$(assets_output_dir) src/$(SUBDIR)/$(assets_rules_dir)
 
 .PHONY: genny-gen-$(SUBDIR)
 genny-gen-$(SUBDIR): install-tools
@@ -344,6 +350,54 @@ endef
 # of metalint and finishes faster.
 $(foreach SUBDIR_TARGET, $(filter-out metalint,$(SUBDIR_TARGETS)), $(eval $(SUBDIR_TARGET_RULE)))
 
+.PHONY: build-ui-ctl
+build-ui-ctl:
+ifeq ($(shell ls ./src/ctl/ui/build 2>/dev/null),)
+	# Need to use subshell output of set-node-version as cannot
+	# set side-effects of nvm to later commands
+	@echo "Building UI components, if npm install or build fails try: npm cache clean"
+	make node-yarn-run \
+		node_version="6" \
+		node_cmd="cd $(m3_package_path)/src/ctl/ui && yarn install && npm run build"
+else
+	@echo "Skip building UI components, already built, to rebuild first make clean"
+endif
+	# Move public assets into public subdirectory so that it can
+	# be included in the single statik package built from ./ui/build
+	rm -rf ./src/ctl/ui/build/public
+	cp -r ./src/ctl/public ./src/ctl/ui/build/public
+
+.PHONY: build-ui-ctl-statik-gen
+build-ui-ctl-statik-gen: build-ui-ctl-statik license-gen-ctl
+
+.PHONY: build-ui-ctl-statik
+build-ui-ctl-statik: build-ui-ctl install-tools
+	mkdir -p ./src/ctl/generated/ui
+	$(retool_bin_path)/statik -f -src ./src/ctl/ui/build -dest ./src/ctl/generated/ui -p statik
+
+.PHONY: node-yarn-run
+node-yarn-run:
+	make node-run \
+		node_version="$(node_version)" \
+		node_cmd="(yarn --version 2>&1 >/dev/null || npm install -g yarn) && $(node_cmd)"
+
+.PHONY: node-run
+node-run:
+ifneq ($(shell brew --prefix nvm 2>/dev/null),)
+	@echo "Using nvm from brew to select node version $(node_version)"
+	source $(shell brew --prefix nvm)/nvm.sh && nvm use $(node_version) && bash -c "$(node_cmd)"
+else ifneq ($(shell type nvm 2>/dev/null),)
+	@echo "Using nvm to select node version $(node_version)"
+	nvm use $(node_version) && bash -c "$(node_cmd)"
+else
+	node --version 2>&1 >/dev/null || \
+		(echo "Trying apt install" && which apt-get && (curl -sL https://deb.nodesource.com/setup_$(node_version).x | bash) && apt-get install -y nodejs) || \
+		(echo "Trying apk install" && which apk && apk add --update nodejs nodejs-npm) || \
+		(echo "No node install or known package manager" && exit 1)
+	@echo "Not using nvm, using node version $(shell node --version)"
+	bash -c "$(node_cmd)"
+endif
+
 .PHONY: metalint
 metalint: install-gometalinter install-linter-badtime install-linter-importorder
 	@echo "--- metalinting src/"
@@ -371,5 +425,6 @@ clean:
 	@rm -f *.html *.xml *.out *.test
 	@rm -rf $(BUILD)
 	@rm -rf $(VENDOR)
+	@rm -rf ./src/ctl/ui/build
 
 .DEFAULT_GOAL := all

glide.yaml

@@ -231,6 +231,9 @@ import:
     repo: https://github.com/go-playground/validator.git
     vcs: git
 
+  - package: github.com/go-playground/universal-translator
+    version: 71201497bace774495daed26a3874fd339e0b538
+
   - package: gopkg.in/yaml.v2
     version: 5420a8b6744d3b0345ab293f6fcba19c978f1183
     repo: https://github.com/go-yaml/yaml.git

src/aggregator/README.md

@@ -1,18 +1,5 @@
-## WARNING: This is Alpha software and not intended for use until a stable release.
-
-# M3Aggregator [![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Coverage Status][cov-img]][cov]
+# M3Aggregator
 
 Metrics aggregation server. Supports multiple metric types including counters, gauges, and timers as
 well as efficient summary metrics for timer data. Supports dynamically changing sampling resolution
 as query and storage requirements change.
-
-<hr>
-
-This project is released under the [Apache License, Version 2.0](LICENSE).
-
-[doc-img]: https://godoc.org/github.com/m3db/m3aggregator?status.svg
-[doc]: https://godoc.org/github.com/m3db/m3aggregator
-[ci-img]: https://travis-ci.org/m3db/m3aggregator.svg?branch=master
-[ci]: https://travis-ci.org/m3db/m3aggregator
-[cov-img]: https://coveralls.io/repos/m3db/m3aggregator/badge.svg?branch=master&service=github
-[cov]: https://coveralls.io/github/m3db/m3aggregator?branch=master

src/ctl/services/r2ctl/config/r2ctl.go → src/cmd/services/m3ctl/config/config.go

@@ -26,13 +26,13 @@ import (
 
 	"github.com/m3db/m3/src/cluster/client/etcd"
 	clusterkv "github.com/m3db/m3/src/cluster/kv"
+	"github.com/m3db/m3/src/ctl/auth"
+	r2store "github.com/m3db/m3/src/ctl/service/r2/store"
+	r2kv "github.com/m3db/m3/src/ctl/service/r2/store/kv"
+	"github.com/m3db/m3/src/ctl/service/r2/store/stub"
 	"github.com/m3db/m3/src/metrics/rules"
 	ruleskv "github.com/m3db/m3/src/metrics/rules/store/kv"
 	"github.com/m3db/m3/src/metrics/rules/validator"
-	"github.com/m3db/m3ctl/auth"
-	r2store "github.com/m3db/m3ctl/service/r2/store"
-	r2kv "github.com/m3db/m3ctl/service/r2/store/kv"
-	"github.com/m3db/m3ctl/service/r2/store/stub"
 	"github.com/m3db/m3x/instrument"
 	"github.com/m3db/m3x/log"
 )

src/ctl/services/r2ctl/config/server.go → src/cmd/services/m3ctl/config/server.go

@@ -23,7 +23,7 @@ package config
 import (
 	"time"
 
-	httpserver "github.com/m3db/m3ctl/server/http"
+	httpserver "github.com/m3db/m3/src/ctl/server/http"
 	"github.com/m3db/m3x/instrument"
 )
 

src/ctl/services/r2ctl/main/main.go → src/cmd/services/m3ctl/main/main.go

@@ -29,11 +29,11 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/m3db/m3ctl/auth"
-	"github.com/m3db/m3ctl/server/http"
-	"github.com/m3db/m3ctl/service/health"
-	"github.com/m3db/m3ctl/service/r2"
-	"github.com/m3db/m3ctl/services/r2ctl/config"
+	"github.com/m3db/m3/src/cmd/services/m3ctl/config"
+	"github.com/m3db/m3/src/ctl/auth"
+	"github.com/m3db/m3/src/ctl/server/http"
+	"github.com/m3db/m3/src/ctl/service/health"
+	"github.com/m3db/m3/src/ctl/service/r2"
 	"github.com/m3db/m3x/clock"
 	xconfig "github.com/m3db/m3x/config"
 	"github.com/m3db/m3x/instrument"
@@ -46,7 +46,7 @@ const (
 )
 
 func main() {
-	configFile := flag.String("f", "config/base.yaml", "configuration file")
+	configFile := flag.String("f", "m3ctl.yml", "configuration file")
 	flag.Parse()
 
 	if len(*configFile) == 0 {
@@ -135,7 +135,7 @@ func main() {
 		logger.Fatalf("could not create new server: %v", err)
 	}
 
-	logger.Infof("starting HTTP server on: %s", listenAddr)
+	logger.Infof("starting http server on: %s", listenAddr)
 	if err := server.ListenAndServe(); err != nil {
 		logger.Fatalf("could not start serving traffic: %v", err)
 	}
@@ -146,7 +146,7 @@ func main() {
 	doneCh := make(chan struct{})
 	go func() {
 		server.Close()
-		logger.Infof("HTTP server closed")
+		logger.Infof("http server closed")
 		doneCh <- struct{}{}
 	}()