Migrate to buf schema v2 and move api related tasks to dedicated Makefile

.github/workflows/main.yml

@@ -69,7 +69,7 @@ jobs:
       - name: Update API compatibility descriptors
         run: |
           # log if descriptors changed, useful for "update descriptors" PRs
-          make descriptors
+          make -C api descriptors
           git diff --text
 
       - name: Run check-license

Makefile.include

@@ -1,5 +1,4 @@
 # vim: ts=8:sw=8:ft=make:noai:noet
-SWAGGER_UI_VERSION?=latest
 
 .PHONY: default init release gen test clean all
 default: help
@@ -29,9 +28,9 @@ release:              ## Build release versions of all components
 	make -C qan-api2 release
 
 gen: clean            ## Generate files
-	make gen-api
-	make gen-alertmanager
-	make clean_swagger
+	make -C api gen
+	make -C api gen-alertmanager
+	make -C api clean-swagger
 
 	make -C agent gen
 	make -C admin gen
@@ -43,102 +42,13 @@ gen: clean            ## Generate files
 	make format ## TODO: One formatting run is not enough, figure out why.
 	go install -v ./...
 
+clean:               ## Remove generated files
+	make -C api clean
+
 gen-mocks:
 	find . -name mock_*.go -delete
 	./bin/mockery --config .mockery.yaml
 
-gen-api:            ## Generate PMM API
-	# generated by descriptors target
-	bin/buf breaking --against descriptor.bin api
-
-	bin/buf generate -v api
-
-	for API in api/agentlocalpb api/serverpb api/inventorypb api/managementpb api/managementpb/dbaas api/managementpb/ia api/managementpb/alerting api/managementpb/backup api/managementpb/dump api/managementpb/azure api/managementpb/role api/qanpb api/managementpb/agent api/managementpb/node api/managementpb/service api/platformpb api/userpb; do \
-		set -x ; \
-		bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
-		bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
-		bin/swagger validate $$API/json/$$(basename $$API).json ; \
-		bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \
-			--additional-initialism=aws \
-			--additional-initialism=db \
-			--additional-initialism=ok \
-			--additional-initialism=pmm \
-			--additional-initialism=psmdb \
-			--additional-initialism=pxc \
-			--additional-initialism=pt \
-			--additional-initialism=qan \
-			--additional-initialism=rds \
-			--additional-initialism=sql \
-			--additional-initialism=ha ; \
-	done
-
-	# generate public API spec, omit agentlocalpb (always private),
-	# as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below)
-	bin/swagger mixin --output=api/swagger/swagger.json \
-		api/swagger/header.json \
-		api/serverpb/json/serverpb.json \
-		api/userpb/json/userpb.json \
-		api/inventorypb/json/inventorypb.json \
-		api/managementpb/json/managementpb.json \
-		api/managementpb/backup/json/backup.json \
-		api/managementpb/alerting/json/alerting.json
-	bin/swagger validate api/swagger/swagger.json
-
-	bin/swagger-order --output=api/swagger/swagger.json api/swagger/swagger.json
-
-	# generate API spec with all PMM Server APIs (omit agentlocalpb)
-	bin/swagger mixin --output=api/swagger/swagger-dev.json \
-		api/swagger/header-dev.json \
-		api/serverpb/json/serverpb.json \
-		api/userpb/json/userpb.json \
-		api/inventorypb/json/inventorypb.json \
-		api/managementpb/json/managementpb.json \
-		api/managementpb/dbaas/json/dbaas.json \
-		api/managementpb/ia/json/ia.json \
-		api/managementpb/alerting/json/alerting.json \
-		api/managementpb/backup/json/backup.json \
-		api/managementpb/dump/json/dump.json \
-		api/managementpb/azure/json/azure.json \
-		api/managementpb/role/json/role.json \
-		api/managementpb/agent/json/agent.json \
-		api/managementpb/node/json/node.json \
-		api/managementpb/service/json/service.json \
-		api/qanpb/json/qanpb.json \
-		api/platformpb/json/platformpb.json
-	bin/swagger validate api/swagger/swagger-dev.json
-
-	bin/swagger-order --output=api/swagger/swagger-dev.json api/swagger/swagger-dev.json
-
-gen-alertmanager:     ## Generate Alertmanager client
-	bin/swagger generate client --model-package=ammodels --client-package=amclient --spec=api/alertmanager/openapi.yaml --target=api/alertmanager
-
-clean_swagger:
-	find api -name '*.swagger.json' -print -delete
-
-
-ifeq ($(shell test "${SWAGGER_UI_VERSION}" = "latest" && echo 1 || echo 0), 1)
-get_swagger_version:
-override SWAGGER_UI_VERSION = $(shell curl --silent --head https://github.com/swagger-api/swagger-ui/releases/latest | grep -E '^[lL]ocation' | sed 's;^.*/;;')
-else
-get_swagger_version:
-endif
-
-update-swagger: get_swagger_version               ## Update Swagger UI in api/swagger, use `SWAGGER_UI_VERSION=xxx make update-swagger` to choose a version other than latest
-	curl --output /tmp/swagger.tar.gz -sSfL "https://github.com/swagger-api/swagger-ui/archive/refs/tags/${SWAGGER_UI_VERSION}.tar.gz"
-	tar xf /tmp/swagger.tar.gz --transform 's;swagger-ui-.*/dist/;api/swagger/;g'
-	rm -f api/swagger/.npmrc
-	sed -i 's/url: ".*"/url: "\/swagger.json",\n    validatorUrl: "none"/' api/swagger/swagger-initializer.js
-
-clean: clean_swagger  ## Remove generated files
-	find api -name '*.pb.go' -print -delete
-	find api -name '*.pb.gw.go' -print -delete
-	find api -name '*.validate.go' -print -delete
-
-	for API in api/agentlocalpb api/serverpb api/inventorypb api/managementpb api/managementpb/dbaas api/managementpb/ia api/managementpb/alerting api/managementpb/backup api/management/dump api/managementpb/role api/managementpb/agent api/managementpb/node api/managementpb/service api/qanpb api/platformpb ; do \
-		rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
-	done
-	rm -f api/swagger/swagger.json api/swagger/swagger-dev.json
-
 test-common:          ## Run tests from API (and other shared) packages only (i.e it ignores directories that are explicitly listed)
 	go test $(shell go list ./... | grep -v -e admin -e agent -e managed -e api-tests -e qan-api2 -e update)
 
@@ -159,14 +69,10 @@ check-all: check-license check    ## Run golangci linter to check for changes ag
 FILES = $(shell find . -type f -name '*.go')
 
 format:               ## Format source code
+	make -C api format
 	bin/gofumpt -l -w $(FILES)
 	bin/goimports -local github.com/percona/pmm -l -w $(FILES)
 	bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES)
-	bin/buf format api -w
 
 serve:                ## Serve API documentation with nginx
 	nginx -p . -c api/nginx/nginx.conf
-
-descriptors:          ## Update API compatibility descriptors
-	#./prototool break descriptor-set . -o api/api.descriptor
-	bin/buf build -o descriptor.bin --as-file-descriptor-set api

api/Makefile

@@ -0,0 +1,107 @@
+SWAGGER_UI_VERSION?=latest
+
+default: help
+
+help:                           ## Display this help message
+	@echo "Please use \`make <target>\` where <target> is one of:"
+	@grep -h '^[a-zA-Z]' $(MAKEFILE_LIST) | \
+		awk -F ':.*?## ' 'NF==2 {printf "  %-26s%s\n", $$1, $$2}'
+	@echo
+
+gen: clean            ## Generate PMM API
+	# generated by descriptors target
+	../bin/buf breaking --against descriptor.bin
+
+	../bin/buf generate -v .
+
+	for API in agentlocalpb serverpb inventorypb managementpb managementpb/dbaas managementpb/ia managementpb/alerting managementpb/backup managementpb/dump managementpb/azure managementpb/role qanpb managementpb/agent managementpb/node managementpb/service platformpb userpb; do \
+		set -x ; \
+		../bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
+		../bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
+		../bin/swagger validate $$API/json/$$(basename $$API).json ; \
+		../bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \
+			--additional-initialism=aws \
+			--additional-initialism=db \
+			--additional-initialism=ok \
+			--additional-initialism=pmm \
+			--additional-initialism=psmdb \
+			--additional-initialism=pxc \
+			--additional-initialism=pt \
+			--additional-initialism=qan \
+			--additional-initialism=rds \
+			--additional-initialism=sql \
+			--additional-initialism=ha ; \
+	done
+
+	# generate public API spec, omit agentlocalpb (always private),
+	# as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below)
+	../bin/swagger mixin --output=swagger/swagger.json \
+		swagger/header.json \
+		serverpb/json/serverpb.json \
+		userpb/json/userpb.json \
+		inventorypb/json/inventorypb.json \
+		managementpb/json/managementpb.json \
+		managementpb/backup/json/backup.json \
+		managementpb/alerting/json/alerting.json
+	../bin/swagger validate swagger/swagger.json
+
+	../bin/swagger-order --output=swagger/swagger.json swagger/swagger.json
+
+	# generate API spec with all PMM Server APIs (omit agentlocalpb)
+	../bin/swagger mixin --output=swagger/swagger-dev.json \
+		swagger/header-dev.json \
+		serverpb/json/serverpb.json \
+		userpb/json/userpb.json \
+		inventorypb/json/inventorypb.json \
+		managementpb/json/managementpb.json \
+		managementpb/dbaas/json/dbaas.json \
+		managementpb/ia/json/ia.json \
+		managementpb/alerting/json/alerting.json \
+		managementpb/backup/json/backup.json \
+		managementpb/dump/json/dump.json \
+		managementpb/azure/json/azure.json \
+		managementpb/role/json/role.json \
+		managementpb/agent/json/agent.json \
+		managementpb/node/json/node.json \
+		managementpb/service/json/service.json \
+		qanpb/json/qanpb.json \
+		platformpb/json/platformpb.json
+	../bin/swagger validate swagger/swagger-dev.json
+
+	../bin/swagger-order --output=swagger/swagger-dev.json swagger/swagger-dev.json
+
+gen-alertmanager:     ## Generate Alertmanager client
+	../bin/swagger generate client --model-package=ammodels --client-package=amclient --spec=alertmanager/openapi.yaml --target=alertmanager
+
+format:               ## Format API definitions
+	../bin/buf format . -w
+
+clean-swagger:
+	find . -name '*.swagger.json' -print -delete
+
+
+ifeq ($(shell test "${SWAGGER_UI_VERSION}" = "latest" && echo 1 || echo 0), 1)
+get_swagger_version:
+override SWAGGER_UI_VERSION = $(shell curl --silent --head https://github.com/swagger-swagger-ui/releases/latest | grep -E '^[lL]ocation' | sed 's;^.*/;;')
+else
+get_swagger_version:
+endif
+
+update-swagger: get_swagger_version               ## Update Swagger UI in swagger, use `SWAGGER_UI_VERSION=xxx make update-swagger` to choose a version other than latest
+	curl --output /tmp/swagger.tar.gz -sSfL "https://github.com/swagger-swagger-ui/archive/refs/tags/${SWAGGER_UI_VERSION}.tar.gz"
+	tar xf /tmp/swagger.tar.gz --transform 's;swagger-ui-.*/dist/;swagger/;g'
+	rm -f swagger/.npmrc
+	sed -i 's/url: ".*"/url: "\/swagger.json",\n    validatorUrl: "none"/' swagger/swagger-initializer.js
+
+clean: clean-swagger  ## Remove generated files
+	find . -name '*.pb.go' -print -delete
+	find . -name '*.pb.gw.go' -print -delete
+	find . -name '*.validate.go' -print -delete
+
+	for API in agentlocalpb serverpb inventorypb managementpb userpb managementpb/azure managementpb/dbaas  managementpb/dump managementpb/ia managementpb/alerting managementpb/backup management/dump managementpb/role managementpb/agent managementpb/node managementpb/service qanpb platformpb ; do \
+		rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
+	done
+	rm -f swagger/swagger.json swagger/swagger-dev.json
+
+descriptors:          ## Update API compatibility descriptors
+	../bin/buf build -o descriptor.bin --as-file-descriptor-set .

api/buf.gen.yaml

@@ -0,0 +1,33 @@
+version: v2
+managed:
+  enabled: true
+  disable:
+    - file_option: go_package
+      module: buf.build/googleapis/googleapis
+    - file_option: go_package
+      module: buf.build/grpc-ecosystem/grpc-gateway
+    - file_option: go_package
+      module: buf.build/envoyproxy/protoc-gen-validate
+  override:
+    - file_option: go_package_prefix
+      value: github.com/percona/pmm/api
+plugins:
+  - local: ../bin/protoc-gen-go
+    out: .
+    opt: paths=source_relative
+  - local: ../bin/protoc-gen-go-grpc
+    out: .
+    opt: paths=source_relative
+  - local: ../bin/protoc-gen-grpc-gateway
+    out: .
+    opt: paths=source_relative
+  - local: ../bin/protoc-gen-validate
+    out: .
+    opt:
+      - paths=source_relative
+      - lang=go
+  - local: ../bin/protoc-gen-openapiv2
+    out: .
+    opt:
+      - simple_operation_ids=true
+      - json_names_for_fields=false

api/buf.lock

@@ -1,18 +1,12 @@
 # Generated by buf. DO NOT EDIT.
-version: v1
+version: v2
 deps:
-  - remote: buf.build
-    owner: envoyproxy
-    repository: protoc-gen-validate
+  - name: buf.build/envoyproxy/protoc-gen-validate
     commit: eac44469a7af47e7839a7f1f3d7ac004
-    digest: shake256:0feabcde01b6b11e3c75a5e3f807968d5995626546f39c37e5d4205892b3a59cced0ed83b35a2eb9e6dddd3309660ad46b737c9dcd224b425de0a6654ce04417
-  - remote: buf.build
-    owner: googleapis
-    repository: googleapis
+    digest: b5:2ef7b25d9671fde82d58278e94d209788fb6a42936cc2d60e42303b67dcdb72b2d0e121238ba44caf1870a34edab55518df5d3f1ee1e62b8d9b6cf98817eae6e
+  - name: buf.build/googleapis/googleapis
     commit: 28151c0d0a1641bf938a7672c500e01d
-    digest: shake256:49215edf8ef57f7863004539deff8834cfb2195113f0b890dd1f67815d9353e28e668019165b9d872395871eeafcbab3ccfdb2b5f11734d3cca95be9e8d139de
-  - remote: buf.build
-    owner: grpc-ecosystem
-    repository: grpc-gateway
+    digest: b5:93b70089baa4fc05a92d3e52db91a4b7812db3b57b9664f6cb301733938cb630e377a938e8a56779388171c749c1d42a2e9a6c6230f2ff45f127a8102a6a27d0
+  - name: buf.build/grpc-ecosystem/grpc-gateway
     commit: 3f42134f4c564983838425bc43c7a65f
-    digest: shake256:3d11d4c0fe5e05fda0131afefbce233940e27f0c31c5d4e385686aea58ccd30f72053f61af432fa83f1fc11cda57f5f18ca3da26a29064f73c5a0d076bba8d92
+    digest: b5:291b947d8ac09492517557e4e72e294788cb8201afc7d0df7bda80fa10931adb60d4d669208a7696bf24f1ecb2a33a16d4c1e766e6f31809248b00343119569b

api/buf.yaml

@@ -1,11 +1,20 @@
-version: v1
-breaking:
-  use:
-    - FILE
-lint:
-  use:
-    - DEFAULT
+version: v2
+modules:
+  - path: .
 deps:
+  - buf.build/envoyproxy/protoc-gen-validate
   - buf.build/googleapis/googleapis
   - buf.build/grpc-ecosystem/grpc-gateway
-  - buf.build/envoyproxy/protoc-gen-validate
+lint:
+  use:
+    - DEFAULT
+  except:
+    - FIELD_NOT_REQUIRED
+    - PACKAGE_NO_IMPORT_CYCLE
+  disallow_comment_ignores: true
+breaking:
+  use:
+    - FILE
+  except:
+    - EXTENSION_NO_DELETE
+    - FIELD_SAME_DEFAULT