CI check to verify that auto generated protobuf files match that of t…

…he proto spec

.github/workflows/test.yml → .github/workflows/ci.yml

@@ -75,4 +75,26 @@ jobs:
 
       - name: Run Tests
         run: |
-          make test-pkg
+          make test-pkg
+  
+  verify_protobufs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.21"
+
+      - name: Cache Go modules
+        uses: actions/cache@v3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      
+      - name: Verify Protobuf Codegen
+        run: |
+          make verify-protocol

Makefile

@@ -48,7 +48,10 @@ stop:
 	cd hack && okteto down --file okteto.yml
 
 protocol:
-	cd proto && ./gen.sh
+	./bin/gen_proto.sh
+
+verify-protocol:
+	./bin/verify_proto.sh
 
 test-pkg:
 	go test -v ./pkg/... -bench=./pkg/..

bin/gen_proto.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Generate code for gateway services
+protoc -I ./pkg/scheduler/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/scheduler/scheduler.proto
+protoc -I ./pkg/worker/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/worker/worker.proto
+
+protoc -I ./pkg/gateway/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/gateway/gateway.proto
+protoc -I ./pkg/gateway/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/  ./pkg/gateway/gateway.proto
+
+# Generate code for abstractions
+protoc -I ./pkg/abstractions/image/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/image/image.proto
+protoc -I ./pkg/abstractions/image/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/  ./pkg/abstractions/image/image.proto
+
+protoc -I ./pkg/abstractions/map/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/map/map.proto
+protoc -I ./pkg/abstractions/map/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/  ./pkg/abstractions/map/map.proto
+
+protoc -I ./pkg/abstractions/function/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/function/function.proto
+protoc -I ./pkg/abstractions/function/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/  ./pkg/abstractions/function/function.proto
+
+protoc -I ./pkg/abstractions/queue/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/queue/queue.proto
+protoc -I ./pkg/abstractions/queue/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/  ./pkg/abstractions/queue/queue.proto
+
+protoc -I ./pkg/abstractions/volume/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/volume/volume.proto
+protoc -I ./pkg/abstractions/volume/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/  ./pkg/abstractions/volume/volume.proto
+
+protoc -I ./pkg/abstractions/taskqueue/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/taskqueue/taskqueue.proto
+protoc -I ./pkg/abstractions/taskqueue/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/  ./pkg/abstractions/taskqueue/taskqueue.proto
+
+protoc -I ./pkg/abstractions/endpoint/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/endpoint/endpoint.proto
+protoc -I ./pkg/abstractions/endpoint/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/  ./pkg/abstractions/endpoint/endpoint.proto
+
+protoc -I ./pkg/abstractions/container/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/container/container.proto
+protoc -I ./pkg/abstractions/container/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/  ./pkg/abstractions/container/container.proto
+
+protoc -I ./pkg/abstractions/output/ --go_out=./proto --go_opt=paths=source_relative --go-grpc_out=./proto --go-grpc_opt=paths=source_relative ./pkg/abstractions/output/output.proto
+protoc -I ./pkg/abstractions/output/ --python_betterproto_beta9_out=./sdk/src/beta9/clients/  ./pkg/abstractions/output/output.proto

bin/verify_proto.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# This scripts is automatically run by CI to prevent pull requests missing running genproto.sh
+# after changing *.proto file.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+tmpWorkDir=$(mktemp -d -t 'twd.XXXXXX')
+mkdir "$tmpWorkDir/beta9"
+tmpWorkDir="$tmpWorkDir/beta9"
+cp -r . "$tmpWorkDir"
+pushd "$tmpWorkDir"
+git add -A
+git commit -m init || true # maybe fail because nothing to commit 
+./bin/gen_proto.sh
+diff=$(git diff --numstat | awk '{print $3}')
+popd
+if [ -z "$diff" ]; then
+  echo "PASSED genproto-verification!"
+  exit 0
+fi
+echo "Failed genproto-verification!" >&2
+printf "* Found changed files:\n%s\n" "$diff" >&2
+echo "* Please rerun genproto.sh after changing *.proto file" >&2
+echo "* Run ./scripts/genproto.sh" >&2
+exit 1