Generate protobuf code for Go and Python services (open-telemetry#1794)

Part of open-telemetry#1787
to generate profobuf files for all services.

This should help unblock
open-telemetry#1754 to allow
dependabot to manage dependecy upgrades for the Go services.

There are new Makefile recipes for managing the protobuf files.

1. docker-generate-protobuf - to generate the protobuf files with docker
   so that the only dependency on the machine needed is docker.
2. clean - to remove the protobuf files generated
3. check-clean-work-tree - to check that the working tree is clean and
   to help with verifying that the protobuf files are updated for all of
   the services when there are changes to the protobuf definition.

There's a new check in the GitHub Actions workflow to verify that the
protobuf code is generated. It is only verifying that the protobuf code
is generated for Go and Python, but other services can apply the same
workflow by updating the docker-gen-proto.sh script to uncomment the
function call for the service.

The dockerfiles for the Go and Python services have been updated to not
generate the protobuf files during the build process. Instead, this
function has been refactored into the genproto directory of each
service.

Signed-off-by: Charlie Le <[email protected]>

Author: CharlieTLe

.github/workflows/component-build-images.yml

@@ -26,8 +26,20 @@ on:
         type: string
 
 jobs:
+  protobufcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Generate
+        run: make clean docker-generate-protobuf
+      - name: Check Clean Work Tree
+        run: make check-clean-work-tree
+
   build_and_push_images:
     runs-on: ubuntu-latest
+    needs: protobufcheck
+
     permissions:
       contents: read
       packages: write

.gitignore

@@ -52,9 +52,6 @@ test/tracetesting/tracetesting-vars.yaml
 /src/frontend/pb/
 /src/frontend/protos/
 /src/paymentservice/demo.proto
-/src/recommendationservice/demo_pb2*.py
 /src/shippingservice/proto/
-/src/productcatalogservice/genproto
 /src/currencyservice/proto
-/src/checkoutservice/genproto
 /src/accountingservice/genproto

.licenserc.json

@@ -45,6 +45,8 @@
         "src/featureflagservice/assets/vendor/",
         "src/featureflagservice/priv/",
         "src/productcatalogservice/genproto/",
+        "src/recommendationservice/demo_pb2.py",
+        "src/recommendationservice/demo_pb2_grpc.py",
         "internal/tools/"
     ]
   }

CHANGELOG.md

@@ -21,6 +21,8 @@ the release.
   ([#1786](https://github.com/open-telemetry/opentelemetry-demo/pull/1786))
 * [chore] Add multi-platform build support
   ([#1785](https://github.com/open-telemetry/opentelemetry-demo/pull/1785))
+* [chore] Generate protobuf code for Go and Python services
+  ([#1794](https://github.com/open-telemetry/opentelemetry-demo/pull/1784))
 
 ## 1.12.0
 

Makefile

@@ -133,6 +133,25 @@ generate-kubernetes-manifests:
 	echo "  name: otel-demo" >> kubernetes/opentelemetry-demo.yaml
 	helm template opentelemetry-demo open-telemetry/opentelemetry-demo --namespace otel-demo | sed '/helm.sh\/chart\:/d' | sed '/helm.sh\/hook/d' | sed '/managed-by\: Helm/d' >> kubernetes/opentelemetry-demo.yaml
 
+.PHONY: docker-generate-protobuf
+docker-generate-protobuf:
+	./docker-gen-proto.sh
+
+.PHONY: clean
+clean:
+	rm -rf ./src/{checkoutservice,productcatalogservice}/genproto/oteldemo/
+	rm -rf ./src/recommendationservice/{demo_pb2,demo_pb2_grpc}.py
+
+.PHONY: check-clean-work-tree
+check-clean-work-tree:
+	@if ! git diff --quiet; then \
+	  echo; \
+	  echo 'Working tree is not clean, did you forget to run "make docker-generate-protobuf"?'; \
+	  echo; \
+	  git status; \
+	  exit 1; \
+	fi
+
 .PHONY: start
 start:
 	$(DOCKER_COMPOSE_CMD) $(DOCKER_COMPOSE_ENV) up --force-recreate --remove-orphans --detach

docker-gen-proto.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+set -e # Exit immediately if a command exits with a non-zero status.
+set -x # Print commands and their arguments as they are executed
+
+# This script is used to generate protobuf files for all services with Docker.
+
+gen_proto_go() {
+  echo "Generating Go protobuf files for $1"
+  docker build -f "src/$1/genproto/Dockerfile" -t "$1-genproto" .
+  docker run --rm -v $(pwd):/build "$1-genproto" \
+    protoc -I /build/pb /build/pb/demo.proto --go_out="./src/$1/" --go-grpc_out="./src/$1/"
+}
+
+gen_proto_python() {
+  echo "Generating Python protobuf files for $1"
+  docker build -f "src/$1/genproto/Dockerfile" -t "$1-genproto" .
+  docker run --rm -v $(pwd):/build "$1-genproto" \
+    python -m grpc_tools.protoc -I /build/pb/ --python_out="./src/$1/" --grpc_python_out="./src/$1/" /build/pb/demo.proto
+}
+
+#gen_proto_dotnet accountingservice
+#gen_proto_java adservice
+#gen_proto_dotnet cartservice
+gen_proto_go checkoutservice
+#gen_proto_cpp currencyservice
+#gen_proto_ruby emailservice
+#gen_proto_ts frontend
+#gen_proto_js paymentservice
+gen_proto_go productcatalogservice
+#gen_proto_php quoteservice
+gen_proto_python recommendationservice
+#gen_proto_rust shippingservice

internal/tools/sanitycheck.py

@@ -84,7 +84,7 @@ def sanitycheck(pattern, allow_utf8 = False, allow_eol = (CRLF, LF), indent = 1)
 retval += sanitycheck('**/*.md', allow_eol = (LF,))
 retval += sanitycheck('**/*.proj', allow_eol = (LF,), indent = 2)
 retval += sanitycheck('**/*.props', allow_eol = (LF,), indent = 2)
-retval += sanitycheck('**/*.py', allow_eol = (LF,), indent = 4)
+retval += sanitycheck('**/[!demo_pb2]*.py', allow_eol = (LF,), indent = 4)
 retval += sanitycheck('**/*.sln', allow_utf8 = True, indent = 4)
 retval += sanitycheck('**/*.targets', allow_eol = (LF,), indent = 2)
 retval += sanitycheck('**/*.xml', allow_eol = (LF,), indent = 2)

src/checkoutservice/Dockerfile

@@ -6,22 +6,15 @@ FROM golang:1.22-alpine AS builder
 
 WORKDIR /usr/src/app/
 
-RUN apk update \
-    && apk add --no-cache make protobuf-dev
-
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=bind,source=./src/checkoutservice/go.sum,target=go.sum \
     --mount=type=bind,source=./src/checkoutservice/go.mod,target=go.mod \
-    --mount=type=bind,source=./src/checkoutservice/tools.go,target=tools.go \
-    go mod download \
-    && go list -e -f '{{range .Imports}}{{.}} {{end}}' tools.go | CGO_ENABLED=0 xargs go install -mod=readonly
+    go mod download
 
 RUN --mount=type=cache,target=/go/pkg/mod/ \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=bind,rw,source=./src/checkoutservice,target=. \
-    --mount=type=bind,rw,source=./pb,target=./pb \
-    protoc -I ./pb ./pb/demo.proto --go_out=./ --go-grpc_out=./ \
-    && go build -ldflags "-s -w" -o /go/bin/checkoutservice/ ./
+    go build -ldflags "-s -w" -o /go/bin/checkoutservice/ ./
 
 FROM alpine
 

src/checkoutservice/README.md

@@ -20,13 +20,10 @@ docker compose build checkoutservice
 
 ## Regenerate protos
 
-> [!NOTE]
-> [`protoc`](https://grpc.io/docs/protoc-installation/) is required.
-
-To regenerate gRPC code run:
+To build the protos, run from the root directory:
 
 ```sh
-go generate
+make docker-generate-protobuf
 ```
 
 ## Bump dependencies

src/checkoutservice/genproto/Dockerfile

@@ -0,0 +1,16 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+FROM golang:1.22-alpine
+
+WORKDIR /build
+
+RUN apk add --no-cache protobuf-dev
+
+COPY ./src/checkoutservice/go.mod ./
+COPY ./src/checkoutservice/go.sum ./
+COPY ./src/checkoutservice/tools.go ./
+
+RUN go env -w GOMODCACHE=/root/.cache/go-build
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    go list -e -f '{{range .Imports}}{{.}} {{end}}' tools.go | CGO_ENABLED=0 xargs go install -mod=readonly