Add sourceinfo plugin

Makefile

@@ -35,9 +35,11 @@ test: build $(BIN)/jv ## Run unit tests
 golden: generate
 	rm -rf internal/testdata/pubsub
 	rm -rf internal/testdata/jsonschema
+	rm -rf internal/testdata/sourceinfo
 	buf build ./internal/proto -o -#format=json > ./internal/testdata/codegenrequest/input.json
 	go run internal/cmd/pubsub-generate-testdata/main.go internal/testdata/pubsub
 	go run internal/cmd/jsonschema-generate-testdata/main.go internal/testdata/jsonschema
+	go run internal/cmd/sourceinfo-generate-testdata/main.go internal/testdata/sourceinfo
 
 .PHONY: build
 build: generate ## Build all packages

README.md

@@ -162,6 +162,19 @@ Results in the following JSON Schema files:
 
 </details>
 
+## SourceInfo
+
+The sourceinfo plugin provides a protoc-gen-sourceinfo command that generates an `.sourceinfo.binpb` file, for every `.proto` file.
+Each generated file contains the associated serialized `google.protobuf.SourceCodeInfo` value.
+These files can then be loaded directly or embedded in a binary.
+
+The following helpers are provided for go via `import "github.com/bufbuild/protoschema-plugins/sourceinfo"`:
+
+- `sourceinfo.GlobalFiles()`: A drop-in replacement for protoregistry.GlobalFiles
+- `sourceinfo.GlobalTypes()`: A drop-in replacement for protogregistry.GlobalTypes
+- `sourceinfo.RegisterAll(root string)`: Registers all `*.sourceinfo.binpb` files found under the root dir.
+- `sourceinfo.RegisterAllFS(fsys fs.FS, root string)`: Same as above, except uses the given file system.
+
 ## Community
 
 For help and discussion around Protobuf, best practices, and more, join us

cmd/protoc-gen-jsonschema/main.go

@@ -16,10 +16,10 @@ package main
 
 import (
 	"github.com/bufbuild/protoplugin"
-	"github.com/bufbuild/protoschema-plugins/internal/protoschema"
+	"github.com/bufbuild/protoschema-plugins/internal"
 	"github.com/bufbuild/protoschema-plugins/internal/protoschema/plugin/pluginjsonschema"
 )
 
 func main() {
-	protoplugin.Main(protoplugin.HandlerFunc(pluginjsonschema.Handle), protoplugin.WithVersion(protoschema.Version()))
+	protoplugin.Main(protoplugin.HandlerFunc(pluginjsonschema.Handle), protoplugin.WithVersion(internal.Version()))
 }

cmd/protoc-gen-pubsub/main.go

@@ -16,10 +16,10 @@ package main
 
 import (
 	"github.com/bufbuild/protoplugin"
-	"github.com/bufbuild/protoschema-plugins/internal/protoschema"
+	"github.com/bufbuild/protoschema-plugins/internal"
 	"github.com/bufbuild/protoschema-plugins/internal/protoschema/plugin/pluginpubsub"
 )
 
 func main() {
-	protoplugin.Main(protoplugin.HandlerFunc(pluginpubsub.Handle), protoplugin.WithVersion(protoschema.Version()))
+	protoplugin.Main(protoplugin.HandlerFunc(pluginpubsub.Handle), protoplugin.WithVersion(internal.Version()))
 }

cmd/protoc-gen-sourceinfo/main.go

@@ -0,0 +1,25 @@
+// Copyright 2024 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/bufbuild/protoplugin"
+	"github.com/bufbuild/protoschema-plugins/internal"
+	"github.com/bufbuild/protoschema-plugins/internal/protoschema/plugin/pluginsourceinfo"
+)
+
+func main() {
+	protoplugin.Main(protoplugin.HandlerFunc(pluginsourceinfo.Handle), protoplugin.WithVersion(internal.Version()))
+}

internal/cmd/jsonschema-generate-testdata/main.go

@@ -1,4 +1,4 @@
-// Copyright 2023 Buf Technologies, Inc.
+// Copyright 2023-2024 Buf Technologies, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

internal/cmd/sourceinfo-generate-testdata/main.go

@@ -0,0 +1,90 @@
+// Copyright 2024 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	_ "github.com/bufbuild/protoschema-plugins/internal/gen/proto/bufext/cel/expr/conformance/proto3"
+	"github.com/bufbuild/protoschema-plugins/internal/protoschema/golden"
+	"github.com/bufbuild/protoschema-plugins/internal/protoschema/plugin/pluginsourceinfo"
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+func main() {
+	if err := run(); err != nil {
+		if errString := err.Error(); errString != "" {
+			_, _ = fmt.Fprintln(os.Stderr, errString)
+		}
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	if len(os.Args) != 2 {
+		return fmt.Errorf("usage: %s [output dir]", os.Args[0])
+	}
+	outputDir := os.Args[1]
+	// Make sure the directory exists
+	if err := os.MkdirAll(outputDir, 0755); err != nil {
+		return err
+	}
+
+	testFiles, err := golden.GetTestFiles(filepath.FromSlash("./internal/testdata"))
+	if err != nil {
+		return err
+	}
+	// TODO: Use the normal plugin to generate golden files
+	includePrefixes := []string{
+		filepath.FromSlash("buf/protoschema/test/"),
+		filepath.FromSlash("bufext/cel/expr/conformance/proto3/"),
+	}
+	testFiles.RangeFiles(func(testDesc protoreflect.FileDescriptor) bool {
+		if !shouldIncludeFile(testDesc, includePrefixes) {
+			return true
+		}
+		fileName := pluginsourceinfo.GetSourceInfoPath(testDesc)
+		filePath := filepath.Join(outputDir, fileName)
+		// Create any missing directories
+		if innerErr := os.MkdirAll(filepath.Dir(filePath), 0755); innerErr != nil {
+			err = fmt.Errorf("failed to create directory for %s: %w", filePath, innerErr)
+			return false
+		}
+		var data []byte
+		data, innerErr := pluginsourceinfo.GenFileContents(testDesc)
+		if innerErr != nil {
+			err = fmt.Errorf("failed to generate source info for %s: %w", testDesc.FullName(), innerErr)
+			return false
+		}
+		if innerErr := os.WriteFile(filePath, data, 0600); innerErr != nil {
+			err = fmt.Errorf("failed to write source info to %s: %w", filePath, innerErr)
+			return false
+		}
+		return true
+	})
+	return err
+}
+
+func shouldIncludeFile(fileDesc protoreflect.FileDescriptor, includePrefixes []string) bool {
+	for _, prefix := range includePrefixes {
+		if strings.HasPrefix(fileDesc.Path(), prefix) {
+			return true
+		}
+	}
+	return false
+}

internal/proto/buf/protoschema/test/v1/test_cases.proto

@@ -19,7 +19,9 @@ package buf.protoschema.test.v1;
 import "buf/validate/validate.proto";
 import "bufext/cel/expr/conformance/proto3/test_all_types.proto";
 
+// A message comment.
 message NestedReference {
+  // A field comment.
   bufext.cel.expr.conformance.proto3.TestAllTypes.NestedMessage nested_message = 1;
 }
 

internal/protoschema/golden/golden.go

@@ -23,25 +23,44 @@ import (
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/reflect/protodesc"
 	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/reflect/protoregistry"
 	"google.golang.org/protobuf/types/descriptorpb"
 	"google.golang.org/protobuf/types/dynamicpb"
 )
 
-// GetTestDescriptors returns the test descriptors that were generated from the ./internal/proto
-// directory.
-func GetTestDescriptors(testdataPath string) ([]protoreflect.MessageDescriptor, error) {
+// GetTestFileDescriptorSet returns the FileDescriptorSet descriptors that were generated from the
+// ./internal/proto directory.
+func GetTestFileDescriptorSet(testdataPath string) (*descriptorpb.FileDescriptorSet, error) {
 	inputPath := filepath.Join(filepath.FromSlash(testdataPath), "codegenrequest", "input.json")
 	input, err := os.ReadFile(inputPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open input file descritpor set at %q: %w", inputPath, err)
 	}
 	fdset := &descriptorpb.FileDescriptorSet{}
-	if err = (&protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(input, fdset); err != nil {
+	if err := (&protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(input, fdset); err != nil {
 		return nil, fmt.Errorf("failed to parse file descriptor set at %q: %w", inputPath, err)
 	}
+	return fdset, nil
+}
+
+// GetTestFiles returns the protoregistry.Files for the test files defined in internal/proto.
+func GetTestFiles(testdataPath string) (*protoregistry.Files, error) {
+	fdset, err := GetTestFileDescriptorSet(testdataPath)
+	if err != nil {
+		return nil, err
+	}
 	files, err := protodesc.NewFiles(fdset)
 	if err != nil {
-		return nil, fmt.Errorf("failed to link file descriptor set at %q: %w", inputPath, err)
+		return nil, fmt.Errorf("failed to link file descriptor set at %q: %w", testdataPath, err)
+	}
+	return files, nil
+}
+
+// GetTestDescriptors returns the descriptors for specific test messages defined in internal/proto.
+func GetTestDescriptors(testdataPath string) ([]protoreflect.MessageDescriptor, error) {
+	files, err := GetTestFiles(testdataPath)
+	if err != nil {
+		return nil, err
 	}
 	types := dynamicpb.NewTypes(files)
 
@@ -87,6 +106,7 @@ func CheckGolden(filePath string, data string) error {
 	return nil
 }
 
+// GenerateGolden writes the given data to the golden file.
 func GenerateGolden(filePath string, data string) error {
 	file, err := os.Create(filePath)
 	if err != nil {

internal/protoschema/jsonschema/jsonschema.go

@@ -33,12 +33,16 @@ const (
 	jsString  = "string"
 )
 
-type FieldVisibility int
+// fieldVisibility is an enumeration of the visibility of a field in the JSON schema.
+type fieldVisibility int
 
 const (
-	FieldVisible FieldVisibility = iota
-	FieldHide
-	FieldIgnore
+	// fieldVisible indicates that the field should be included in the schema.
+	fieldVisible fieldVisibility = iota
+	// fieldHide indicates that the field should be included in the schema, but not with a primary name.
+	fieldHide
+	// fieldIgnore indicates that the field should be excluded from the schema.
+	fieldIgnore
 )
 
 type GeneratorOption func(*jsonSchemaGenerator)
@@ -100,7 +104,7 @@ func (p *jsonSchemaGenerator) generateDefault(result map[string]interface{}, des
 	for i := range desc.Fields().Len() {
 		field := desc.Fields().Get(i)
 		visibility := p.shouldIgnoreField(field)
-		if visibility == FieldIgnore {
+		if visibility == fieldIgnore {
 			continue
 		}
 
@@ -111,7 +115,7 @@ func (p *jsonSchemaGenerator) generateDefault(result map[string]interface{}, des
 		aliases := make([]string, 0, 1)
 
 		switch {
-		case visibility == FieldHide:
+		case visibility == fieldHide:
 			aliases = append(aliases, string(field.Name()))
 			if field.JSONName() != string(field.Name()) {
 				aliases = append(aliases, field.JSONName())
@@ -324,18 +328,18 @@ func (p *jsonSchemaGenerator) makeWktGenerators() map[protoreflect.FullName]func
 	return result
 }
 
-func (p *jsonSchemaGenerator) shouldIgnoreField(fdesc protoreflect.FieldDescriptor) FieldVisibility {
+func (p *jsonSchemaGenerator) shouldIgnoreField(fdesc protoreflect.FieldDescriptor) fieldVisibility {
 	const ignoreComment = "jsonschema:ignore"
 	const hideComment = "jsonschema:hide"
 	srcLoc := fdesc.ParentFile().SourceLocations().ByDescriptor(fdesc)
 	switch {
 	case strings.Contains(srcLoc.LeadingComments, ignoreComment),
 		strings.Contains(srcLoc.TrailingComments, ignoreComment):
-		return FieldIgnore
+		return fieldIgnore
 	case strings.Contains(srcLoc.LeadingComments, hideComment),
 		strings.Contains(srcLoc.TrailingComments, hideComment):
-		return FieldHide
+		return fieldHide
 	default:
-		return FieldVisible
+		return fieldVisible
 	}
 }

internal/protoschema/plugin/pluginsourceinfo/pluginsourceinfo.go

@@ -0,0 +1,68 @@
+// Copyright 2024 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pluginsourceinfo
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/bufbuild/protoplugin"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protodesc"
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// FileExtension is the file extension for the source info files.
+const FileExtension = ".sourceinfo.binpb"
+
+// Handle implements protoplugin.Handler and is the main entry point for the plugin.
+func Handle(
+	_ context.Context,
+	_ protoplugin.PluginEnv,
+	responseWriter protoplugin.ResponseWriter,
+	request protoplugin.Request,
+) error {
+	fileDescriptors, err := request.FileDescriptorsToGenerate()
+	if err != nil {
+		return err
+	}
+	for _, fileDescriptor := range fileDescriptors {
+		// Write the YAML string to the response.
+		data, err := GenFileContents(fileDescriptor)
+		if err != nil {
+			return err
+		}
+		name := GetSourceInfoPath(fileDescriptor)
+		responseWriter.AddFile(name, string(data))
+	}
+
+	responseWriter.SetFeatureProto3Optional()
+	return nil
+}
+
+// GetSourceInfoPath returns the path to the source info file for the given file descriptor.
+func GetSourceInfoPath(fileDescriptor protoreflect.FileDescriptor) string {
+	path := fileDescriptor.Path()
+	path = strings.TrimSuffix(path, ".proto")
+	return fmt.Sprintf("%s%s", path, FileExtension)
+}
+
+// GenFileContents marshals the `SourceCodeInfo` field of the given file descriptor to protobuf bytes.
+func GenFileContents(fileDescriptor protoreflect.FileDescriptor) ([]byte, error) {
+	// Convert the file descriptor to a descriptorpb.FileDescriptorProto.
+	fileDescProto := protodesc.ToFileDescriptorProto(fileDescriptor)
+	return proto.Marshal(fileDescProto.SourceCodeInfo)
+}