[protobuf] Preserve documentation comments (#2283)

This change preserves `@doc` comments when emitting protobuf schemas. We
convert the protobuf documentation comments into a format that will feel
familiar to Protobuf developers and that Protobuf tooling will be able
to parse. I've never used a documentation generator with Protobuf, but I
made sure it works with pseudomuto/protoc-gen-doc, as that seems to be
the most recommended community project for doc generation. I also made
sure that the protoc Java compiler emits readable documentation based on
the example spec in the tests.

The rules for comments are:
- On _field_ and _enum variant_ declarations, we will emit a trailing
`//` comment if the length of the line overall will not be longer than
80 characters.
- In all other cases, we will emit a block of `//` comments aligned with
the beginning of the declaration.

Closes #1878

---------

Co-authored-by: Will Temple <[email protected]>
Co-authored-by: Timothee Guerin <[email protected]>

common/changes/@typespec/protobuf/protobuf-emit-doc-comments_2023-08-10-18-18.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@typespec/protobuf",
+      "comment": "Added support for emitting documentation comments in protobuf specifications.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@typespec/protobuf"
+}

packages/protobuf/lib/proto.tsp

@@ -74,6 +74,7 @@ namespace WellKnown {
  * Uses variable-length encoding. These more efficiently encode negative numbers than regular int32s.
  */
 scalar sint32 extends int32;
+
 /**
  * A signed 64-bit integer that will use the `sint64` encoding when used in a Protobuf message.
  *
@@ -82,6 +83,7 @@ scalar sint32 extends int32;
  * Uses variable-length encoding. These more efficiently encode negative numbers than regular `int64s`.
  */
 scalar sint64 extends int64;
+
 /**
  * A signed 32-bit integer that will use the `sfixed32` encoding when used in a Protobuf message.
  *
@@ -90,6 +92,7 @@ scalar sint64 extends int64;
  * Always four bytes.
  */
 scalar sfixed32 extends int32;
+
 /**
  * A signed 64-bit integer that will use the `sfixed64` encoding when used in a Protobuf message.
  *
@@ -98,6 +101,7 @@ scalar sfixed32 extends int32;
  * Always eight bytes.
  */
 scalar sfixed64 extends int64;
+
 /**
  * An unsigned 32-bit integer that will use the `fixed32` encoding when used in a Protobuf message.
  *
@@ -106,6 +110,7 @@ scalar sfixed64 extends int64;
  * Always four bytes. More efficient than `uint32` if values are often greater than 2<sup>28</sup>.
  */
 scalar fixed32 extends uint32;
+
 /**
  * An unsigned 64-bit integer that will use the `fixed64` encoding when used in a Protobuf message.
  *

packages/protobuf/src/ast.ts

@@ -35,6 +35,11 @@ export interface ProtoFile {
    * The original namespace node from which this ProtoFile originated.
    */
   source: Namespace;
+
+  /**
+   * The package-level documentation comment, if any.
+   */
+  doc?: string | undefined;
 }
 
 /**
@@ -66,7 +71,8 @@ export type ProtoDeclaration =
   | ProtoFieldDeclaration
   | ProtoOneOfDeclaration
   | ProtoEnumDeclaration
-  | ProtoMethodDeclaration;
+  | ProtoMethodDeclaration
+  | ProtoEnumVariantDeclaration;
 
 /**
  * A Protobuf scalar type.
@@ -187,10 +193,20 @@ export function matchType<Result>(type: ProtoType, pattern: ProtoTypeMatchPatter
   }
 }
 
+/**
+ * Elements common to all protobuf declarations.
+ */
+export interface ProtoDeclarationCommon {
+  /**
+   * Documentation comment text, if any.
+   */
+  doc?: string | undefined;
+}
+
 /**
  * A `service` declaration.
  */
-export interface ProtoServiceDeclaration {
+export interface ProtoServiceDeclaration extends ProtoDeclarationCommon {
   kind: "service";
   name: string;
   operations: ProtoMethodDeclaration[];
@@ -209,7 +225,7 @@ export const enum StreamingMode {
 /**
  * An `rfc` method declaration.
  */
-export interface ProtoMethodDeclaration {
+export interface ProtoMethodDeclaration extends ProtoDeclarationCommon {
   kind: "method";
   stream: StreamingMode;
   name: string;
@@ -229,7 +245,7 @@ export type ProtoMessageBodyDeclaration =
 /**
  * A `message` declaration.
  */
-export interface ProtoMessageDeclaration {
+export interface ProtoMessageDeclaration extends ProtoDeclarationCommon {
   kind: "message";
   name: string;
   declarations: Array<ProtoMessageBodyDeclaration>;
@@ -239,7 +255,7 @@ export interface ProtoMessageDeclaration {
 /**
  * A field declaration within a message.
  */
-export interface ProtoFieldDeclaration {
+export interface ProtoFieldDeclaration extends ProtoDeclarationCommon {
   kind: "field";
   name: string;
   /**
@@ -262,7 +278,7 @@ export interface DefaultFieldOptions {
 /**
  * A `one_of` declaration.
  */
-export interface ProtoOneOfDeclaration {
+export interface ProtoOneOfDeclaration extends ProtoDeclarationCommon {
   kind: "oneof";
   name: string;
   declarations: ProtoFieldDeclaration[];
@@ -271,9 +287,18 @@ export interface ProtoOneOfDeclaration {
 /**
  * An `enum` declaration.
  */
-export interface ProtoEnumDeclaration {
+export interface ProtoEnumDeclaration extends ProtoDeclarationCommon {
   kind: "enum";
   name: string;
   allowAlias?: boolean;
-  variants: [string, number][];
+  variants: ProtoEnumVariantDeclaration[];
+}
+
+/**
+ * A variant within an `enum` declaration.
+ */
+export interface ProtoEnumVariantDeclaration extends ProtoDeclarationCommon {
+  kind: "variant";
+  name: string;
+  value: number;
 }

packages/protobuf/src/transform/index.ts

@@ -5,6 +5,7 @@ import {
   DiagnosticTarget,
   Enum,
   formatDiagnostic,
+  getDoc,
   getEffectiveModelType,
   getTypeName,
   Interface,
@@ -26,6 +27,7 @@ import {
   map,
   matchType,
   ProtoEnumDeclaration,
+  ProtoEnumVariantDeclaration,
   ProtoFieldDeclaration,
   ProtoFile,
   ProtoMap,
@@ -206,6 +208,8 @@ function tspToProto(program: Program, emitterOptions: ProtobufEmitterOptions): P
 
       declarations: declarationMap.get(namespace),
       source: namespace,
+
+      doc: getDoc(program, namespace),
     } as ProtoFile;
   });
 
@@ -261,6 +265,7 @@ function tspToProto(program: Program, emitterOptions: ProtobufEmitterOptions): P
         name: iface.name,
         // The service's methods are just projections of the interface operations.
         operations: [...iface.operations.values()].map(toMethodFromOperation),
+        doc: getDoc(program, iface),
       });
     }
   }
@@ -297,6 +302,7 @@ function tspToProto(program: Program, emitterOptions: ProtobufEmitterOptions): P
         operation,
         operation.returnType as NamespaceTraversable
       ),
+      doc: getDoc(program, operation),
     };
   }
 
@@ -680,6 +686,7 @@ function tspToProto(program: Program, emitterOptions: ProtobufEmitterOptions): P
       name: model.name,
       reservations: program.stateMap(state.reserve).get(model),
       declarations: [...model.properties.values()].map((f) => toMessageBodyDeclaration(f, model)),
+      doc: getDoc(program, model),
     };
   }
 
@@ -775,6 +782,7 @@ function tspToProto(program: Program, emitterOptions: ProtobufEmitterOptions): P
         property.type as NamespaceTraversable
       ),
       index: program.stateMap(state.fieldIndex).get(property),
+      doc: getDoc(program, property),
     };
 
     // Determine if the property type is an array
@@ -796,7 +804,15 @@ function tspToProto(program: Program, emitterOptions: ProtobufEmitterOptions): P
       kind: "enum",
       name: e.name,
       allowAlias: needsAlias,
-      variants: [...e.members.values()].map(({ name, value }) => [name, value as number]),
+      variants: [...e.members.values()].map(
+        (variant): ProtoEnumVariantDeclaration => ({
+          kind: "variant",
+          name: variant.name,
+          value: variant.value as number,
+          doc: getDoc(program, variant),
+        })
+      ),
+      doc: getDoc(program, e),
     };
   }