feat: added normalize transform (#1168)

This pull request introduces a new feature to normalize OpenAPI
documents to a more supported structure. The main changes include adding
a new command for normalization, implementing the normalization logic,
and adding tests to ensure the feature works correctly.

New normalization command:

*
[`cmd/openapi/transform.go`](diffhunk://#diff-f618debf6dbd5b8b72bdb3a3772aff1757bd578d0810643b6821fb87c7c6f864L17-R17):
Added `normalizeCmd` to the list of commands and defined its usage,
flags, and run function.
[[1]](diffhunk://#diff-f618debf6dbd5b8b72bdb3a3772aff1757bd578d0810643b6821fb87c7c6f864L17-R17)
[[2]](diffhunk://#diff-f618debf6dbd5b8b72bdb3a3772aff1757bd578d0810643b6821fb87c7c6f864R100-R130)

Normalization logic implementation:

*
[`internal/transform/normalize.go`](diffhunk://#diff-4887fac2f0911fe4a69c0adb49004936df059ada94f745076066bff118f079c7R1-R130):
Implemented the `NormalizeDocument` function and supporting functions to
handle the normalization process, including walking through the YAML
nodes and modifying them based on the provided options.

Integration test resources:

*
[`integration/resources/normalize-input.yaml`](diffhunk://#diff-a3d9287327c7665602fc8bb09bc7b2c454f85b4c6ab26d19b9e5680135edf5aaR1-R27):
Added a sample input OpenAPI document for testing the normalization
feature.
*
[`integration/resources/normalize-output.yaml`](diffhunk://#diff-eba744aa9c9a8bfbd48c4a2a91270c201ed13b329411811f9f3df291157641bfR1-R17):
Added the expected output OpenAPI document after normalization.

Tests for normalization:

*
[`internal/transform/normalize_test.go`](diffhunk://#diff-bdcde49821ec149ef12c3d38ca3ade5075d3d8bc8c3c71096445a410bd04089cR1-R99):
Added tests to verify the normalization feature, including cases with
and without the `prefixItems` option.

cmd/openapi/transform.go

@@ -14,7 +14,7 @@ import (
 var transformCmd = &model.CommandGroup{
 	Usage:    "transform",
 	Short:    "Transform an OpenAPI spec using a well-defined function",
-	Commands: []model.Command{removeUnusedCmd, filterOperationsCmd, cleanupCmd, formatCmd, convertSwaggerCmd},
+	Commands: []model.Command{removeUnusedCmd, filterOperationsCmd, cleanupCmd, formatCmd, convertSwaggerCmd, normalizeCmd},
 }
 
 type basicFlagsI struct {
@@ -97,6 +97,37 @@ var formatCmd = &model.ExecutableCommand[basicFlagsI]{
 	Flags: basicFlags,
 }
 
+var normalizeCmd = &model.ExecutableCommand[normalizeFlags]{
+	Usage: "normalize",
+	Short: "Normalize an OpenAPI document to be more human-readable",
+	Run:   runNormalize,
+	Flags: append(basicFlags, []flag.Flag{
+		flag.BooleanFlag{
+			Name:         "prefixItems",
+			Description:  "Normalize prefixItems to be a simple string",
+			DefaultValue: false,
+		},
+	}...),
+}
+
+type normalizeFlags struct {
+	Schema      string `json:"schema"`
+	Out         string `json:"out"`
+	PrefixItems bool   `json:"prefixItems"`
+}
+
+func runNormalize(ctx context.Context, flags normalizeFlags) error {
+	out, yamlOut, err := setupOutput(ctx, flags.Out)
+	defer out.Close()
+	if err != nil {
+		return err
+	}
+
+	return transform.NormalizeDocument(ctx, flags.Schema, transform.NormalizeOptions{
+		PrefixItems: flags.PrefixItems,
+	}, yamlOut, out)
+}
+
 func runRemoveUnused(ctx context.Context, flags basicFlagsI) error {
 	out, yamlOut, err := setupOutput(ctx, flags.Out)
 	defer out.Close()

integration/resources/normalize-input.yaml

@@ -0,0 +1,27 @@
+openapi: 3.1.0
+info:
+  title: Normalize-Test
+  version: 1.0.0
+paths:
+  /test:
+    get:
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                prefixItems:
+                  - type: string
+                  - type: string
+                minItems: 1
+                maxItems: 1
+                type: array
+components:
+  schemas:
+    test:
+      prefixItems:
+        - type: string
+        - type: string
+      minItems: 1
+      maxItems: 1
+      type: array

integration/resources/normalize-output.yaml

@@ -0,0 +1,17 @@
+openapi: 3.1.0
+info:
+  title: Normalize-Test
+  version: 1.0.0
+paths:
+  /test:
+    get:
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                type: string
+components:
+  schemas:
+    test:
+      type: string

internal/transform/normalize.go

@@ -0,0 +1,130 @@
+package transform
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/pb33f/libopenapi"
+	v3 "github.com/pb33f/libopenapi/datamodel/high/v3"
+	"github.com/speakeasy-api/speakeasy-core/openapi"
+	"gopkg.in/yaml.v3"
+)
+
+type normalizeArgs struct {
+	schemaPath       string
+	normalizeOptions NormalizeOptions
+}
+
+func NormalizeDocument(ctx context.Context, schemaPath string, normalizeOptions NormalizeOptions, yamlOut bool, w io.Writer) error {
+	return transformer[normalizeArgs]{
+		schemaPath:  schemaPath,
+		transformFn: Normalize,
+		w:           w,
+		jsonOut:     !yamlOut,
+		args: normalizeArgs{
+			schemaPath:       schemaPath,
+			normalizeOptions: normalizeOptions,
+		},
+	}.Do(ctx)
+}
+
+func NormalizeFromReader(ctx context.Context, schema io.Reader, schemaPath string, normalizeOptions NormalizeOptions, w io.Writer, yamlOut bool) error {
+	return transformer[normalizeArgs]{
+		r:           schema,
+		schemaPath:  schemaPath,
+		transformFn: Normalize,
+		w:           w,
+		jsonOut:     !yamlOut,
+		args: normalizeArgs{
+			schemaPath:       schemaPath,
+			normalizeOptions: normalizeOptions,
+		},
+	}.Do(ctx)
+}
+
+func Normalize(ctx context.Context, doc libopenapi.Document, model *libopenapi.DocumentModel[v3.Document], args normalizeArgs) (libopenapi.Document, *libopenapi.DocumentModel[v3.Document], error) {
+	root := model.Index.GetRootNode()
+
+	walkAndNormalizeDocument(root, args.normalizeOptions)
+
+	updatedDoc, err := yaml.Marshal(root)
+	if err != nil {
+		return doc, model, fmt.Errorf("failed to marshal document: %w", err)
+	}
+
+	docNew, model, err := openapi.Load(updatedDoc, doc.GetConfiguration().BasePath)
+	if err != nil {
+		return doc, model, fmt.Errorf("failed to reload document: %w", err)
+	}
+
+	return *docNew, model, nil
+}
+
+type NormalizeOptions struct {
+	PrefixItems bool
+}
+
+func walkAndNormalizeDocument(node *yaml.Node, options NormalizeOptions) {
+
+	switch node.Kind {
+	case yaml.MappingNode, yaml.DocumentNode:
+		for _, child := range node.Content {
+			if options.PrefixItems && child.Value == "prefixItems" {
+				normalizePrefixItems(node)
+				break
+			}
+			walkAndNormalizeDocument(child, options)
+		}
+	case yaml.SequenceNode:
+		for _, child := range node.Content {
+			walkAndNormalizeDocument(child, options)
+		}
+	}
+}
+
+func removeKeys(mapNode *yaml.Node, keys ...string) {
+	keysToRemove := make(map[string]struct{})
+	for _, k := range keys {
+		keysToRemove[k] = struct{}{}
+	}
+
+	newContent := make([]*yaml.Node, 0, len(mapNode.Content))
+	for i := 0; i < len(mapNode.Content); i += 2 {
+		kNode := mapNode.Content[i]
+		vNode := mapNode.Content[i+1]
+		if _, found := keysToRemove[kNode.Value]; !found {
+			// Keep this pair
+			newContent = append(newContent, kNode, vNode)
+		}
+	}
+	mapNode.Content = newContent
+}
+
+func addKeyValue(mapNode *yaml.Node, key, value string) {
+	// Create a key node
+	keyNode := &yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Tag:   "!!str",
+		Value: key,
+	}
+
+	// Create a value node
+	valNode := &yaml.Node{
+		Kind:  yaml.ScalarNode,
+		Tag:   "!!str",
+		Value: value,
+	}
+
+	// Append them to the map content
+	mapNode.Content = append(mapNode.Content, keyNode, valNode)
+}
+
+// Take the given yaml OpenAPI node, remove the prefixItems key change the type key to string, remove the minItems and maxItems keys
+func normalizePrefixItems(node *yaml.Node) {
+
+	keysToRemove := []string{"prefixItems", "minItems", "maxItems", "type"}
+	removeKeys(node, keysToRemove...)
+
+	addKeyValue(node, "type", "string")
+}

internal/transform/normalize_test.go

@@ -0,0 +1,99 @@
+package transform
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"os"
+	"testing"
+
+	"github.com/pb33f/libopenapi"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func TestNormalize(t *testing.T) {
+
+	// Create a buffer to store the normalized spec
+	var testInput bytes.Buffer
+	var testOutput bytes.Buffer
+
+	// Call FormatDocument to format the spec
+	err := NormalizeDocument(context.Background(), "../../integration/resources/normalize-input.yaml", NormalizeOptions{
+		PrefixItems: true,
+	}, true, &testInput)
+	require.NoError(t, err)
+
+	// Parse the normalized spec
+	normalizedDoc, err := libopenapi.NewDocument(testInput.Bytes())
+	require.NoError(t, err)
+
+	// Check that the normalized spec is valid
+	_, errors := normalizedDoc.BuildV3Model()
+	require.Empty(t, errors)
+
+	// Open the spec we expect to see to compare
+	file, err := os.Open("../../integration/resources/normalize-output.yaml")
+	require.NoError(t, err)
+	defer file.Close()
+
+	// Read the expected spec into a buffer
+	reader := bufio.NewReader(file)
+	testOutput.ReadFrom(reader)
+	require.NoError(t, err)
+
+	var actual yaml.Node
+	var expected yaml.Node
+
+	err = yaml.Unmarshal(testInput.Bytes(), &actual)
+	require.NoError(t, err)
+
+	err = yaml.Unmarshal(testOutput.Bytes(), &expected)
+	require.NoError(t, err)
+
+	// Require the pre-normalized spec matches the expected spec
+	require.Equal(t, expected, actual)
+}
+
+func TestNormalizeNoPrefixItems(t *testing.T) {
+
+	// Create a buffer to store the normalized spec
+	var testInput bytes.Buffer
+	var testOutput bytes.Buffer
+
+	// Call FormatDocument to format the spec
+	err := NormalizeDocument(context.Background(), "../../integration/resources/normalize-input.yaml", NormalizeOptions{
+		PrefixItems: false,
+	}, true, &testInput)
+	require.NoError(t, err)
+
+	// Parse the normalized spec
+	normalizedDoc, err := libopenapi.NewDocument(testInput.Bytes())
+	require.NoError(t, err)
+
+	// Check that the normalized spec is valid
+	_, errors := normalizedDoc.BuildV3Model()
+	require.Empty(t, errors)
+
+	// Open the spec we expect to see to compare
+	file, err := os.Open("../../integration/resources/normalize-input.yaml")
+	require.NoError(t, err)
+	defer file.Close()
+
+	// Read the expected spec into a buffer
+	reader := bufio.NewReader(file)
+	testOutput.ReadFrom(reader)
+	require.NoError(t, err)
+
+	var actual yaml.Node
+	var expected yaml.Node
+
+	err = yaml.Unmarshal(testInput.Bytes(), &actual)
+	require.NoError(t, err)
+
+	err = yaml.Unmarshal(testOutput.Bytes(), &expected)
+	require.NoError(t, err)
+
+	// Require the pre-normalized spec matches the expected spec
+	require.Equal(t, expected, actual)
+}