Generate API doc examples (#60)

* Generate API doc examples

* Github actions to init submodules

.github/workflows/master.yml

@@ -124,6 +124,8 @@ jobs:
           repo: pulumi/pulumictl
       - name: Install Pulumi CLI
         uses: pulumi/[email protected]
+      - name: Initialize submodules
+        run: make init_submodules
       - name: Build codegen binaries
         run: make codegen
       - name: Build Schema

.github/workflows/pull_request.yml

@@ -41,6 +41,8 @@ jobs:
           repo: pulumi/pulumictl
       - name: Install Pulumi CLI
         uses: pulumi/[email protected]
+      - name: Initialize submodules
+        run: make init_submodules
       - name: Build codegen binaries
         run: make codegen
       - name: Build Schema

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "aws-cloudformation-user-guide"]
+	path = aws-cloudformation-user-guide
+	url = https://github.com/awsdocs/aws-cloudformation-user-guide

Makefile

@@ -17,11 +17,27 @@ CFN_SCHEMA_URL      ?= https://cfn-resource-specifications-${CFN_SCHEMA_REGION}-
 CFN_SCHEMA_DIR      := provider/cmd/pulumi-gen-${PACK}
 CFN_SCHEMA_FILE     := ${CFN_SCHEMA_DIR}/cfn-spec-${CFN_SCHEMA_REGION}.json
 
-discovery::codegen
+init_submodules::
+	@for submodule in $$(git submodule status | awk {'print $$2'}); do \
+		if [ ! -f "$$submodule/.git" ]; then \
+			echo "Initializing submodule $$submodule" ; \
+			(cd $$submodule && git submodule update --init); \
+		fi; \
+	done
+
+update_submodules:: init_submodules
+	@for submodule in $$(git submodule status | awk {'print $$2'}); do \
+		echo "Updating submodule $$submodule" ; \
+		(cd $$submodule && git checkout main && git pull origin main); \
+	done
+	rm ./azure-provider-versions/provider_list.json
+	az provider list >> ./azure-provider-versions/provider_list.json
+
+discovery:: update_submodules codegen
 	curl -s -L $(CFN_SCHEMA_URL) | gzip -d > $(CFN_SCHEMA_FILE)
 	$(WORKING_DIR)/bin/$(CODEGEN) discovery $(CFN_SCHEMA_FILE) ${VERSION}
 
-ensure::
+ensure:: init_submodules
 	@echo "GO111MODULE=on go mod tidy"
 	cd aws-sdk-go-v2-cf-preview && GO111MODULE=on go mod tidy
 	cd provider && GO111MODULE=on go mod tidy

provider/cmd/pulumi-gen-aws-native/examples.go

@@ -0,0 +1,293 @@
+// Copyright 2016-2021, Pulumi Corporation.
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"github.com/blang/semver"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/pkg/errors"
+	"github.com/pulumi/pulumi-aws-native/provider/pkg/cf2pulumi"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/dotnet"
+	gogen "github.com/pulumi/pulumi/pkg/v3/codegen/go"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/hcl2"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/hcl2/syntax"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/nodejs"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/python"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"text/template"
+)
+
+func generateExamples(pkgSpec *schema.PackageSpec, languages []string) error {
+	// Find all snippets in the AWS CloudFormation Docs repo.
+	folder := path.Join(".", "aws-cloudformation-user-guide", "doc_source")
+	examples, err := findAllExamples(folder)
+	if err != nil {
+		return err
+	}
+
+	// Cache to speed up code generation.
+	hcl2Cache := hcl2.Cache(hcl2.NewPackageCache())
+	pkg, err := schema.ImportSpec(*pkgSpec, nil)
+	if err != nil {
+		return err
+	}
+	loaderOption := hcl2.Loader(inMemoryPackageLoader(map[string]*schema.Package{
+		"aws-native": pkg,
+	}))
+
+	// Render examples to SDK languages.
+	examplesRenderData := map[string][]exampleRenderData{}
+	for _, yaml := range examples {
+		example, err := generateExample(yaml, languages, hcl2Cache, loaderOption)
+		if err != nil {
+			// Skip all snippets that don't produce valid examples.
+			continue
+		}
+		var existing []exampleRenderData
+		if other, ok := examplesRenderData[example.ResourceType]; ok {
+			existing = other
+		}
+		examplesRenderData[example.ResourceType] = append(existing, exampleRenderData{
+			ExampleDescription:       "Example",
+			LanguageToExampleProgram: example.LanguageToExampleProgram,
+		})
+	}
+
+	// Write examples to the schema docs.
+	for resourceType, data := range examplesRenderData {
+		err = renderExampleToSchema(pkgSpec, resourceType, data)
+		if err != nil {
+			return errors.Wrapf(err, "rendering %s %+v", resourceType, data)
+		}
+	}
+	return nil
+}
+
+func generateExample(yaml string, languages []string, bindOpts ...hcl2.BindOption) (*resourceExample, error) {
+	body, err := cf2pulumi.RenderText(yaml)
+	if err != nil {
+		return nil, errors.Wrapf(err, "rendering YAML")
+	}
+
+	cf2pulumi.FormatBody(body)
+	text := fmt.Sprintf("%v", body)
+
+	parser := syntax.NewParser()
+	if err := parser.ParseFile(strings.NewReader(text), "program.pp"); err != nil {
+		return nil, errors.Wrapf(err, "parsing IR")
+	}
+	if parser.Diagnostics.HasErrors() {
+		buf := new(bytes.Buffer)
+		_ = parser.NewDiagnosticWriter(buf, 0, false).WriteDiagnostics(parser.Diagnostics)
+		return nil, errors.Errorf("parser diagnostic errors: %s", buf)
+	}
+
+	program, diags, err := hcl2.BindProgram(parser.Files, bindOpts...)
+	if err != nil {
+		return nil, errors.Wrapf(err, "binding program")
+	}
+	if diags.HasErrors() {
+		buf := new(bytes.Buffer)
+		_ = program.NewDiagnosticWriter(buf, 0, false).WriteDiagnostics(diags)
+		return nil, errors.Errorf("bind diagnostic errors: %s", buf)
+	}
+
+	resourceType := ""
+	for _, node := range program.Nodes {
+		if res, ok := node.(*hcl2.Resource); ok {
+			resourceType = res.Token
+			break
+		}
+	}
+	if resourceType == "" {
+		return nil, errors.New("no resource node found")
+	}
+
+	perLanguage := languageToExampleProgram{}
+	for _, target := range languages {
+		var files map[string][]byte
+		switch target {
+		case "dotnet":
+			files, diags, err = dotnet.GenerateProgram(program)
+		case "go":
+			files, diags, err = recoverableProgramGen(program, gogen.GenerateProgram)
+		case "nodejs":
+			files, diags, err = nodejs.GenerateProgram(program)
+		case "python":
+			files, diags, err = python.GenerateProgram(program)
+		}
+		if err != nil {
+			return nil, errors.Wrapf(err, "generating program")
+		}
+		if diags.HasErrors() {
+			buf := new(bytes.Buffer)
+			_ = program.NewDiagnosticWriter(buf, 0, true).WriteDiagnostics(diags)
+			return nil, errors.Errorf("generate diagnostic errors: %s", buf)
+		}
+
+		var sb strings.Builder
+		for _, f := range files {
+			sb.WriteString(string(f))
+		}
+		perLanguage[language(target)] = programText(sb.String())
+	}
+	return &resourceExample{
+		ResourceType:             resourceType,
+		LanguageToExampleProgram: perLanguage,
+	}, nil
+}
+
+func findAllExamples(folder string) ([]string, error) {
+	var fileNames, result []string
+	err := filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
+		if !info.IsDir() && strings.Contains(path, "aws-resource-") {
+			fileNames = append(fileNames, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	for _, fileName := range fileNames {
+		examples, err := findExamples(fileName)
+		if err != nil {
+			return nil, err
+		}
+		result = append(result, examples...)
+	}
+	return result, nil
+}
+
+func findExamples(fileName string) ([]string, error) {
+	docFile, err := os.Open(fileName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "opening file")
+	}
+	defer func() {
+		_ = docFile.Close()
+	}()
+
+	var result []string
+	var buf strings.Builder
+	snippet := false
+
+	scanner := bufio.NewScanner(docFile)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "```") {
+			if snippet && buf.Len() > 0 {
+				result = append(result, buf.String())
+			}
+			snippet = !snippet
+			continue
+		}
+		if snippet {
+			buf.WriteString(line)
+			buf.WriteRune('\n')
+		} else {
+			buf.Reset()
+		}
+	}
+	return result, scanner.Err()
+}
+
+type programGenFn func(*hcl2.Program) (map[string][]byte, hcl.Diagnostics, error)
+
+func recoverableProgramGen(program *hcl2.Program, fn programGenFn) (files map[string][]byte, d hcl.Diagnostics, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("panic recovered during generation: %v", r)
+		}
+	}()
+
+	return fn(program)
+}
+
+type programText string
+type language string
+
+type languageToExampleProgram map[language]programText
+type exampleRenderData struct {
+	ExampleDescription       string
+	LanguageToExampleProgram languageToExampleProgram
+}
+type resourceExample struct {
+	ResourceType             string
+	LanguageToExampleProgram languageToExampleProgram
+}
+
+func renderExampleToSchema(pkgSpec *schema.PackageSpec, resourceName string,
+	examplesRenderData []exampleRenderData) error {
+	const tmpl = `
+
+{{"{{% examples %}}"}}
+## Example Usage
+{{- range . }}
+{{ "{{% example %}}" }}
+### {{ .ExampleDescription }}
+
+{{- range $lang, $example := .LanguageToExampleProgram }}
+{{ beginLanguage $lang }}
+{{ $example }}
+{{ endLanguage }}
+{{ end }}
+{{"{{% /example %}}"}}
+{{- end }}
+{{"{{% /examples %}}"}}
+`
+	res, ok := pkgSpec.Resources[resourceName]
+	if !ok {
+		return fmt.Errorf("missing resource from schema: %s", resourceName)
+	}
+
+	t, err := template.New("examples").Funcs(template.FuncMap{
+		"beginLanguage": func(lang interface{}) string {
+			l := fmt.Sprintf("%s", lang)
+			switch l {
+			case "nodejs":
+				l = "typescript"
+			case "dotnet":
+				l = "csharp"
+			}
+			return fmt.Sprintf("```%s", l)
+		},
+		"endLanguage": func() string {
+			return "```"
+		},
+	}).Parse(tmpl)
+	if err != nil {
+		return err
+	}
+	b := strings.Builder{}
+	if err = t.Execute(&b, examplesRenderData); err != nil {
+		return err
+	}
+	res.Description += b.String()
+	pkgSpec.Resources[resourceName] = res
+	return nil
+}
+
+// inMemoryPackageLoader prevents having to fetch the schema from
+// the provider every time which significantly speeds up codegen.
+func inMemoryPackageLoader(pkgs map[string]*schema.Package) schema.Loader {
+	return &inMemoryLoader{pkgs: pkgs}
+}
+
+type inMemoryLoader struct {
+	pkgs map[string]*schema.Package
+}
+
+func (l *inMemoryLoader) LoadPackage(pkg string, _ *semver.Version) (*schema.Package, error) {
+	if p, ok := l.pkgs[pkg]; ok {
+		return p, nil
+	}
+
+	return nil, errors.Errorf("package %s not found in the in-memory map", pkg)
+}

provider/cmd/pulumi-gen-aws-native/main.go

@@ -74,6 +74,10 @@ func main() {
 		case "go":
 			writeGoSDK(ppkg, outdir)
 		case "schema":
+			err := generateExamples(&pkgSpec, []string{"nodejs","python","dotnet","go"})
+			if err != nil {
+				panic(fmt.Sprintf("error generating examples: %v", err))
+			}
 			writePulumiSchema(pkgSpec, providerDir)
 		default:
 			panic(fmt.Sprintf("Unrecognized language '%s'", language))