Skip to content
This repository has been archived by the owner on Dec 15, 2022. It is now read-only.

Commit

Permalink
Add example manifest generation pipeline
Browse files Browse the repository at this point in the history
Signed-off-by: Alper Rifat Ulucinar <[email protected]>
  • Loading branch information
ulucinar committed Jan 3, 2022
1 parent 2c54e84 commit 6a039a9
Show file tree
Hide file tree
Showing 8 changed files with 393 additions and 19 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ require (
k8s.io/apimachinery v0.21.3
k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471
sigs.k8s.io/controller-runtime v0.9.6
sigs.k8s.io/yaml v1.2.0
)

// This is a temporary workaround until https://github.com/crossplane/terrajet/issues/131
Expand Down
28 changes: 28 additions & 0 deletions pkg/config/provider.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
Copyright 2021 The Crossplane Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
Expand Down Expand Up @@ -93,6 +109,10 @@ type Provider struct {
// resource name.
Resources map[string]*Resource

// ProviderMetadataPath is the scraped provider metadata file path
// from Terraform registry
ProviderMetadataPath string

// resourceConfigurators is a map holding resource configurators where key
// is Terraform resource name.
resourceConfigurators map[string]ResourceConfiguratorChain
Expand Down Expand Up @@ -143,6 +163,14 @@ func WithDefaultResourceFn(f DefaultResourceFn) ProviderOption {
}
}

// WithProviderMetadata configures the Terraform metadata file scraped
// from the Terraform registry
func WithProviderMetadata(metadataPath string) ProviderOption {
return func(p *Provider) {
p.ProviderMetadataPath = metadataPath
}
}

// NewProvider builds and returns a new Provider.
func NewProvider(resourceMap map[string]*schema.Resource, prefix string, modulePath string, opts ...ProviderOption) *Provider {
p := &Provider{
Expand Down
3 changes: 3 additions & 0 deletions pkg/pipeline/crd.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ type CRDGenerator struct {
Group string
ProviderShortName string
LicenseHeaderPath string
Generated *tjtypes.Generated

pkg *types.Package
}
Expand All @@ -77,6 +78,8 @@ func (cg *CRDGenerator) Generate(cfg *config.Resource) (string, error) {
if err != nil {
return "", errors.Wrapf(err, "cannot build types for %s", cfg.Kind)
}
cg.Generated = &gen

// TODO(muvaf): TypePrinter uses the given scope to see if the type exists
// before printing. We should ideally load the package in file system but
// loading the local package will result in error if there is
Expand Down
297 changes: 297 additions & 0 deletions pkg/pipeline/example.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,297 @@
/*
Copyright 2021 The Crossplane Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pipeline

import (
"bytes"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strings"

"github.com/crossplane/crossplane-runtime/pkg/fieldpath"
"github.com/pkg/errors"
"sigs.k8s.io/yaml"

"github.com/crossplane/terrajet/pkg/config"
"github.com/crossplane/terrajet/pkg/resource/json"
tjtypes "github.com/crossplane/terrajet/pkg/types"
)

var (
reRef = regexp.MustCompile(`\${(.+)}`)
)

type pavedWithManifest struct {
manifestPath string
paved *fieldpath.Paved
}

// ResourceExample represents the scraped example HCL configuration
// for a Terraform resource
type ResourceExample struct {
Manifest string `yaml:"manifest"`
References map[string]string `yaml:"references,omitempty"`
}

// Resource represents the scraped metadata for a Terraform resource
type Resource struct {
SubCategory string `yaml:"subCategory"`
Description string `yaml:"description,omitempty"`
Name string `yaml:"name"`
TitleName string `yaml:"titleName"`
Examples []ResourceExample `yaml:"examples,omitempty"`
ArgumentDocs map[string]string `yaml:"argumentDocs"`
ImportStatements []string `yaml:"importStatements"`
}

// ProviderMetadata metadata for a Terraform native provider
type ProviderMetadata struct {
Name string `yaml:"name"`
Resources map[string]*Resource `yaml:"resources"`
}

// NewProviderMetadataFromFile loads metadata from the specified YAML-formatted file
func NewProviderMetadataFromFile(path string) (*ProviderMetadata, error) {
buff, err := ioutil.ReadFile(filepath.Clean(path))
if err != nil {
return nil, errors.Wrapf(err, "failed to read metadata file %q", path)
}

metadata := &ProviderMetadata{}
return metadata, errors.Wrap(yaml.Unmarshal(buff, metadata), "failed to unmarshal provider metadata")
}

// ExampleGenerator represents a pipeline for generating example manifests.
// Generates example manifests for Terraform resources under examples-generated.
type ExampleGenerator struct {
rootDir string
resourceMeta map[string]*Resource
resources map[string]*pavedWithManifest
}

// NewExampleGenerator returns a configured ExampleGenerator
func NewExampleGenerator(rootDir string, resourceMeta map[string]*Resource) *ExampleGenerator {
return &ExampleGenerator{
rootDir: rootDir,
resourceMeta: resourceMeta,
resources: make(map[string]*pavedWithManifest),
}
}

// StoreExamples stores the generated example manifests under examples-generated in
// their respective API groups.
func (eg *ExampleGenerator) StoreExamples() error {
for n, pm := range eg.resources {
if err := eg.resolveReferences(pm.paved.UnstructuredContent()); err != nil {
return errors.Wrapf(err, "cannot resolve references for resource: %s", n)
}
buff, err := yaml.Marshal(pm.paved.UnstructuredContent())
if err != nil {
return errors.Wrapf(err, "cannot marshal example manifest for resource: %s", n)
}
manifestDir := filepath.Dir(pm.manifestPath)
if err := os.MkdirAll(manifestDir, 0750); err != nil {
return errors.Wrapf(err, "cannot mkdir %s", manifestDir)
}

b := bytes.Buffer{}
b.WriteString("# This example manifest is auto-generated, and has not been tested.\n")
b.WriteString("# Please make the necessary adjustments before using it.\n")
b.Write(buff)
// no sensitive info in the example manifest
if err := ioutil.WriteFile(pm.manifestPath, b.Bytes(), 0644); err != nil { // nolint:gosec
return errors.Wrapf(err, "cannot write example manifest file %s for resource %s", pm.manifestPath, n)
}
}
return nil
}

func (eg *ExampleGenerator) resolveReferences(params map[string]interface{}) error { // nolint:gocyclo
for k, v := range params {
switch t := v.(type) {
case map[string]interface{}:
if err := eg.resolveReferences(t); err != nil {
return err
}

case []interface{}:
for _, e := range t {
eM, ok := e.(map[string]interface{})
if !ok {
continue
}
if err := eg.resolveReferences(eM); err != nil {
return err
}
}

case string:
g := reRef.FindStringSubmatch(t)
if len(g) != 2 {
continue
}
path := strings.Split(g[1], ".")
// expected reference format is <resource type>.<resource name>.<field name>
if len(path) < 3 {
continue
}
pm := eg.resources[path[0]]
if pm == nil || pm.paved == nil {
continue
}
pathStr := strings.Join(append([]string{"spec", "forProvider"}, path[2:]...), ".")
s, err := pm.paved.GetString(pathStr)
if fieldpath.IsNotFound(err) {
continue
}
if err != nil {
return errors.Wrapf(err, "cannot get string value from paved: %s", pathStr)
}
params[k] = s
}
}
return nil
}

// Generate generates an example manifest for the specified Terraform resource.
func (eg *ExampleGenerator) Generate(group, version string, r *config.Resource, fieldTransformations map[string]tjtypes.Transformation) error {
rm := eg.resourceMeta[r.Name]
if rm == nil || len(rm.Examples) == 0 {
return nil
}
var exampleParams map[string]interface{}
if err := json.TFParser.Unmarshal([]byte(rm.Examples[0].Manifest), &exampleParams); err != nil {
return errors.Wrapf(err, "cannot unmarshal example manifest for resource: %s", r.Name)
}
transformRefFields(exampleParams, r.ExternalName.OmittedFields, fieldTransformations, "")

example := map[string]interface{}{
"apiVersion": fmt.Sprintf("%s/%s", group, version),
"kind": r.Kind,
"metadata": map[string]interface{}{
"name": "example",
},
"spec": map[string]interface{}{
"forProvider": exampleParams,
"providerConfigRef": map[string]interface{}{
"name": "example",
},
},
}
manifestDir := filepath.Join(eg.rootDir, "examples-generated", strings.ToLower(strings.Split(group, ".")[0]))
eg.resources[r.Name] = &pavedWithManifest{
manifestPath: filepath.Join(manifestDir, fmt.Sprintf("%s.yaml", strings.ToLower(r.Kind))),
paved: fieldpath.Pave(example),
}
return nil
}

func getHierarchicalName(prefix, name string) string {
if prefix == "" {
return name
}
return fmt.Sprintf("%s.%s", prefix, name)
}

func transformRefFields(params map[string]interface{}, omittedFields []string, t map[string]tjtypes.Transformation, namePrefix string) { // nolint:gocyclo
for _, hn := range omittedFields {
for n := range params {
if hn == getHierarchicalName(namePrefix, n) {
delete(params, n)
break
}
}
}

for n, v := range params {
switch pT := v.(type) {
case map[string]interface{}:
transformRefFields(pT, omittedFields, t, getHierarchicalName(namePrefix, n))

case []interface{}:
for _, e := range pT {
eM, ok := e.(map[string]interface{})
if !ok {
continue
}
transformRefFields(eM, omittedFields, t, getHierarchicalName(namePrefix, n))
}
}
}

for hn, transform := range t {
for n, v := range params {
if hn == getHierarchicalName(namePrefix, n) {
delete(params, n)
if transform.IsRef {
if !transform.IsSensitive {
params[transform.TransformedName] = getRefField(v,
map[string]interface{}{
"name": "example",
})
} else {
secretName, secretKey := getSecretRef(v)
params[transform.TransformedName] = getRefField(v,
map[string]interface{}{
"name": secretName,
"namespace": "crossplane-system",
"key": secretKey,
})
}
} else {
params[transform.TransformedName] = v
}
break
}
}
}
}

func getRefField(v interface{}, ref map[string]interface{}) interface{} {
switch v.(type) {
case []interface{}:
return []interface{}{
ref,
}

default:
return ref
}
}

func getSecretRef(v interface{}) (string, string) {
secretName := "example-secret"
secretKey := "example-key"
s, ok := v.(string)
if !ok {
return secretName, secretKey
}
g := reRef.FindStringSubmatch(s)
if len(g) != 2 {
return secretName, secretKey
}
parts := strings.Split(g[1], ".")
if len(parts) < 3 {
return secretName, secretKey
}
secretName = fmt.Sprintf("example-%s", strings.Join(strings.Split(parts[0], "_")[1:], "-"))
secretKey = fmt.Sprintf("attribute.%s", strings.Join(parts[2:], "."))
return secretName, secretKey
}
Loading

0 comments on commit 6a039a9

Please sign in to comment.