From 7a5693d18bd15a9136ab9aabedb21d3d086298bd Mon Sep 17 00:00:00 2001 From: Ramon Quitales Date: Mon, 30 Sep 2024 19:47:11 -0700 Subject: [PATCH] Resolve package collisions (#153) ## Proposed Changes This PR ensures the correct CRD versions are used when parsing. Previously, the Swagger version generated by `builder.BuildOpenAPIV2` defaulted to `0.1.0`, but we should be pulling the version directly from the CRD spec itself. This change ensures that SDKs are generated for all CRD versions. Additionally, this PR incorporates the latest updates from p-k to prevent package collisions. ## Related Issues (optional) Closes: #152 --- CHANGELOG.md | 5 + go.mod | 12 +- go.sum | 24 +- pkg/codegen/customresourcegenerator.go | 13 +- pkg/codegen/customresourcegenerator_test.go | 233 ++++++++++++++++++++ tests/crds_test.go | 12 +- 6 files changed, 268 insertions(+), 31 deletions(-) create mode 100644 pkg/codegen/customresourcegenerator_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 98eeb16..d36a5fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +## 1.5.3 (2024-09-30) + +- Fix crd2pulumi not generating all CRD versions. [#152](https://github.com/pulumi/crd2pulumi/issues/152) +- Fix crd2pulumi generating packages and types with incorrect group names. [#152](https://github.com/pulumi/crd2pulumi/issues/152) + ## 1.5.2 (2024-09-16) - Set the pulumi-kubernetes dependency for Python packages to v4.18.0. [#148](https://github.com/pulumi/crd2pulumi/issues/148) diff --git a/go.mod b/go.mod index 234b20c..e2216cd 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,10 @@ toolchain go1.22.7 require ( github.com/go-openapi/jsonreference v0.21.0 github.com/iancoleman/strcase v0.3.0 - github.com/pulumi/pulumi-java/pkg v0.14.0 - github.com/pulumi/pulumi-kubernetes/provider/v4 v4.0.0-20240916231657-74673fdf488b - github.com/pulumi/pulumi/pkg/v3 v3.130.0 - github.com/pulumi/pulumi/sdk/v3 v3.130.0 + github.com/pulumi/pulumi-java/pkg v0.16.1 + github.com/pulumi/pulumi-kubernetes/provider/v4 v4.0.0-20240930223531-b7dae6b4ea56 + github.com/pulumi/pulumi/pkg/v3 v3.134.1 + github.com/pulumi/pulumi/sdk/v3 v3.134.1 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 golang.org/x/text v0.17.0 @@ -41,7 +41,6 @@ require ( github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect github.com/agext/levenshtein v1.2.3 // indirect - github.com/ahmetb/go-linq v3.0.0+incompatible // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect @@ -181,7 +180,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect github.com/pulumi/cloud-ready-checks v1.1.1-0.20240731201114-3a703c6bee71 // indirect - github.com/pulumi/esc v0.9.1 // indirect + github.com/pulumi/esc v0.10.0 // indirect github.com/pulumi/inflector v0.1.1 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect @@ -197,7 +196,6 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/texttheater/golang-levenshtein v1.0.1 // indirect - github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7 // indirect github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/go.sum b/go.sum index bb5ee66..f5334fe 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,6 @@ github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmH github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= -github.com/ahmetb/go-linq v3.0.0+incompatible h1:qQkjjOXKrKOTy83X8OpRmnKflXKQIL/mC/gMVVDMhOA= -github.com/ahmetb/go-linq v3.0.0+incompatible/go.mod h1:PFffvbdbtw+QTB0WKRP0cNht7vnCfnGlEpak/DVg5cY= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= @@ -469,18 +467,18 @@ github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 h1:vkHw5I/plNdTr435 github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231/go.mod h1:murToZ2N9hNJzewjHBgfFdXhZKjY3z5cYC1VXk+lbFE= github.com/pulumi/cloud-ready-checks v1.1.1-0.20240731201114-3a703c6bee71 h1:r0nNcIW3Y+wvXBfstXoD+ALyqJnIbcCxbDz0pHME+pQ= github.com/pulumi/cloud-ready-checks v1.1.1-0.20240731201114-3a703c6bee71/go.mod h1:Rkj3dUQiwjbCpuk+6d2fAtSzSbDiaJVx2pI4kBB/HkY= -github.com/pulumi/esc v0.9.1 h1:HH5eEv8sgyxSpY5a8yePyqFXzA8cvBvapfH8457+mIs= -github.com/pulumi/esc v0.9.1/go.mod h1:oEJ6bOsjYlQUpjf70GiX+CXn3VBmpwFDxUTlmtUN84c= +github.com/pulumi/esc v0.10.0 h1:jzBKzkLVW0mePeanDRfqSQoCJ5yrkux0jIwAkUxpRKE= +github.com/pulumi/esc v0.10.0/go.mod h1:2Bfa+FWj/xl8CKqRTWbWgDX0SOD4opdQgvYSURTGK2c= github.com/pulumi/inflector v0.1.1 h1:dvlxlWtXwOJTUUtcYDvwnl6Mpg33prhK+7mzeF+SobA= github.com/pulumi/inflector v0.1.1/go.mod h1:HUFCjcPTz96YtTuUlwG3i3EZG4WlniBvR9bd+iJxCUY= -github.com/pulumi/pulumi-java/pkg v0.14.0 h1:CKL7lLF81Fq6VRhA5TNFsSMnHraTNCUzIhqCzYX8Wzk= -github.com/pulumi/pulumi-java/pkg v0.14.0/go.mod h1:VybuJMWJtJc9ZNbt1kcYH4TbpocMx9mEi7YWL2Co99c= -github.com/pulumi/pulumi-kubernetes/provider/v4 v4.0.0-20240916231657-74673fdf488b h1:Eb5/zL+RY2TnHn0zciQ+3lylAuvsWoi8XvLKRpc9xGE= -github.com/pulumi/pulumi-kubernetes/provider/v4 v4.0.0-20240916231657-74673fdf488b/go.mod h1:glS84cIQOYkQBbAtHQjlNnx28XAZrPMwlPah4+Izj2U= -github.com/pulumi/pulumi/pkg/v3 v3.130.0 h1:lS51XeCnhg72LXkMiw2FP1cGP+Y85wYD3quWhCPD5+M= -github.com/pulumi/pulumi/pkg/v3 v3.130.0/go.mod h1:jhZ1Ug5Rl1qivexgEWvmwSWYIT/jRnKSFhLwwv6PrZ0= -github.com/pulumi/pulumi/sdk/v3 v3.130.0 h1:gGJNd+akPqhZ+vrsZmAjSNJn6kGJkitjjkwrmIQMmn8= -github.com/pulumi/pulumi/sdk/v3 v3.130.0/go.mod h1:p1U24en3zt51agx+WlNboSOV8eLlPWYAkxMzVEXKbnY= +github.com/pulumi/pulumi-java/pkg v0.16.1 h1:orHnDWFbpOERwaBLry9f+6nqPX7x0MsrIkaa5QDGAns= +github.com/pulumi/pulumi-java/pkg v0.16.1/go.mod h1:QH0DihZkWYle9XFc+LJ76m4hUo+fA3RdyaM90pqOaSM= +github.com/pulumi/pulumi-kubernetes/provider/v4 v4.0.0-20240930223531-b7dae6b4ea56 h1:r0lPAtVXpold10wxlSz8YkMM2w336xKMzxkeUUNP+h4= +github.com/pulumi/pulumi-kubernetes/provider/v4 v4.0.0-20240930223531-b7dae6b4ea56/go.mod h1:caVn9OjrsHrldGBZ4/VvZreVCV46asZ0DRnkONEja38= +github.com/pulumi/pulumi/pkg/v3 v3.134.1 h1:iGKvaSHEoPCGBqDoIGQUXUm3qkrawfd513lL0I9vnNQ= +github.com/pulumi/pulumi/pkg/v3 v3.134.1/go.mod h1:1iCee1QIwXYvkIQJ/HnBjsPsmYJ/arBPWX6hAao/Pro= +github.com/pulumi/pulumi/sdk/v3 v3.134.1 h1:v1zd0d+B9gpUhsdJ483YUMHwHXqDvXvZ+mh/A4HhPWg= +github.com/pulumi/pulumi/sdk/v3 v3.134.1/go.mod h1:J5kQEX8v87aeUhk6NdQXnjCo1DbiOnOiL3Sf2DuDda8= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= @@ -536,8 +534,6 @@ github.com/texttheater/golang-levenshtein v1.0.1 h1:+cRNoVrfiwufQPhoMzB6N0Yf/Mqa github.com/texttheater/golang-levenshtein v1.0.1/go.mod h1:PYAKrbF5sAiq9wd+H82hs7gNaen0CplQ9uvm6+enD/8= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= -github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7 h1:X9dsIWPuuEJlPX//UmRKophhOKCGXc46RVIGuttks68= -github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7/go.mod h1:UxoP3EypF8JfGEjAII8jx1q8rQyDnX8qdTCs/UQBVIE= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= diff --git a/pkg/codegen/customresourcegenerator.go b/pkg/codegen/customresourcegenerator.go index b9302dc..ba36b1c 100644 --- a/pkg/codegen/customresourcegenerator.go +++ b/pkg/codegen/customresourcegenerator.go @@ -214,15 +214,12 @@ func sanitizeReferenceName(fieldName string) string { } // crdToOpenAPI generates the OpenAPI specs for a given CRD manifest. -func crdToOpenAPI(crd *extensionv1.CustomResourceDefinition) ([]*spec.Swagger, error) { - var openAPIManifests []*spec.Swagger +func crdToOpenAPI(crd *extensionv1.CustomResourceDefinition) (map[string]*spec.Swagger, error) { + openAPIManifests := make(map[string]*spec.Swagger) setCRDDefaults(crd) for _, v := range crd.Spec.Versions { - if !v.Served { - continue - } // Defaults are not pruned here, but before being served. sw, err := builder.BuildOpenAPIV2(crd, v.Name, builder.Options{V2: true, StripValueValidation: true, StripNullable: true, AllowNonStructural: true, IncludeSelectableFields: true}) if err != nil { @@ -234,7 +231,7 @@ func crdToOpenAPI(crd *extensionv1.CustomResourceDefinition) ([]*spec.Swagger, e return nil, fmt.Errorf("error flattening OpenAPI spec: %w", err) } - openAPIManifests = append(openAPIManifests, sw) + openAPIManifests[v.Name] = sw } return openAPIManifests, nil @@ -261,8 +258,8 @@ func NewCustomResourceGenerator(crd extensionv1.CustomResourceDefinition) (Custo return CustomResourceGenerator{}, fmt.Errorf("could not generate OpenAPI spec for CRD: %w", err) } - for _, sw := range swagger { - schemas[sw.Info.Version] = *sw + for version, sw := range swagger { + schemas[version] = *sw } kind := crd.Spec.Names.Kind diff --git a/pkg/codegen/customresourcegenerator_test.go b/pkg/codegen/customresourcegenerator_test.go new file mode 100644 index 0000000..f112c59 --- /dev/null +++ b/pkg/codegen/customresourcegenerator_test.go @@ -0,0 +1,233 @@ +// Copyright 2016-2024, Pulumi Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codegen + +import ( + "testing" + + extensionv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestSetCRDDefaults(t *testing.T) { + tests := []struct { + name string + crd extensionv1.CustomResourceDefinition + expected extensionv1.CustomResourceDefinition + }{ + { + name: "Singular and ListKind are empty", + crd: extensionv1.CustomResourceDefinition{ + Spec: extensionv1.CustomResourceDefinitionSpec{ + Names: extensionv1.CustomResourceDefinitionNames{ + Kind: "TestKind", + }, + }, + }, + expected: extensionv1.CustomResourceDefinition{ + Spec: extensionv1.CustomResourceDefinitionSpec{ + Names: extensionv1.CustomResourceDefinitionNames{ + Kind: "TestKind", + Singular: "testkind", + ListKind: "TestKindList", + }, + }, + }, + }, + { + name: "Singular is set, ListKind is empty", + crd: extensionv1.CustomResourceDefinition{ + Spec: extensionv1.CustomResourceDefinitionSpec{ + Names: extensionv1.CustomResourceDefinitionNames{ + Kind: "TestKind", + Singular: "customsingular", + }, + }, + }, + expected: extensionv1.CustomResourceDefinition{ + Spec: extensionv1.CustomResourceDefinitionSpec{ + Names: extensionv1.CustomResourceDefinitionNames{ + Kind: "TestKind", + Singular: "customsingular", + ListKind: "TestKindList", + }, + }, + }, + }, + { + name: "Singular is empty, ListKind is set", + crd: extensionv1.CustomResourceDefinition{ + Spec: extensionv1.CustomResourceDefinitionSpec{ + Names: extensionv1.CustomResourceDefinitionNames{ + Kind: "TestKind", + ListKind: "CustomListKind", + }, + }, + }, + expected: extensionv1.CustomResourceDefinition{ + Spec: extensionv1.CustomResourceDefinitionSpec{ + Names: extensionv1.CustomResourceDefinitionNames{ + Kind: "TestKind", + Singular: "testkind", + ListKind: "CustomListKind", + }, + }, + }, + }, + { + name: "Singular and ListKind are set", + crd: extensionv1.CustomResourceDefinition{ + Spec: extensionv1.CustomResourceDefinitionSpec{ + Names: extensionv1.CustomResourceDefinitionNames{ + Kind: "TestKind", + Singular: "customsingular", + ListKind: "CustomListKind", + }, + }, + }, + expected: extensionv1.CustomResourceDefinition{ + Spec: extensionv1.CustomResourceDefinitionSpec{ + Names: extensionv1.CustomResourceDefinitionNames{ + Kind: "TestKind", + Singular: "customsingular", + ListKind: "CustomListKind", + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + setCRDDefaults(&tt.crd) + if tt.crd.Spec.Names.Singular != tt.expected.Spec.Names.Singular { + t.Errorf("expected Singular %s, got %s", tt.expected.Spec.Names.Singular, tt.crd.Spec.Names.Singular) + } + if tt.crd.Spec.Names.ListKind != tt.expected.Spec.Names.ListKind { + t.Errorf("expected ListKind %s, got %s", tt.expected.Spec.Names.ListKind, tt.crd.Spec.Names.ListKind) + } + }) + } +} +func TestNewCustomResourceGenerator(t *testing.T) { + tests := []struct { + name string + crd extensionv1.CustomResourceDefinition + expected CustomResourceGenerator + wantErr bool + }{ + { + name: "Valid CRD", + crd: extensionv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apiextensions.k8s.io/v1", + }, + Spec: extensionv1.CustomResourceDefinitionSpec{ + Group: "example.com", + Names: extensionv1.CustomResourceDefinitionNames{ + Kind: "TestKind", + Plural: "testkinds", + }, + Versions: []extensionv1.CustomResourceDefinitionVersion{ + {Name: "v1"}, + }, + }, + }, + expected: CustomResourceGenerator{ + APIVersion: "apiextensions.k8s.io/v1", + Kind: "TestKind", + Plural: "testkinds", + Group: "example.com", + Versions: []string{"v1"}, + GroupVersions: []string{ + "example.com/v1", + }, + ResourceTokens: []string{ + "example.com:TestKind:v1", + }, + }, + wantErr: false, + }, + { + name: "Valid CRD with multiple versions", + crd: extensionv1.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apiextensions.k8s.io/v1", + }, + Spec: extensionv1.CustomResourceDefinitionSpec{ + Group: "example.com", + Names: extensionv1.CustomResourceDefinitionNames{ + Kind: "TestKind", + Plural: "testkinds", + }, + Versions: []extensionv1.CustomResourceDefinitionVersion{ + {Name: "v1"}, + {Name: "v1alpha1"}, + }, + }, + }, + expected: CustomResourceGenerator{ + APIVersion: "apiextensions.k8s.io/v1", + Kind: "TestKind", + Plural: "testkinds", + Group: "example.com", + Versions: []string{"v1alpha1", "v1"}, + GroupVersions: []string{ + "example.com/v1", + "example.com/v1alpha1", + }, + ResourceTokens: []string{ + "example.com:TestKind:v1", + "example.com:TestKind:v1alpha1", + }, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewCustomResourceGenerator(tt.crd) + if (err != nil) != tt.wantErr { + t.Errorf("NewCustomResourceGenerator() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr != false { + return + } + if got.APIVersion != tt.expected.APIVersion { + t.Errorf("expected APIVersion %s, got %s", tt.expected.APIVersion, got.APIVersion) + } + if got.Kind != tt.expected.Kind { + t.Errorf("expected Kind %s, got %s", tt.expected.Kind, got.Kind) + } + if got.Plural != tt.expected.Plural { + t.Errorf("expected Plural %s, got %s", tt.expected.Plural, got.Plural) + } + if got.Group != tt.expected.Group { + t.Errorf("expected Group %s, got %s", tt.expected.Group, got.Group) + } + if len(got.Versions) != len(tt.expected.Versions) { + t.Errorf("expected Versions %v, got %v", tt.expected.Versions, got.Versions) + } + if len(got.GroupVersions) != len(tt.expected.GroupVersions) { + t.Errorf("expected GroupVersions %v, got %v", tt.expected.GroupVersions, got.GroupVersions) + } + if len(got.ResourceTokens) != len(tt.expected.ResourceTokens) { + t.Errorf("expected ResourceTokens %v, got %v", tt.expected.ResourceTokens, got.ResourceTokens) + } + }) + } +} diff --git a/tests/crds_test.go b/tests/crds_test.go index 9bfda95..f94129a 100644 --- a/tests/crds_test.go +++ b/tests/crds_test.go @@ -189,9 +189,17 @@ func TestCRDsFromUrl(t *testing.T) { t.Run(tt.name, func(t *testing.T) { for _, lang := range languages { t.Run(lang, func(t *testing.T) { - if lang == "dotnet" && (tt.name == "CertManager" || tt.name == "GKEManagedCerts") { - t.Skip("Skipping compilation for dotnet. See https://github.com/pulumi/crd2pulumi/issues/17") + if lang == "dotnet" { + if tt.name == "CertManager" || tt.name == "GKEManagedCerts" { + t.Skip("Skipping compilation for dotnet. See https://github.com/pulumi/crd2pulumi/issues/17") + } + + if tt.name == "Percona" { + t.Skip("Skipping dotnet compilation for Percona as we generate invalid code with hyphens that are not allowed in C# identifiers.") + } + } + execCrd2Pulumi(t, lang, tt.url, compileValidationFn[lang]) }) }