Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor Encryption logic for improved testing #645

Merged
merged 8 commits into from
Jan 14, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions e2e/backup/backup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@ import (
"sync"
"time"

"github.com/rancher/backup-restore-operator/pkg/util/encryptionconfig"

. "github.com/kralicky/kmatch"
"github.com/minio/minio-go/v7"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/rancher/backup-restore-operator/e2e/test"
backupv1 "github.com/rancher/backup-restore-operator/pkg/apis/resources.cattle.io/v1"
backuputil "github.com/rancher/backup-restore-operator/pkg/util"
"github.com/rancher/wrangler/v3/pkg/condition"
"github.com/samber/lo"
"github.com/testcontainers/testcontainers-go"
Expand Down Expand Up @@ -301,7 +302,7 @@ var _ = Describe("Backup e2e local driver", Ordered, Label("integration"), func(
Namespace: ts.ChartNamespace,
},
Data: map[string][]byte{
backuputil.EncryptionProviderConfigKey: payload,
encryptionconfig.EncryptionProviderConfigKey: payload,
},
}
o.Add(secret)
Expand Down
5 changes: 3 additions & 2 deletions e2e/backup/setup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"fmt"

"github.com/rancher/backup-restore-operator/pkg/util/encryptionconfig"

. "github.com/kralicky/kmatch"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
Expand All @@ -12,7 +14,6 @@ import (
"github.com/rancher/backup-restore-operator/e2e/test"
backupv1 "github.com/rancher/backup-restore-operator/pkg/apis/resources.cattle.io/v1"
"github.com/rancher/backup-restore-operator/pkg/operator"
backuputil "github.com/rancher/backup-restore-operator/pkg/util"
"github.com/testcontainers/testcontainers-go"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -49,7 +50,7 @@ func SetupEncryption(o *ObjectTracker) {
Namespace: ts.ChartNamespace,
},
Data: map[string][]byte{
backuputil.EncryptionProviderConfigKey: payload,
encryptionconfig.EncryptionProviderConfigKey: payload,
},
}
o.Add(encsecret)
Expand Down
13 changes: 9 additions & 4 deletions pkg/controllers/backup/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
backupControllers "github.com/rancher/backup-restore-operator/pkg/generated/controllers/resources.cattle.io/v1"
"github.com/rancher/backup-restore-operator/pkg/resourcesets"
"github.com/rancher/backup-restore-operator/pkg/util"
"github.com/rancher/backup-restore-operator/pkg/util/encryptionconfig"
"github.com/rancher/wrangler/v3/pkg/condition"
v1core "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
"github.com/rancher/wrangler/v3/pkg/genericcondition"
Expand All @@ -22,8 +23,7 @@ import (

"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/storage/value"
k8sEncryptionconfig "k8s.io/apiserver/pkg/server/options/encryptionconfig"
"k8s.io/client-go/discovery"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/util/retry"
Expand Down Expand Up @@ -215,10 +215,15 @@ func (h *handler) OnBackupChange(_ string, backup *v1.Backup) (*v1.Backup, error

func (h *handler) performBackup(backup *v1.Backup, tmpBackupPath, backupFileName string) error {
var err error
transformerMap := make(map[schema.GroupResource]value.Transformer)
transformerMap := k8sEncryptionconfig.StaticTransformers{}
if backup.Spec.EncryptionConfigSecretName != "" {
logrus.Infof("Processing encryption config %v for backup CR %v", backup.Spec.EncryptionConfigSecretName, backup.Name)
transformerMap, err = util.GetEncryptionTransformersFromSecret(backup.Spec.EncryptionConfigSecretName, h.secrets)
encryptionConfigSecret, err := encryptionconfig.GetEncryptionConfigSecret(h.secrets, backup.Spec.EncryptionConfigSecretName)
if err != nil {
return err
}

transformerMap, err = encryptionconfig.GetEncryptionTransformersFromSecret(h.ctx, encryptionConfigSecret)
if err != nil {
return err
}
Expand Down
13 changes: 10 additions & 3 deletions pkg/controllers/restore/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
v1 "github.com/rancher/backup-restore-operator/pkg/apis/resources.cattle.io/v1"
restoreControllers "github.com/rancher/backup-restore-operator/pkg/generated/controllers/resources.cattle.io/v1"
"github.com/rancher/backup-restore-operator/pkg/util"
"github.com/rancher/backup-restore-operator/pkg/util/encryptionconfig"
lasso "github.com/rancher/lasso/pkg/client"
"github.com/rancher/wrangler/v3/pkg/condition"
v1core "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
Expand All @@ -28,7 +29,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/storage/value"
k8sEncryptionconfig "k8s.io/apiserver/pkg/server/options/encryptionconfig"
"k8s.io/client-go/discovery"
"k8s.io/client-go/dynamic"
coordinationclientv1 "k8s.io/client-go/kubernetes/typed/coordination/v1"
Expand Down Expand Up @@ -156,11 +157,17 @@ func (h *handler) OnRestoreChange(_ string, restore *v1.Restore) (*v1.Restore, e
backupResourceSet: v1.ResourceSet{},
}

transformerMap := make(map[schema.GroupResource]value.Transformer)
transformerMap := k8sEncryptionconfig.StaticTransformers{}
var err error
if restore.Spec.EncryptionConfigSecretName != "" {
logrus.Infof("Processing encryption config %v for restore CR %v", restore.Spec.EncryptionConfigSecretName, restore.Name)
transformerMap, err = util.GetEncryptionTransformersFromSecret(restore.Spec.EncryptionConfigSecretName, h.secrets)
encryptionConfigSecret, err := encryptionconfig.GetEncryptionConfigSecret(h.secrets, restore.Spec.EncryptionConfigSecretName)
if err != nil {
logrus.Errorf("Error fetching encryption config secret: %v", err)
return h.setReconcilingCondition(restore, err)
}

transformerMap, err = encryptionconfig.GetEncryptionTransformersFromSecret(h.ctx, encryptionConfigSecret)
if err != nil {
logrus.Errorf("Error processing encryption config: %v", err)
return h.setReconcilingCondition(restore, err)
Expand Down
15 changes: 7 additions & 8 deletions pkg/controllers/restore/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ import (
"strings"

v1 "github.com/rancher/backup-restore-operator/pkg/apis/resources.cattle.io/v1"
"github.com/rancher/backup-restore-operator/pkg/util/encryptionconfig"

"github.com/rancher/backup-restore-operator/pkg/objectstore"
"github.com/rancher/backup-restore-operator/pkg/util"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
k8sEncryptionconfig "k8s.io/apiserver/pkg/server/options/encryptionconfig"
"k8s.io/apiserver/pkg/storage/value"
)

Expand All @@ -41,7 +41,7 @@ func (h *handler) downloadFromS3(restore *v1.Restore, objStore *v1.S3ObjectStore
}

// very initial parts: https://medium.com/@skdomino/taring-untaring-files-in-go-6b07cf56bc07
func (h *handler) LoadFromTarGzip(tarGzFilePath string, transformerMap map[schema.GroupResource]value.Transformer,
func (h *handler) LoadFromTarGzip(tarGzFilePath string, transformerMap k8sEncryptionconfig.StaticTransformers,
cr *ObjectsFromBackupCR) error {
r, err := os.Open(tarGzFilePath)
if err != nil {
Expand Down Expand Up @@ -87,7 +87,7 @@ func (h *handler) LoadFromTarGzip(tarGzFilePath string, transformerMap map[schem
}

func (h *handler) loadDataFromFile(tarContent *tar.Header, readData []byte,
transformerMap map[schema.GroupResource]value.Transformer, cr *ObjectsFromBackupCR) error {
transformerMap k8sEncryptionconfig.StaticTransformers, cr *ObjectsFromBackupCR) error {
var name, namespace, additionalAuthenticatedData string

cr.resourcesFromBackup[tarContent.Name] = true
Expand All @@ -106,9 +106,8 @@ func (h *handler) loadDataFromFile(tarContent *tar.Header, readData []byte,
gvrStr := splitPath[0]
gvr := getGVR(gvrStr)

var staticTransformers encryptionconfig.StaticTransformers = transformerMap
decryptionTransformer := staticTransformers.TransformerForResource(gvr.GroupResource())
if decryptionTransformer != nil && !util.IsDefaultEncryptionTransformer(decryptionTransformer) {
decryptionTransformer := transformerMap.TransformerForResource(gvr.GroupResource())
if decryptionTransformer != nil && !encryptionconfig.IsDefaultEncryptionTransformer(decryptionTransformer) {
var encryptedBytes []byte
if err := json.Unmarshal(readData, &encryptedBytes); err != nil {
logrus.Errorf("Error unmarshaling encrypted data for resource [%v]: %v", gvr.GroupResource(), err)
Expand Down
4 changes: 2 additions & 2 deletions pkg/controllers/restore/prune.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/storage/value"
k8sEncryptionconfig "k8s.io/apiserver/pkg/server/options/encryptionconfig"
"k8s.io/client-go/dynamic"
)

Expand All @@ -24,7 +24,7 @@ type pruneResourceInfo struct {
gvr schema.GroupVersionResource
}

func (h *handler) prune(resourceSelectors []v1.ResourceSelector, transformerMap map[schema.GroupResource]value.Transformer,
func (h *handler) prune(resourceSelectors []v1.ResourceSelector, transformerMap k8sEncryptionconfig.StaticTransformers,
cr ObjectsFromBackupCR, deleteTimeout int) error {
var resourcesToDelete []pruneResourceInfo
rh := resourcesets.ResourceHandler{
Expand Down
12 changes: 6 additions & 6 deletions pkg/resourcesets/collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@ import (
"strings"

v1 "github.com/rancher/backup-restore-operator/pkg/apis/resources.cattle.io/v1"
"github.com/rancher/backup-restore-operator/pkg/util"
"github.com/rancher/backup-restore-operator/pkg/util/encryptionconfig"

"github.com/sirupsen/logrus"
apierrors "k8s.io/apimachinery/pkg/api/errors"
k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
k8sEncryptionconfig "k8s.io/apiserver/pkg/server/options/encryptionconfig"
"k8s.io/apiserver/pkg/storage/value"
"k8s.io/client-go/discovery"
"k8s.io/client-go/dynamic"
Expand All @@ -33,7 +34,7 @@ type GVResource struct {
type ResourceHandler struct {
DiscoveryClient discovery.DiscoveryInterface
DynamicClient dynamic.Interface
TransformerMap map[schema.GroupResource]value.Transformer
TransformerMap k8sEncryptionconfig.StaticTransformers
GVResourceToObjects map[GVResource][]unstructured.Unstructured
Ctx context.Context
}
Expand Down Expand Up @@ -389,8 +390,7 @@ func (h *ResourceHandler) WriteBackupObjects(backupPath string) error {
}

gr := schema.ParseGroupResource(gvResource.Name + "." + gv.Group)
var staticTransformers encryptionconfig.StaticTransformers = h.TransformerMap
encryptionTransformer := staticTransformers.TransformerForResource(gr)
encryptionTransformer := h.TransformerMap.TransformerForResource(gr)
additionalAuthenticatedData := objName
if gvResource.Namespaced {
additionalAuthenticatedData = fmt.Sprintf("%s#%s", metadata["namespace"].(string), additionalAuthenticatedData)
Expand Down Expand Up @@ -436,7 +436,7 @@ func writeToBackup(ctx context.Context, resource map[string]interface{}, backupP
if err != nil {
return fmt.Errorf("error converting resource to JSON: %v", err)
}
if transformer != nil && !util.IsDefaultEncryptionTransformer(transformer) {
if transformer != nil && !encryptionconfig.IsDefaultEncryptionTransformer(transformer) {
encrypted, err := transformer.TransformToStorage(ctx, resourceBytes, value.DefaultContext(additionalAuthenticatedData))
if err != nil {
return fmt.Errorf("error converting resource to JSON: %v", err)
Expand Down
78 changes: 78 additions & 0 deletions pkg/util/encryptionconfig/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package encryptionconfig

import (
"context"
"fmt"
"os"

"github.com/rancher/backup-restore-operator/pkg/util"
v1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"

"github.com/sirupsen/logrus"
v1core "k8s.io/api/core/v1"
v2 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8sEncryptionconfig "k8s.io/apiserver/pkg/server/options/encryptionconfig"
storagevalue "k8s.io/apiserver/pkg/storage/value"
"k8s.io/apiserver/pkg/storage/value/encrypt/identity"
)

type contextKey string

var tempConfigPathKey = contextKey("tmpConfigPath")

const EncryptionProviderConfigKey = "encryption-provider-config.yaml"

func GetEncryptionConfigSecret(secrets v1.SecretController, encryptionConfigSecretName string) (*v1core.Secret, error) {
// EncryptionConfig secret ns is hardcoded to ns of controller in chart's ns
// kubectl create secret generic test-encryptionconfig --from-file=./encryption-provider-config.yaml
logrus.Infof("Get encryption config from namespace %v", util.GetChartNamespace())
encryptionConfigSecret, err := secrets.Get(util.GetChartNamespace(), encryptionConfigSecretName, v2.GetOptions{})
if err != nil {
return nil, err
}

return encryptionConfigSecret, nil
}

func GetEncryptionTransformersFromSecret(ctx context.Context, encryptionConfigSecret *v1core.Secret) (k8sEncryptionconfig.StaticTransformers, error) {
fileHandle, err := PrepareEncryptionConfigSecretTempConfig(encryptionConfigSecret)
// we defer file removal till here to ensure it's around for all of PrepareEncryptionTransformersFromConfig
defer os.Remove(EncryptionProviderConfigKey)
if err != nil {
return nil, err
}
ctx = context.WithValue(ctx, tempConfigPathKey, fileHandle.Name())
return PrepareEncryptionTransformersFromConfig(ctx, EncryptionProviderConfigKey)
}

func PrepareEncryptionConfigSecretTempConfig(encryptionConfigSecret *v1core.Secret) (*os.File, error) {
encryptionConfigBytes, ok := encryptionConfigSecret.Data[EncryptionProviderConfigKey]
if !ok {
return nil, fmt.Errorf("no encryptionConfig provided")
}
err := os.WriteFile(EncryptionProviderConfigKey, encryptionConfigBytes, os.ModePerm)
if err != nil {
return nil, err
}

// Open the file for reading (or other operations) and return the handle
file, err := os.Open(EncryptionProviderConfigKey)
if err != nil {
return nil, err
}

Comment on lines +58 to +63
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't notice this before, but you we should be careful about opening files inside a function and expecting others to close said file.

Can lead to concurrent closes or file descriptor leaks.
We should look into alternative return types instead of os.File. I also don't see this being used by anything other than the tests at the moment.

Will we need this inside the controller logic?

return file, nil
}

func PrepareEncryptionTransformersFromConfig(ctx context.Context, encryptionProviderPath string) (k8sEncryptionconfig.StaticTransformers, error) {
apiServerID := ""
encryptionConfig, err := k8sEncryptionconfig.LoadEncryptionConfig(ctx, encryptionProviderPath, false, apiServerID)
if err != nil {
return nil, err
}
return encryptionConfig.Transformers, nil
}

func IsDefaultEncryptionTransformer(transformer storagevalue.Transformer) bool {
return transformer == identity.NewEncryptCheckTransformer()
}
Loading
Loading