Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Provisioner - fully remove obsolete code for runtime registration/deregistration in compass #3411

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 1 addition & 48 deletions components/provisioner/cmd/init.go
Original file line number Diff line number Diff line change
@@ -1,25 +1,17 @@
package main

import (
"crypto/tls"
"fmt"
"net/http"
"os"
"time"

"k8s.io/apimachinery/pkg/util/yaml"

"github.com/kyma-project/control-plane/components/provisioner/internal/operations/queue"

"github.com/kyma-project/control-plane/components/provisioner/internal/provisioning/persistence/dbsession"

"github.com/kyma-project/control-plane/components/provisioner/internal/director"
"github.com/kyma-project/control-plane/components/provisioner/internal/gardener"
"github.com/kyma-project/control-plane/components/provisioner/internal/graphql"
"github.com/kyma-project/control-plane/components/provisioner/internal/oauth"
"github.com/kyma-project/control-plane/components/provisioner/internal/provisioning"
"github.com/kyma-project/control-plane/components/provisioner/internal/uuid"
"github.com/pkg/errors"
ctrl "sigs.k8s.io/controller-runtime"

restclient "k8s.io/client-go/rest"
Expand All @@ -38,15 +30,13 @@ func newProvisioningService(
gardenerProject string,
provisioner provisioning.Provisioner,
dbsFactory dbsession.Factory,
directorService director.DirectorClient,
shootProvider gardener.ShootProvider,
provisioningQueue queue.OperationQueue,
deprovisioningQueue queue.OperationQueue,
shootUpgradeQueue queue.OperationQueue,
defaultEnableKubernetesVersionAutoUpdate,
defaultEnableMachineImageVersionAutoUpdate bool,
defaultEnableIMDSv2 bool,
runtimeRegistrationEnabled bool,
dynamicKubeconfigProvider DynamicKubeconfigProvider) provisioning.Service {

uuidGenerator := uuid.NewUUIDGenerator()
Expand All @@ -56,42 +46,14 @@ func newProvisioningService(
return provisioning.NewProvisioningService(
inputConverter,
graphQLConverter,
directorService,
dbsFactory,
provisioner,
uuidGenerator,
shootProvider,
provisioningQueue,
deprovisioningQueue,
shootUpgradeQueue,
dynamicKubeconfigProvider,
runtimeRegistrationEnabled)
}

func newDirectorClient(config config) (director.DirectorClient, error) {
file, err := os.ReadFile(config.DirectorOAuthPath)
if err != nil {
return nil, errors.Wrap(err, "Failed to open director config")
}

cfg := DirectorOAuth{}
err = yaml.Unmarshal(file, &cfg)
if err != nil {
return nil, errors.Wrap(err, "Failed to unmarshal director config")
}

gqlClient := graphql.NewGraphQLClient(config.DirectorURL, true, config.SkipDirectorCertVerification)
oauthClient := oauth.NewOauthClient(newHTTPClient(config.SkipDirectorCertVerification), cfg.Data.ClientID, cfg.Data.ClientSecret, cfg.Data.TokensEndpoint)

return director.NewDirectorClient(gqlClient, oauthClient), nil
}

type DirectorOAuth struct {
Data struct {
ClientID string `json:"client_id"`
ClientSecret string `json:"client_secret"`
TokensEndpoint string `json:"tokens_endpoint"`
} `json:"data"`
dynamicKubeconfigProvider)
}

func newShootController(gardenerNamespace string, gardenerClusterCfg *restclient.Config, dbsFactory dbsession.Factory, auditLogTenantConfigPath string) (*gardener.ShootController, error) {
Expand Down Expand Up @@ -119,12 +81,3 @@ func newGardenerClusterConfig(cfg config) (*restclient.Config, error) {

return gardenerClusterConfig, nil
}

func newHTTPClient(skipCertVerification bool) *http.Client {
return &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: skipCertVerification},
},
Timeout: 30 * time.Second,
}
}
71 changes: 8 additions & 63 deletions components/provisioner/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import (
provisioningStages "github.com/kyma-project/control-plane/components/provisioner/internal/operations/stages/provisioning"
"github.com/kyma-project/control-plane/components/provisioner/internal/persistence/database"
"github.com/kyma-project/control-plane/components/provisioner/internal/provisioning/persistence/dbsession"
"github.com/kyma-project/control-plane/components/provisioner/internal/runtime"
"github.com/kyma-project/control-plane/components/provisioner/internal/util/k8s"
"github.com/kyma-project/control-plane/components/provisioner/pkg/gqlschema"
"github.com/pkg/errors"
Expand All @@ -40,14 +39,9 @@ import (
const connStringFormat string = "host=%s port=%s user=%s password=%s dbname=%s sslmode=%s sslrootcert=%s"

type config struct {
Address string `envconfig:"default=127.0.0.1:3000"`
APIEndpoint string `envconfig:"default=/graphql"`
PlaygroundAPIEndpoint string `envconfig:"default=/graphql"`
DirectorURL string `envconfig:"default=http://compass-director.compass-system.svc.cluster.local:3000/graphql"`
SkipDirectorCertVerification bool `envconfig:"default=false"`
DirectorOAuthPath string `envconfig:"APP_DIRECTOR_OAUTH_PATH,default=./dev/director.yaml"`
RuntimeRegistrationEnabled bool `envconfig:"default=true"`
RuntimeDeregistrationEnabled bool `envconfig:"default=true"`
Address string `envconfig:"default=127.0.0.1:3000"`
APIEndpoint string `envconfig:"default=/graphql"`
PlaygroundAPIEndpoint string `envconfig:"default=/graphql"`

Database struct {
User string `envconfig:"default=postgres"`
Expand Down Expand Up @@ -78,9 +72,6 @@ type config struct {
DefaultEnableIMDSv2 bool `envconfig:"default=false"`
}

LatestDownloadedReleases int `envconfig:"default=5"`
DownloadPreReleases bool `envconfig:"default=true"`

EnqueueInProgressOperations bool `envconfig:"default=true"`

MetricsAddress string `envconfig:"default=127.0.0.1:9000"`
Expand All @@ -89,34 +80,26 @@ type config struct {
}

func (c *config) String() string {
return fmt.Sprintf("Address: %s, APIEndpoint: %s, DirectorURL: %s, "+
"SkipDirectorCertVerification: %v, DirectorOAuthPath: %s, "+
"RuntimeRegistrationEnabled %v, RuntimeDeregistrationEnabled %v, "+
return fmt.Sprintf("Address: %s, APIEndpoint: %s, "+
"DatabaseUser: %s, DatabaseHost: %s, DatabasePort: %s, "+
"DatabaseName: %s, DatabaseSSLMode: %s, "+
"ProvisioningTimeoutClusterCreation: %s "+
"ProvisioningTimeoutInstallation: %s, ProvisioningTimeoutUpgrade: %s, "+
"ProvisioningTimeoutAgentConfiguration: %s, ProvisioningTimeoutAgentConnection: %s, "+
"DeprovisioningNoInstallTimeoutClusterDeletion: %s, DeprovisioningNoInstallTimeoutWaitingForClusterDeletion: %s "+
"ShootUpgradeTimeout: %s, "+
"OperatorRoleBindingL2SubjectName: %s, OperatorRoleBindingL3SubjectName: %s, OperatorRoleBindingCreatingForAdmin: %t "+
"GardenerProject: %s, GardenerKubeconfigPath: %s, GardenerAuditLogsPolicyConfigMap: %s, AuditLogsTenantConfigPath: %s, DefaultEnableIMDSv2: %v"+
"LatestDownloadedReleases: %d, DownloadPreReleases: %v, "+
"EnqueueInProgressOperations: %v"+
"LogLevel: %s",
c.Address, c.APIEndpoint, c.DirectorURL,
c.SkipDirectorCertVerification, c.DirectorOAuthPath,
c.RuntimeDeregistrationEnabled, c.RuntimeDeregistrationEnabled,
c.Address, c.APIEndpoint,
c.Database.User, c.Database.Host, c.Database.Port,
c.Database.Name, c.Database.SSLMode,
c.ProvisioningTimeout.ClusterCreation.String(),
c.ProvisioningTimeout.Installation.String(), c.ProvisioningTimeout.Upgrade.String(),
c.ProvisioningTimeout.AgentConfiguration.String(), c.ProvisioningTimeout.AgentConnection.String(),
c.DeprovisioningTimeout.ClusterDeletion.String(), c.DeprovisioningTimeout.WaitingForClusterDeletion.String(),
c.ProvisioningTimeout.ShootUpgrade.String(),
c.OperatorRoleBinding.L2SubjectName, c.OperatorRoleBinding.L3SubjectName, c.OperatorRoleBinding.CreatingForAdmin,
c.Gardener.Project, c.Gardener.KubeconfigPath, c.Gardener.AuditLogsPolicyConfigMap, c.Gardener.AuditLogsTenantConfigPath, c.Gardener.DefaultEnableIMDSv2,
c.LatestDownloadedReleases, c.DownloadPreReleases,
c.EnqueueInProgressOperations,
c.LogLevel)
}
Expand Down Expand Up @@ -169,50 +152,14 @@ func main() {

shootClient := gardenerClientSet.Shoots(gardenerNamespace)

directorClient, err := newDirectorClient(cfg)
exitOnError(err, "Failed to initialize Director client")

k8sClientProvider := k8s.NewK8sClientProvider()

runtimeConfigurator := runtime.NewRuntimeConfigurator(k8sClientProvider, directorClient)
adminKubeconfigRequest := gardenerClient.SubResource("adminkubeconfig")
kubeconfigProvider := gardener.NewKubeconfigProvider(shootClient, adminKubeconfigRequest, secretsInterface)

var provisioningQueue queue.OperationQueue
var shootUpgradeQueue queue.OperationQueue

if cfg.RuntimeRegistrationEnabled {
provisioningQueue = queue.CreateProvisioningQueue(
cfg.ProvisioningTimeout,
dbsFactory,
directorClient,
shootClient,
cfg.OperatorRoleBinding,
k8sClientProvider,
runtimeConfigurator,
kubeconfigProvider)

shootUpgradeQueue = queue.CreateShootUpgradeQueue(cfg.ProvisioningTimeout, dbsFactory, directorClient, shootClient, cfg.OperatorRoleBinding, k8sClientProvider, kubeconfigProvider)

} else {
provisioningQueue = queue.CreateProvisioningQueueWithoutRegistration(
cfg.ProvisioningTimeout,
dbsFactory,
shootClient,
cfg.OperatorRoleBinding,
k8sClientProvider,
kubeconfigProvider)

shootUpgradeQueue = queue.CreateShootUpgradeQueue(cfg.ProvisioningTimeout, dbsFactory, nil, shootClient, cfg.OperatorRoleBinding, k8sClientProvider, kubeconfigProvider)
}

var deprovisioningQueue queue.OperationQueue

if cfg.RuntimeDeregistrationEnabled {
deprovisioningQueue = queue.CreateDeprovisioningQueue(cfg.DeprovisioningTimeout, dbsFactory, directorClient, shootClient)
} else {
deprovisioningQueue = queue.CreateDeprovisioningQueue(cfg.DeprovisioningTimeout, dbsFactory, nil, shootClient)
}
provisioningQueue := queue.CreateProvisioningQueue(cfg.ProvisioningTimeout, dbsFactory, shootClient, cfg.OperatorRoleBinding, k8sClientProvider, kubeconfigProvider)
shootUpgradeQueue := queue.CreateShootUpgradeQueue(cfg.ProvisioningTimeout, dbsFactory, shootClient, cfg.OperatorRoleBinding, k8sClientProvider, kubeconfigProvider)
deprovisioningQueue := queue.CreateDeprovisioningQueue(cfg.DeprovisioningTimeout, dbsFactory, shootClient)

provisioner := gardener.NewProvisioner(gardenerNamespace, shootClient, dbsFactory, cfg.Gardener.AuditLogsPolicyConfigMap, cfg.Gardener.MaintenanceWindowConfigPath)
shootController, err := newShootController(gardenerNamespace, gardenerClusterConfig, dbsFactory, cfg.Gardener.AuditLogsTenantConfigPath)
Expand All @@ -226,15 +173,13 @@ func main() {
cfg.Gardener.Project,
provisioner,
dbsFactory,
directorClient,
gardener.NewShootProvider(shootClient),
provisioningQueue,
deprovisioningQueue,
shootUpgradeQueue,
cfg.Gardener.DefaultEnableKubernetesVersionAutoUpdate,
cfg.Gardener.DefaultEnableMachineImageVersionAutoUpdate,
cfg.Gardener.DefaultEnableIMDSv2,
cfg.RuntimeRegistrationEnabled,
kubeconfigProvider,
)

Expand Down
15 changes: 0 additions & 15 deletions components/provisioner/e2e_test/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,6 @@ has_to_succeed

wait_for postgres 5432

export APP_DIRECTOR_URL=https://compass-gateway-auth-oauth.cmp-main.dev.kyma.cloud.sap/director/graphql
if [[ -z "$APP_DIRECTOR_OAUTH_PATH" ]]; then
export APP_DIRECTOR_OAUTH_PATH=/compass-director-secret/secret.yaml
fi

if ! [[ -f "$APP_DIRECTOR_OAUTH_PATH" ]]; then
echo "APP_DIRECTOR_OAUTH_PATH is not set or file doesn't exist $APP_DIRECTOR_OAUTH_PATH"
exit 1
fi

export APP_GARDENER_KUBECONFIG_PATH=${APP_GARDENER_KUBECONFIG_PATH:-$GARDENER_KYMA_PROW_KUBECONFIG}
export APP_GARDENER_PROJECT=${GARDENER_KYMA_PROW_PROJECT_NAME:-$APP_GARDENER_PROJECT}
Expand All @@ -102,18 +93,12 @@ export APP_DATABASE_USER=postgres

export APP_PROVISIONING_TIMEOUT_INSTALLATION=90m
export APP_PROVISIONING_TIMEOUT_UPGRADE=90m
export APP_PROVISIONING_TIMEOUT_AGENT_CONFIGURATION=90m
export APP_PROVISIONING_NO_INSTALL_TIMEOUT_AGENT_CONFIGURATION=90m
export APP_PROVISIONING_TIMEOUT_AGENT_CONNECTION=90m
export APP_PROVISIONING_TIMEOUT_CLUSTER_CREATION=90m
export APP_PROVISIONING_TIMEOUT_CLUSTER_DOMAINS=20m
export APP_PROVISIONING_NO_INSTALL_TIMEOUT_CLUSTER_CREATION=90m
export APP_PROVISIONING_TIMEOUT_UPGRADE_TRIGGERING=90m

# for testing disabled Compass registration/deregistration
export APP_RUNTIME_REGISTRATION_ENABLED="false"
export APP_RUNTIME_DEREGISTRATION_ENABLED="false"


printf '\n########## SETTING UP THE DB ##########\n\n'
go run ./pgsetup.go
Expand Down
47 changes: 10 additions & 37 deletions components/provisioner/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -32,37 +32,29 @@ require (

require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/semver/v3 v3.2.0 // indirect
github.com/Masterminds/sprig/v3 v3.2.3 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Microsoft/hcsshim v0.9.6 // indirect
github.com/agnivade/levenshtein v1.1.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/containerd/containerd v1.6.8 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker v20.10.24+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
Expand All @@ -73,22 +65,12 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.14 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kyma-incubator/compass/components/hydrator v0.0.0-20240311150451-fb47652e9f70 // indirect
github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
github.com/lestrrat-go/blackmagic v1.0.2 // indirect
github.com/lestrrat-go/httpcc v1.0.1 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
github.com/lestrrat-go/jwx v1.2.29 // indirect
github.com/lestrrat-go/option v1.0.1 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/sys/mount v0.3.3 // indirect
github.com/moby/sys/mountinfo v0.7.1 // indirect
github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
Expand All @@ -104,27 +86,18 @@ require (
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/sosodev/duration v1.2.0 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/tidwall/gjson v1.17.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
github.com/urfave/cli/v2 v2.27.1 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/mod v0.16.0 // indirect
golang.org/x/net v0.22.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/term v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.19.0 // indirect
Expand All @@ -149,11 +122,11 @@ require (
replace (
github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.14
github.com/containerd/containerd => github.com/containerd/containerd v1.6.18
golang.org/x/crypto => golang.org/x/crypto v0.16.0
golang.org/x/net => golang.org/x/net v0.19.0
golang.org/x/sys => golang.org/x/sys v0.15.0
golang.org/x/crypto => golang.org/x/crypto v0.22.0
golang.org/x/net => golang.org/x/net v0.24.0
golang.org/x/sys => golang.org/x/sys v0.19.0
golang.org/x/text => golang.org/x/text v0.14.0
golang.org/x/tools => golang.org/x/tools v0.16.0
golang.org/x/tools => golang.org/x/tools v0.20.0
k8s.io/client-go => k8s.io/client-go v0.26.1
sourcegraph.com/sourcegraph/appdash-data => github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67
)
Loading
Loading