Merge pull request #437 from YashwantGohokar/oke_oss_release_1.26

OKE OSS Release 1.26.1

Dockerfile

@@ -37,4 +37,11 @@ RUN yum install -y util-linux \
   && yum install -y xfsprogs \
   && yum clean all
 
+COPY scripts/encrypt-mount /sbin/encrypt-mount
+COPY scripts/encrypt-umount /sbin/encrypt-umount
+COPY scripts/rpm-host /sbin/rpm-host
+RUN chmod 755 /sbin/encrypt-mount
+RUN chmod 755 /sbin/encrypt-umount
+RUN chmod 755 /sbin/rpm-host
+
 COPY --from=0 /go/src/github.com/oracle/oci-cloud-controller-manager/dist/* /usr/local/bin/

Dockerfile_arm_all

@@ -21,4 +21,11 @@ RUN yum install -y util-linux \
   && yum clean all
  \
 
+COPY scripts/encrypt-mount /sbin/encrypt-mount
+COPY scripts/encrypt-umount /sbin/encrypt-umount
+COPY scripts/rpm-host /sbin/rpm-host
+RUN chmod 755 /sbin/encrypt-mount
+RUN chmod 755 /sbin/encrypt-umount
+RUN chmod 755 /sbin/rpm-host
+
 COPY --from=0 /go/src/github.com/oracle/oci-cloud-controller-manager/dist/arm/* /usr/local/bin/

README.md

@@ -31,14 +31,15 @@ cloud-provider specific code out of the Kubernetes codebase.
 | v1.22.0   | v1.22                  | -                      |
 | v1.23.0   | v1.23                  | -                      |
 | v1.24.2   | v1.24                  | -                      |
-| v1.25.1   | v1.25                  | -                      |
-| v1.26.0   | v1.26                  | -                      |
+| v1.25.2   | v1.25                  | -                      |
+| v1.26.1   | v1.26                  | -                      |
+| v1.27.0   | v1.27                  | -                      |
 
 
 
 
 Note: 
-Versions older than v1.24.2 are no longer supported, new features / bug fixes will be available in v1.24.1 and later. 
+Versions older than v1.25.2 are no longer supported, new features / bug fixes will be available in v1.25.2 and later. 
 
 ## Implementation
  Currently `oci-cloud-controller-manager` implements:

THIRD_PARTY_LICENSES.txt

@@ -6,12 +6,15 @@ SPDX:Apache-2.0
 ---------------------------------- Copyright -----------------------------------
 Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
 Copyright 2014 The Kubernetes Authors.
+Copyright 2016 The Kubernetes Authors.
 Copyright 2017 Oracle and/or its affiliates. All rights reserved.
 Copyright 2017 The Kubernetes Authors.
 Copyright 2018 Oracle and/or its affiliates. All rights reserved.
 Copyright 2019 Oracle and/or its affiliates. All rights reserved.
 Copyright 2020 Oracle and/or its affiliates. All rights reserved.
 Copyright 2021 Oracle and/or its affiliates. All rights reserved.
+Copyright 2022 Oracle and/or its affiliates. All rights reserved.
+Copyright 2023 Oracle and/or its affiliates. All rights reserved.
 
 -------------------------- Fourth Party Dependencies ---------------------------
 
@@ -3902,4 +3905,4 @@ the Mozilla Public License, v. 2.0.
 
 === ATTRIBUTION-HELPER-GENERATED:
 === Attribution helper version: {Major:0 Minor:11 GitVersion: GitCommit: GitTreeState:dirty BuildDate:1970-01-01T00:00:00Z GoVersion:go1.19.3 Compiler:gc Platform:darwin/arm64}
-=== License file based on go.mod with md5 sum:  44a3b09805c5f60c83e1b48d2c1b752f
+=== License file based on go.mod with md5 sum:  e957b8b0f5b20080cd769d0650a881eb

container-storage-interface.md

@@ -130,7 +130,7 @@ $ kubectl describe pvc/oci-bv-claim
 
 # Troubleshoot
 
-### FsGroup policy not propagated from pod security context
+## FsGroup policy not propagated from pod security context
 
 If your fsGroup is not being applied on the files in your volume.
 
@@ -183,6 +183,21 @@ spec:
 ```
 `File` - Indicates that the CSI volume driver supports volume ownership and permission change via fsGroup, and Kubernetes may use fsGroup to change permissions and ownership of the volume to match user requested fsGroup in the pod's SecurityPolicy regardless of fstype or access mode.
 
+## Calico-node pods fail readiness health checks when oci-fss-utils is installed on node
+
+If you are using Calico CNI, the following error is thrown by the Calico daemonset pods that are scheduled on the node which has oci-fss-utils package installed:
+```
+Readiness probe failed: 2023-10-12 08:51:08.172 [INFO][346] confd/health.go 180: Number of node(s) with BGP peering established = 0 calico/node is not ready: BIRD is not ready: BGP not established with 10.0.0.5,10.0.76.88,10.0.66.239
+```
+
+### Solution:
+
+Add the following env setting to the calico-node container in the calico-node daemonset manifest.
+```yaml
+- name: IP_AUTODETECTION_METHOD
+  value: 'skip-interface=v-eth.*'
+```
+
 [1]: https://docs.us-phoenix-1.oraclecloud.com/Content/Block/Concepts/overview.htm
 [2]: https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/
 [3]: https://kubernetes.io/docs/admin/authorization/rbac/

go.mod

@@ -44,16 +44,15 @@ require (
 	github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.24.1
-	github.com/oracle/oci-go-sdk/v65 v65.40.1
+	github.com/oracle/oci-go-sdk/v65 v65.49.2
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.14.0
 	github.com/spf13/cobra v1.6.1 // indirect
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.8.1
-	github.com/stretchr/testify v1.8.1 // indirect
 	go.uber.org/zap v1.24.0
 	golang.org/x/net v0.7.0
-	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/sys v0.8.0 // indirect
 	google.golang.org/grpc v1.52.3
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -75,6 +74,7 @@ require (
 )
 
 require (
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	google.golang.org/protobuf v1.28.1
 	k8s.io/apiextensions-apiserver v0.26.2
 )
@@ -160,7 +160,6 @@ require (
 	golang.org/x/crypto v0.1.0 // indirect
 	golang.org/x/mod v0.6.0 // indirect
 	golang.org/x/oauth2 v0.2.0 // indirect
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
 	golang.org/x/term v0.5.0 // indirect
 	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

go.sum

@@ -365,8 +365,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU=
 github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
-github.com/oracle/oci-go-sdk/v65 v65.40.1 h1:nukjC4GfrpOxOEoGvqg8y31/11VtaeSnejF7icyMKJg=
-github.com/oracle/oci-go-sdk/v65 v65.40.1/go.mod h1:MXMLMzHnnd9wlpgadPkdlkZ9YrwQmCOmbX5kjVEJodw=
+github.com/oracle/oci-go-sdk/v65 v65.49.2 h1:optOfjGIVmZZMT3a/8ri/CVV1loDG0ab1p2tEpNW5ro=
+github.com/oracle/oci-go-sdk/v65 v65.49.2/go.mod h1:IBEV9l1qBzUpo7zgGaRUhbB05BVfcDGYRFBCPlTcPp0=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
@@ -429,8 +429,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
@@ -682,8 +682,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

hack/test-canary.sh

@@ -16,8 +16,6 @@
 
 # A small script to run the CCM ginkgo 'Canary' e2e tests, and, generate the
 # defined canary test response file.
-#
-# https://confluence.oci.oraclecorp.com/display/BRISTOL/OKE+Canary+Test+Image+Contract
 
 # Functions *******************************************************************
 #

hack/validate-canary.sh

@@ -16,8 +16,6 @@
 
 
 # A small script to validate the CCM 'Canary' test image works as expected. 
-#
-# https://confluence.oci.oraclecorp.com/display/BRISTOL/OKE+Canary+Test+Image+Contract
 
 
 # Helper Functions ************************************************************

manifests/cloud-controller-manager/oci-cloud-controller-manager-rbac.yaml

@@ -34,6 +34,7 @@ rules:
   - list
   - watch
   - patch
+  - get
 
 - apiGroups:
   - ""
@@ -130,6 +131,9 @@ rules:
   - serviceaccounts
   verbs:
   - create
+  - list
+  - get
+  - watch
 - apiGroups:
   - ""
   resources:

manifests/cloud-controller-manager/oci-cloud-controller-manager.yaml

@@ -42,12 +42,13 @@ spec:
             path: /etc/kubernetes
       containers:
         - name: oci-cloud-controller-manager
-          image: ghcr.io/oracle/cloud-provider-oci:latest
+          image: ghcr.io/oracle/cloud-provider-oci:v1.26.1
           command: ["/usr/local/bin/oci-cloud-controller-manager"]
           args:
             - --cloud-config=/etc/oci/cloud-provider.yaml
             - --cloud-provider=oci
             - --leader-elect-resource-lock=configmapsleases
+            - --concurrent-service-syncs=3
             - --v=2
           volumeMounts:
             - name: cfg

manifests/container-storage-interface/oci-csi-controller-driver.yaml

@@ -52,7 +52,7 @@ spec:
             - mountPath: /var/run/shared-tmpfs
               name: shared-tmpfs
         - name: csi-attacher
-          image: k8s.gcr.io/sig-storage/csi-attacher:v4.2.0
+          image: registry.k8s.io/sig-storage/csi-attacher:v4.2.0
           args:
             - --csi-address=/var/run/shared-tmpfs/csi.sock
             - --timeout=120s
@@ -65,7 +65,7 @@ spec:
             - mountPath: /var/run/shared-tmpfs
               name: shared-tmpfs
         - name: csi-resizer
-          image: k8s.gcr.io/sig-storage/csi-resizer:v1.7.0
+          image: registry.k8s.io/sig-storage/csi-resizer:v1.7.0
           args:
             - --csi-address=/var/run/shared-tmpfs/csi.sock
             - --leader-election
@@ -96,7 +96,7 @@ spec:
             - --fss-csi-endpoint=unix://var/run/shared-tmpfs/csi-fss.sock
           command:
             - /usr/local/bin/oci-csi-controller-driver
-          image: ghcr.io/oracle/cloud-provider-oci:latest
+          image: ghcr.io/oracle/cloud-provider-oci:v1.26.1
           imagePullPolicy: IfNotPresent
           volumeMounts:
             - name: config

manifests/container-storage-interface/oci-csi-node-driver.yaml

@@ -117,7 +117,7 @@ spec:
                   fieldPath: spec.nodeName
             - name: PATH
               value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/host/usr/bin:/host/sbin
-          image: ghcr.io/oracle/cloud-provider-oci:latest
+          image: ghcr.io/oracle/cloud-provider-oci:v1.26.1
           securityContext:
             privileged: true
           volumeMounts:
@@ -131,6 +131,7 @@ spec:
             - mountPath: /dev
               name: device-dir
             - mountPath: /host
+              mountPropagation: HostToContainer
               name: host-root
             - mountPath: /sbin/iscsiadm
               name: chroot-iscsiadm
@@ -151,7 +152,7 @@ spec:
           args:
             - --csi-address=/csi/csi.sock
             - --kubelet-registration-path=/var/lib/kubelet/plugins/blockvolume.csi.oraclecloud.com/csi.sock
-          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.1
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
           securityContext:
             privileged: true
           lifecycle:
@@ -170,7 +171,7 @@ spec:
           args:
             - --csi-address=/fss/csi.sock
             - --kubelet-registration-path=/var/lib/kubelet/plugins/fss.csi.oraclecloud.com/csi.sock
-          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.0
           securityContext:
             privileged: true
           lifecycle:

manifests/flexvolume-driver/oci-flexvolume-driver.yaml

@@ -40,7 +40,7 @@ spec:
             secretName: oci-flexvolume-driver
       containers:
         - name: oci-flexvolume-driver
-          image: ghcr.io/oracle/cloud-provider-oci:latest
+          image: ghcr.io/oracle/cloud-provider-oci:v1.26.1
           command: ["/usr/local/bin/install.py", "-c", "/tmp/config.yaml"]
           securityContext:
             privileged: true
@@ -76,7 +76,7 @@ spec:
             type: DirectoryOrCreate
       containers:
         - name: oci-flexvolume-driver
-          image: ghcr.io/oracle/cloud-provider-oci:latest
+          image: ghcr.io/oracle/cloud-provider-oci:v1.26.1
           command: ["/usr/local/bin/install.py"]
           securityContext:
             privileged: true

manifests/volume-provisioner/oci-volume-provisioner-fss.yaml

@@ -35,7 +35,7 @@ spec:
             secretName: oci-volume-provisioner
       containers:
         - name: oci-volume-provisioner
-          image: ghcr.io/oracle/cloud-provider-oci:latest
+          image: ghcr.io/oracle/cloud-provider-oci:v1.26.1
           command: ["/usr/local/bin/oci-volume-provisioner"]
           env:
             - name: NODE_NAME

manifests/volume-provisioner/oci-volume-provisioner.yaml

@@ -35,7 +35,7 @@ spec:
             secretName: oci-volume-provisioner
       containers:
         - name: oci-volume-provisioner
-          image: ghcr.io/oracle/cloud-provider-oci:latest
+          image: ghcr.io/oracle/cloud-provider-oci:v1.26.1
           command: ["/usr/local/bin/oci-volume-provisioner"]
           env:
             - name: NODE_NAME

pkg/cloudprovider/providers/oci/ccm.go

@@ -60,6 +60,10 @@ type CloudProvider struct {
 	// we use the node lister to go from IP -> node / provider id -> ... -> subnet
 	NodeLister listersv1.NodeLister
 
+	// ServiceAccountLister provides a cache to lookup Service Accounts to exchange
+	// with Worker Identity which then can be used to communicate with OCI services.
+	ServiceAccountLister listersv1.ServiceAccountLister
+
 	client     client.Interface
 	kubeclient clientset.Interface
 
@@ -69,6 +73,8 @@ type CloudProvider struct {
 	logger        *zap.SugaredLogger
 	instanceCache cache.Store
 	metricPusher  *metrics.MetricPusher
+
+	lbLocks *loadBalancerLocks
 }
 
 func (cp *CloudProvider) InstancesV2() (cloudprovider.InstancesV2, bool) {
@@ -136,6 +142,7 @@ func NewCloudProvider(config *providercfg.Config) (cloudprovider.Interface, erro
 		logger:        logger.Sugar(),
 		instanceCache: cache.NewTTLStore(instanceCacheKeyFn, time.Duration(24)*time.Hour),
 		metricPusher:  metricPusher,
+		lbLocks:       NewLoadBalancerLocks(),
 	}, nil
 }
 
@@ -175,8 +182,13 @@ func (cp *CloudProvider) Initialize(clientBuilder cloudprovider.ControllerClient
 
 	nodeInformer := factory.Core().V1().Nodes()
 	go nodeInformer.Informer().Run(wait.NeverStop)
+
 	serviceInformer := factory.Core().V1().Services()
 	go serviceInformer.Informer().Run(wait.NeverStop)
+
+	serviceAccountInformer := factory.Core().V1().ServiceAccounts()
+	go serviceAccountInformer.Informer().Run(wait.NeverStop)
+
 	go nodeInfoController.Run(wait.NeverStop)
 
 	cp.logger.Info("Waiting for node informer cache to sync")
@@ -185,6 +197,8 @@ func (cp *CloudProvider) Initialize(clientBuilder cloudprovider.ControllerClient
 	}
 	cp.NodeLister = nodeInformer.Lister()
 
+	cp.ServiceAccountLister = serviceAccountInformer.Lister()
+
 	cp.securityListManagerFactory = func(mode string) securityListManager {
 		if cp.config.LoadBalancer.Disabled {
 			return newSecurityListManagerNOOP()