Skip to content

Commit

Permalink
Update coreDNS coremap (#628)
Browse files Browse the repository at this point in the history 
If there is stale Lighhouse DNS Service IP in in-cluster CoreDNS config
map it will be updated with new serviceIP

Fixes: #620

Signed-off-by: Aswin Surayanarayanan <[email protected]>
Co-authored-by: Thomas Pantelis <[email protected]>
Co-authored-by: Vishal Thapar <[email protected]>
  • Loading branch information
@aswinsuryan @tpantelis @vthapar
3 people authored Sep 23, 2020
1 parent 29508ae commit 072d3b8
Show file tree
Hide file tree
Showing 2 changed files with 142 additions and 31 deletions.
57 changes: 35 additions & 22 deletions pkg/controller/servicediscovery/servicediscovery_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ const (
lighthouseCoreDNSName = "submariner-lighthouse-coredns"
defaultOpenShiftDNSController = "default"
lighthouseForwardPluginName = "lighthouse"
coreDNSNamespace = "kube-system"
coreDNSName = "coredns"
)

const (
Expand Down Expand Up @@ -163,7 +165,7 @@ func (r *ReconcileServiceDiscovery) Reconcile(request reconcile.Request) (reconc
return reconcile.Result{}, err
}
}
err = updateDNSConfigMap(r.client, r.k8sClientSet, instance)
err = updateDNSConfigMap(r.client, r.k8sClientSet, instance, reqLogger)
if err != nil {
// Try to update Openshift-DNS
return reconcile.Result{}, updateOpenshiftClusterDNSOperator(instance, r.client, r.operatorClientSet, reqLogger)
Expand Down Expand Up @@ -355,11 +357,12 @@ func newLigthhouseCoreDNSService(cr *submarinerv1alpha1.ServiceDiscovery) *corev
}
}

func updateDNSConfigMap(client controllerClient.Client, k8sclientSet clientset.Interface, cr *submarinerv1alpha1.ServiceDiscovery) error {
configMaps := k8sclientSet.CoreV1().ConfigMaps("kube-system")
func updateDNSConfigMap(client controllerClient.Client, k8sclientSet clientset.Interface, cr *submarinerv1alpha1.ServiceDiscovery,
reqLogger logr.Logger) error {
retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
configMap, err := configMaps.Get("coredns", metav1.GetOptions{})
configMap, err := k8sclientSet.CoreV1().ConfigMaps(coreDNSNamespace).Get(coreDNSName, metav1.GetOptions{})
if err != nil {
reqLogger.Error(err, "Error retrieving 'coredns' ConfigMap")
return err
}
/* This entry will be added to config map
Expand All @@ -371,33 +374,43 @@ func updateDNSConfigMap(client controllerClient.Client, k8sclientSet clientset.I
forward . 2.2.2.2:5353
}
*/
corefile := configMap.Data["Corefile"]
if strings.Contains(corefile, "lighthouse") {
// Assume this means we've already set the ConfigMap up
return nil
}
lighthouseDnsService := &corev1.Service{}
err = client.Get(context.TODO(), types.NamespacedName{Name: lighthouseCoreDNSName, Namespace: cr.Namespace}, lighthouseDnsService)
if err != nil || lighthouseDnsService.Spec.ClusterIP == "" {
lighthouseClusterIp := lighthouseDnsService.Spec.ClusterIP
if err != nil || lighthouseClusterIp == "" {
return goerrors.New("lighthouseDnsService ClusterIp should be available")
}
expectedCorefile := `#lighthouse
clusterset.local:53 {
forward . `
expectedCorefile = expectedCorefile + lighthouseDnsService.Spec.ClusterIP + "\n" + "}\n"
superclusterCorefile := `supercluster.local:53 {
forward . `
expectedCorefile = expectedCorefile + superclusterCorefile + lighthouseDnsService.Spec.ClusterIP + "\n" + "}\n"

coreFile := configMap.Data["Corefile"]
if strings.Contains(coreFile, "clusterset") {
if strings.Contains(coreFile, "clusterset.local") {
// Assume this means we've already set the ConfigMap up
return nil
reqLogger.Info("coredns configmap has lighthouse configuration hence updating")
lines := strings.Split(coreFile, "\n")
for i, line := range lines {
if strings.Contains(line, "clusterset.local") || strings.Contains(line, "supercluster.local") {
if strings.Contains(lines[i+1], lighthouseClusterIp) {
return nil
}

lines[i+1] = " forward . " + lighthouseClusterIp
}
}
coreFile = strings.Join(lines, "\n")
} else {
reqLogger.Info("coredns configmap does not have lighthouse configuration hence creating")
expectedCorefile := `#lighthouse
clusterset.local:53 {
forward . `
expectedCorefile = expectedCorefile + lighthouseClusterIp + "\n" + "}\n"
superclusterCorefile := `supercluster.local:53 {
forward . `
expectedCorefile = expectedCorefile + superclusterCorefile + lighthouseClusterIp + "\n" + "}\n"
coreFile = expectedCorefile + coreFile
}
coreFile = expectedCorefile + coreFile
log.Info("Updated coredns CoreFile " + coreFile)
log.Info("Updated coredns ConfigMap " + coreFile)
configMap.Data["Corefile"] = coreFile
// Potentially retried
_, err = configMaps.Update(configMap)
_, err = k8sclientSet.CoreV1().ConfigMaps(coreDNSNamespace).Update(configMap)
return err
})
return retryErr
Expand Down
116 changes: 107 additions & 9 deletions pkg/controller/servicediscovery/servicediscovery_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,14 @@ const (
submarinerNamespace = "submariner-operator"
)

const (
clusterlocalConfig = `#lighthouse
clusterset.local:53 {
forward . `
superClusterlocalConfig = `supercluster.local:53 {
forward . `
)

var _ = BeforeSuite(func() {
err := submariner_v1.AddToScheme(scheme.Scheme)
Expect(err).To(Succeed())
Expand Down Expand Up @@ -66,11 +74,6 @@ func testReconciliation() {
if fakeClient == nil {
fakeClient = newClient()
}

if fakeK8sClient == nil {
fakeK8sClient = fakeKubeClient.NewSimpleClientset()
}

controller = &ReconcileServiceDiscovery{
client: fakeClient,
scheme: scheme.Scheme,
Expand All @@ -84,7 +87,7 @@ func testReconciliation() {
}})
})

When("the lighthouseDNS service IP is updated", func() {
When("ClusterDNS operator should be updated when the lighthouseDNS service IP is updated", func() {
var dnsconfig *operatorv1.DNS
var lighthouseDNSService *corev1.Service
oldClusterIp := "10.10.10.10"
Expand All @@ -93,9 +96,10 @@ func testReconciliation() {
dnsconfig = newDNSConfig(oldClusterIp)
lighthouseDNSService = newDNSService(updatedClusterIp)
initClientObjs = append(initClientObjs, dnsconfig, lighthouseDNSService)
fakeK8sClient = fakeKubeClient.NewSimpleClientset()
})

It("should update the DNS operator config", func() {
It("ClusterDNS operator not be updated when the lighthouseDNS service IP is not updated", func() {
Expect(reconcileErr).To(Succeed())
Expect(reconcileResult.Requeue).To(BeFalse())

Expand All @@ -112,14 +116,15 @@ func testReconciliation() {
})
})

When("the lighthouseDNS service IP is not updated", func() {
When("ClusterDNS operator should not be updated when the lighthouseDNS service IP is not updated", func() {
var dnsconfig *operatorv1.DNS
var lighthouseDNSService *corev1.Service
clusterIp := "10.10.10.10"
BeforeEach(func() {
dnsconfig = newDNSConfig(clusterIp)
lighthouseDNSService = newDNSService(clusterIp)
initClientObjs = append(initClientObjs, dnsconfig, lighthouseDNSService)
fakeK8sClient = fakeKubeClient.NewSimpleClientset()
})

It("the DNS config should not be updated", func() {
Expand All @@ -138,6 +143,63 @@ func testReconciliation() {
Expect(expectDNSConfigUpdated(defaultOpenShiftDNSController, fakeClient).Spec).To(Equal(newDNSConfig(clusterIp).Spec))
})
})

When("The coreDNS configmap should be updated if the lighthouse clusterIP is not configured", func() {
var lighthouseDNSService *corev1.Service
clusterIp := "10.10.10.10"
BeforeEach(func() {
lighthouseDNSService = newDNSService(clusterIp)
configMap := newConfigMap("")
initClientObjs = append(initClientObjs, lighthouseDNSService)
fakeK8sClient = fakeKubeClient.NewSimpleClientset(configMap)
})

It("the coreDNS config map should be updated", func() {
Expect(reconcileErr).To(Succeed())
Expect(reconcileResult.Requeue).To(BeFalse())

Expect(fakeClient.Update(context.TODO(), serviceDiscovery)).To(Succeed())

reconcileResult, reconcileErr = controller.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{
Namespace: submarinerNamespace,
Name: submarinerName,
}})

Expect(reconcileErr).To(Succeed())
Expect(reconcileResult.Requeue).To(BeFalse())
Expect(expectCoreMapUpdated(fakeK8sClient).Data["Corefile"]).To(ContainSubstring(clusterlocalConfig + clusterIp + "\n}"))
Expect(expectCoreMapUpdated(fakeK8sClient).Data["Corefile"]).To(ContainSubstring(superClusterlocalConfig + clusterIp + "\n}"))
})
})
When("The coreDNS configmap should be updated if the lighthouse clusterIP is already configured", func() {
var lighthouseDNSService *corev1.Service
clusterIp := "10.10.10.10"
updatedClusterIp := "10.10.10.11"
BeforeEach(func() {
lighthouseDNSService = newDNSService(updatedClusterIp)
lightHouseConfig := clusterlocalConfig + clusterIp + "\n}" + superClusterlocalConfig + clusterIp + "\n}"
configMap := newConfigMap(lightHouseConfig)
initClientObjs = append(initClientObjs, lighthouseDNSService)
fakeK8sClient = fakeKubeClient.NewSimpleClientset(configMap)
})

It("the coreDNS config map should be updated", func() {
Expect(reconcileErr).To(Succeed())
Expect(reconcileResult.Requeue).To(BeFalse())

Expect(fakeClient.Update(context.TODO(), serviceDiscovery)).To(Succeed())

reconcileResult, reconcileErr = controller.Reconcile(reconcile.Request{NamespacedName: types.NamespacedName{
Namespace: submarinerNamespace,
Name: submarinerName,
}})

Expect(reconcileErr).To(Succeed())
Expect(reconcileResult.Requeue).To(BeFalse())
Expect(expectCoreMapUpdated(fakeK8sClient).Data["Corefile"]).To(ContainSubstring(clusterlocalConfig + updatedClusterIp))
Expect(expectCoreMapUpdated(fakeK8sClient).Data["Corefile"]).To(ContainSubstring(superClusterlocalConfig + updatedClusterIp))
})
})
}

func newServiceDiscovery() *submariner_v1.ServiceDiscovery {
Expand All @@ -157,11 +219,41 @@ func newServiceDiscovery() *submariner_v1.ServiceDiscovery {
ClusterID: "east",
Namespace: "submariner_ns",
Debug: true,
CustomDomains: []string{"domain1", "domain2"},
},
}
}

func newConfigMap(lighthouseConfig string) *corev1.ConfigMap {
corefile := lighthouseConfig + `.:53 {
errors
health {
lameduck 5s
}
ready
kubernetes cluster1.local in-addr.arpa ip6.arpa {
pods insecure
fallthrough in-addr.arpa ip6.arpa
ttl 30
}
prometheus :9153
forward . /etc/resolv.conf
cache 30
loop
reload
loadbalance
}`
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: coreDNSName,
Namespace: coreDNSNamespace,
},
Data: map[string]string{
"Corefile": corefile,
},
BinaryData: nil,
}
}

func newDNSConfig(clusterIp string) *operatorv1.DNS {
return &operatorv1.DNS{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -211,3 +303,9 @@ func expectDNSConfigUpdated(name string, client controllerClient.Client) *operat
Expect(err).To(Succeed())
return foundDNSConfig
}

func expectCoreMapUpdated(client clientset.Interface) *corev1.ConfigMap {
foundCoreMap, err := client.CoreV1().ConfigMaps(coreDNSNamespace).Get(coreDNSName, metav1.GetOptions{})
Expect(err).To(Succeed())
return foundCoreMap
}

0 comments on commit 072d3b8

Please sign in to comment.