Configurable cron schedule for reconciler

.github/workflows/build.yml

@@ -4,7 +4,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         #goarch: [386, amd64, arm, ppc64le, arm64]
         goarch: [amd64, arm64]
         os: [ubuntu-latest] #, macos-latest, windows-latest]

.github/workflows/test.yml

@@ -4,7 +4,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -74,7 +74,7 @@ jobs:
       - name: Set up Go version
         uses: actions/setup-go@v1
         with:
-          go-version: 1.19.x
+          go-version: 1.20.x
 
       - name: Checkout code into the Go module directory
         uses: actions/checkout@v2

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19
+FROM golang:1.20
 ADD . /usr/src/whereabouts
 RUN mkdir -p $GOPATH/src/github.com/k8snetworkplumbingwg/whereabouts
 WORKDIR $GOPATH/src/github.com/k8snetworkplumbingwg/whereabouts

Dockerfile.arm64

@@ -1,4 +1,4 @@
-FROM golang:1.19
+FROM golang:1.20
 ADD . /usr/src/whereabouts
 
 ENV GOARCH "arm64"

Dockerfile.openshift

@@ -1,5 +1,5 @@
 # This dockerfile is used for building for OpenShift
-FROM registry.ci.openshift.org/openshift/release:golang-1.19 as builder
+FROM registry.ci.openshift.org/openshift/release:golang-1.20 as builder
 ADD . /go/src/github.com/k8snetworkplumbingwg/whereabouts
 WORKDIR /go/src/github.com/k8snetworkplumbingwg/whereabouts
 ENV CGO_ENABLED=1

cmd/controlloop/controlloop.go

@@ -5,9 +5,11 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"strings"
 	"time"
 
-	gocron "github.com/go-co-op/gocron"
+	"github.com/fsnotify/fsnotify"
+	"github.com/go-co-op/gocron/v2"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -28,15 +30,19 @@ import (
 )
 
 const (
-	allNamespaces  = ""
-	controllerName = "pod-ip-controlloop"
+	allNamespaces               = ""
+	controllerName              = "pod-ip-controlloop"
+	reconcilerCronConfiguration = "/cron-schedule/config"
 )
 
 const (
-	couldNotCreateController = 1
-	couldNotReadFlatfile     = 1
-	couldNotGetFlatIPAM      = 1
-	cronExpressionError      = 1
+	_ int = iota
+	couldNotCreateController
+	couldNotGetFlatIPAM
+	cronExpressionError
+	cronSchedulerCreationError
+	fileWatcherError
+	fileWatcherAddWatcherError
 )
 
 const (
@@ -65,24 +71,46 @@ func main() {
 	networkController.Start(stopChan)
 	defer networkController.Shutdown()
 
-	s := gocron.NewScheduler(time.UTC)
-	schedule := cronExpressionFromFlatFile()
-
-	_, err = s.Cron(schedule).Do(func() { // user configurable cron expression in install-cni.sh
-		reconciler.ReconcileIPs(errorChan)
-	})
+	s, err := gocron.NewScheduler(gocron.WithLocation(time.UTC))
+	if err != nil {
+		os.Exit(cronSchedulerCreationError)
+	}
+	schedule := determineCronExpression()
+
+	job, err := s.NewJob(
+		gocron.CronJob(schedule, false),
+		gocron.NewTask(func() {
+			reconciler.ReconcileIPs(errorChan)
+		}),
+	)
 	if err != nil {
 		_ = logging.Errorf("error with cron expression schedule: %v", err)
 		os.Exit(cronExpressionError)
 	}
 
-	s.StartAsync()
+	logging.Verbosef("started cron with job ID: %q", job.ID().String())
+	s.Start()
+
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		_ = logging.Errorf("error creating configuration watcher: %v", err)
+		os.Exit(fileWatcherError)
+	}
+	defer watcher.Close()
+
+	go syncConfiguration(watcher, s, job, errorChan)
+	if err := watcher.Add(reconcilerCronConfiguration); err != nil {
+		_ = logging.Errorf("error adding watcher to config %q: %v", reconcilerCronConfiguration, err)
+		os.Exit(fileWatcherAddWatcherError)
+	}
 
 	for {
 		select {
 		case <-stopChan:
 			logging.Verbosef("shutting down network controller")
-			s.Stop()
+			if err := s.Shutdown(); err != nil {
+				_ = logging.Errorf("error shutting : %v", err)
+			}
 			return
 		case err := <-errorChan:
 			if err == nil {
@@ -164,11 +192,63 @@ func newEventRecorder(broadcaster record.EventBroadcaster) record.EventRecorder
 	return broadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerName})
 }
 
-func cronExpressionFromFlatFile() string {
+func determineCronExpression() string {
 	flatipam, _, err := config.GetFlatIPAM(true, &types.IPAMConfig{}, "")
 	if err != nil {
-		_ = logging.Errorf("could not get flatipam: %v", err)
+		_ = logging.Errorf("could not get flatipam config: %v", err)
 		os.Exit(couldNotGetFlatIPAM)
 	}
-	return flatipam.IPAM.ReconcilerCronExpression
+
+	// We read the expression from a file if present, otherwise we use ReconcilerCronExpression
+	fileContents, err := os.ReadFile(reconcilerCronConfiguration)
+	if err != nil {
+		_ = logging.Errorf("could not read file: %v, using expression from flatfile: %v", err, flatipam.IPAM.ReconcilerCronExpression)
+		return flatipam.IPAM.ReconcilerCronExpression
+	}
+	logging.Verbosef("using expression: %v", strings.TrimSpace(string(fileContents))) // do i need to trim spaces? idk i think the file would JUST be the expression?
+	return strings.TrimSpace(string(fileContents))
+}
+
+func syncConfiguration(
+	watcher *fsnotify.Watcher,
+	scheduler gocron.Scheduler,
+	job gocron.Job,
+	errorChannel chan error,
+) {
+	for {
+		select {
+		case event, ok := <-watcher.Events:
+			if !ok {
+				return
+			}
+
+			updatedSchedule := determineCronExpression()
+			logging.Verbosef(
+				"configuration updated to file %q. New cron expression: %s",
+				event.Name,
+				updatedSchedule,
+			)
+			updatedJob, err := scheduler.Update(
+				job.ID(),
+				gocron.CronJob(updatedSchedule, false),
+				gocron.NewTask(func() {
+					reconciler.ReconcileIPs(errorChannel)
+				}),
+			)
+			if err != nil {
+				_ = logging.Errorf("error updating job %q configuration: %v", job.ID().String(), err)
+			}
+
+			logging.Verbosef(
+				"successfully updated CRON configuration id %q - new cron expression: %s",
+				updatedJob.ID().String(),
+				updatedSchedule,
+			)
+		case err, ok := <-watcher.Errors:
+			_ = logging.Errorf("error when listening to config changes: %v", err)
+			if !ok {
+				return
+			}
+		}
+	}
 }

doc/crds/daemonset-install.yaml

@@ -70,6 +70,18 @@ rules:
   - patch
   - update
   - get
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: whereabouts-config
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: |
+      Configmap containing user customizable cronjob schedule
+data:
+  cron-expression: "30 4 * * *" # Default schedule is once per day at 4:30am. Users may configure this value to their liking.
 ---
 apiVersion: apps/v1
 kind: DaemonSet
@@ -130,10 +142,19 @@ spec:
           mountPath: /host/opt/cni/bin
         - name: cni-net-dir
           mountPath: /host/etc/cni/net.d
+        - name: cron-scheduler-configmap
+          mountPath: /cron-schedule
       volumes:
         - name: cnibin
           hostPath:
             path: /opt/cni/bin
         - name: cni-net-dir
           hostPath:
             path: /etc/cni/net.d
+        - name: cron-scheduler-configmap
+          configMap:
+            name: "whereabouts-config"
+            defaultMode: 0744
+            items:
+            - key: "cron-expression"
+              path: "config"

doc/extended-configuration.md

@@ -134,16 +134,33 @@ spec:
 
 You'll note that in the `ipam` section there's a lot less parameters than are used in the previous examples.
 
-### Reconciler Cron Expression Configuration (optional)
+## Reconciler Cron Expression configuration for clusters via flatfile (optional)
 
-You may want to provide a cron expression to configure how frequently the ip-reconciler runs. This is done via the flatfile.
+*NOTE: configuring cron expression prior to cluster launch will only work for **non-Openshift** Kubernetes clusters, such as a vanilla Kubernetes cluster. Skip to the next section if you have an Openshift cluster or a cluster that has already launched.*
+
+Yuki~ do we even need to support configuring via flatfile? Leaving it for now, but hey, food for thought.
+
+You may want to provide a cron expression to configure how frequently the ip-reconciler runs. For clusters that have not yet been launched, this can be configured via the flatfile.
 
 You can speficy the `WHEREABOUTS_RECONCILER_CRON` environment variable in your daemonset definition file to override the default cron expression:
 ```yaml
        env:
         - name: WHEREABOUTS_RECONCILER_CRON
           value: 30 * * * *
 ```
+
+## Reconciler Cron Expression Configuration for live clusters via configmap (optional)
+
+Yuki~ README: this section may belong in the CNO since the steps outlined are not technically part of whereabouts. I'm not sure, and suggestions are welcome.
+
+You may want to provide a cron expression to configure how frequently the ip-reconciler runs. For **Openshift** Kubernetes clusters, this is done via updating the cron-scheduler-configmap. 
+
+You can check that the cron-scheduler-configmap is present by running `oc get configmaps` in the openshift-multus namespace.
+
+To update the cron-scheduler-configmap, run `oc edit configmap cron-scheduler-configmap` and adjust the value to a valid cron expression of your liking. Shortly after, the reconciler schedule will update.
+
+If you are using a non-Openshift cluster, you can do the same steps, but you will need to look for the configmap in the kube-system namespace.
+
 ## Installing etcd. (optional)
 
 etcd installation is optional. By default, we recommend the custom resource backend (given in the first example configuration).

go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/containernetworking/cni v0.8.1
 	github.com/containernetworking/plugins v0.8.2
-	github.com/go-co-op/gocron v1.13.0
 	github.com/imdario/mergo v0.3.12
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20210510153419-66a699ae3b05
@@ -21,17 +20,20 @@ require (
 	k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6
 )
 
+require github.com/go-co-op/gocron/v2 v2.1.0
+
 require (
+	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
-	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect
 	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful v2.16.0+incompatible // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/fsnotify/fsnotify v1.5.4
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
@@ -42,22 +44,22 @@ require (
 	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.5.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
-	golang.org/x/tools v0.6.0 // indirect
+	golang.org/x/tools v0.16.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect