diff --git a/.github/workflows/pr-make.yml b/.github/workflows/pr-make.yml index fa4d2d731..c81015c6f 100644 --- a/.github/workflows/pr-make.yml +++ b/.github/workflows/pr-make.yml @@ -8,7 +8,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: 1.21 + go-version: 1.22 - name: Setup dependencies run: | sudo apt-get update && sudo apt-get install -y libgpgme-dev libdevmapper-dev btrfs-progs libbtrfs-dev diff --git a/Makefile b/Makefile index b050d4646..eacadad42 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ ifeq (, $(shell which controller-gen)) CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ cd $$CONTROLLER_GEN_TMP_DIR ;\ go mod init tmp ;\ - go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.0 ;\ + go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0 ;\ rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ } CONTROLLER_GEN=$(GOBIN)/controller-gen diff --git a/config/crds/migration.openshift.io_directvolumemigrations.yaml b/config/crds/migration.openshift.io_directvolumemigrations.yaml index 57efcb6f4..1c302ef79 100644 --- a/config/crds/migration.openshift.io_directvolumemigrations.yaml +++ b/config/crds/migration.openshift.io_directvolumemigrations.yaml @@ -107,6 +107,13 @@ spec: type: string type: object x-kubernetes-map-type: atomic + liveMigrate: + description: Specifies if any volumes associated with a VM should + be live storage migrated instead of offline migrated + type: boolean + migrationType: + description: Specifies if this is the final DVM in the migration plan + type: string persistentVolumeClaims: description: Holds all the PVCs that are to be migrated with direct volume migration @@ -272,6 +279,86 @@ spec: items: type: string type: array + failedLiveMigration: + items: + properties: + lastObservedProgressPercent: + type: string + lastObservedTransferRate: + type: string + message: + type: string + pvcRef: + description: "ObjectReference contains enough information to + let you inspect or modify the referred object. --- New uses + of this type are discouraged because of difficulty describing + its usage when embedded in APIs. 1. Ignored fields. It includes + many fields which are not generally honored. For instance, + ResourceVersion and FieldPath are both very rarely valid in + actual usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must + be restricted\". Those cannot be well described when embedded. + 3. Inconsistent validation. Because the usages are different, + the validation rules are different by usage, which makes it + hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type + will affect numerous schemas. Don't make new APIs embed an + underspecified API type they do not control. \n Instead of + using this type, create a locally provided and used type that + is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + totalElapsedTime: + type: string + vmName: + type: string + vmNamespace: + type: string + type: object + type: array failedPods: items: properties: @@ -383,37 +470,14 @@ spec: type: string observedDigest: type: string - pendingPods: + pendingLiveMigration: items: properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string lastObservedProgressPercent: type: string lastObservedTransferRate: type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + message: type: string pvcRef: description: "ObjectReference contains enough information to @@ -478,36 +542,73 @@ spec: type: string type: object x-kubernetes-map-type: atomic - resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string totalElapsedTime: type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + vmName: + type: string + vmNamespace: type: string type: object - x-kubernetes-map-type: atomic type: array - phase: - type: string - phaseDescription: - type: string - rsyncOperations: + pendingPods: items: - description: RsyncOperation defines observed state of an Rsync Operation properties: - currentAttempt: - description: CurrentAttempt current ongoing attempt of an Rsync - operation - type: integer - failed: - description: Failed whether operation as a whole failed - type: boolean - pvcReference: - description: PVCReference pvc to which this Rsync operation - corresponds to + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + lastObservedProgressPercent: + type: string + lastObservedTransferRate: + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + pvcRef: + description: "ObjectReference contains enough information to + let you inspect or modify the referred object. --- New uses + of this type are discouraged because of difficulty describing + its usage when embedded in APIs. 1. Ignored fields. It includes + many fields which are not generally honored. For instance, + ResourceVersion and FieldPath are both very rarely valid in + actual usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must + be restricted\". Those cannot be well described when embedded. + 3. Inconsistent validation. Because the usages are different, + the validation rules are different by usage, which makes it + hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type + will affect numerous schemas. Don't make new APIs embed an + underspecified API type they do not control. \n Instead of + using this type, create a locally provided and used type that + is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." properties: apiVersion: description: API version of the referent. @@ -544,12 +645,19 @@ spec: type: string type: object x-kubernetes-map-type: atomic - succeeded: - description: Succeeded whether operation as a whole succeded - type: boolean + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + totalElapsedTime: + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string type: object + x-kubernetes-map-type: atomic type: array - runningPods: + pendingSinceTimeLimitPods: items: properties: apiVersion: @@ -656,10 +764,447 @@ spec: type: object x-kubernetes-map-type: atomic type: array - startTimestamp: - format: date-time + phase: type: string - successfulPods: + phaseDescription: + type: string + rsyncOperations: + items: + description: RsyncOperation defines observed state of an Rsync Operation + properties: + currentAttempt: + description: CurrentAttempt current ongoing attempt of an Rsync + operation + type: integer + failed: + description: Failed whether operation as a whole failed + type: boolean + pvcReference: + description: PVCReference pvc to which this Rsync operation + corresponds to + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + succeeded: + description: Succeeded whether operation as a whole succeded + type: boolean + type: object + type: array + runningLiveMigration: + items: + properties: + lastObservedProgressPercent: + type: string + lastObservedTransferRate: + type: string + message: + type: string + pvcRef: + description: "ObjectReference contains enough information to + let you inspect or modify the referred object. --- New uses + of this type are discouraged because of difficulty describing + its usage when embedded in APIs. 1. Ignored fields. It includes + many fields which are not generally honored. For instance, + ResourceVersion and FieldPath are both very rarely valid in + actual usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must + be restricted\". Those cannot be well described when embedded. + 3. Inconsistent validation. Because the usages are different, + the validation rules are different by usage, which makes it + hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type + will affect numerous schemas. Don't make new APIs embed an + underspecified API type they do not control. \n Instead of + using this type, create a locally provided and used type that + is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + totalElapsedTime: + type: string + vmName: + type: string + vmNamespace: + type: string + type: object + type: array + runningPods: + items: + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + lastObservedProgressPercent: + type: string + lastObservedTransferRate: + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + pvcRef: + description: "ObjectReference contains enough information to + let you inspect or modify the referred object. --- New uses + of this type are discouraged because of difficulty describing + its usage when embedded in APIs. 1. Ignored fields. It includes + many fields which are not generally honored. For instance, + ResourceVersion and FieldPath are both very rarely valid in + actual usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must + be restricted\". Those cannot be well described when embedded. + 3. Inconsistent validation. Because the usages are different, + the validation rules are different by usage, which makes it + hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type + will affect numerous schemas. Don't make new APIs embed an + underspecified API type they do not control. \n Instead of + using this type, create a locally provided and used type that + is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + totalElapsedTime: + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + type: array + skippedVolumes: + items: + type: string + type: array + startTimestamp: + format: date-time + type: string + successfulLiveMigration: + items: + properties: + lastObservedProgressPercent: + type: string + lastObservedTransferRate: + type: string + message: + type: string + pvcRef: + description: "ObjectReference contains enough information to + let you inspect or modify the referred object. --- New uses + of this type are discouraged because of difficulty describing + its usage when embedded in APIs. 1. Ignored fields. It includes + many fields which are not generally honored. For instance, + ResourceVersion and FieldPath are both very rarely valid in + actual usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must + be restricted\". Those cannot be well described when embedded. + 3. Inconsistent validation. Because the usages are different, + the validation rules are different by usage, which makes it + hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type + will affect numerous schemas. Don't make new APIs embed an + underspecified API type they do not control. \n Instead of + using this type, create a locally provided and used type that + is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + totalElapsedTime: + type: string + vmName: + type: string + vmNamespace: + type: string + type: object + type: array + successfulPods: + items: + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + lastObservedProgressPercent: + type: string + lastObservedTransferRate: + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + pvcRef: + description: "ObjectReference contains enough information to + let you inspect or modify the referred object. --- New uses + of this type are discouraged because of difficulty describing + its usage when embedded in APIs. 1. Ignored fields. It includes + many fields which are not generally honored. For instance, + ResourceVersion and FieldPath are both very rarely valid in + actual usage. 2. Invalid usage help. It is impossible to + add specific help for individual usage. In most embedded + usages, there are particular restrictions like, \"must refer + only to types A and B\" or \"UID not honored\" or \"name must + be restricted\". Those cannot be well described when embedded. + 3. Inconsistent validation. Because the usages are different, + the validation rules are different by usage, which makes it + hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency + is on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type + will affect numerous schemas. Don't make new APIs embed an + underspecified API type they do not control. \n Instead of + using this type, create a locally provided and used type that + is well-focused on your reference. For example, ServiceReferences + for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + totalElapsedTime: + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + x-kubernetes-map-type: atomic + type: array + unknownPods: items: properties: apiVersion: diff --git a/config/crds/migration.openshift.io_migplans.yaml b/config/crds/migration.openshift.io_migplans.yaml index 05194a234..46ab17692 100644 --- a/config/crds/migration.openshift.io_migplans.yaml +++ b/config/crds/migration.openshift.io_migplans.yaml @@ -275,6 +275,11 @@ spec: type: object type: object x-kubernetes-map-type: atomic + liveMigrate: + description: LiveMigrate optional flag to enable live migration of + VMs during direct volume migration Only running VMs when the plan + is executed will be live migrated + type: boolean migStorageRef: description: "ObjectReference contains enough information to let you inspect or modify the referred object. --- New uses of this type diff --git a/go.mod b/go.mod index 888e70984..8978e0299 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/konveyor/mig-controller -go 1.21 +go 1.22.0 + +toolchain go1.22.5 require ( cloud.google.com/go/storage v1.30.1 @@ -17,7 +19,7 @@ require ( github.com/google/uuid v1.6.0 github.com/joho/godotenv v1.3.0 github.com/konveyor/controller v0.12.0 - github.com/konveyor/crane-lib v0.1.2 + github.com/konveyor/crane-lib v0.1.3 github.com/konveyor/openshift-velero-plugin v0.0.0-20210729141849-876132e34f3d github.com/mattn/go-sqlite3 v1.14.22 github.com/onsi/ginkgo v1.16.4 @@ -27,6 +29,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/common v0.53.0 github.com/uber/jaeger-client-go v2.25.0+incompatible github.com/vmware-tanzu/velero v1.7.1 go.uber.org/zap v1.27.0 @@ -36,7 +39,7 @@ require ( k8s.io/apimachinery v0.30.0 k8s.io/client-go v1.5.2 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 - kubevirt.io/api v1.2.0 + kubevirt.io/api v1.3.0 kubevirt.io/containerized-data-importer-api v1.59.0 sigs.k8s.io/controller-runtime v0.18.1 ) @@ -156,7 +159,6 @@ require ( github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/proglottis/gpgme v0.1.3 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.53.0 // indirect github.com/prometheus/procfs v0.14.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect @@ -208,7 +210,7 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.29.4 // indirect + k8s.io/apiextensions-apiserver v0.30.0 // indirect k8s.io/component-base v0.30.0 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f // indirect diff --git a/go.sum b/go.sum index 37fe20f5a..871ca4cca 100644 --- a/go.sum +++ b/go.sum @@ -1542,6 +1542,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -1585,8 +1586,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konveyor/controller v0.12.0 h1:TYEGrb6TxegMqMH1ZPy/TMyOeecH+JHKQNqKvSUGWkM= github.com/konveyor/controller v0.12.0/go.mod h1:kGFv+5QxjuRo1wUO+bO/HGpULkdkR1pb1tEsVMPAz3s= -github.com/konveyor/crane-lib v0.1.2 h1:HSQd2u7UJB0vV8c75Gxd5KqDm9gevpT9bsB02P9MoZ4= -github.com/konveyor/crane-lib v0.1.2/go.mod h1:oSqLMhvUa3kNC/IaKXdGV/Tfs2bdwoZpYrbiV4KpVdY= +github.com/konveyor/crane-lib v0.1.3 h1:dlVe8uGfLhu5cs9GFFikXmTL1ysukHopv5dba54iBcM= +github.com/konveyor/crane-lib v0.1.3/go.mod h1:oSqLMhvUa3kNC/IaKXdGV/Tfs2bdwoZpYrbiV4KpVdY= github.com/konveyor/openshift-velero-plugin v0.0.0-20210729141849-876132e34f3d h1:tETgPq+JxXhVhnrcLc7rcW9BURax36VUsix8DAU0wuY= github.com/konveyor/openshift-velero-plugin v0.0.0-20210729141849-876132e34f3d/go.mod h1:Yk0xQ4N5rwE1+NWLSFQUQLgNT1/8p4Uor60LlgQfymg= github.com/konveyor/velero v0.10.2-0.20220124204642-f91d69bb9a5e h1:cjeGzY/zgPJCSLpC6Yurpn1li9ERrMtrF0nixFz2fmc= @@ -1719,6 +1720,7 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= @@ -3151,8 +3153,8 @@ k8s.io/api v0.29.4/go.mod h1:DetSv0t4FBTcEpfA84NJV3g9a7+rSzlUHk5ADAYHUv0= k8s.io/apiextensions-apiserver v0.17.1/go.mod h1:DRIFH5x3jalE4rE7JP0MQKby9zdYk9lUJQuMmp+M/L0= k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= -k8s.io/apiextensions-apiserver v0.29.4 h1:M7hbuHU/ckbibR7yPbe6DyNWgTFKNmZDbdZKD8q1Smk= -k8s.io/apiextensions-apiserver v0.29.4/go.mod h1:TTDC9fB+0kHY2rogf5hgBR03KBKCwED+GHUsXGpR7SM= +k8s.io/apiextensions-apiserver v0.30.0 h1:jcZFKMqnICJfRxTgnC4E+Hpcq8UEhT8B2lhBcQ+6uAs= +k8s.io/apiextensions-apiserver v0.30.0/go.mod h1:N9ogQFGcrbWqAY9p2mUAL5mGxsLqwgtUce127VtRX5Y= k8s.io/apimachinery v0.29.4 h1:RaFdJiDmuKs/8cm1M6Dh1Kvyh59YQFDcFuFTSmXes6Q= k8s.io/apimachinery v0.29.4/go.mod h1:i3FJVwhvSp/6n8Fl4K97PJEP8C+MM+aoDq4+ZJBf70Y= k8s.io/apiserver v0.17.1/go.mod h1:BQEUObJv8H6ZYO7DeKI5vb50tjk6paRJ4ZhSyJsiSco= @@ -3217,8 +3219,8 @@ k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kubevirt.io/api v1.2.0 h1:1f8XQLPl4BuHPsc6SHTPnYSYeDxucKCQGa8CdrGJSRc= -kubevirt.io/api v1.2.0/go.mod h1:SbeR9ma4EwnaOZEUkh/lNz0kzYm5LPpEDE30vKXC5Zg= +kubevirt.io/api v1.3.0 h1:9sGElMmnRU50pGED+MPPD2OwQl4S5lvjCUjm+t0mI90= +kubevirt.io/api v1.3.0/go.mod h1:e6LkElYZZm8NcP2gKlFVHZS9pgNhIARHIjSBSfeiP1s= kubevirt.io/containerized-data-importer-api v1.59.0 h1:GdDt9BlR0qHejpMaPfASbsG8JWDmBf1s7xZBj5W9qn0= kubevirt.io/containerized-data-importer-api v1.59.0/go.mod h1:4yOGtCE7HvgKp7wftZZ3TBvDJ0x9d6N6KaRjRYcUFpE= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= diff --git a/pkg/apis/migration/v1alpha1/directvolumemigration_types.go b/pkg/apis/migration/v1alpha1/directvolumemigration_types.go index bfd8c2348..4bf7115a6 100644 --- a/pkg/apis/migration/v1alpha1/directvolumemigration_types.go +++ b/pkg/apis/migration/v1alpha1/directvolumemigration_types.go @@ -18,6 +18,7 @@ package v1alpha1 import ( "fmt" + "slices" kapi "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -65,22 +66,43 @@ type DirectVolumeMigrationSpec struct { // Specifies if progress reporting CRs needs to be deleted or not DeleteProgressReportingCRs bool `json:"deleteProgressReportingCRs,omitempty"` + + // Specifies if any volumes associated with a VM should be live storage migrated instead of offline migrated + LiveMigrate *bool `json:"liveMigrate,omitempty"` + + // Specifies if this is the final DVM in the migration plan + MigrationType *DirectVolumeMigrationType `json:"migrationType,omitempty"` } +type DirectVolumeMigrationType string + +const ( + MigrationTypeStage DirectVolumeMigrationType = "Stage" + MigrationTypeFinal DirectVolumeMigrationType = "CutOver" + MigrationTypeRollback DirectVolumeMigrationType = "Rollback" +) + // DirectVolumeMigrationStatus defines the observed state of DirectVolumeMigration type DirectVolumeMigrationStatus struct { - Conditions `json:","` - ObservedDigest string `json:"observedDigest"` - StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` - PhaseDescription string `json:"phaseDescription"` - Phase string `json:"phase,omitempty"` - Itinerary string `json:"itinerary,omitempty"` - Errors []string `json:"errors,omitempty"` - SuccessfulPods []*PodProgress `json:"successfulPods,omitempty"` - FailedPods []*PodProgress `json:"failedPods,omitempty"` - RunningPods []*PodProgress `json:"runningPods,omitempty"` - PendingPods []*PodProgress `json:"pendingPods,omitempty"` - RsyncOperations []*RsyncOperation `json:"rsyncOperations,omitempty"` + Conditions `json:","` + ObservedDigest string `json:"observedDigest"` + StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"` + PhaseDescription string `json:"phaseDescription"` + Phase string `json:"phase,omitempty"` + Itinerary string `json:"itinerary,omitempty"` + Errors []string `json:"errors,omitempty"` + SuccessfulPods []*PodProgress `json:"successfulPods,omitempty"` + FailedPods []*PodProgress `json:"failedPods,omitempty"` + RunningPods []*PodProgress `json:"runningPods,omitempty"` + PendingPods []*PodProgress `json:"pendingPods,omitempty"` + UnknownPods []*PodProgress `json:"unknownPods,omitempty"` + PendingSinceTimeLimitPods []*PodProgress `json:"pendingSinceTimeLimitPods,omitempty"` + SuccessfulLiveMigrations []*LiveMigrationProgress `json:"successfulLiveMigration,omitempty"` + RunningLiveMigrations []*LiveMigrationProgress `json:"runningLiveMigration,omitempty"` + PendingLiveMigrations []*LiveMigrationProgress `json:"pendingLiveMigration,omitempty"` + FailedLiveMigrations []*LiveMigrationProgress `json:"failedLiveMigration,omitempty"` + RsyncOperations []*RsyncOperation `json:"rsyncOperations,omitempty"` + SkippedVolumes []string `json:"skippedVolumes,omitempty"` } // GetRsyncOperationStatusForPVC returns RsyncOperation from status for matching PVC, creates new one if doesn't exist already @@ -117,6 +139,42 @@ func (ds *DirectVolumeMigrationStatus) AddRsyncOperation(podStatus *RsyncOperati ds.RsyncOperations = append(ds.RsyncOperations, podStatus) } +func (dvm *DirectVolumeMigration) IsCompleted() bool { + return len(dvm.Status.SuccessfulPods)+ + len(dvm.Status.FailedPods)+ + len(dvm.Status.SkippedVolumes)+ + len(dvm.Status.SuccessfulLiveMigrations)+ + len(dvm.Status.FailedLiveMigrations) == len(dvm.Spec.PersistentVolumeClaims) +} + +func (dvm *DirectVolumeMigration) IsCutover() bool { + return dvm.Spec.MigrationType != nil && *dvm.Spec.MigrationType == MigrationTypeFinal +} + +func (dvm *DirectVolumeMigration) IsRollback() bool { + return dvm.Spec.MigrationType != nil && *dvm.Spec.MigrationType == MigrationTypeRollback +} + +func (dvm *DirectVolumeMigration) IsStage() bool { + return dvm.Spec.MigrationType != nil && *dvm.Spec.MigrationType == MigrationTypeStage +} + +func (dvm *DirectVolumeMigration) SkipVolume(volumeName, namespace string) { + dvm.Status.SkippedVolumes = append(dvm.Status.SkippedVolumes, fmt.Sprintf("%s/%s", namespace, volumeName)) +} + +func (dvm *DirectVolumeMigration) IsLiveMigrate() bool { + return dvm.Spec.LiveMigrate != nil && *dvm.Spec.LiveMigrate +} + +func (dvm *DirectVolumeMigration) AllReportingCompleted() bool { + isCompleted := dvm.IsCompleted() + isAnyPending := len(dvm.Status.PendingPods) > 0 || len(dvm.Status.PendingLiveMigrations) > 0 + isAnyRunning := len(dvm.Status.RunningPods) > 0 || len(dvm.Status.RunningLiveMigrations) > 0 + isAnyUnknown := len(dvm.Status.UnknownPods) > 0 + return !isAnyRunning && !isAnyPending && !isAnyUnknown && isCompleted +} + // TODO: Explore how to reliably get stunnel+rsync logs/status reported back to // DirectVolumeMigrationStatus @@ -151,6 +209,16 @@ type PodProgress struct { TotalElapsedTime *metav1.Duration `json:"totalElapsedTime,omitempty"` } +type LiveMigrationProgress struct { + VMName string `json:"vmName,omitempty"` + VMNamespace string `json:"vmNamespace,omitempty"` + PVCReference *kapi.ObjectReference `json:"pvcRef,omitempty"` + LastObservedProgressPercent string `json:"lastObservedProgressPercent,omitempty"` + LastObservedTransferRate string `json:"lastObservedTransferRate,omitempty"` + TotalElapsedTime *metav1.Duration `json:"totalElapsedTime,omitempty"` + Message string `json:"message,omitempty"` +} + // RsyncOperation defines observed state of an Rsync Operation type RsyncOperation struct { // PVCReference pvc to which this Rsync operation corresponds to @@ -205,6 +273,26 @@ func (r *DirectVolumeMigration) GetMigrationForDVM(client k8sclient.Client) (*Mi return GetMigrationForDVM(client, r.OwnerReferences) } +func (r *DirectVolumeMigration) GetSourceNamespaces() []string { + namespaces := []string{} + for _, pvc := range r.Spec.PersistentVolumeClaims { + if pvc.Namespace != "" && !slices.Contains(namespaces, pvc.Namespace) { + namespaces = append(namespaces, pvc.Namespace) + } + } + return namespaces +} + +func (r *DirectVolumeMigration) GetDestinationNamespaces() []string { + namespaces := []string{} + for _, pvc := range r.Spec.PersistentVolumeClaims { + if pvc.TargetNamespace != "" && !slices.Contains(namespaces, pvc.TargetNamespace) { + namespaces = append(namespaces, pvc.TargetNamespace) + } + } + return namespaces +} + // Add (de-duplicated) errors. func (r *DirectVolumeMigration) AddErrors(errors []string) { m := map[string]bool{} diff --git a/pkg/apis/migration/v1alpha1/migcluster_types.go b/pkg/apis/migration/v1alpha1/migcluster_types.go index 31cbb6966..d4492f2c1 100644 --- a/pkg/apis/migration/v1alpha1/migcluster_types.go +++ b/pkg/apis/migration/v1alpha1/migcluster_types.go @@ -804,6 +804,13 @@ var accessModeList = []provisionerAccessModes{ kapi.PersistentVolumeBlock: {kapi.ReadWriteOnce, kapi.ReadOnlyMany, kapi.ReadWriteMany}, }, }, + provisionerAccessModes{ + Provisioner: "csi.kubevirt.io", + AccessModes: map[kapi.PersistentVolumeMode][]kapi.PersistentVolumeAccessMode{ + kapi.PersistentVolumeFilesystem: {kapi.ReadWriteOnce}, + kapi.PersistentVolumeBlock: {kapi.ReadWriteOnce}, + }, + }, } // Get the list of k8s StorageClasses from the cluster. diff --git a/pkg/apis/migration/v1alpha1/migplan_types.go b/pkg/apis/migration/v1alpha1/migplan_types.go index 4c4f18616..cc0b46148 100644 --- a/pkg/apis/migration/v1alpha1/migplan_types.go +++ b/pkg/apis/migration/v1alpha1/migplan_types.go @@ -115,6 +115,11 @@ type MigPlanSpec struct { // LabelSelector optional label selector on the included resources in Velero Backup // +kubebuilder:validation:Optional LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"` + + // LiveMigrate optional flag to enable live migration of VMs during direct volume migration + // Only running VMs when the plan is executed will be live migrated + // +kubebuilder:validation:Optional + LiveMigrate *bool `json:"liveMigrate,omitempty"` } // MigPlanStatus defines the observed state of MigPlan @@ -218,6 +223,10 @@ type PlanResources struct { DestMigCluster *MigCluster } +func (r *MigPlan) LiveMigrationChecked() bool { + return r.Spec.LiveMigrate != nil && *r.Spec.LiveMigrate +} + // GetRefResources gets referenced resources from a MigPlan. func (r *MigPlan) GetRefResources(client k8sclient.Client) (*PlanResources, error) { isIntraCluster, err := r.IsIntraCluster(client) diff --git a/pkg/apis/migration/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/migration/v1alpha1/zz_generated.deepcopy.go index 164fd1378..2f571abf1 100644 --- a/pkg/apis/migration/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/migration/v1alpha1/zz_generated.deepcopy.go @@ -570,6 +570,16 @@ func (in *DirectVolumeMigrationSpec) DeepCopyInto(out *DirectVolumeMigrationSpec (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.LiveMigrate != nil { + in, out := &in.LiveMigrate, &out.LiveMigrate + *out = new(bool) + **out = **in + } + if in.MigrationType != nil { + in, out := &in.MigrationType, &out.MigrationType + *out = new(DirectVolumeMigrationType) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DirectVolumeMigrationSpec. @@ -639,6 +649,72 @@ func (in *DirectVolumeMigrationStatus) DeepCopyInto(out *DirectVolumeMigrationSt } } } + if in.UnknownPods != nil { + in, out := &in.UnknownPods, &out.UnknownPods + *out = make([]*PodProgress, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(PodProgress) + (*in).DeepCopyInto(*out) + } + } + } + if in.PendingSinceTimeLimitPods != nil { + in, out := &in.PendingSinceTimeLimitPods, &out.PendingSinceTimeLimitPods + *out = make([]*PodProgress, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(PodProgress) + (*in).DeepCopyInto(*out) + } + } + } + if in.SuccessfulLiveMigrations != nil { + in, out := &in.SuccessfulLiveMigrations, &out.SuccessfulLiveMigrations + *out = make([]*LiveMigrationProgress, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(LiveMigrationProgress) + (*in).DeepCopyInto(*out) + } + } + } + if in.RunningLiveMigrations != nil { + in, out := &in.RunningLiveMigrations, &out.RunningLiveMigrations + *out = make([]*LiveMigrationProgress, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(LiveMigrationProgress) + (*in).DeepCopyInto(*out) + } + } + } + if in.PendingLiveMigrations != nil { + in, out := &in.PendingLiveMigrations, &out.PendingLiveMigrations + *out = make([]*LiveMigrationProgress, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(LiveMigrationProgress) + (*in).DeepCopyInto(*out) + } + } + } + if in.FailedLiveMigrations != nil { + in, out := &in.FailedLiveMigrations, &out.FailedLiveMigrations + *out = make([]*LiveMigrationProgress, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(LiveMigrationProgress) + (*in).DeepCopyInto(*out) + } + } + } if in.RsyncOperations != nil { in, out := &in.RsyncOperations, &out.RsyncOperations *out = make([]*RsyncOperation, len(*in)) @@ -650,6 +726,11 @@ func (in *DirectVolumeMigrationStatus) DeepCopyInto(out *DirectVolumeMigrationSt } } } + if in.SkippedVolumes != nil { + in, out := &in.SkippedVolumes, &out.SkippedVolumes + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DirectVolumeMigrationStatus. @@ -749,6 +830,31 @@ func (in *IncompatibleNamespace) DeepCopy() *IncompatibleNamespace { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LiveMigrationProgress) DeepCopyInto(out *LiveMigrationProgress) { + *out = *in + if in.PVCReference != nil { + in, out := &in.PVCReference, &out.PVCReference + *out = new(v1.ObjectReference) + **out = **in + } + if in.TotalElapsedTime != nil { + in, out := &in.TotalElapsedTime, &out.TotalElapsedTime + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LiveMigrationProgress. +func (in *LiveMigrationProgress) DeepCopy() *LiveMigrationProgress { + if in == nil { + return nil + } + out := new(LiveMigrationProgress) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MigAnalytic) DeepCopyInto(out *MigAnalytic) { *out = *in @@ -1416,6 +1522,11 @@ func (in *MigPlanSpec) DeepCopyInto(out *MigPlanSpec) { *out = new(metav1.LabelSelector) (*in).DeepCopyInto(*out) } + if in.LiveMigrate != nil { + in, out := &in.LiveMigrate, &out.LiveMigrate + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MigPlanSpec. diff --git a/pkg/compat/fake/client.go b/pkg/compat/fake/client.go index 05afc3e03..8307501e8 100644 --- a/pkg/compat/fake/client.go +++ b/pkg/compat/fake/client.go @@ -15,6 +15,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" virtv1 "kubevirt.io/api/core/v1" + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -106,5 +107,8 @@ func getSchemeForFakeClient() (*runtime.Scheme, error) { if err := virtv1.AddToScheme(scheme.Scheme); err != nil { return nil, err } + if err := cdiv1.AddToScheme(scheme.Scheme); err != nil { + return nil, err + } return scheme.Scheme, nil } diff --git a/pkg/controller/directvolumemigration/directvolumemigration_controller.go b/pkg/controller/directvolumemigration/directvolumemigration_controller.go index f9e0bd91b..7499b4704 100644 --- a/pkg/controller/directvolumemigration/directvolumemigration_controller.go +++ b/pkg/controller/directvolumemigration/directvolumemigration_controller.go @@ -22,10 +22,15 @@ import ( "github.com/konveyor/controller/pkg/logging" migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + "github.com/konveyor/mig-controller/pkg/compat" migref "github.com/konveyor/mig-controller/pkg/reference" "github.com/opentracing/opentracing-go" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -34,6 +39,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" ) +const ( + dvmFinalizer = "migration.openshift.io/directvolumemigrationfinalizer" +) + var ( sink = logging.WithName("directvolume") log = sink.Real @@ -47,7 +56,7 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &ReconcileDirectVolumeMigration{Client: mgr.GetClient(), scheme: mgr.GetScheme()} + return &ReconcileDirectVolumeMigration{Config: mgr.GetConfig(), Client: mgr.GetClient(), scheme: mgr.GetScheme()} } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -86,6 +95,7 @@ var _ reconcile.Reconciler = &ReconcileDirectVolumeMigration{} // ReconcileDirectVolumeMigration reconciles a DirectVolumeMigration object type ReconcileDirectVolumeMigration struct { + *rest.Config client.Client scheme *runtime.Scheme tracer opentracing.Tracer @@ -104,6 +114,8 @@ func (r *ReconcileDirectVolumeMigration) Reconcile(ctx context.Context, request // Set values tracer := logging.WithName("directvolume", "dvm", request.Name) log := tracer.Real + // Default to PollReQ, can be overridden by r.migrate phase-specific ReQ interval + requeueAfter := time.Duration(PollReQ) // Fetch the DirectVolumeMigration instance direct := &migapi.DirectVolumeMigration{} @@ -120,6 +132,9 @@ func (r *ReconcileDirectVolumeMigration) Reconcile(ctx context.Context, request // Set MigMigration name key on logger migration, err := direct.GetMigrationForDVM(r) + if err != nil { + return reconcile.Result{}, err + } if migration != nil { log = log.WithValues("migMigration", migration.Name) } @@ -131,8 +146,32 @@ func (r *ReconcileDirectVolumeMigration) Reconcile(ctx context.Context, request defer reconcileSpan.Finish() } - // Check if completed - if direct.Status.Phase == Completed { + if direct.DeletionTimestamp != nil { + sourceDeleted, err := r.cleanupSourceResourcesInNamespace(direct) + if err != nil { + return reconcile.Result{}, err + } + targetDeleted, err := r.cleanupTargetResourcesInNamespaces(direct) + if err != nil { + return reconcile.Result{}, err + } + if sourceDeleted && targetDeleted { + log.V(5).Info("DirectVolumeMigration resources deleted. removing finalizer") + // Remove finalizer + RemoveFinalizer(direct, dvmFinalizer) + } else { + // Requeue + log.V(5).Info("Requeing waiting for cleanup", "after", requeueAfter) + return reconcile.Result{RequeueAfter: requeueAfter}, nil + } + } else { + // Add finalizer + AddFinalizer(direct, dvmFinalizer) + } + + // Check if completed and return if not deleted, otherwise need to update to + // remove the finalizer. + if direct.Status.Phase == Completed && direct.DeletionTimestamp == nil { return reconcile.Result{Requeue: false}, nil } @@ -146,10 +185,7 @@ func (r *ReconcileDirectVolumeMigration) Reconcile(ctx context.Context, request return reconcile.Result{Requeue: true}, nil } - // Default to PollReQ, can be overridden by r.migrate phase-specific ReQ interval - requeueAfter := time.Duration(PollReQ) - - if !direct.Status.HasBlockerCondition() { + if !direct.Status.HasBlockerCondition() && direct.DeletionTimestamp == nil { requeueAfter, err = r.migrate(ctx, direct) if err != nil { tracer.Trace(err) @@ -182,3 +218,140 @@ func (r *ReconcileDirectVolumeMigration) Reconcile(ctx context.Context, request // Done return reconcile.Result{Requeue: false}, nil } + +// AddFinalizer adds a finalizer to a resource +func AddFinalizer(obj metav1.Object, name string) { + if HasFinalizer(obj, name) { + return + } + + obj.SetFinalizers(append(obj.GetFinalizers(), name)) +} + +// RemoveFinalizer removes a finalizer from a resource +func RemoveFinalizer(obj metav1.Object, name string) { + if !HasFinalizer(obj, name) { + return + } + + var finalizers []string + for _, f := range obj.GetFinalizers() { + if f != name { + finalizers = append(finalizers, f) + } + } + + obj.SetFinalizers(finalizers) +} + +// HasFinalizer returns true if a resource has a specific finalizer +func HasFinalizer(object metav1.Object, value string) bool { + for _, f := range object.GetFinalizers() { + if f == value { + return true + } + } + return false +} + +func (r *ReconcileDirectVolumeMigration) cleanupSourceResourcesInNamespace(direct *migapi.DirectVolumeMigration) (bool, error) { + sourceCluster, err := direct.GetSourceCluster(r) + if err != nil { + return false, err + } + + // Cleanup source resources + client, err := sourceCluster.GetClient(r) + if err != nil { + return false, err + } + + liveMigrationCanceled, err := r.cancelLiveMigrations(client, direct) + if err != nil { + return false, err + } + + completed, err := r.cleanupResourcesInNamespaces(client, direct.GetUID(), direct.GetSourceNamespaces()) + return completed && liveMigrationCanceled, err +} + +func (r *ReconcileDirectVolumeMigration) cancelLiveMigrations(client compat.Client, direct *migapi.DirectVolumeMigration) (bool, error) { + if direct.IsCutover() && direct.IsLiveMigrate() { + // Cutover and live migration is enabled, attempt to cancel migrations in progress. + namespaces := direct.GetSourceNamespaces() + allLiveMigrationCompleted := true + for _, ns := range namespaces { + volumeVmMap, err := getRunningVmVolumeMap(client, ns) + if err != nil { + return false, err + } + vmVolumeMap := make(map[string]*vmVolumes) + for i := range direct.Spec.PersistentVolumeClaims { + sourceName := direct.Spec.PersistentVolumeClaims[i].Name + targetName := direct.Spec.PersistentVolumeClaims[i].TargetName + vmName, found := volumeVmMap[sourceName] + if !found { + continue + } + volumes := vmVolumeMap[vmName] + if volumes == nil { + volumes = &vmVolumes{ + sourceVolumes: []string{sourceName}, + targetVolumes: []string{targetName}, + } + } else { + volumes.sourceVolumes = append(volumes.sourceVolumes, sourceName) + volumes.targetVolumes = append(volumes.targetVolumes, targetName) + } + vmVolumeMap[vmName] = volumes + } + vmNames := make([]string, 0) + for vmName, volumes := range vmVolumeMap { + if err := cancelLiveMigration(client, vmName, ns, volumes, log); err != nil { + return false, err + } + vmNames = append(vmNames, vmName) + } + migrated, err := liveMigrationsCompleted(client, ns, vmNames) + if err != nil { + return false, err + } + allLiveMigrationCompleted = allLiveMigrationCompleted && migrated + } + return allLiveMigrationCompleted, nil + } + return true, nil +} + +func (r *ReconcileDirectVolumeMigration) cleanupTargetResourcesInNamespaces(direct *migapi.DirectVolumeMigration) (bool, error) { + destinationCluster, err := direct.GetDestinationCluster(r) + if err != nil { + return false, err + } + + // Cleanup source resources + client, err := destinationCluster.GetClient(r) + if err != nil { + return false, err + } + return r.cleanupResourcesInNamespaces(client, direct.GetUID(), direct.GetDestinationNamespaces()) +} + +func (r *ReconcileDirectVolumeMigration) cleanupResourcesInNamespaces(client compat.Client, uid types.UID, namespaces []string) (bool, error) { + selector := labels.SelectorFromSet(map[string]string{ + "directvolumemigration": string(uid), + }) + + sourceDeleted := true + for _, ns := range namespaces { + if err := findAndDeleteNsResources(client, ns, selector, log); err != nil { + return false, err + } + err, deleted := areRsyncNsResourcesDeleted(client, ns, selector, log) + if err != nil { + return false, err + } + sourceDeleted = sourceDeleted && deleted + } + return sourceDeleted, nil +} diff --git a/pkg/controller/directvolumemigration/directvolumemigration_controller_test.go b/pkg/controller/directvolumemigration/directvolumemigration_controller_test.go index 39f9b6f6a..912e30b91 100644 --- a/pkg/controller/directvolumemigration/directvolumemigration_controller_test.go +++ b/pkg/controller/directvolumemigration/directvolumemigration_controller_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" migrationv1alpha1 "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" "github.com/onsi/gomega" "golang.org/x/net/context" @@ -27,6 +28,8 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -82,3 +85,208 @@ func TestReconcile(t *testing.T) { defer c.Delete(context.TODO(), instance) g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest))) } + +func TestCleanupTargetResourcesInNamespaces(t *testing.T) { + g := gomega.NewGomegaWithT(t) + reconciler := &ReconcileDirectVolumeMigration{} + instance := &migrationv1alpha1.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: migrationv1alpha1.OpenshiftMigrationNamespace, + UID: "1234", + }, + } + client := getFakeClientWithObjs( + createDVMPod("namespace1", string(instance.GetUID())), + createDVMSecret("namespace1", string(instance.GetUID())), + createDVMSecret("namespace2", string(instance.GetUID())), + ) + + _, err := reconciler.cleanupResourcesInNamespaces(client, instance.GetUID(), []string{"namespace1", "namespace2"}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // ensure the pod is gone + pod := &kapi.Pod{} + err = client.Get(context.TODO(), types.NamespacedName{Name: "dvm-pod", Namespace: "namespace1"}, pod) + g.Expect(apierrors.IsNotFound(err)).To(gomega.BeTrue()) + + // ensure the secrets are gone + secret := &kapi.Secret{} + err = client.Get(context.TODO(), types.NamespacedName{Name: "dvm-secret", Namespace: "namespace1"}, secret) + g.Expect(apierrors.IsNotFound(err)).To(gomega.BeTrue()) + secret = &kapi.Secret{} + err = client.Get(context.TODO(), types.NamespacedName{Name: "dvm-secret", Namespace: "namespace2"}, secret) + g.Expect(apierrors.IsNotFound(err)).To(gomega.BeTrue()) +} + +func TestCancelLiveMigrationsStage(t *testing.T) { + g := gomega.NewGomegaWithT(t) + reconciler := &ReconcileDirectVolumeMigration{} + instance := &migrationv1alpha1.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: migrationv1alpha1.OpenshiftMigrationNamespace, + UID: "1234", + }, + Spec: migrationv1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeStage), + }, + } + client := getFakeClientWithObjs() + allCompleted, err := reconciler.cancelLiveMigrations(client, instance) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(allCompleted).To(gomega.BeTrue()) +} + +func TestCancelLiveMigrationsCutoverAllCompleted(t *testing.T) { + g := gomega.NewGomegaWithT(t) + reconciler := &ReconcileDirectVolumeMigration{} + instance := &migrationv1alpha1.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: migrationv1alpha1.OpenshiftMigrationNamespace, + UID: "1234", + }, + Spec: migrationv1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeFinal), + LiveMigrate: ptr.To[bool](true), + PersistentVolumeClaims: []migapi.PVCToMigrate{ + { + ObjectReference: &kapi.ObjectReference{ + Namespace: testNamespace, + Name: "pvc1", + }, + TargetNamespace: testNamespace, + TargetName: "target-pvc1", + }, + { + ObjectReference: &kapi.ObjectReference{ + Namespace: testNamespace, + Name: "pvc2", + }, + TargetNamespace: testNamespace, + TargetName: "target-pvc2", + }, + { + ObjectReference: &kapi.ObjectReference{ + Namespace: testNamespace, + Name: "pvc3", + }, + TargetNamespace: testNamespace, + TargetName: "target-pvc3", + }, + { + ObjectReference: &kapi.ObjectReference{ + Namespace: testNamespace, + Name: "pvc4", + }, + TargetNamespace: testNamespace, + TargetName: "target-pvc4", + }, + }, + }, + } + client := getFakeClientWithObjs( + createVirtualMachineWithVolumes("vm1", testNamespace, []virtv1.Volume{ + { + Name: "pvc1", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "pvc1", + }, + }, + }, + }), + createVirtlauncherPod("vm1", testNamespace, []string{"pvc1"}), + createVirtualMachineWithVolumes("vm2", testNamespace, []virtv1.Volume{ + { + Name: "pvc2", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "pvc2", + }, + }, + }, + { + Name: "pvc3", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "pvc3", + }, + }, + }, + }), + createVirtlauncherPod("vm2", testNamespace, []string{"pvc2", "pvc3"}), + createVirtualMachine("vm3", testNamespace), + ) + allCompleted, err := reconciler.cancelLiveMigrations(client, instance) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(allCompleted).To(gomega.BeTrue()) +} + +func TestCancelLiveMigrationsCutoverNotCompleted(t *testing.T) { + g := gomega.NewGomegaWithT(t) + reconciler := &ReconcileDirectVolumeMigration{} + instance := &migrationv1alpha1.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: migrationv1alpha1.OpenshiftMigrationNamespace, + UID: "1234", + }, + Spec: migrationv1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeFinal), + LiveMigrate: ptr.To[bool](true), + PersistentVolumeClaims: []migapi.PVCToMigrate{ + { + ObjectReference: &kapi.ObjectReference{ + Namespace: testNamespace, + Name: "pvc1", + }, + TargetNamespace: testNamespace, + TargetName: "target-pvc1", + }, + }, + }, + } + client := getFakeClientWithObjs( + createVirtualMachineWithVolumes("vm1", testNamespace, []virtv1.Volume{ + { + Name: "pvc1", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "pvc1", + }, + }, + }, + }), + createVirtlauncherPod("vm1", testNamespace, []string{"pvc1"}), + createInProgressVirtualMachineMigration("vmim", testNamespace, "vm1"), + ) + allCompleted, err := reconciler.cancelLiveMigrations(client, instance) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(allCompleted).To(gomega.BeFalse()) +} + +func createDVMPod(namespace, uid string) *kapi.Pod { + return &kapi.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dvm-pod", + Namespace: namespace, + Labels: map[string]string{ + "directvolumemigration": uid, + }, + }, + } +} + +func createDVMSecret(namespace, uid string) *kapi.Secret { + return &kapi.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dvm-secret", + Namespace: namespace, + Labels: map[string]string{ + "directvolumemigration": uid, + }, + }, + } +} diff --git a/pkg/controller/directvolumemigration/pvcs_test.go b/pkg/controller/directvolumemigration/pvcs_test.go index 05735b63c..e001720bf 100644 --- a/pkg/controller/directvolumemigration/pvcs_test.go +++ b/pkg/controller/directvolumemigration/pvcs_test.go @@ -25,7 +25,7 @@ const ( func Test_CreateDestinationPVCsNewPVCMigrationOwner(t *testing.T) { RegisterTestingT(t) migPlan := createMigPlan() - sourcePVC := createSourcePvc("pvc1", testNamespace) + sourcePVC := createPvc("pvc1", testNamespace) client := getFakeCompatClient(&migapi.MigCluster{ ObjectMeta: metav1.ObjectMeta{ Name: "cluster", @@ -68,7 +68,7 @@ func Test_CreateDestinationPVCsNewPVCMigrationOwner(t *testing.T) { func Test_CreateDestinationPVCsNewPVCDVMOwnerOtherNS(t *testing.T) { RegisterTestingT(t) migPlan := createMigPlan() - sourcePVC := createSourcePvc("pvc1", testNamespace) + sourcePVC := createPvc("pvc1", testNamespace) client := getFakeCompatClient(&migapi.MigCluster{ ObjectMeta: metav1.ObjectMeta{ Name: "cluster", @@ -110,7 +110,7 @@ func Test_CreateDestinationPVCsExpandPVSize(t *testing.T) { settings.Settings.DvmOpts.EnablePVResizing = true defer func() { settings.Settings.DvmOpts.EnablePVResizing = false }() migPlan := createMigPlan() - sourcePVC := createSourcePvc("pvc1", testNamespace) + sourcePVC := createPvc("pvc1", testNamespace) migPlan.Spec.PersistentVolumes.List = append(migPlan.Spec.PersistentVolumes.List, migapi.PV{ PVC: migapi.PVC{ Namespace: testNamespace, @@ -154,7 +154,7 @@ func Test_CreateDestinationPVCsExpandProposedPVSize(t *testing.T) { settings.Settings.DvmOpts.EnablePVResizing = true defer func() { settings.Settings.DvmOpts.EnablePVResizing = false }() migPlan := createMigPlan() - sourcePVC := createSourcePvc("pvc1", testNamespace) + sourcePVC := createPvc("pvc1", testNamespace) migPlan.Spec.PersistentVolumes.List = append(migPlan.Spec.PersistentVolumes.List, migapi.PV{ PVC: migapi.PVC{ Namespace: testNamespace, @@ -197,7 +197,7 @@ func Test_CreateDestinationPVCsExpandProposedPVSize(t *testing.T) { func Test_CreateDestinationPVCsExistingTargetPVC(t *testing.T) { RegisterTestingT(t) migPlan := createMigPlan() - sourcePVC := createSourcePvc("pvc1", testNamespace) + sourcePVC := createPvc("pvc1", testNamespace) client := getFakeCompatClient(&migapi.MigCluster{ ObjectMeta: metav1.ObjectMeta{ Name: "cluster", @@ -246,7 +246,7 @@ func createMigPlan() *migapi.MigPlan { } } -func createSourcePvc(name, namespace string) *kapi.PersistentVolumeClaim { +func createPvc(name, namespace string) *kapi.PersistentVolumeClaim { return &kapi.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: name, diff --git a/pkg/controller/directvolumemigration/rsync.go b/pkg/controller/directvolumemigration/rsync.go index 66762aa88..59cc96764 100644 --- a/pkg/controller/directvolumemigration/rsync.go +++ b/pkg/controller/directvolumemigration/rsync.go @@ -10,10 +10,12 @@ import ( "path" "reflect" "regexp" + "slices" "strconv" "strings" "time" + "github.com/go-logr/logr" liberr "github.com/konveyor/controller/pkg/error" "github.com/konveyor/crane-lib/state_transfer/endpoint" routeendpoint "github.com/konveyor/crane-lib/state_transfer/endpoint/route" @@ -36,6 +38,8 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/set" + virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -63,11 +67,6 @@ const ( // ensureRsyncEndpoints ensures that new Endpoints are created for Rsync and Blockrsync Transfers func (t *Task) ensureRsyncEndpoints() error { - destClient, err := t.getDestinationClient() - if err != nil { - return liberr.Wrap(err) - } - dvmLabels := t.buildDVMLabels() dvmLabels["purpose"] = DirectVolumeMigrationRsync blockdvmLabels := t.buildDVMLabels() @@ -76,7 +75,8 @@ func (t *Task) ensureRsyncEndpoints() error { hostnames := []string{} if t.EndpointType == migapi.NodePort { - hostnames, err = getWorkerNodeHostnames(destClient) + var err error + hostnames, err = getWorkerNodeHostnames(t.destinationClient) if err != nil { return liberr.Wrap(err) } @@ -120,7 +120,7 @@ func (t *Task) ensureRsyncEndpoints() error { if err != nil { t.Log.Info("failed to get cluster_subdomain, attempting to get cluster's ingress domain", "error", err) ingressConfig := &configv1.Ingress{} - err = destClient.Get(context.TODO(), types.NamespacedName{Name: "cluster"}, ingressConfig) + err = t.destinationClient.Get(context.TODO(), types.NamespacedName{Name: "cluster"}, ingressConfig) if err != nil { t.Log.Error(err, "failed to retrieve cluster's ingress domain, extremely long namespace names will cause route creation failure") } else { @@ -148,12 +148,10 @@ func (t *Task) ensureRsyncEndpoints() error { ) } - err = endpoint.Create(destClient) - if err != nil { + if err := endpoint.Create(t.destinationClient); err != nil { return liberr.Wrap(err) } - err = blockEndpoint.Create(destClient) - if err != nil { + if err := blockEndpoint.Create(t.destinationClient); err != nil { return liberr.Wrap(err) } } @@ -250,6 +248,9 @@ func (t *Task) getRsyncClientMutations(srcClient compat.Client, destClient compa if err != nil { return nil, liberr.Wrap(err) } + if migration == nil { + return transferOptions, nil + } containerMutation.SecurityContext, err = t.getSecurityContext(srcClient, namespace, migration) if err != nil { @@ -281,6 +282,9 @@ func (t *Task) getRsyncTransferServerMutations(client compat.Client, namespace s if err != nil { return nil, liberr.Wrap(err) } + if migration == nil { + return transferOptions, nil + } containerMutation.SecurityContext, err = t.getSecurityContext(client, namespace, migration) if err != nil { @@ -402,22 +406,12 @@ func (t *Task) getSecurityContext(client compat.Client, namespace string, migrat // ensureRsyncTransferServer ensures that server component of the Transfer is created func (t *Task) ensureRsyncTransferServer() error { - destClient, err := t.getDestinationClient() - if err != nil { - return liberr.Wrap(err) - } - - srcClient, err := t.getSourceClient() - if err != nil { - return liberr.Wrap(err) - } - nsMap, err := t.getNamespacedPVCPairs() if err != nil { return liberr.Wrap(err) } - err = t.buildDestinationLimitRangeMap(nsMap, destClient) + err = t.buildDestinationLimitRangeMap(nsMap, t.destinationClient) if err != nil { return liberr.Wrap(err) } @@ -427,16 +421,16 @@ func (t *Task) ensureRsyncTransferServer() error { return liberr.Wrap(err) } - if err := t.ensureFilesystemRsyncTransferServer(srcClient, destClient, nsMap, transportOptions); err != nil { + if err := t.ensureFilesystemRsyncTransferServer(nsMap, transportOptions); err != nil { return err } - if err := t.ensureBlockRsyncTransferServer(srcClient, destClient, nsMap, transportOptions); err != nil { + if err := t.ensureBlockRsyncTransferServer(nsMap, transportOptions); err != nil { return err } return nil } -func (t *Task) ensureFilesystemRsyncTransferServer(srcClient, destClient compat.Client, nsMap map[string][]transfer.PVCPair, transportOptions *transport.Options) error { +func (t *Task) ensureFilesystemRsyncTransferServer(nsMap map[string][]transfer.PVCPair, transportOptions *transport.Options) error { for bothNs, pvcPairs := range nsMap { srcNs := getSourceNs(bothNs) destNs := getDestNs(bothNs) @@ -444,12 +438,12 @@ func (t *Task) ensureFilesystemRsyncTransferServer(srcClient, destClient compat. types.NamespacedName{Name: DirectVolumeMigrationRsyncTransfer, Namespace: srcNs}, types.NamespacedName{Name: DirectVolumeMigrationRsyncTransfer, Namespace: destNs}, ) - endpoints, err := t.getEndpoints(destClient, destNs) + endpoints, err := t.getEndpoints(t.destinationClient, destNs) if err != nil { return liberr.Wrap(err) } stunnelTransport, err := stunneltransport.GetTransportFromKubeObjects( - srcClient, destClient, "fs", nnPair, endpoints[0], transportOptions) + t.sourceClient, t.destinationClient, "fs", nnPair, endpoints[0], transportOptions) if err != nil { return liberr.Wrap(err) } @@ -464,21 +458,21 @@ func (t *Task) ensureFilesystemRsyncTransferServer(srcClient, destClient compat. if err != nil { return liberr.Wrap(err) } - mutations, err := t.getRsyncTransferServerMutations(destClient, destNs) + mutations, err := t.getRsyncTransferServerMutations(t.destinationClient, destNs) if err != nil { return liberr.Wrap(err) } rsyncOptions = append(rsyncOptions, mutations...) rsyncOptions = append(rsyncOptions, rsynctransfer.WithDestinationPodLabels(labels)) transfer, err := rsynctransfer.NewTransfer( - stunnelTransport, endpoints[0], srcClient, destClient, filesystemPvcList, t.Log, rsyncOptions...) + stunnelTransport, endpoints[0], t.sourceClient, t.destinationClient, filesystemPvcList, t.Log, rsyncOptions...) if err != nil { return liberr.Wrap(err) } if transfer == nil { return fmt.Errorf("transfer %s/%s not found", nnPair.Source().Namespace, nnPair.Source().Name) } - err = transfer.CreateServer(destClient) + err = transfer.CreateServer(t.destinationClient) if err != nil { return liberr.Wrap(err) } @@ -487,7 +481,7 @@ func (t *Task) ensureFilesystemRsyncTransferServer(srcClient, destClient compat. return nil } -func (t *Task) ensureBlockRsyncTransferServer(srcClient, destClient compat.Client, nsMap map[string][]transfer.PVCPair, transportOptions *transport.Options) error { +func (t *Task) ensureBlockRsyncTransferServer(nsMap map[string][]transfer.PVCPair, transportOptions *transport.Options) error { for bothNs, pvcPairs := range nsMap { srcNs := getSourceNs(bothNs) destNs := getDestNs(bothNs) @@ -495,19 +489,26 @@ func (t *Task) ensureBlockRsyncTransferServer(srcClient, destClient compat.Clien types.NamespacedName{Name: DirectVolumeMigrationRsyncTransfer, Namespace: srcNs}, types.NamespacedName{Name: DirectVolumeMigrationRsyncTransfer, Namespace: destNs}, ) - endpoints, err := t.getEndpoints(destClient, destNs) + endpoints, err := t.getEndpoints(t.destinationClient, destNs) if err != nil { return liberr.Wrap(err) } stunnelTransports, err := stunneltransport.GetTransportFromKubeObjects( - srcClient, destClient, "block", nnPair, endpoints[1], transportOptions) + t.sourceClient, t.destinationClient, "block", nnPair, endpoints[1], transportOptions) if err != nil { return liberr.Wrap(err) } + blockOrVMPvcList, err := transfer.NewBlockOrVMDiskPVCPairList(pvcPairs...) if err != nil { return liberr.Wrap(err) } + if t.PlanResources.MigPlan.LiveMigrationChecked() { + blockOrVMPvcList, err = t.filterRunningVMs(blockOrVMPvcList) + if err != nil { + return liberr.Wrap(err) + } + } if len(blockOrVMPvcList) > 0 { labels := t.buildDVMLabels() labels["app"] = DirectVolumeMigrationRsyncTransferBlock @@ -521,14 +522,14 @@ func (t *Task) ensureBlockRsyncTransferServer(srcClient, destClient compat.Clien } transfer, err := blockrsynctransfer.NewTransfer( - stunnelTransports, endpoints[1], srcClient, destClient, blockOrVMPvcList, t.Log, transferOptions) + stunnelTransports, endpoints[1], t.sourceClient, t.destinationClient, blockOrVMPvcList, t.Log, transferOptions) if err != nil { return liberr.Wrap(err) } if transfer == nil { return fmt.Errorf("transfer %s/%s not found", nnPair.Source().Namespace, nnPair.Source().Name) } - err = transfer.CreateServer(destClient) + err = transfer.CreateServer(t.destinationClient) if err != nil { return liberr.Wrap(err) } @@ -537,9 +538,29 @@ func (t *Task) ensureBlockRsyncTransferServer(srcClient, destClient compat.Clien return nil } +// Return only PVCPairs that are NOT associated with a running VM +func (t *Task) filterRunningVMs(unfilteredPVCPairs []transfer.PVCPair) ([]transfer.PVCPair, error) { + runningVMPairs := []transfer.PVCPair{} + ns := set.New[string]() + for _, pvcPair := range unfilteredPVCPairs { + ns.Insert(pvcPair.Source().Claim().Namespace) + } + nsVolumes, err := t.getRunningVMVolumes(ns.SortedList()) + if err != nil { + return nil, err + } + + for _, pvcPair := range unfilteredPVCPairs { + if !slices.Contains(nsVolumes, fmt.Sprintf("%s/%s", pvcPair.Source().Claim().Namespace, pvcPair.Source().Claim().Name)) { + runningVMPairs = append(runningVMPairs, pvcPair) + } + } + return runningVMPairs, nil +} + func (t *Task) createRsyncTransferClients(srcClient compat.Client, - destClient compat.Client, nsMap map[string][]transfer.PVCPair) (*rsyncClientOperationStatusList, error) { - statusList := &rsyncClientOperationStatusList{} + destClient compat.Client, nsMap map[string][]transfer.PVCPair) (*migrationOperationStatusList, error) { + statusList := &migrationOperationStatusList{} pvcNodeMap, err := t.getPVCNodeNameMap(srcClient) if err != nil { @@ -624,7 +645,7 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, ) if lastObservedOperationStatus.IsComplete() { statusList.Add( - rsyncClientOperationStatus{ + migrationOperationStatus{ failed: lastObservedOperationStatus.Failed, succeeded: lastObservedOperationStatus.Succeeded, operation: lastObservedOperationStatus, @@ -634,7 +655,7 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, } newOperation := lastObservedOperationStatus - currentStatus := rsyncClientOperationStatus{ + currentStatus := migrationOperationStatus{ operation: newOperation, } pod, err := t.getLatestPodForOperation(srcClient, *lastObservedOperationStatus) @@ -654,11 +675,20 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, } blockOrVMPvcList, err := transfer.NewBlockOrVMDiskPVCPairList(pvc) if err != nil { - t.Log.Error(err, "failed creating PVC pair", "pvc", newOperation) + t.Log.Error(err, "failed creating block PVC pair", "pvc", newOperation) currentStatus.AddError(err) statusList.Add(currentStatus) continue } + if t.PlanResources.MigPlan.LiveMigrationChecked() { + blockOrVMPvcList, err = t.filterRunningVMs(blockOrVMPvcList) + if err != nil { + t.Log.Error(err, "failed filtering block PVC pairs", "pvc", newOperation) + currentStatus.AddError(err) + statusList.Add(currentStatus) + continue + } + } // Force schedule Rsync Pod on the application node nodeName := pvcNodeMap[fmt.Sprintf("%s/%s", srcNs, pvc.Source().Claim().Name)] @@ -699,7 +729,7 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, if pod != nil { newOperation.CurrentAttempt, _ = strconv.Atoi(pod.Labels[RsyncAttemptLabel]) updateOperationStatus(¤tStatus, pod) - if currentStatus.failed && currentStatus.operation.CurrentAttempt < GetRsyncPodBackOffLimit(*t.Owner) { + if currentStatus.failed && currentStatus.operation.CurrentAttempt < GetRsyncPodBackOffLimit(t.Owner) { // since we have not yet attempted all retries, // reset the failed status and set the pending status currentStatus.failed = false @@ -726,6 +756,7 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, SourcePodMeta: transfer.ResourceMetadata{ Labels: labels, }, + NodeName: nodeName, } transfer, err := blockrsynctransfer.NewTransfer( blockStunnelTransport, endpoints[1], srcClient, destClient, blockOrVMPvcList, t.Log, &transferOptions) @@ -793,6 +824,8 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, transferOptions.SourcePodMeta = transfer.ResourceMetadata{ Labels: labels, } + transferOptions.NodeName = nodeName + transfer, err := blockrsynctransfer.NewTransfer( blockStunnelTransport, endpoints[1], srcClient, destClient, blockOrVMPvcList, t.Log, transferOptions) if err != nil { @@ -809,7 +842,6 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, } } statusList.Add(currentStatus) - t.Log.Info("adding status of pvc", "pvc", currentStatus.operation, "errors", currentStatus.errors) } } return statusList, nil @@ -818,7 +850,7 @@ func (t *Task) createRsyncTransferClients(srcClient compat.Client, func (t *Task) createClientPodForTransfer( name, namespace string, tr transfer.Transfer, - currentStatus *rsyncClientOperationStatus) *rsyncClientOperationStatus { + currentStatus *migrationOperationStatus) *migrationOperationStatus { if tr == nil { currentStatus.AddError( fmt.Errorf("transfer %s/%s not found", namespace, name)) @@ -833,12 +865,6 @@ func (t *Task) createClientPodForTransfer( } func (t *Task) areRsyncTransferPodsRunning() (arePodsRunning bool, nonRunningPods []*corev1.Pod, e error) { - // Get client for destination - destClient, err := t.getDestinationClient() - if err != nil { - return false, nil, err - } - pvcMap := t.getPVCNamespaceMap() dvmLabels := t.buildDVMLabels() dvmLabels["purpose"] = DirectVolumeMigrationRsync @@ -847,7 +873,7 @@ func (t *Task) areRsyncTransferPodsRunning() (arePodsRunning bool, nonRunningPod for bothNs, _ := range pvcMap { ns := getDestNs(bothNs) pods := corev1.PodList{} - err = destClient.List( + err := t.destinationClient.List( context.TODO(), &pods, &k8sclient.ListOptions{ @@ -861,7 +887,7 @@ func (t *Task) areRsyncTransferPodsRunning() (arePodsRunning bool, nonRunningPod if pod.Status.Phase != corev1.PodRunning { // Log abnormal events for Rsync transfer Pod if any are found migevent.LogAbnormalEventsForResource( - destClient, t.Log, + t.destinationClient, t.Log, "Found abnormal event for Rsync transfer Pod on destination cluster", types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, pod.UID, "Pod") @@ -960,16 +986,6 @@ func (p pvcWithSecurityContextInfo) Get(srcClaimName string, srcClaimNamespace s // With namespace mapping, the destination cluster namespace may be different than that in the source cluster. // This function maps PVCs to the appropriate src:dest namespace pairs. func (t *Task) getNamespacedPVCPairs() (map[string][]transfer.PVCPair, error) { - srcClient, err := t.getSourceClient() - if err != nil { - return nil, err - } - - destClient, err := t.getDestinationClient() - if err != nil { - return nil, err - } - nsMap := map[string][]transfer.PVCPair{} for _, pvc := range t.Owner.Spec.PersistentVolumeClaims { srcNs := pvc.Namespace @@ -978,12 +994,12 @@ func (t *Task) getNamespacedPVCPairs() (map[string][]transfer.PVCPair, error) { destNs = pvc.TargetNamespace } srcPvc := corev1.PersistentVolumeClaim{} - err := srcClient.Get(context.TODO(), types.NamespacedName{Name: pvc.Name, Namespace: srcNs}, &srcPvc) + err := t.sourceClient.Get(context.TODO(), types.NamespacedName{Name: pvc.Name, Namespace: srcNs}, &srcPvc) if err != nil { return nil, err } destPvc := corev1.PersistentVolumeClaim{} - err = destClient.Get(context.TODO(), types.NamespacedName{Name: pvc.TargetName, Namespace: destNs}, &destPvc) + err = t.destinationClient.Get(context.TODO(), types.NamespacedName{Name: pvc.TargetName, Namespace: destNs}, &destPvc) if err != nil { return nil, err } @@ -1062,10 +1078,6 @@ func getDestNs(bothNs string) string { func (t *Task) areRsyncRoutesAdmitted() (bool, []string, error) { messages := []string{} // Get client for destination - destClient, err := t.getDestinationClient() - if err != nil { - return false, messages, err - } nsMap := t.getPVCNamespaceMap() for bothNs := range nsMap { namespace := getDestNs(bothNs) @@ -1075,13 +1087,12 @@ func (t *Task) areRsyncRoutesAdmitted() (bool, []string, error) { route := routev1.Route{} key := types.NamespacedName{Name: DirectVolumeMigrationRsyncTransferRoute, Namespace: namespace} - err = destClient.Get(context.TODO(), key, &route) - if err != nil { + if err := t.destinationClient.Get(context.TODO(), key, &route); err != nil { return false, messages, err } // Logs abnormal events related to route if any are found migevent.LogAbnormalEventsForResource( - destClient, t.Log, + t.destinationClient, t.Log, "Found abnormal event for Rsync Route on destination cluster", types.NamespacedName{Namespace: route.Namespace, Name: route.Name}, route.UID, "Route") @@ -1110,8 +1121,7 @@ func (t *Task) areRsyncRoutesAdmitted() (bool, []string, error) { messages = append(messages, message) } default: - _, err = t.getEndpoints(destClient, namespace) - if err != nil { + if _, err := t.getEndpoints(t.destinationClient, namespace); err != nil { t.Log.Info("rsync transfer service is not healthy", "namespace", namespace) messages = append(messages, fmt.Sprintf("rsync transfer service is not healthy in namespace %s", namespace)) } @@ -1318,7 +1328,17 @@ func (t *Task) createPVProgressCR() error { labels := t.Owner.GetCorrelationLabels() for bothNs, vols := range pvcMap { ns := getSourceNs(bothNs) + volumeNames, err := t.getRunningVMVolumes([]string{ns}) + if err != nil { + return liberr.Wrap(err) + } for _, vol := range vols { + matchString := fmt.Sprintf("%s/%s", ns, vol.Name) + if t.PlanResources.MigPlan.LiveMigrationChecked() && + slices.Contains(volumeNames, matchString) { + t.Log.V(3).Info("Skipping Rsync Progress CR creation for running VM", "volume", matchString) + continue + } dvmp := migapi.DirectVolumeMigrationProgress{ ObjectMeta: metav1.ObjectMeta{ Name: getMD5Hash(t.Owner.Name + vol.Name + ns), @@ -1365,79 +1385,189 @@ func getMD5Hash(s string) string { // hasAllProgressReportingCompleted reads DVMP CR and status of Rsync Operations present for all PVCs and generates progress information in CR status // returns True when progress reporting for all Rsync Pods is complete func (t *Task) hasAllProgressReportingCompleted() (bool, error) { - t.Owner.Status.RunningPods = []*migapi.PodProgress{} - t.Owner.Status.FailedPods = []*migapi.PodProgress{} - t.Owner.Status.SuccessfulPods = []*migapi.PodProgress{} - t.Owner.Status.PendingPods = []*migapi.PodProgress{} - unknownPods := []*migapi.PodProgress{} - var pendingSinceTimeLimitPods []string + // Keep current progress in case looking up progress fails, this way we don't wipe out + // the progress until next time the update succeeds. + currentProgress := t.getCurrentLiveMigrationProgress() + t.resetProgressCounters() pvcMap := t.getPVCNamespaceMap() for bothNs, vols := range pvcMap { ns := getSourceNs(bothNs) + if err := t.populateVMMappings(ns); err != nil { + return false, err + } for _, vol := range vols { - operation := t.Owner.Status.GetRsyncOperationStatusForPVC(&corev1.ObjectReference{ - Namespace: ns, - Name: vol.Name, - }) - dvmp := migapi.DirectVolumeMigrationProgress{} - err := t.Client.Get(context.TODO(), types.NamespacedName{ - Name: getMD5Hash(t.Owner.Name + vol.Name + ns), - Namespace: migapi.OpenshiftMigrationNamespace, - }, &dvmp) - if err != nil { - return false, err + matchString := fmt.Sprintf("%s/%s", ns, vol.Name) + if t.PlanResources.MigPlan.LiveMigrationChecked() && + slices.Contains(t.VirtualMachineMappings.runningVMVolumeNames, matchString) { + // Only count skipped during staging. During cutover we need to live migrate + // PVCs for running VMs. For reporting purposes we won't count skipped PVCs + // here since they will get reported with the live migration status. + if !(t.Owner.IsCutover() || t.Owner.IsRollback()) { + t.Owner.SkipVolume(vol.Name, ns) + } else { + if err := t.updateVolumeLiveMigrationProgressStatus(vol.Name, ns, currentProgress); err != nil { + return false, err + } + } + } else { + // On rollback we are only interested in live migration volumes, skip the rest + if t.Owner.IsRollback() { + t.Owner.SkipVolume(vol.Name, ns) + } + if err := t.updateRsyncProgressStatus(vol.Name, ns); err != nil { + return false, err + } } - podProgress := &migapi.PodProgress{ - ObjectReference: &corev1.ObjectReference{ - Namespace: ns, - Name: dvmp.Status.PodName, - }, - PVCReference: &corev1.ObjectReference{ - Namespace: ns, - Name: vol.Name, - }, - LastObservedProgressPercent: dvmp.Status.TotalProgressPercentage, - LastObservedTransferRate: dvmp.Status.LastObservedTransferRate, - TotalElapsedTime: dvmp.Status.RsyncElapsedTime, + } + } + + return t.Owner.AllReportingCompleted(), nil +} + +func (t *Task) updateVolumeLiveMigrationProgressStatus(volumeName, namespace string, currentProgress map[string]*migapi.LiveMigrationProgress) error { + matchString := fmt.Sprintf("%s/%s", namespace, volumeName) + + liveMigrationProgress := &migapi.LiveMigrationProgress{ + VMName: "", + VMNamespace: namespace, + PVCReference: &corev1.ObjectReference{ + Namespace: namespace, + Name: volumeName, + }, + TotalElapsedTime: nil, + LastObservedProgressPercent: "", + } + // Look up VirtualMachineInstanceMigration CR to get the status of the migration + if vmim, exists := t.VirtualMachineMappings.volumeVMIMMap[matchString]; exists { + liveMigrationProgress.VMName = vmim.Spec.VMIName + elapsedTime := getVMIMElapsedTime(vmim) + liveMigrationProgress.TotalElapsedTime = &elapsedTime + if vmim.Status.MigrationState.FailureReason != "" { + liveMigrationProgress.Message = vmim.Status.MigrationState.FailureReason + } + switch vmim.Status.Phase { + case virtv1.MigrationSucceeded: + liveMigrationProgress.LastObservedProgressPercent = "100%" + t.Owner.Status.SuccessfulLiveMigrations = append(t.Owner.Status.SuccessfulLiveMigrations, liveMigrationProgress) + case virtv1.MigrationFailed: + + t.Owner.Status.FailedLiveMigrations = append(t.Owner.Status.FailedLiveMigrations, liveMigrationProgress) + case virtv1.MigrationRunning: + progressPercent, err := t.getLastObservedProgressPercent(vmim.Spec.VMIName, namespace, currentProgress) + if err != nil { + return err } - switch { - case dvmp.Status.PodPhase == corev1.PodRunning: - t.Owner.Status.RunningPods = append(t.Owner.Status.RunningPods, podProgress) - case operation.Failed: - t.Owner.Status.FailedPods = append(t.Owner.Status.FailedPods, podProgress) - case dvmp.Status.PodPhase == corev1.PodSucceeded: - t.Owner.Status.SuccessfulPods = append(t.Owner.Status.SuccessfulPods, podProgress) - case dvmp.Status.PodPhase == corev1.PodPending: - t.Owner.Status.PendingPods = append(t.Owner.Status.PendingPods, podProgress) - if dvmp.Status.CreationTimestamp != nil { - if time.Now().UTC().Sub(dvmp.Status.CreationTimestamp.Time.UTC()) > PendingPodWarningTimeLimit { - pendingSinceTimeLimitPods = append(pendingSinceTimeLimitPods, fmt.Sprintf("%s/%s", podProgress.Namespace, podProgress.Name)) + liveMigrationProgress.LastObservedProgressPercent = progressPercent + t.Owner.Status.RunningLiveMigrations = append(t.Owner.Status.RunningLiveMigrations, liveMigrationProgress) + case virtv1.MigrationPending: + t.Owner.Status.PendingLiveMigrations = append(t.Owner.Status.PendingLiveMigrations, liveMigrationProgress) + } + } else { + // VMIM doesn't exist, check if the VMI is in error. + vmName := t.VirtualMachineMappings.volumeVMNameMap[volumeName] + message, err := virtualMachineMigrationStatus(t.sourceClient, vmName, namespace, t.Log) + if err != nil { + return err + } + liveMigrationProgress.VMName = vmName + if message != "" { + vmMatchString := fmt.Sprintf("%s/%s", namespace, vmName) + liveMigrationProgress.Message = message + if currentProgress[vmMatchString] != nil && currentProgress[vmMatchString].TotalElapsedTime != nil { + liveMigrationProgress.TotalElapsedTime = currentProgress[vmMatchString].TotalElapsedTime + } else { + if t.Owner.Status.StartTimestamp != nil { + dvmStart := *t.Owner.Status.StartTimestamp + liveMigrationProgress.TotalElapsedTime = &metav1.Duration{ + Duration: time.Since(dvmStart.Time), } } - case dvmp.Status.PodPhase == "": - unknownPods = append(unknownPods, podProgress) - case !operation.Failed: - t.Owner.Status.RunningPods = append(t.Owner.Status.RunningPods, podProgress) } + t.Owner.Status.FailedLiveMigrations = append(t.Owner.Status.FailedLiveMigrations, liveMigrationProgress) + } else { + t.Owner.Status.PendingLiveMigrations = append(t.Owner.Status.PendingLiveMigrations, liveMigrationProgress) } } + return nil +} - isCompleted := len(t.Owner.Status.SuccessfulPods)+len(t.Owner.Status.FailedPods) == len(t.Owner.Spec.PersistentVolumeClaims) - isAnyPending := len(t.Owner.Status.PendingPods) > 0 - isAnyRunning := len(t.Owner.Status.RunningPods) > 0 - isAnyUnknown := len(unknownPods) > 0 - if len(pendingSinceTimeLimitPods) > 0 { - pendingMessage := fmt.Sprintf("Rsync Client Pods [%s] are stuck in Pending state for more than 10 mins", strings.Join(pendingSinceTimeLimitPods[:], ", ")) - t.Log.Info(pendingMessage) - t.Owner.Status.SetCondition(migapi.Condition{ - Type: RsyncClientPodsPending, - Status: migapi.True, - Reason: "PodStuckInContainerCreating", - Category: migapi.Warn, - Message: pendingMessage, - }) +func (t *Task) updateRsyncProgressStatus(volumeName, namespace string) error { + operation := t.Owner.Status.GetRsyncOperationStatusForPVC(&corev1.ObjectReference{ + Namespace: namespace, + Name: volumeName, + }) + dvmp := migapi.DirectVolumeMigrationProgress{} + err := t.Client.Get(context.TODO(), types.NamespacedName{ + Name: getMD5Hash(t.Owner.Name + volumeName + namespace), + Namespace: migapi.OpenshiftMigrationNamespace, + }, &dvmp) + if err != nil && !k8serror.IsNotFound(err) { + return err + } else if k8serror.IsNotFound(err) { + return nil + } + podProgress := &migapi.PodProgress{ + ObjectReference: &corev1.ObjectReference{ + Namespace: namespace, + Name: dvmp.Status.PodName, + }, + PVCReference: &corev1.ObjectReference{ + Namespace: namespace, + Name: volumeName, + }, + LastObservedProgressPercent: dvmp.Status.TotalProgressPercentage, + LastObservedTransferRate: dvmp.Status.LastObservedTransferRate, + TotalElapsedTime: dvmp.Status.RsyncElapsedTime, + } + switch { + case dvmp.Status.PodPhase == corev1.PodRunning: + t.Owner.Status.RunningPods = append(t.Owner.Status.RunningPods, podProgress) + case operation.Failed: + t.Owner.Status.FailedPods = append(t.Owner.Status.FailedPods, podProgress) + case dvmp.Status.PodPhase == corev1.PodSucceeded: + t.Owner.Status.SuccessfulPods = append(t.Owner.Status.SuccessfulPods, podProgress) + case dvmp.Status.PodPhase == corev1.PodPending: + t.Owner.Status.PendingPods = append(t.Owner.Status.PendingPods, podProgress) + if dvmp.Status.CreationTimestamp != nil { + if time.Now().UTC().Sub(dvmp.Status.CreationTimestamp.Time.UTC()) > PendingPodWarningTimeLimit { + t.Owner.Status.PendingSinceTimeLimitPods = append(t.Owner.Status.PendingSinceTimeLimitPods, podProgress) + } + } + case dvmp.Status.PodPhase == "": + t.Owner.Status.UnknownPods = append(t.Owner.Status.UnknownPods, podProgress) + case !operation.Failed: + t.Owner.Status.RunningPods = append(t.Owner.Status.RunningPods, podProgress) + } + return nil +} + +func (t *Task) resetProgressCounters() { + t.Owner.Status.RunningPods = []*migapi.PodProgress{} + t.Owner.Status.FailedPods = []*migapi.PodProgress{} + t.Owner.Status.SuccessfulPods = []*migapi.PodProgress{} + t.Owner.Status.PendingPods = []*migapi.PodProgress{} + t.Owner.Status.UnknownPods = []*migapi.PodProgress{} + t.Owner.Status.RunningLiveMigrations = []*migapi.LiveMigrationProgress{} + t.Owner.Status.FailedLiveMigrations = []*migapi.LiveMigrationProgress{} + t.Owner.Status.SuccessfulLiveMigrations = []*migapi.LiveMigrationProgress{} + t.Owner.Status.PendingLiveMigrations = []*migapi.LiveMigrationProgress{} +} + +func (t *Task) getCurrentLiveMigrationProgress() map[string]*migapi.LiveMigrationProgress { + currentProgress := make(map[string]*migapi.LiveMigrationProgress) + for _, progress := range t.Owner.Status.RunningLiveMigrations { + currentProgress[fmt.Sprintf("%s/%s", progress.VMNamespace, progress.VMName)] = progress + } + for _, progress := range t.Owner.Status.FailedLiveMigrations { + currentProgress[fmt.Sprintf("%s/%s", progress.VMNamespace, progress.VMName)] = progress } - return !isAnyRunning && !isAnyPending && !isAnyUnknown && isCompleted, nil + for _, progress := range t.Owner.Status.SuccessfulLiveMigrations { + currentProgress[fmt.Sprintf("%s/%s", progress.VMNamespace, progress.VMName)] = progress + } + for _, progress := range t.Owner.Status.PendingLiveMigrations { + currentProgress[fmt.Sprintf("%s/%s", progress.VMNamespace, progress.VMName)] = progress + } + return currentProgress } func (t *Task) hasAllRsyncClientPodsTimedOut() (bool, error) { @@ -1489,29 +1619,17 @@ func (t *Task) isAllRsyncClientPodsNoRouteToHost() (bool, error) { // Delete rsync resources func (t *Task) deleteRsyncResources() error { - // Get client for source + destination - srcClient, err := t.getSourceClient() - if err != nil { - return err - } - destClient, err := t.getDestinationClient() - if err != nil { - return err - } - t.Log.Info("Checking for stale Rsync resources on source MigCluster", "migCluster", path.Join(t.Owner.Spec.SrcMigClusterRef.Namespace, t.Owner.Spec.SrcMigClusterRef.Name)) t.Log.Info("Checking for stale Rsync resources on destination MigCluster", "migCluster", path.Join(t.Owner.Spec.DestMigClusterRef.Namespace, t.Owner.Spec.DestMigClusterRef.Name)) - err = t.findAndDeleteResources(srcClient, destClient, t.getPVCNamespaceMap()) - if err != nil { + if err := t.findAndDeleteResources(t.sourceClient, t.destinationClient, t.getPVCNamespaceMap()); err != nil { return err } - err = t.deleteRsyncPassword() - if err != nil { + if err := t.deleteRsyncPassword(); err != nil { return err } @@ -1521,68 +1639,49 @@ func (t *Task) deleteRsyncResources() error { t.Log.Info("Checking for stale DVMP resources on host MigCluster", "migCluster", "host") - err = t.deleteProgressReportingCRs(t.Client) - if err != nil { + if err := t.deleteProgressReportingCRs(t.Client); err != nil { return err } return nil } -func (t *Task) waitForRsyncResourcesDeleted() (error, bool) { - srcClient, err := t.getSourceClient() - if err != nil { - return err, false - } - destClient, err := t.getDestinationClient() - if err != nil { - return err, false - } +func (t *Task) waitForRsyncResourcesDeleted() (bool, error) { t.Log.Info("Checking if Rsync resource deletion has completed on source and destination MigClusters") - err, deleted := t.areRsyncResourcesDeleted(srcClient, destClient, t.getPVCNamespaceMap()) - if err != nil { - return err, false - } - if !deleted { - return nil, false - } - return nil, true -} - -func (t *Task) areRsyncResourcesDeleted(srcClient, destClient compat.Client, pvcMap map[string][]pvcMapElement) (error, bool) { + pvcMap := t.getPVCNamespaceMap() selector := labels.SelectorFromSet(map[string]string{ "app": DirectVolumeMigrationRsyncTransfer, }) - for bothNs, _ := range pvcMap { + for bothNs := range pvcMap { srcNs := getSourceNs(bothNs) destNs := getDestNs(bothNs) t.Log.Info("Searching source namespace for leftover Rsync Pods, ConfigMaps, "+ "Services, Secrets, Routes with label.", "searchNamespace", srcNs, "labelSelector", selector) - err, areDeleted := t.areRsyncNsResourcesDeleted(srcClient, srcNs, selector) + err, areDeleted := areRsyncNsResourcesDeleted(t.sourceClient, srcNs, selector, t.Log) if err != nil { - return err, false + return false, err } if !areDeleted { - return nil, false + return false, nil } t.Log.Info("Searching destination namespace for leftover Rsync Pods, ConfigMaps, "+ "Services, Secrets, Routes with label.", "searchNamespace", destNs, "labelSelector", selector) - err, areDeleted = t.areRsyncNsResourcesDeleted(destClient, destNs, selector) + err, areDeleted = areRsyncNsResourcesDeleted(t.destinationClient, destNs, selector, t.Log) if err != nil { - return err, false + return false, err } if !areDeleted { - return nil, false + return false, nil } } - return nil, true + return true, nil } -func (t *Task) areRsyncNsResourcesDeleted(client compat.Client, ns string, selector labels.Selector) (error, bool) { +func areRsyncNsResourcesDeleted(client compat.Client, ns string, selector labels.Selector, log logr.Logger) (error, bool) { podList := corev1.PodList{} cmList := corev1.ConfigMapList{} svcList := corev1.ServiceList{} @@ -1601,7 +1700,7 @@ func (t *Task) areRsyncNsResourcesDeleted(client compat.Client, ns string, selec return err, false } if len(podList.Items) > 0 { - t.Log.Info("Found stale Rsync Pod.", + log.Info("Found stale Rsync Pod.", "pod", path.Join(podList.Items[0].Namespace, podList.Items[0].Name), "podPhase", podList.Items[0].Status.Phase) return nil, false @@ -1618,7 +1717,7 @@ func (t *Task) areRsyncNsResourcesDeleted(client compat.Client, ns string, selec return err, false } if len(secretList.Items) > 0 { - t.Log.Info("Found stale Rsync Secret.", + log.Info("Found stale Rsync Secret.", "secret", path.Join(secretList.Items[0].Namespace, secretList.Items[0].Name)) return nil, false } @@ -1634,7 +1733,7 @@ func (t *Task) areRsyncNsResourcesDeleted(client compat.Client, ns string, selec return err, false } if len(cmList.Items) > 0 { - t.Log.Info("Found stale Rsync ConfigMap.", + log.Info("Found stale Rsync ConfigMap.", "configMap", path.Join(cmList.Items[0].Namespace, cmList.Items[0].Name)) return nil, false } @@ -1650,7 +1749,7 @@ func (t *Task) areRsyncNsResourcesDeleted(client compat.Client, ns string, selec return err, false } if len(svcList.Items) > 0 { - t.Log.Info("Found stale Rsync Service.", + log.Info("Found stale Rsync Service.", "service", path.Join(svcList.Items[0].Namespace, svcList.Items[0].Name)) return nil, false } @@ -1667,7 +1766,7 @@ func (t *Task) areRsyncNsResourcesDeleted(client compat.Client, ns string, selec return err, false } if len(routeList.Items) > 0 { - t.Log.Info("Found stale Rsync Route.", + log.Info("Found stale Rsync Route.", "route", path.Join(routeList.Items[0].Namespace, routeList.Items[0].Name)) return nil, false } @@ -1685,11 +1784,11 @@ func (t *Task) findAndDeleteResources(srcClient, destClient compat.Client, pvcMa for bothNs := range pvcMap { srcNs := getSourceNs(bothNs) destNs := getDestNs(bothNs) - err := t.findAndDeleteNsResources(srcClient, srcNs, selector) + err := findAndDeleteNsResources(srcClient, srcNs, selector, t.Log) if err != nil { return err } - err = t.findAndDeleteNsResources(destClient, destNs, selector) + err = findAndDeleteNsResources(destClient, destNs, selector, t.Log) if err != nil { return err } @@ -1697,7 +1796,7 @@ func (t *Task) findAndDeleteResources(srcClient, destClient compat.Client, pvcMa return nil } -func (t *Task) findAndDeleteNsResources(client compat.Client, ns string, selector labels.Selector) error { +func findAndDeleteNsResources(client compat.Client, ns string, selector labels.Selector, log logr.Logger) error { podList := corev1.PodList{} cmList := corev1.ConfigMapList{} svcList := corev1.ServiceList{} @@ -1765,7 +1864,7 @@ func (t *Task) findAndDeleteNsResources(client compat.Client, ns string, selecto // Delete pods for _, pod := range podList.Items { - t.Log.Info("Deleting stale DVM Pod", + log.Info("Deleting stale DVM Pod", "pod", path.Join(pod.Namespace, pod.Name)) err = client.Delete(context.TODO(), &pod, k8sclient.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil && !k8serror.IsNotFound(err) { @@ -1775,7 +1874,7 @@ func (t *Task) findAndDeleteNsResources(client compat.Client, ns string, selecto // Delete secrets for _, secret := range secretList.Items { - t.Log.Info("Deleting stale DVM Secret", + log.Info("Deleting stale DVM Secret", "secret", path.Join(secret.Namespace, secret.Name)) err = client.Delete(context.TODO(), &secret, k8sclient.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil && !k8serror.IsNotFound(err) { @@ -1785,7 +1884,7 @@ func (t *Task) findAndDeleteNsResources(client compat.Client, ns string, selecto // Delete routes for _, route := range routeList.Items { - t.Log.Info("Deleting stale DVM Route", + log.Info("Deleting stale DVM Route", "route", path.Join(route.Namespace, route.Name)) err = client.Delete(context.TODO(), &route, k8sclient.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil && !k8serror.IsNotFound(err) { @@ -1795,7 +1894,7 @@ func (t *Task) findAndDeleteNsResources(client compat.Client, ns string, selecto // Delete svcs for _, svc := range svcList.Items { - t.Log.Info("Deleting stale DVM Service", + log.Info("Deleting stale DVM Service", "service", path.Join(svc.Namespace, svc.Name)) err = client.Delete(context.TODO(), &svc, k8sclient.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil && !k8serror.IsNotFound(err) { @@ -1805,7 +1904,7 @@ func (t *Task) findAndDeleteNsResources(client compat.Client, ns string, selecto // Delete configmaps for _, cm := range cmList.Items { - t.Log.Info("Deleting stale DVM ConfigMap", + log.Info("Deleting stale DVM ConfigMap", "configMap", path.Join(cm.Namespace, cm.Name)) err = client.Delete(context.TODO(), &cm, k8sclient.PropagationPolicy(metav1.DeletePropagationBackground)) if err != nil && !k8serror.IsNotFound(err) { @@ -1855,7 +1954,7 @@ func isPrivilegedLabelPresent(client compat.Client, namespace string) (bool, err return false, nil } -func GetRsyncPodBackOffLimit(dvm migapi.DirectVolumeMigration) int { +func GetRsyncPodBackOffLimit(dvm *migapi.DirectVolumeMigration) int { overriddenBackOffLimit := settings.Settings.DvmOpts.RsyncOpts.BackOffLimit // when both the spec and the overridden backoff limits are not set, use default if dvm.Spec.BackOffLimit == 0 && overriddenBackOffLimit == 0 { @@ -1873,41 +1972,160 @@ func GetRsyncPodBackOffLimit(dvm migapi.DirectVolumeMigration) int { // returns whether or not all operations are completed, whether any of the operation is failed, and a list of failure reasons func (t *Task) runRsyncOperations() (bool, bool, []string, error) { var failureReasons []string - destClient, err := t.getDestinationClient() - if err != nil { - return false, false, failureReasons, liberr.Wrap(err) - } - srcClient, err := t.getSourceClient() - if err != nil { - return false, false, failureReasons, liberr.Wrap(err) - } pvcMap, err := t.getNamespacedPVCPairs() if err != nil { - return false, false, failureReasons, liberr.Wrap(err) + return false, false, failureReasons, err } - err = t.buildSourceLimitRangeMap(pvcMap, srcClient) + err = t.buildSourceLimitRangeMap(pvcMap, t.sourceClient) if err != nil { - return false, false, failureReasons, liberr.Wrap(err) + return false, false, failureReasons, err } - status, err := t.createRsyncTransferClients(srcClient, destClient, pvcMap) - if err != nil { - return false, false, failureReasons, liberr.Wrap(err) + var ( + status *migrationOperationStatusList + progressCompleted, + rsyncOperationsCompleted, + anyRsyncFailed bool + ) + + if !t.Owner.IsRollback() { + t.Log.V(3).Info("Creating Rsync Transfer Clients") + status, err = t.createRsyncTransferClients(t.sourceClient, t.destinationClient, pvcMap) + if err != nil { + return false, false, failureReasons, err + } + t.podPendingSinceTimeLimit() } // report progress of pods - progressCompleted, err := t.hasAllProgressReportingCompleted() - if err != nil { - return false, false, failureReasons, liberr.Wrap(err) + progressCompleted, err = t.hasAllProgressReportingCompleted() + if err != nil { + return false, false, failureReasons, err + } + migrationOperationsCompleted := true + anyMigrationFailed := false + if t.Owner.IsCutover() || t.Owner.IsRollback() { + liveMigrationPVCMap := pvcMap + if t.Owner.IsRollback() { + // Swap source and destination PVCs for rollback + liveMigrationPVCMap = swapSourceDestination(pvcMap) + } + var migrationFailureReasons []string + if t.Owner.IsLiveMigrate() { + t.Log.V(3).Info("Starting live migrations") + // Doing a cutover or rollback, start any live migrations if needed. + failureReasons, err = t.startLiveMigrations(liveMigrationPVCMap) + if err != nil { + return false, len(failureReasons) > 0, failureReasons, err + } + migrationOperationsCompleted, anyMigrationFailed, migrationFailureReasons, err = t.processMigrationOperationStatus(pvcMap, t.sourceClient) + if err != nil { + return false, len(migrationFailureReasons) > 0, migrationFailureReasons, err + } + failureReasons = append(failureReasons, migrationFailureReasons...) + } } - operationsCompleted, anyFailed, failureReasons, err := t.processRsyncOperationStatus(status, []error{}) - if err != nil { - return false, false, failureReasons, liberr.Wrap(err) + + if !t.Owner.IsRollback() { + var rsyncFailureReasons []string + rsyncOperationsCompleted, anyRsyncFailed, rsyncFailureReasons, err = t.processRsyncOperationStatus(status, []error{}) + if err != nil { + return false, len(failureReasons) > 0, failureReasons, err + } + failureReasons = append(failureReasons, rsyncFailureReasons...) + } else { + rsyncOperationsCompleted = true + } + t.Log.V(3).Info("Migration Operations Completed", "MigrationOperationsCompleted", migrationOperationsCompleted, "RsyncOperationsCompleted", rsyncOperationsCompleted, "ProgressCompleted", progressCompleted) + return migrationOperationsCompleted && rsyncOperationsCompleted && progressCompleted, anyRsyncFailed || anyMigrationFailed, failureReasons, nil +} + +func swapSourceDestination(pvcMap map[string][]transfer.PVCPair) map[string][]transfer.PVCPair { + swappedMap := make(map[string][]transfer.PVCPair) + for bothNs, volumes := range pvcMap { + swappedVolumes := make([]transfer.PVCPair, 0) + for _, volume := range volumes { + swappedVolumes = append(swappedVolumes, transfer.NewPVCPair(volume.Destination().Claim(), volume.Source().Claim())) + } + ns := getSourceNs(bothNs) + destNs := getDestNs(bothNs) + swappedMap[fmt.Sprintf("%s:%s", destNs, ns)] = swappedVolumes } - return operationsCompleted && progressCompleted, anyFailed, failureReasons, nil + return swappedMap +} + +func (t *Task) podPendingSinceTimeLimit() { + if len(t.Owner.Status.PendingSinceTimeLimitPods) > 0 { + pendingPods := make([]string, 0) + for _, pod := range t.Owner.Status.PendingSinceTimeLimitPods { + pendingPods = append(pendingPods, fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)) + } + + pendingMessage := fmt.Sprintf("Rsync Client Pods [%s] are stuck in Pending state for more than 10 mins", strings.Join(pendingPods[:], ", ")) + t.Owner.Status.SetCondition(migapi.Condition{ + Type: RsyncClientPodsPending, + Status: migapi.True, + Reason: "PodStuckInContainerCreating", + Category: migapi.Warn, + Message: pendingMessage, + }) + } +} + +// Count the number of VMs, and the number of migrations in error/completed. If they match total isComplete needs to be true +// returns: +// isComplete: whether all migrations are completed, false if no pvc pairs are found +// anyFailed: whether any of the migrations failed +// failureReasons: list of failure reasons +// error: error if any +func (t *Task) processMigrationOperationStatus(nsMap map[string][]transfer.PVCPair, sourceClient k8sclient.Client) (bool, bool, []string, error) { + isComplete, anyFailed, failureReasons := false, false, make([]string, 0) + vmVolumeMap := make(map[string]vmVolumes) + + for k, v := range nsMap { + namespace, err := getNamespace(k) + if err != nil { + failureReasons = append(failureReasons, err.Error()) + return isComplete, anyFailed, failureReasons, err + } + volumeVmMap, err := getRunningVmVolumeMap(sourceClient, namespace) + if err != nil { + failureReasons = append(failureReasons, err.Error()) + return isComplete, anyFailed, failureReasons, err + } + for _, pvcPair := range v { + if vmName, found := volumeVmMap[pvcPair.Source().Claim().Name]; found { + vmVolumeMap[vmName] = vmVolumes{ + sourceVolumes: append(vmVolumeMap[vmName].sourceVolumes, pvcPair.Source().Claim().Name), + targetVolumes: append(vmVolumeMap[vmName].targetVolumes, pvcPair.Destination().Claim().Name), + } + } + } + isComplete = true + completeCount := 0 + failedCount := 0 + for vmName := range vmVolumeMap { + message, err := virtualMachineMigrationStatus(sourceClient, vmName, namespace, t.Log) + if err != nil { + failureReasons = append(failureReasons, message) + return isComplete, anyFailed, failureReasons, err + } + if message == "" { + // Completed + completeCount++ + } else { + // Failed + failedCount++ + anyFailed = true + failureReasons = append(failureReasons, message) + } + } + isComplete = true + } + return isComplete, anyFailed, failureReasons, nil } // processRsyncOperationStatus processes status of Rsync operations by reading the status list // returns whether all operations are completed and whether any of the operation is failed -func (t *Task) processRsyncOperationStatus(status *rsyncClientOperationStatusList, garbageCollectionErrors []error) (bool, bool, []string, error) { +func (t *Task) processRsyncOperationStatus(status *migrationOperationStatusList, garbageCollectionErrors []error) (bool, bool, []string, error) { isComplete, anyFailed, failureReasons := false, false, make([]string, 0) if status.AllCompleted() { isComplete = true @@ -1916,7 +2134,7 @@ func (t *Task) processRsyncOperationStatus(status *rsyncClientOperationStatusLis if status.Failed() > 0 { anyFailed = true // attempt to categorize failures in any of the special failure categories we defined - failureReasons, err := t.reportAdvancedErrorHeuristics() + failureReasons, err := t.reportAdvancedRsyncErrorHeuristics() if err != nil { return isComplete, anyFailed, failureReasons, liberr.Wrap(err) } @@ -1964,7 +2182,7 @@ func (t *Task) processRsyncOperationStatus(status *rsyncClientOperationStatusLis // for all errored pods, attempts to determine whether the errors fall into any // of the special categories we can identify and reports them as conditions // returns reasons and error for reconcile decisions -func (t *Task) reportAdvancedErrorHeuristics() ([]string, error) { +func (t *Task) reportAdvancedRsyncErrorHeuristics() ([]string, error) { reasons := make([]string, 0) // check if the pods are failing due to a network misconfiguration causing Stunnel to timeout isStunnelTimeout, err := t.hasAllRsyncClientPodsTimedOut() @@ -2007,8 +2225,8 @@ func (t *Task) reportAdvancedErrorHeuristics() ([]string, error) { return reasons, nil } -// rsyncClientOperationStatus defines status of one Rsync operation -type rsyncClientOperationStatus struct { +// migrationOperationStatus defines status of one Rsync operation +type migrationOperationStatus struct { operation *migapi.RsyncOperation // When set,.means that all attempts have been exhausted resulting in a failure failed bool @@ -2024,33 +2242,33 @@ type rsyncClientOperationStatus struct { // HasErrors Checks whether there were errors in processing this operation // presence of errors indicates that the status information may not be accurate, demands a retry -func (e *rsyncClientOperationStatus) HasErrors() bool { +func (e *migrationOperationStatus) HasErrors() bool { return len(e.errors) > 0 } -func (e *rsyncClientOperationStatus) AddError(err error) { +func (e *migrationOperationStatus) AddError(err error) { if e.errors == nil { e.errors = make([]error, 0) } e.errors = append(e.errors, err) } -// rsyncClientOperationStatusList managed list of all ongoing Rsync operations -type rsyncClientOperationStatusList struct { +// migrationOperationStatusList managed list of all ongoing Rsync operations +type migrationOperationStatusList struct { // ops list of operations - ops []rsyncClientOperationStatus + ops []migrationOperationStatus } -func (r *rsyncClientOperationStatusList) Add(s rsyncClientOperationStatus) { +func (r *migrationOperationStatusList) Add(s migrationOperationStatus) { if r.ops == nil { - r.ops = make([]rsyncClientOperationStatus, 0) + r.ops = make([]migrationOperationStatus, 0) } r.ops = append(r.ops, s) } // AllCompleted checks whether all of the Rsync attempts are in a terminal state // If true, reconcile can move to next phase -func (r *rsyncClientOperationStatusList) AllCompleted() bool { +func (r *migrationOperationStatusList) AllCompleted() bool { for _, attempt := range r.ops { if attempt.pending || attempt.running || attempt.HasErrors() { return false @@ -2060,7 +2278,7 @@ func (r *rsyncClientOperationStatusList) AllCompleted() bool { } // AnyErrored checks whether any of the operation is resulting in an error -func (r *rsyncClientOperationStatusList) AnyErrored() bool { +func (r *migrationOperationStatusList) AnyErrored() bool { for _, attempt := range r.ops { if attempt.HasErrors() { return true @@ -2070,7 +2288,7 @@ func (r *rsyncClientOperationStatusList) AnyErrored() bool { } // Failed returns number of failed operations -func (r *rsyncClientOperationStatusList) Failed() int { +func (r *migrationOperationStatusList) Failed() int { i := 0 for _, attempt := range r.ops { if attempt.failed { @@ -2081,7 +2299,7 @@ func (r *rsyncClientOperationStatusList) Failed() int { } // Succeeded returns number of failed operations -func (r *rsyncClientOperationStatusList) Succeeded() int { +func (r *migrationOperationStatusList) Succeeded() int { i := 0 for _, attempt := range r.ops { if attempt.succeeded { @@ -2092,7 +2310,7 @@ func (r *rsyncClientOperationStatusList) Succeeded() int { } // Pending returns number of pending operations -func (r *rsyncClientOperationStatusList) Pending() int { +func (r *migrationOperationStatusList) Pending() int { i := 0 for _, attempt := range r.ops { if attempt.pending { @@ -2103,7 +2321,7 @@ func (r *rsyncClientOperationStatusList) Pending() int { } // Running returns number of running operations -func (r *rsyncClientOperationStatusList) Running() int { +func (r *migrationOperationStatusList) Running() int { i := 0 for _, attempt := range r.ops { if attempt.running { @@ -2161,7 +2379,7 @@ func (t *Task) getLatestPodForOperation(client compat.Client, operation migapi.R } // updateOperationStatus given a Rsync Pod and operation status, updates operation status with pod status -func updateOperationStatus(status *rsyncClientOperationStatus, pod *corev1.Pod) { +func updateOperationStatus(status *migrationOperationStatus, pod *corev1.Pod) { switch pod.Status.Phase { case corev1.PodFailed: status.failed = true diff --git a/pkg/controller/directvolumemigration/rsync_test.go b/pkg/controller/directvolumemigration/rsync_test.go index e7face64e..a1085cb58 100644 --- a/pkg/controller/directvolumemigration/rsync_test.go +++ b/pkg/controller/directvolumemigration/rsync_test.go @@ -18,16 +18,24 @@ import ( fakecompat "github.com/konveyor/mig-controller/pkg/compat/fake" configv1 "github.com/openshift/api/config/v1" routev1 "github.com/openshift/api/route/v1" + prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/rand" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + testVolume = "test-volume" + testDVM = "test-dvm" +) + func getTestRsyncPodForPVC(podName string, pvcName string, ns string, attemptNo string, timestamp time.Time) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -668,7 +676,7 @@ func TestTask_processRsyncOperationStatus(t *testing.T) { Owner *migapi.DirectVolumeMigration } type args struct { - status rsyncClientOperationStatusList + status migrationOperationStatusList garbageCollectionErrors []error } tests := []struct { @@ -688,7 +696,7 @@ func TestTask_processRsyncOperationStatus(t *testing.T) { Client: getFakeCompatClient(), Owner: &migapi.DirectVolumeMigration{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-dvm", Namespace: "openshift-migration", + Name: testDVM, Namespace: "openshift-migration", }, Status: migapi.DirectVolumeMigrationStatus{ Conditions: migapi.Conditions{ @@ -700,8 +708,8 @@ func TestTask_processRsyncOperationStatus(t *testing.T) { }, }, args: args{ - status: rsyncClientOperationStatusList{ - ops: []rsyncClientOperationStatus{ + status: migrationOperationStatusList{ + ops: []migrationOperationStatus{ {errors: []error{fmt.Errorf("failed creating pod")}}, }, }, @@ -718,7 +726,7 @@ func TestTask_processRsyncOperationStatus(t *testing.T) { Client: getFakeCompatClient(), Owner: &migapi.DirectVolumeMigration{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-dvm", Namespace: "openshift-migration", + Name: testDVM, Namespace: "openshift-migration", }, Status: migapi.DirectVolumeMigrationStatus{ Conditions: migapi.Conditions{ @@ -730,8 +738,8 @@ func TestTask_processRsyncOperationStatus(t *testing.T) { }, }, args: args{ - status: rsyncClientOperationStatusList{ - ops: []rsyncClientOperationStatus{ + status: migrationOperationStatusList{ + ops: []migrationOperationStatus{ {errors: []error{fmt.Errorf("failed creating pod")}}, }, }, @@ -770,6 +778,926 @@ func TestTask_processRsyncOperationStatus(t *testing.T) { } } +func TestTask_processMigrationOperationStatus(t *testing.T) { + tests := []struct { + name string + nsMap map[string][]transfer.PVCPair + expectCompleted bool + expectFailed bool + expectedFailureReasons []string + client compat.Client + wantErr bool + }{ + { + name: "empty namespace pair", + nsMap: map[string][]transfer.PVCPair{}, + expectedFailureReasons: []string{}, + client: getFakeCompatClient(), + }, + { + name: "invalid namespace pair", + nsMap: map[string][]transfer.PVCPair{ + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + client: getFakeCompatClient(), + expectedFailureReasons: []string{"invalid namespace pair: test-namespace"}, + wantErr: true, + }, + { + name: "no running VMs", + nsMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + client: getFakeCompatClient(), + expectedFailureReasons: []string{}, + expectCompleted: true, + }, + { + name: "running VMs, no matching volumes", + nsMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + client: getFakeCompatClient(createVirtualMachine("vm", testNamespace), createVirtlauncherPod("vm", testNamespace, []string{"dv"})), + expectedFailureReasons: []string{}, + expectCompleted: true, + }, + { + name: "running VMs, no matching volumes", + nsMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + client: getFakeCompatClient( + createVirtualMachine("vm", testNamespace), + createVirtlauncherPod("vm", testNamespace, []string{"pvc1"}), + createVirtualMachineInstance("vm", testNamespace, virtv1.Running), + ), + expectedFailureReasons: []string{}, + expectCompleted: true, + }, + { + name: "failed migration, no matching volumes", + nsMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + client: getFakeCompatClient( + createVirtualMachine("vm", testNamespace), + createVirtlauncherPod("vm", testNamespace, []string{"pvc1"}), + createVirtualMachineInstance("vm", testNamespace, virtv1.Failed), + createCanceledVirtualMachineMigration("vmim", testNamespace, "vm", virtv1.MigrationAbortSucceeded), + ), + expectCompleted: true, + expectFailed: true, + expectedFailureReasons: []string{"Migration canceled"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{} + task.sourceClient = tt.client + isComplete, anyFailed, failureReasons, err := task.processMigrationOperationStatus(tt.nsMap, task.sourceClient) + if err != nil && !tt.wantErr { + t.Errorf("Unexpected() error = %v", err) + t.FailNow() + } else if err == nil && tt.wantErr { + t.Errorf("Expected error, got nil") + t.FailNow() + } + if isComplete != tt.expectCompleted { + t.Errorf("Expected completed to be %t, got %t", tt.expectCompleted, isComplete) + t.FailNow() + } + if anyFailed != tt.expectFailed { + t.Errorf("Expected failed to be %t, got %t", tt.expectFailed, anyFailed) + t.FailNow() + } + if !reflect.DeepEqual(failureReasons, tt.expectedFailureReasons) { + t.Errorf("Unexpected() got = %v, want %v", failureReasons, tt.expectedFailureReasons) + t.FailNow() + } + }) + } +} + +func TestTask_podPendingSinceTimeLimit(t *testing.T) { + tests := []struct { + name string + dvm *migapi.DirectVolumeMigration + expectedCondition *migapi.Condition + }{ + { + name: "No pending pods", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + PendingSinceTimeLimitPods: []*migapi.PodProgress{}, + }, + }, + expectedCondition: nil, + }, + { + name: "Pending pods", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + PendingSinceTimeLimitPods: []*migapi.PodProgress{ + { + ObjectReference: &corev1.ObjectReference{ + Name: "test-pod", + Namespace: testNamespace, + }, + }, + }, + }, + }, + expectedCondition: &migapi.Condition{ + Type: RsyncClientPodsPending, + Status: migapi.True, + Reason: "PodStuckInContainerCreating", + Category: migapi.Warn, + Message: "", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{ + Owner: tt.dvm, + } + task.podPendingSinceTimeLimit() + if tt.expectedCondition != nil { + if !tt.dvm.Status.HasCondition(tt.expectedCondition.Type) { + t.Errorf("Condition %s not found", tt.expectedCondition.Type) + } + } else { + if tt.dvm.Status.HasCondition(RsyncClientPodsPending) { + t.Errorf("Condition %s found", RsyncClientPodsPending) + } + } + }) + } +} + +func TestTask_swapSourceDestination(t *testing.T) { + tests := []struct { + name string + pvcMap map[string][]transfer.PVCPair + expectedMap map[string][]transfer.PVCPair + }{ + { + name: "empty map", + pvcMap: map[string][]transfer.PVCPair{}, + expectedMap: map[string][]transfer.PVCPair{}, + }, + { + name: "one namespace, one pair", + pvcMap: map[string][]transfer.PVCPair{ + "foo:bar": {transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace))}, + }, + expectedMap: map[string][]transfer.PVCPair{ + "bar:foo": {transfer.NewPVCPair(createPvc("pvc2", testNamespace), createPvc("pvc1", testNamespace))}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := swapSourceDestination(tt.pvcMap) + if !reflect.DeepEqual(out, tt.expectedMap) { + t.Errorf("swapSourceDestination() = %v, want %v", out, tt.expectedMap) + } + }) + } +} + +func TestTask_runRsyncOperations(t *testing.T) { + tests := []struct { + name string + client compat.Client + dvm *migapi.DirectVolumeMigration + expectComplete bool + expectFailed bool + expectedReasons []string + }{ + { + name: "no PVCs, stage migration type", + client: getFakeCompatClient(), + dvm: &migapi.DirectVolumeMigration{ + Spec: migapi.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []migapi.PVCToMigrate{}, + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeStage), + }, + }, + expectComplete: true, + }, + { + name: "no PVCs, cutover migration type, no live migration", + client: getFakeCompatClient(), + dvm: &migapi.DirectVolumeMigration{ + Spec: migapi.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []migapi.PVCToMigrate{}, + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeFinal), + }, + }, + expectComplete: true, + }, + { + name: "no PVCs, cutover migration type, live migration", + client: getFakeCompatClient(), + dvm: &migapi.DirectVolumeMigration{ + Spec: migapi.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []migapi.PVCToMigrate{}, + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeFinal), + LiveMigrate: ptr.To[bool](true), + }, + }, + }, + { + name: "no PVCs, rollback migration type, live migration", + client: getFakeCompatClient(), + dvm: &migapi.DirectVolumeMigration{ + Spec: migapi.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []migapi.PVCToMigrate{}, + MigrationType: ptr.To[migapi.DirectVolumeMigrationType](migapi.MigrationTypeRollback), + LiveMigrate: ptr.To[bool](true), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{} + task.sourceClient = tt.client + task.destinationClient = tt.client + task.Client = tt.client + task.Owner = tt.dvm + completed, failed, reasons, err := task.runRsyncOperations() + if err != nil { + t.Errorf("runRsyncOperations() error = %v", err) + t.FailNow() + } + if completed != tt.expectComplete { + t.Errorf("Expected completed to be %t, got %t", tt.expectComplete, completed) + t.FailNow() + } + if failed != tt.expectFailed { + t.Errorf("Expected failed to be %t, got %t", tt.expectFailed, failed) + t.FailNow() + } + if len(reasons) != len(tt.expectedReasons) { + t.Errorf("%v is not the same length as %v", reasons, tt.expectedReasons) + t.FailNow() + } + for i, s := range reasons { + if s != tt.expectedReasons[i] { + t.Errorf("%s is not equal to %s", s, tt.expectedReasons[i]) + t.FailNow() + } + } + }) + } +} + +func TestTask_getCurrentLiveMigrationProgress(t *testing.T) { + tests := []struct { + name string + dvm *migapi.DirectVolumeMigration + expectedProgress map[string]*migapi.LiveMigrationProgress + }{ + { + name: "no progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{}, + }, + expectedProgress: map[string]*migapi.LiveMigrationProgress{}, + }, + { + name: "live migration progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + RunningLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + { + VMName: "test-vm-2", + VMNamespace: testNamespace, + }, + }, + }, + }, + expectedProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/test-vm", testNamespace): { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + fmt.Sprintf("%s/test-vm-2", testNamespace): { + VMName: "test-vm-2", + VMNamespace: testNamespace, + }, + }, + }, + { + name: "failed live migration progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + FailedLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + }, + }, + }, + expectedProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/test-vm", testNamespace): { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + }, + }, + { + name: "successful live migration progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + SuccessfulLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + }, + }, + }, + expectedProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/test-vm", testNamespace): { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + }, + }, + { + name: "pending live migration progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + PendingLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + }, + }, + }, + expectedProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/test-vm", testNamespace): { + VMName: "test-vm", + VMNamespace: testNamespace, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{} + task.Owner = tt.dvm + progress := task.getCurrentLiveMigrationProgress() + if len(progress) != len(tt.expectedProgress) { + t.Errorf("getCurrentLiveMigrationProgress() = %v, want %v", progress, tt.expectedProgress) + t.FailNow() + } + for k, v := range progress { + if !reflect.DeepEqual(v, tt.expectedProgress[k]) { + t.Errorf("getCurrentLiveMigrationProgress() = %v, want %v", progress, tt.expectedProgress) + t.FailNow() + } + } + }) + } +} + +func TestTask_updateRsyncProgressStatus(t *testing.T) { + tests := []struct { + name string + dvm *migapi.DirectVolumeMigration + volumeName string + client compat.Client + expectedSuccessfulPods []*migapi.PodProgress + expectedFailedPods []*migapi.PodProgress + expectedRunningPods []*migapi.PodProgress + expectedPendingPods []*migapi.PodProgress + expectedUnknownPods []*migapi.PodProgress + expectedPendingSince []*migapi.PodProgress + }{ + { + name: "no progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(), + }, + { + name: "running pod progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(createDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, corev1.PodRunning)), + expectedRunningPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + { + name: "failed pod progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{ + RsyncOperations: []*migapi.RsyncOperation{ + { + Failed: true, + PVCReference: &corev1.ObjectReference{ + Namespace: testNamespace, + Name: testVolume, + }, + }, + }, + }, + }, + volumeName: testVolume, + client: getFakeCompatClient(createDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, corev1.PodFailed)), + expectedFailedPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + { + name: "failed pod, operation hasn't failed progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(createDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, corev1.PodFailed)), + expectedRunningPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + { + name: "pending pod, progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(createDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, corev1.PodPending)), + expectedPendingPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + { + name: "pending pod older than 10 minutes, progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(createOldDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, corev1.PodPending)), + expectedPendingPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + expectedPendingSince: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + { + name: "unknown pod, progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(createDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, "")), + expectedUnknownPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + { + name: "successful pod, progress exists", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: testNamespace}, + Status: migapi.DirectVolumeMigrationStatus{}, + }, + volumeName: testVolume, + client: getFakeCompatClient(createDirectVolumeMigrationProgress(testDVM, testVolume, testNamespace, corev1.PodSucceeded)), + expectedSuccessfulPods: []*migapi.PodProgress{ + createExpectedPodProgress(), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{} + task.Client = tt.client + task.Owner = tt.dvm + err := task.updateRsyncProgressStatus(tt.volumeName, testNamespace) + if err != nil { + t.Errorf("Unexpected error: %v", err) + t.FailNow() + } + if len(tt.expectedSuccessfulPods) != len(tt.dvm.Status.SuccessfulPods) { + t.Errorf("Expected %d successful pods, got %d", len(tt.expectedSuccessfulPods), len(tt.dvm.Status.SuccessfulPods)) + t.FailNow() + } + if len(tt.expectedFailedPods) != len(tt.dvm.Status.FailedPods) { + t.Errorf("Expected %d failed pods, got %d", len(tt.expectedFailedPods), len(tt.dvm.Status.FailedPods)) + t.FailNow() + } + if len(tt.expectedRunningPods) != len(tt.dvm.Status.RunningPods) { + t.Errorf("Expected %d running pods, got %d", len(tt.expectedRunningPods), len(tt.dvm.Status.RunningPods)) + t.FailNow() + } + if len(tt.expectedPendingPods) != len(tt.dvm.Status.PendingPods) { + t.Errorf("Expected %d pending pods, got %d", len(tt.expectedPendingPods), len(tt.dvm.Status.PendingPods)) + t.FailNow() + } + if len(tt.expectedUnknownPods) != len(tt.dvm.Status.UnknownPods) { + t.Errorf("Expected %d unknown pods, got %d", len(tt.expectedUnknownPods), len(tt.dvm.Status.UnknownPods)) + t.FailNow() + } + if len(tt.expectedPendingSince) != len(tt.dvm.Status.PendingSinceTimeLimitPods) { + t.Errorf("Expected %d pending since pods, got %d", len(tt.expectedPendingSince), len(tt.dvm.Status.PendingSinceTimeLimitPods)) + t.FailNow() + } + }) + } +} + +func TestTask_updateVolumeLiveMigrationProgressStatus(t *testing.T) { + tests := []struct { + name string + dvm *migapi.DirectVolumeMigration + volumeName string + client compat.Client + virtualMachineMappings VirtualMachineMappings + existingProgress map[string]*migapi.LiveMigrationProgress + expectedSuccessfulLiveMigrations []*migapi.LiveMigrationProgress + expectedFailedLiveMigrations []*migapi.LiveMigrationProgress + expectedRunningLiveMigrations []*migapi.LiveMigrationProgress + expectedPendingLiveMigrations []*migapi.LiveMigrationProgress + promQuery func(context.Context, string, time.Time, ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) + }{ + { + name: "no progress", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(), + existingProgress: map[string]*migapi.LiveMigrationProgress{}, + expectedFailedLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + }, + { + name: "failed vmim with failure reason", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(), + existingProgress: map[string]*migapi.LiveMigrationProgress{}, + expectedFailedLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMIMMap: map[string]*virtv1.VirtualMachineInstanceMigration{ + fmt.Sprintf("%s/%s", testNamespace, testVolume): { + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + FailureReason: "test failure", + }, + Phase: virtv1.MigrationFailed, + }, + }, + }, + }, + }, + { + name: "successful vmim without failure reason", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(), + existingProgress: map[string]*migapi.LiveMigrationProgress{}, + expectedSuccessfulLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMIMMap: map[string]*virtv1.VirtualMachineInstanceMigration{ + fmt.Sprintf("%s/%s", testNamespace, testVolume): { + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{}, + Phase: virtv1.MigrationSucceeded, + }, + }, + }, + }, + }, + { + name: "running vmim without failure reason", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(), + existingProgress: map[string]*migapi.LiveMigrationProgress{}, + expectedRunningLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMIMMap: map[string]*virtv1.VirtualMachineInstanceMigration{ + fmt.Sprintf("%s/%s", testNamespace, testVolume): { + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{}, + Phase: virtv1.MigrationRunning, + }, + }, + }, + }, + promQuery: func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) { + return &model.String{ + Value: "=> 59.3 @", + }, nil, nil + }, + }, + { + name: "pending vmim without failure reason", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(), + existingProgress: map[string]*migapi.LiveMigrationProgress{}, + expectedPendingLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMIMMap: map[string]*virtv1.VirtualMachineInstanceMigration{ + fmt.Sprintf("%s/%s", testNamespace, testVolume): { + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{}, + Phase: virtv1.MigrationPending, + }, + }, + }, + }, + }, + { + name: "no vmim, but VMI exists", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(createVirtualMachineInstance("test-vm", testNamespace, virtv1.Running)), + existingProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/%s", testNamespace, "test-vm"): createExpectedLiveMigrationProgress(nil), + }, + expectedPendingLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMNameMap: map[string]string{ + testVolume: "test-vm", + }, + }, + }, + { + name: "no vmim, but VMI exists, unable to live migrate", + dvm: &migapi.DirectVolumeMigration{}, + volumeName: testVolume, + client: getFakeCompatClient(createVirtualMachineInstanceWithConditions("test-vm", testNamespace, []virtv1.VirtualMachineInstanceCondition{ + { + Type: virtv1.VirtualMachineInstanceVolumesChange, + Status: corev1.ConditionTrue, + }, + { + Type: virtv1.VirtualMachineInstanceIsMigratable, + Status: corev1.ConditionFalse, + Message: "Unable to live migrate because of the test reason", + }, + })), + existingProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/%s", testNamespace, "test-vm"): createExpectedLiveMigrationProgress(&metav1.Duration{ + Duration: time.Second * 10, + }), + }, + expectedFailedLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMNameMap: map[string]string{ + testVolume: "test-vm", + }, + }, + }, + { + name: "no vmim, but VMI exists, unable to live migrate", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + StartTimestamp: &metav1.Time{Time: time.Now().Add(-time.Minute * 11)}, + }, + }, + volumeName: testVolume, + client: getFakeCompatClient(createVirtualMachineInstanceWithConditions("test-vm", testNamespace, []virtv1.VirtualMachineInstanceCondition{ + { + Type: virtv1.VirtualMachineInstanceVolumesChange, + Status: corev1.ConditionTrue, + }, + { + Type: virtv1.VirtualMachineInstanceIsMigratable, + Status: corev1.ConditionFalse, + Message: "Unable to live migrate because of the test reason", + }, + })), + existingProgress: map[string]*migapi.LiveMigrationProgress{ + fmt.Sprintf("%s/%s", testNamespace, "test-vm"): createExpectedLiveMigrationProgress(nil), + }, + expectedFailedLiveMigrations: []*migapi.LiveMigrationProgress{ + createExpectedLiveMigrationProgress(nil), + }, + virtualMachineMappings: VirtualMachineMappings{ + volumeVMNameMap: map[string]string{ + testVolume: "test-vm", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{} + task.sourceClient = tt.client + task.Client = tt.client + task.Owner = tt.dvm + task.PrometheusAPI = prometheusv1.NewAPI(nil) + task.PromQuery = tt.promQuery + task.VirtualMachineMappings = tt.virtualMachineMappings + err := task.updateVolumeLiveMigrationProgressStatus(tt.volumeName, testNamespace, tt.existingProgress) + if err != nil { + t.Errorf("Unexpected error: %v", err) + t.FailNow() + } + if len(tt.expectedSuccessfulLiveMigrations) != len(tt.dvm.Status.SuccessfulLiveMigrations) { + t.Errorf("Expected %d successful live migrations, got %d", len(tt.expectedSuccessfulLiveMigrations), len(tt.dvm.Status.SuccessfulLiveMigrations)) + t.FailNow() + } + if len(tt.expectedFailedLiveMigrations) != len(tt.dvm.Status.FailedLiveMigrations) { + t.Errorf("Expected %d failed live migrations, got %d", len(tt.expectedFailedLiveMigrations), len(tt.dvm.Status.FailedLiveMigrations)) + t.FailNow() + } + if len(tt.expectedRunningLiveMigrations) != len(tt.dvm.Status.RunningLiveMigrations) { + t.Errorf("Expected %d running live migrations, got %d", len(tt.expectedRunningLiveMigrations), len(tt.dvm.Status.RunningPods)) + t.FailNow() + } + if len(tt.expectedPendingLiveMigrations) != len(tt.dvm.Status.PendingLiveMigrations) { + t.Errorf("Expected %d pending live migrations, got %d", len(tt.expectedPendingLiveMigrations), len(tt.dvm.Status.PendingLiveMigrations)) + t.FailNow() + } + }) + } +} + +func TestTask_filterRunningVMs(t *testing.T) { + tests := []struct { + name string + client compat.Client + input []transfer.PVCPair + expected []transfer.PVCPair + }{ + { + name: "no input", + client: getFakeCompatClient(), + input: []transfer.PVCPair{}, + expected: []transfer.PVCPair{}, + }, + { + name: "single PVCPair with running VM", + client: getFakeCompatClient(createVirtualMachineWithVolumes("test-vm", testNamespace, []virtv1.Volume{ + { + Name: "source", + VolumeSource: virtv1.VolumeSource{ + PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "source", + }, + }, + }, + }, + }), + createVirtlauncherPod("test-vm", testNamespace, []string{"source"}), + ), + input: []transfer.PVCPair{ + transfer.NewPVCPair(createPvc("source", testNamespace), createPvc("target", testNamespace)), + }, + expected: []transfer.PVCPair{}, + }, + { + name: "two PVCPairs one running VM, one without", + client: getFakeCompatClient(createVirtualMachineWithVolumes("test-vm", testNamespace, []virtv1.Volume{ + { + Name: "source", + VolumeSource: virtv1.VolumeSource{ + PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "source", + }, + }, + }, + }, + }), + createVirtlauncherPod("test-vm", testNamespace, []string{"source"}), + ), + input: []transfer.PVCPair{ + transfer.NewPVCPair(createPvc("source", testNamespace), createPvc("target", testNamespace)), + transfer.NewPVCPair(createPvc("source-remain", testNamespace), createPvc("target-remain", testNamespace)), + }, + expected: []transfer.PVCPair{ + transfer.NewPVCPair(createPvc("source-remain", testNamespace), createPvc("target-remain", testNamespace)), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{} + task.sourceClient = tt.client + out, err := task.filterRunningVMs(tt.input) + if err != nil { + t.Errorf("Unexpected error: %v", err) + t.FailNow() + } + if len(out) != len(tt.expected) { + t.Errorf("Expected %d, got %d", len(tt.expected), len(out)) + t.FailNow() + } + for i, p := range out { + if p.Source().Claim().Name != tt.expected[i].Source().Claim().Name { + t.Errorf("Expected %s, got %s", tt.expected[i].Source().Claim().Name, p.Source().Claim().Name) + t.FailNow() + } + if p.Destination().Claim().Name != tt.expected[i].Destination().Claim().Name { + t.Errorf("Expected %s, got %s", tt.expected[i].Destination().Claim().Name, p.Destination().Claim().Name) + t.FailNow() + } + } + }) + } +} + +func createExpectedLiveMigrationProgress(elapsedTime *metav1.Duration) *migapi.LiveMigrationProgress { + return &migapi.LiveMigrationProgress{ + VMName: "test-vm", + VMNamespace: testNamespace, + TotalElapsedTime: elapsedTime, + } +} + +func createExpectedPodProgress() *migapi.PodProgress { + return &migapi.PodProgress{ + ObjectReference: &corev1.ObjectReference{ + Namespace: testNamespace, + Name: "test-pod", + }, + PVCReference: &corev1.ObjectReference{ + Namespace: testNamespace, + Name: testVolume, + }, + LastObservedProgressPercent: "23%", + LastObservedTransferRate: "10MiB/s", + TotalElapsedTime: &metav1.Duration{Duration: time.Second * 10}, + } +} + +func createDirectVolumeMigrationProgress(dvmName, volumeName, namespace string, podPhase corev1.PodPhase) *migapi.DirectVolumeMigrationProgress { + return &migapi.DirectVolumeMigrationProgress{ + ObjectMeta: metav1.ObjectMeta{ + Name: getMD5Hash(dvmName + volumeName + namespace), + Namespace: migapi.OpenshiftMigrationNamespace, + }, + Spec: migapi.DirectVolumeMigrationProgressSpec{}, + Status: migapi.DirectVolumeMigrationProgressStatus{ + RsyncPodStatus: migapi.RsyncPodStatus{ + PodPhase: podPhase, + PodName: "test-pod", + LastObservedTransferRate: "10MiB/s", + }, + TotalProgressPercentage: "23", + RsyncElapsedTime: &metav1.Duration{ + Duration: time.Second * 10, + }, + }, + } +} + +func createOldDirectVolumeMigrationProgress(dvmName, volumeName, namespace string, podPhase corev1.PodPhase) *migapi.DirectVolumeMigrationProgress { + dvmp := createDirectVolumeMigrationProgress(dvmName, volumeName, namespace, podPhase) + dvmp.Status.CreationTimestamp = &metav1.Time{Time: time.Now().Add(-time.Minute * 11)} + return dvmp +} + func getPVCPair(name string, namespace string, volumeMode corev1.PersistentVolumeMode) transfer.PVCPair { pvcPair := transfer.NewPVCPair( &corev1.PersistentVolumeClaim{ @@ -916,31 +1844,31 @@ func TestTask_createRsyncTransferClients(t *testing.T) { fields fields wantPods []*corev1.Pod wantErr bool - wantReturn rsyncClientOperationStatusList + wantReturn migrationOperationStatusList wantCRStatus []*migapi.RsyncOperation }{ { name: "when there are 0 existing Rsync Pods in source and no PVCs to migrate, status list should be empty", fields: fields{ - SrcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - DestClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + SrcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + DestClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, wantPods: []*corev1.Pod{}, wantErr: false, - wantReturn: rsyncClientOperationStatusList{}, + wantReturn: migrationOperationStatusList{}, wantCRStatus: []*migapi.RsyncOperation{}, }, { name: "when there are 0 existing Rsync Pods in source and 1 new fs PVC is provided as input, 1 Rsync Pod must be created in source namespace", fields: fields{ - SrcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - DestClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + SrcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + DestClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, PVCPairMap: map[string][]transfer.PVCPair{ @@ -953,16 +1881,16 @@ func TestTask_createRsyncTransferClients(t *testing.T) { getTestRsyncPodForPVC("pod-0", "pvc-0", "test-ns", "1", metav1.Now().Time), }, wantErr: false, - wantReturn: rsyncClientOperationStatusList{}, + wantReturn: migrationOperationStatusList{}, wantCRStatus: []*migapi.RsyncOperation{}, }, { name: "when there are 0 existing Rsync Pods in source and 1 new block PVC is provided as input, 1 Rsync Pod must be created in source namespace", fields: fields{ - SrcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - DestClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + SrcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + DestClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, PVCPairMap: map[string][]transfer.PVCPair{ @@ -975,16 +1903,16 @@ func TestTask_createRsyncTransferClients(t *testing.T) { getTestRsyncPodForPVC("pod-0", "pvc-0", "test-ns", "1", metav1.Now().Time), }, wantErr: false, - wantReturn: rsyncClientOperationStatusList{}, + wantReturn: migrationOperationStatusList{}, wantCRStatus: []*migapi.RsyncOperation{}, }, { name: "when there are 0 existing Rsync Pods in source and 2 new fs PVCs and one block PVC are provided as input, 3 Rsync Pod must be created in source namespace", fields: fields{ - SrcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - DestClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + SrcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + DestClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, PVCPairMap: map[string][]transfer.PVCPair{ @@ -1001,7 +1929,7 @@ func TestTask_createRsyncTransferClients(t *testing.T) { getTestRsyncPodForPVC("pod-0", "pvc-0", "ns-02", "1", metav1.Now().Time), }, wantErr: false, - wantReturn: rsyncClientOperationStatusList{}, + wantReturn: migrationOperationStatusList{}, wantCRStatus: []*migapi.RsyncOperation{}, }, } @@ -1011,6 +1939,13 @@ func TestTask_createRsyncTransferClients(t *testing.T) { Log: log.WithName("test-logger"), Client: tt.fields.DestClient, Owner: tt.fields.Owner, + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + LiveMigrate: ptr.To[bool](false), + }, + }, + }, } got, err := tr.createRsyncTransferClients(tt.fields.SrcClient, tt.fields.DestClient, tt.fields.PVCPairMap) if (err != nil) != tt.wantErr { @@ -1085,7 +2020,7 @@ func Test_getSecurityContext(t *testing.T) { fields: fields{ client: getFakeCompatClientWithVersion(1, 24), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, Migration: &migapi.MigMigration{ @@ -1118,7 +2053,7 @@ func Test_getSecurityContext(t *testing.T) { fields: fields{ client: getFakeCompatClientWithVersion(1, 24), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, Migration: &migapi.MigMigration{ @@ -1149,7 +2084,7 @@ func Test_getSecurityContext(t *testing.T) { fields: fields{ client: getFakeCompatClientWithVersion(1, 24), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, Migration: &migapi.MigMigration{ @@ -1180,7 +2115,7 @@ func Test_getSecurityContext(t *testing.T) { fields: fields{ client: getFakeCompatClientWithVersion(1, 24), Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, Migration: &migapi.MigMigration{ @@ -1238,8 +2173,10 @@ func Test_ensureRsyncEndpoints(t *testing.T) { checkRoute bool }{ { - name: "error getting destination client", - fields: fields{}, + name: "error getting destination client", + fields: fields{ + owner: &migapi.DirectVolumeMigration{}, + }, wantErr: true, }, { @@ -1249,7 +2186,7 @@ func Test_ensureRsyncEndpoints(t *testing.T) { destClient: getFakeCompatClient(createNode("worker1"), createNode("worker2")), endpointType: migapi.NodePort, owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{ BackOffLimit: 2, PersistentVolumeClaims: []migapi.PVCToMigrate{ @@ -1269,7 +2206,16 @@ func Test_ensureRsyncEndpoints(t *testing.T) { destClient: getFakeCompatClient(createMigCluster("test-cluster"), createClusterIngress()), endpointType: migapi.Route, owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{ + Name: testDVM, + Namespace: migapi.OpenshiftMigrationNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "migmigration", + }, + }, + UID: "test-uid", + }, Spec: migapi.DirectVolumeMigrationSpec{ BackOffLimit: 2, PersistentVolumeClaims: []migapi.PVCToMigrate{ @@ -1292,7 +2238,16 @@ func Test_ensureRsyncEndpoints(t *testing.T) { destClient: getFakeCompatClient(createMigCluster("test-cluster")), endpointType: migapi.Route, owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{ + Name: testDVM, + Namespace: migapi.OpenshiftMigrationNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "migmigration", + }, + }, + UID: "test-uid", + }, Spec: migapi.DirectVolumeMigrationSpec{ BackOffLimit: 2, PersistentVolumeClaims: []migapi.PVCToMigrate{ @@ -1315,7 +2270,16 @@ func Test_ensureRsyncEndpoints(t *testing.T) { destClient: getFakeCompatClientWithSubdomain(createMigCluster("test-cluster")), endpointType: migapi.Route, owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{ + Name: testDVM, + Namespace: migapi.OpenshiftMigrationNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "migmigration", + }, + }, + UID: "test-uid", + }, Spec: migapi.DirectVolumeMigrationSpec{ BackOffLimit: 2, PersistentVolumeClaims: []migapi.PVCToMigrate{ @@ -1363,7 +2327,7 @@ func Test_ensureRsyncEndpoints(t *testing.T) { } } -func verifyServiceHealthy(name, namespace, appLabel string, c client.Client, t *testing.T) { +func verifyServiceHealthy(name, namespace, appLabel string, c k8sclient.Client, t *testing.T) { service := &corev1.Service{} if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, service); err != nil { t.Fatalf("ensureRsyncEndpoints() failed to get rsync service in namespace %s", namespace) @@ -1377,7 +2341,7 @@ func verifyServiceHealthy(name, namespace, appLabel string, c client.Client, t * } } -func verifyRouteHealthy(name, namespace, appLabel string, c client.Client, t *testing.T) { +func verifyRouteHealthy(name, namespace, appLabel string, c k8sclient.Client, t *testing.T) { route := &routev1.Route{} if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, route); err != nil { if !k8serrors.IsNotFound(err) { @@ -1417,15 +2381,15 @@ func Test_ensureFilesystemRsyncTransferServer(t *testing.T) { { name: "get single pvc filesystem server pod", fields: fields{ - srcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - destClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + srcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + destClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), PVCPairMap: map[string][]transfer.PVCPair{ "test-ns": { getPVCPair("pvc", "test-ns", corev1.PersistentVolumeFilesystem), }, }, Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, @@ -1434,15 +2398,15 @@ func Test_ensureFilesystemRsyncTransferServer(t *testing.T) { { name: "no filesystem pod available, no server pods", fields: fields{ - srcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - destClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + srcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + destClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), PVCPairMap: map[string][]transfer.PVCPair{ "test-ns": { getPVCPair("pvc", "test-ns", corev1.PersistentVolumeBlock), }, }, Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, @@ -1450,15 +2414,15 @@ func Test_ensureFilesystemRsyncTransferServer(t *testing.T) { { name: "error with invalid PVCPair", fields: fields{ - srcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - destClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + srcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + destClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), PVCPairMap: map[string][]transfer.PVCPair{ "test-ns": { invalidPVCPair{}, }, }, Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, @@ -1470,10 +2434,11 @@ func Test_ensureFilesystemRsyncTransferServer(t *testing.T) { tr := &Task{ Log: log.WithName("test-logger"), destinationClient: tt.fields.destClient, + sourceClient: tt.fields.srcClient, Owner: tt.fields.Owner, Client: tt.fields.destClient, } - if err := tr.ensureFilesystemRsyncTransferServer(tt.fields.srcClient, tt.fields.destClient, tt.fields.PVCPairMap, tt.fields.transportOptions); err != nil && !tt.wantErr { + if err := tr.ensureFilesystemRsyncTransferServer(tt.fields.PVCPairMap, tt.fields.transportOptions); err != nil && !tt.wantErr { t.Fatalf("ensureFilesystemRsyncTransferServer() error = %v, wantErr %v", err, tt.wantErr) } // Verify the server pod is created in the destination namespace @@ -1512,15 +2477,15 @@ func Test_ensureBlockRsyncTransferServer(t *testing.T) { { name: "get single pvc block server pod", fields: fields{ - srcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - destClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + srcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + destClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), PVCPairMap: map[string][]transfer.PVCPair{ "test-ns": { getPVCPair("pvc", "test-ns", corev1.PersistentVolumeBlock), }, }, Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, @@ -1529,15 +2494,15 @@ func Test_ensureBlockRsyncTransferServer(t *testing.T) { { name: "no block pod available, no server pods", fields: fields{ - srcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - destClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + srcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + destClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), PVCPairMap: map[string][]transfer.PVCPair{ "test-ns": { getPVCPair("pvc", "test-ns", corev1.PersistentVolumeFilesystem), }, }, Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, @@ -1545,15 +2510,15 @@ func Test_ensureBlockRsyncTransferServer(t *testing.T) { { name: "error with invalid PVCPair", fields: fields{ - srcClient: getFakeCompatClient(getDependencies("test-ns", "test-dvm")...), - destClient: getFakeCompatClient(append(getDependencies("test-ns", "test-dvm"), migration)...), + srcClient: getFakeCompatClient(getDependencies("test-ns", testDVM)...), + destClient: getFakeCompatClient(append(getDependencies("test-ns", testDVM), migration)...), PVCPairMap: map[string][]transfer.PVCPair{ "test-ns": { invalidPVCPair{}, }, }, Owner: &migapi.DirectVolumeMigration{ - ObjectMeta: metav1.ObjectMeta{Name: "test-dvm", Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, + ObjectMeta: metav1.ObjectMeta{Name: testDVM, Namespace: migapi.OpenshiftMigrationNamespace, OwnerReferences: []metav1.OwnerReference{{Name: "migmigration"}}}, Spec: migapi.DirectVolumeMigrationSpec{BackOffLimit: 2}, }, }, @@ -1565,10 +2530,18 @@ func Test_ensureBlockRsyncTransferServer(t *testing.T) { tr := &Task{ Log: log.WithName("test-logger"), destinationClient: tt.fields.destClient, + sourceClient: tt.fields.srcClient, Owner: tt.fields.Owner, Client: tt.fields.destClient, + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + LiveMigrate: ptr.To[bool](false), + }, + }, + }, } - if err := tr.ensureBlockRsyncTransferServer(tt.fields.srcClient, tt.fields.destClient, tt.fields.PVCPairMap, tt.fields.transportOptions); err != nil && !tt.wantErr { + if err := tr.ensureBlockRsyncTransferServer(tt.fields.PVCPairMap, tt.fields.transportOptions); err != nil && !tt.wantErr { t.Fatalf("ensureBlockRsyncTransferServer() error = %v, wantErr %v", err, tt.wantErr) } // Verify the server pod is created in the destination namespace diff --git a/pkg/controller/directvolumemigration/task.go b/pkg/controller/directvolumemigration/task.go index c4a75ea0a..27567093f 100644 --- a/pkg/controller/directvolumemigration/task.go +++ b/pkg/controller/directvolumemigration/task.go @@ -12,7 +12,11 @@ import ( migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" "github.com/konveyor/mig-controller/pkg/compat" "github.com/opentracing/opentracing-go" + prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/rest" + virtv1 "kubevirt.io/api/core/v1" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -23,27 +27,28 @@ var NoReQ = time.Duration(0) // Phases const ( - Created = "" - Started = "Started" - Prepare = "Prepare" - CleanStaleRsyncResources = "CleanStaleRsyncResources" - CreateDestinationNamespaces = "CreateDestinationNamespaces" - DestinationNamespacesCreated = "DestinationNamespacesCreated" - CreateDestinationPVCs = "CreateDestinationPVCs" - DestinationPVCsCreated = "DestinationPVCsCreated" - CreateStunnelConfig = "CreateStunnelConfig" - CreateRsyncConfig = "CreateRsyncConfig" - CreateRsyncRoute = "CreateRsyncRoute" - EnsureRsyncRouteAdmitted = "EnsureRsyncRouteAdmitted" - CreateRsyncTransferPods = "CreateRsyncTransferPods" - WaitForRsyncTransferPodsRunning = "WaitForRsyncTransferPodsRunning" - CreatePVProgressCRs = "CreatePVProgressCRs" - RunRsyncOperations = "RunRsyncOperations" - DeleteRsyncResources = "DeleteRsyncResources" - WaitForRsyncResourcesTerminated = "WaitForRsyncResourcesTerminated" - WaitForStaleRsyncResourcesTerminated = "WaitForStaleRsyncResourcesTerminated" - Completed = "Completed" - MigrationFailed = "MigrationFailed" + Created = "" + Started = "Started" + Prepare = "Prepare" + CleanStaleRsyncResources = "CleanStaleRsyncResources" + CreateDestinationNamespaces = "CreateDestinationNamespaces" + DestinationNamespacesCreated = "DestinationNamespacesCreated" + CreateDestinationPVCs = "CreateDestinationPVCs" + DestinationPVCsCreated = "DestinationPVCsCreated" + CreateStunnelConfig = "CreateStunnelConfig" + CreateRsyncConfig = "CreateRsyncConfig" + CreateRsyncRoute = "CreateRsyncRoute" + EnsureRsyncRouteAdmitted = "EnsureRsyncRouteAdmitted" + CreateRsyncTransferPods = "CreateRsyncTransferPods" + WaitForRsyncTransferPodsRunning = "WaitForRsyncTransferPodsRunning" + CreatePVProgressCRs = "CreatePVProgressCRs" + RunRsyncOperations = "RunRsyncOperations" + DeleteRsyncResources = "DeleteRsyncResources" + WaitForRsyncResourcesTerminated = "WaitForRsyncResourcesTerminated" + WaitForStaleRsyncResourcesTerminated = "WaitForStaleRsyncResourcesTerminated" + Completed = "Completed" + MigrationFailed = "MigrationFailed" + DeleteStaleVirtualMachineInstanceMigrations = "DeleteStaleVirtualMachineInstanceMigrations" ) // labels @@ -67,6 +72,13 @@ const ( MigratedByDirectVolumeMigration = "migration.openshift.io/migrated-by-directvolumemigration" // (dvm UID) ) +// Itinerary names +const ( + VolumeMigrationItinerary = "VolumeMigration" + VolumeMigrationFailedItinerary = "VolumeMigrationFailed" + VolumeMigrationRollbackItinerary = "VolumeMigrationRollback" +) + // Flags // TODO: are there any phases to skip? /*const ( @@ -104,7 +116,7 @@ func (r Itinerary) progressReport(phase string) (string, int, int) { } var VolumeMigration = Itinerary{ - Name: "VolumeMigration", + Name: VolumeMigrationItinerary, Steps: []Step{ {phase: Created}, {phase: Started}, @@ -115,6 +127,7 @@ var VolumeMigration = Itinerary{ {phase: DestinationNamespacesCreated}, {phase: CreateDestinationPVCs}, {phase: DestinationPVCsCreated}, + {phase: DeleteStaleVirtualMachineInstanceMigrations}, {phase: CreateRsyncRoute}, {phase: EnsureRsyncRouteAdmitted}, {phase: CreateRsyncConfig}, @@ -130,13 +143,27 @@ var VolumeMigration = Itinerary{ } var FailedItinerary = Itinerary{ - Name: "VolumeMigrationFailed", + Name: VolumeMigrationFailedItinerary, Steps: []Step{ {phase: MigrationFailed}, {phase: Completed}, }, } +var RollbackItinerary = Itinerary{ + Name: VolumeMigrationRollbackItinerary, + Steps: []Step{ + {phase: Created}, + {phase: Started}, + {phase: Prepare}, + {phase: CleanStaleRsyncResources}, + {phase: DeleteStaleVirtualMachineInstanceMigrations}, + {phase: WaitForStaleRsyncResourcesTerminated}, + {phase: RunRsyncOperations}, + {phase: Completed}, + }, +} + // A task that provides the complete migration workflow. // Log - A controller's logger. // Client - A controller's (local) client. @@ -149,8 +176,10 @@ var FailedItinerary = Itinerary{ type Task struct { Log logr.Logger Client k8sclient.Client + PrometheusAPI prometheusv1.API sourceClient compat.Client destinationClient compat.Client + restConfig *rest.Config Owner *migapi.DirectVolumeMigration SSHKeys *sshKeys EndpointType migapi.EndpointType @@ -164,9 +193,10 @@ type Task struct { SparseFileMap sparseFilePVCMap SourceLimitRangeMapping limitRangeMap DestinationLimitRangeMapping limitRangeMap - - Tracer opentracing.Tracer - ReconcileSpan opentracing.Span + VirtualMachineMappings VirtualMachineMappings + PromQuery func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) + Tracer opentracing.Tracer + ReconcileSpan opentracing.Span } type limitRangeMap map[string]corev1.LimitRange @@ -176,17 +206,51 @@ type sshKeys struct { PrivateKey *rsa.PrivateKey } +type VirtualMachineMappings struct { + volumeVMIMMap map[string]*virtv1.VirtualMachineInstanceMigration + volumeVMNameMap map[string]string + runningVMVolumeNames []string +} + func (t *Task) init() error { t.RsyncRoutes = make(map[string]string) t.Requeue = FastReQ if t.failed() { t.Itinerary = FailedItinerary + } else if t.Owner.IsRollback() { + t.Itinerary = RollbackItinerary } else { t.Itinerary = VolumeMigration } if t.Itinerary.Name != t.Owner.Status.Itinerary { t.Phase = t.Itinerary.Steps[0].phase } + // Initialize the source and destination clients + _, err := t.getSourceClient() + if err != nil { + return err + } + _, err = t.getDestinationClient() + return err +} + +func (t *Task) populateVMMappings(namespace string) error { + t.VirtualMachineMappings = VirtualMachineMappings{} + volumeVMIMMap, err := t.getVolumeVMIMInNamespaces([]string{namespace}) + if err != nil { + return err + } + volumeVMMap, err := getRunningVmVolumeMap(t.sourceClient, namespace) + if err != nil { + return err + } + volumeNames, err := t.getRunningVMVolumes([]string{namespace}) + if err != nil { + return err + } + t.VirtualMachineMappings.volumeVMIMMap = volumeVMIMMap + t.VirtualMachineMappings.volumeVMNameMap = volumeVMMap + t.VirtualMachineMappings.runningVMVolumeNames = volumeNames return nil } @@ -209,11 +273,7 @@ func (t *Task) Run(ctx context.Context) error { // Run the current phase. switch t.Phase { - case Created, Started: - if err = t.next(); err != nil { - return liberr.Wrap(err) - } - case Prepare: + case Created, Started, Prepare: if err = t.next(); err != nil { return liberr.Wrap(err) } @@ -377,7 +437,7 @@ func (t *Task) Run(ctx context.Context) error { msg := fmt.Sprintf("Rsync Transfer Pod(s) on destination cluster have not started Running within 3 minutes. "+ "Run these command(s) to check Pod warning events: [%s]", - fmt.Sprintf("%s", strings.Join(nonRunningPodStrings, ", "))) + strings.Join(nonRunningPodStrings, ", ")) t.Log.Info(msg) t.Owner.Status.SetCondition( @@ -425,8 +485,15 @@ func (t *Task) Run(ctx context.Context) error { if err = t.next(); err != nil { return liberr.Wrap(err) } + case DeleteStaleVirtualMachineInstanceMigrations: + if err := t.deleteStaleVirtualMachineInstanceMigrations(); err != nil { + return liberr.Wrap(err) + } + if err = t.next(); err != nil { + return liberr.Wrap(err) + } case WaitForStaleRsyncResourcesTerminated, WaitForRsyncResourcesTerminated: - err, deleted := t.waitForRsyncResourcesDeleted() + deleted, err := t.waitForRsyncResourcesDeleted() if err != nil { return liberr.Wrap(err) } @@ -436,6 +503,7 @@ func (t *Task) Run(ctx context.Context) error { return liberr.Wrap(err) } } + t.Log.Info("Stale Rsync resources are still terminating. Waiting.") t.Requeue = PollReQ case Completed: @@ -528,6 +596,7 @@ func (t *Task) getSourceClient() (compat.Client, error) { if err != nil { return nil, err } + t.sourceClient = client return client, nil } @@ -547,6 +616,7 @@ func (t *Task) getDestinationClient() (compat.Client, error) { if err != nil { return nil, err } + t.destinationClient = client return client, nil } diff --git a/pkg/controller/directvolumemigration/validation.go b/pkg/controller/directvolumemigration/validation.go index 0636cc7da..2bd1f16a9 100644 --- a/pkg/controller/directvolumemigration/validation.go +++ b/pkg/controller/directvolumemigration/validation.go @@ -28,6 +28,7 @@ const ( Running = "Running" Failed = "Failed" RsyncClientPodsPending = "RsyncClientPodsPending" + LiveMigrationsPending = "LiveMigrationsPending" Succeeded = "Succeeded" SourceToDestinationNetworkError = "SourceToDestinationNetworkError" FailedCreatingRsyncPods = "FailedCreatingRsyncPods" @@ -43,6 +44,7 @@ const ( NotReady = "NotReady" RsyncTimeout = "RsyncTimedOut" RsyncNoRouteToHost = "RsyncNoRouteToHost" + NotSupported = "NotSupported" ) // Messages @@ -75,28 +77,25 @@ const ( ) // Validate the direct resource -func (r ReconcileDirectVolumeMigration) validate(ctx context.Context, direct *migapi.DirectVolumeMigration) error { +func (r *ReconcileDirectVolumeMigration) validate(ctx context.Context, direct *migapi.DirectVolumeMigration) error { if opentracing.SpanFromContext(ctx) != nil { var span opentracing.Span span, ctx = opentracing.StartSpanFromContextWithTracer(ctx, r.tracer, "validate") defer span.Finish() } - err := r.validateSrcCluster(ctx, direct) - if err != nil { + if err := r.validateSrcCluster(ctx, direct); err != nil { return liberr.Wrap(err) } - err = r.validateDestCluster(ctx, direct) - if err != nil { + if err := r.validateDestCluster(ctx, direct); err != nil { return liberr.Wrap(err) } - err = r.validatePVCs(ctx, direct) - if err != nil { + if err := r.validatePVCs(ctx, direct); err != nil { return liberr.Wrap(err) } return nil } -func (r ReconcileDirectVolumeMigration) validateSrcCluster(ctx context.Context, direct *migapi.DirectVolumeMigration) error { +func (r *ReconcileDirectVolumeMigration) validateSrcCluster(ctx context.Context, direct *migapi.DirectVolumeMigration) error { if opentracing.SpanFromContext(ctx) != nil { span, _ := opentracing.StartSpanFromContextWithTracer(ctx, r.tracer, "validateSrcCluster") defer span.Finish() @@ -146,7 +145,7 @@ func (r ReconcileDirectVolumeMigration) validateSrcCluster(ctx context.Context, return nil } -func (r ReconcileDirectVolumeMigration) validateDestCluster(ctx context.Context, direct *migapi.DirectVolumeMigration) error { +func (r *ReconcileDirectVolumeMigration) validateDestCluster(ctx context.Context, direct *migapi.DirectVolumeMigration) error { if opentracing.SpanFromContext(ctx) != nil { span, _ := opentracing.StartSpanFromContextWithTracer(ctx, r.tracer, "validateDestCluster") defer span.Finish() @@ -198,11 +197,11 @@ func (r ReconcileDirectVolumeMigration) validateDestCluster(ctx context.Context, // TODO: Validate that storage class mappings have valid storage class selections // Leaving as TODO because this is technically already validated from the // migplan, so not necessary from directvolumemigration controller to be fair -func (r ReconcileDirectVolumeMigration) validateStorageClassMappings(direct *migapi.DirectVolumeMigration) error { +func (r *ReconcileDirectVolumeMigration) validateStorageClassMappings(direct *migapi.DirectVolumeMigration) error { return nil } -func (r ReconcileDirectVolumeMigration) validatePVCs(ctx context.Context, direct *migapi.DirectVolumeMigration) error { +func (r *ReconcileDirectVolumeMigration) validatePVCs(ctx context.Context, direct *migapi.DirectVolumeMigration) error { if opentracing.SpanFromContext(ctx) != nil { span, _ := opentracing.StartSpanFromContextWithTracer(ctx, r.tracer, "validatePVCs") defer span.Finish() diff --git a/pkg/controller/directvolumemigration/vm.go b/pkg/controller/directvolumemigration/vm.go new file mode 100644 index 000000000..bc45bdb05 --- /dev/null +++ b/pkg/controller/directvolumemigration/vm.go @@ -0,0 +1,577 @@ +package directvolumemigration + +import ( + "context" + "errors" + "fmt" + "net/url" + "reflect" + "regexp" + "slices" + "strings" + "time" + + "github.com/go-logr/logr" + "github.com/konveyor/crane-lib/state_transfer/transfer" + migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + routev1 "github.com/openshift/api/route/v1" + prometheusapi "github.com/prometheus/client_golang/api" + prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + prometheusURLKey = "PROMETHEUS_URL" + prometheusRoute = "prometheus-k8s" + progressQuery = "kubevirt_vmi_migration_data_processed_bytes{name=\"%s\"} / (kubevirt_vmi_migration_data_processed_bytes{name=\"%s\"} + kubevirt_vmi_migration_data_remaining_bytes{name=\"%s\"}) * 100" +) + +var ErrVolumesDoNotMatch = errors.New("volumes do not match") + +type vmVolumes struct { + sourceVolumes []string + targetVolumes []string +} + +func (t *Task) startLiveMigrations(nsMap map[string][]transfer.PVCPair) ([]string, error) { + reasons := []string{} + vmVolumeMap := make(map[string]vmVolumes) + sourceClient := t.sourceClient + if t.Owner.IsRollback() { + sourceClient = t.destinationClient + } + for k, v := range nsMap { + namespace, err := getNamespace(k) + if err != nil { + reasons = append(reasons, err.Error()) + return reasons, err + } + volumeVmMap, err := getRunningVmVolumeMap(sourceClient, namespace) + if err != nil { + reasons = append(reasons, err.Error()) + return reasons, err + } + for _, pvcPair := range v { + if vmName, found := volumeVmMap[pvcPair.Source().Claim().Name]; found { + vmVolumeMap[vmName] = vmVolumes{ + sourceVolumes: append(vmVolumeMap[vmName].sourceVolumes, pvcPair.Source().Claim().Name), + targetVolumes: append(vmVolumeMap[vmName].targetVolumes, pvcPair.Destination().Claim().Name), + } + } + } + for vmName, volumes := range vmVolumeMap { + if err := t.storageLiveMigrateVM(vmName, namespace, &volumes); err != nil { + switch err { + case ErrVolumesDoNotMatch: + reasons = append(reasons, fmt.Sprintf("source and target volumes do not match for VM %s", vmName)) + continue + default: + reasons = append(reasons, err.Error()) + return reasons, err + } + } + } + } + return reasons, nil +} + +func getNamespace(colonDelimitedString string) (string, error) { + namespacePair := strings.Split(colonDelimitedString, ":") + if len(namespacePair) != 2 { + return "", fmt.Errorf("invalid namespace pair: %s", colonDelimitedString) + } + if namespacePair[0] != namespacePair[1] && namespacePair[0] != "" { + return "", fmt.Errorf("source and target namespaces must match: %s", colonDelimitedString) + } + return namespacePair[0], nil +} + +// Return a list of namespace/volume combinations that are currently in use by running VMs +func (t *Task) getRunningVMVolumes(namespaces []string) ([]string, error) { + runningVMVolumes := []string{} + + for _, ns := range namespaces { + volumesVmMap, err := getRunningVmVolumeMap(t.sourceClient, ns) + if err != nil { + return nil, err + } + for volume := range volumesVmMap { + runningVMVolumes = append(runningVMVolumes, fmt.Sprintf("%s/%s", ns, volume)) + } + } + return runningVMVolumes, nil +} + +func getRunningVmVolumeMap(client k8sclient.Client, namespace string) (map[string]string, error) { + volumesVmMap := make(map[string]string) + vmMap, err := getVMNamesInNamespace(client, namespace) + if err != nil { + return nil, err + } + for vmName := range vmMap { + podList := corev1.PodList{} + client.List(context.TODO(), &podList, k8sclient.InNamespace(namespace)) + for _, pod := range podList.Items { + for _, owner := range pod.OwnerReferences { + if owner.Name == vmName && owner.Kind == "VirtualMachineInstance" { + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + volumesVmMap[volume.PersistentVolumeClaim.ClaimName] = vmName + } + } + } + } + } + } + return volumesVmMap, nil +} + +func getVMNamesInNamespace(client k8sclient.Client, namespace string) (map[string]interface{}, error) { + vms := make(map[string]interface{}) + vmList := virtv1.VirtualMachineList{} + err := client.List(context.TODO(), &vmList, k8sclient.InNamespace(namespace)) + if err != nil { + if meta.IsNoMatchError(err) { + return nil, nil + } + return nil, err + } + for _, vm := range vmList.Items { + vms[vm.Name] = nil + } + return vms, nil +} + +func (t *Task) storageLiveMigrateVM(vmName, namespace string, volumes *vmVolumes) error { + sourceClient := t.sourceClient + if t.Owner.IsRollback() { + sourceClient = t.destinationClient + } + vm := &virtv1.VirtualMachine{} + if err := sourceClient.Get(context.TODO(), k8sclient.ObjectKey{Namespace: namespace, Name: vmName}, vm); err != nil { + return err + } + // Check if the source volumes match before attempting to migrate. + if !t.compareVolumes(vm, volumes.sourceVolumes) { + // Check if the target volumes match, if so, the migration is already in progress or complete. + if t.compareVolumes(vm, volumes.targetVolumes) { + t.Log.V(5).Info("Volumes already updated for VM", "vm", vmName) + return nil + } else { + return ErrVolumesDoNotMatch + } + } + + // Volumes match, create patch to update the VM with the target volumes + return updateVM(sourceClient, vm, volumes.sourceVolumes, volumes.targetVolumes, t.Log) +} + +func (t *Task) compareVolumes(vm *virtv1.VirtualMachine, volumes []string) bool { + foundCount := 0 + if vm.Spec.Template == nil { + return true + } + for _, vmVolume := range vm.Spec.Template.Spec.Volumes { + if vmVolume.PersistentVolumeClaim == nil && vmVolume.DataVolume == nil { + // Skip all non PVC or DataVolume volumes + continue + } + if vmVolume.PersistentVolumeClaim != nil { + if slices.Contains(volumes, vmVolume.PersistentVolumeClaim.ClaimName) { + foundCount++ + } + } else if vmVolume.DataVolume != nil { + if slices.Contains(volumes, vmVolume.DataVolume.Name) { + foundCount++ + } + } + } + return foundCount == len(volumes) +} + +func findVirtualMachineInstanceMigration(client k8sclient.Client, vmName, namespace string) (*virtv1.VirtualMachineInstanceMigration, error) { + vmimList := &virtv1.VirtualMachineInstanceMigrationList{} + if err := client.List(context.TODO(), vmimList, k8sclient.InNamespace(namespace)); err != nil { + return nil, err + } + for _, vmim := range vmimList.Items { + if vmim.Spec.VMIName == vmName { + return &vmim, nil + } + } + return nil, nil +} + +func virtualMachineMigrationStatus(client k8sclient.Client, vmName, namespace string, log logr.Logger) (string, error) { + log.Info("Checking migration status for VM", "vm", vmName) + vmim, err := findVirtualMachineInstanceMigration(client, vmName, namespace) + if err != nil { + return err.Error(), err + } + if vmim != nil { + log.V(5).Info("Found VMIM", "vmim", vmim.Name, "namespace", vmim.Namespace) + if vmim.Status.MigrationState != nil { + if vmim.Status.MigrationState.Failed { + return vmim.Status.MigrationState.FailureReason, nil + } else if vmim.Status.MigrationState.AbortStatus == virtv1.MigrationAbortSucceeded { + return "Migration canceled", nil + } else if vmim.Status.MigrationState.AbortStatus == virtv1.MigrationAbortFailed { + return "Migration canceled failed", nil + } else if vmim.Status.MigrationState.AbortStatus == virtv1.MigrationAbortInProgress { + return "Migration cancel in progress", nil + } + if vmim.Status.MigrationState.Completed { + return "", nil + } + } + } + + vmi := &virtv1.VirtualMachineInstance{} + if err := client.Get(context.TODO(), k8sclient.ObjectKey{Namespace: namespace, Name: vmName}, vmi); err != nil { + if k8serrors.IsNotFound(err) { + return fmt.Sprintf("VMI %s not found in namespace %s", vmName, namespace), nil + } else { + return err.Error(), err + } + } + volumeChange := false + liveMigrateable := false + liveMigrateableMessage := "" + for _, condition := range vmi.Status.Conditions { + if condition.Type == virtv1.VirtualMachineInstanceVolumesChange { + volumeChange = condition.Status == corev1.ConditionTrue + } + if condition.Type == virtv1.VirtualMachineInstanceIsMigratable { + liveMigrateable = condition.Status == corev1.ConditionTrue + liveMigrateableMessage = condition.Message + } + } + if volumeChange && !liveMigrateable { + // Unable to storage live migrate because something else is preventing migration + return liveMigrateableMessage, nil + } + return "", nil +} + +func cancelLiveMigration(client k8sclient.Client, vmName, namespace string, volumes *vmVolumes, log logr.Logger) error { + vm := &virtv1.VirtualMachine{} + if err := client.Get(context.TODO(), k8sclient.ObjectKey{Namespace: namespace, Name: vmName}, vm); err != nil { + return err + } + + log.V(3).Info("Canceling live migration", "vm", vmName) + if err := updateVM(client, vm, volumes.targetVolumes, volumes.sourceVolumes, log); err != nil { + return err + } + return nil +} + +func liveMigrationsCompleted(client k8sclient.Client, namespace string, vmNames []string) (bool, error) { + vmim := &virtv1.VirtualMachineInstanceMigrationList{} + if err := client.List(context.TODO(), vmim, k8sclient.InNamespace(namespace)); err != nil { + return false, err + } + completed := true + for _, migration := range vmim.Items { + if slices.Contains(vmNames, migration.Spec.VMIName) { + if migration.Status.Phase != virtv1.MigrationSucceeded { + completed = false + break + } + } + } + return completed, nil +} + +func updateVM(client k8sclient.Client, vm *virtv1.VirtualMachine, sourceVolumes, targetVolumes []string, log logr.Logger) error { + if vm == nil || vm.Name == "" { + return nil + } + vmCopy := vm.DeepCopy() + // Ensure the VM migration strategy is set properly. + log.V(5).Info("Setting volume migration strategy to migration", "vm", vmCopy.Name) + vmCopy.Spec.UpdateVolumesStrategy = ptr.To[virtv1.UpdateVolumesStrategy](virtv1.UpdateVolumesStrategyMigration) + + for i := 0; i < len(sourceVolumes); i++ { + // Check if we need to update DataVolumeTemplates. + for j, dvTemplate := range vmCopy.Spec.DataVolumeTemplates { + if dvTemplate.Name == sourceVolumes[i] { + log.V(5).Info("Updating DataVolumeTemplate", "source", sourceVolumes[i], "target", targetVolumes[i]) + vmCopy.Spec.DataVolumeTemplates[j].Name = targetVolumes[i] + } + } + for j, volume := range vm.Spec.Template.Spec.Volumes { + if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == sourceVolumes[i] { + log.V(5).Info("Updating PersistentVolumeClaim", "source", sourceVolumes[i], "target", targetVolumes[i]) + vmCopy.Spec.Template.Spec.Volumes[j].PersistentVolumeClaim.ClaimName = targetVolumes[i] + } + if volume.DataVolume != nil && volume.DataVolume.Name == sourceVolumes[i] { + log.V(5).Info("Updating DataVolume", "source", sourceVolumes[i], "target", targetVolumes[i]) + if err := CreateNewDataVolume(client, sourceVolumes[i], targetVolumes[i], vmCopy.Namespace, log); err != nil { + return err + } + vmCopy.Spec.Template.Spec.Volumes[j].DataVolume.Name = targetVolumes[i] + } + } + } + if !reflect.DeepEqual(vm, vmCopy) { + log.V(5).Info("Calling VM update", "vm", vm.Name) + if err := client.Update(context.TODO(), vmCopy); err != nil { + return err + } + } else { + log.V(5).Info("No changes to VM", "vm", vm.Name) + } + return nil +} + +func CreateNewDataVolume(client k8sclient.Client, sourceDvName, targetDvName, ns string, log logr.Logger) error { + log.V(3).Info("Create new adopting datavolume from source datavolume", "namespace", ns, "source name", sourceDvName, "target name", targetDvName) + originalDv := &cdiv1.DataVolume{} + if err := client.Get(context.TODO(), k8sclient.ObjectKey{Namespace: ns, Name: sourceDvName}, originalDv); err != nil { + log.Error(err, "Failed to get source datavolume", "namespace", ns, "name", sourceDvName) + return err + } + + // Create adopting datavolume. + adoptingDV := originalDv.DeepCopy() + adoptingDV.Name = targetDvName + if adoptingDV.Annotations == nil { + adoptingDV.Annotations = make(map[string]string) + } + adoptingDV.Annotations["cdi.kubevirt.io/allowClaimAdoption"] = "true" + adoptingDV.ResourceVersion = "" + adoptingDV.ManagedFields = nil + adoptingDV.UID = "" + adoptingDV.Spec.Source = &cdiv1.DataVolumeSource{ + Blank: &cdiv1.DataVolumeBlankImage{}, + } + err := client.Create(context.TODO(), adoptingDV) + if err != nil && !k8serrors.IsAlreadyExists(err) { + log.Error(err, "Failed to create adopting datavolume", "namespace", ns, "name", targetDvName) + return err + } + return nil +} + +// Gets all the VirtualMachineInstanceMigration objects by volume in the passed in namespace. +func (t *Task) getVolumeVMIMInNamespaces(namespaces []string) (map[string]*virtv1.VirtualMachineInstanceMigration, error) { + vmimMap := make(map[string]*virtv1.VirtualMachineInstanceMigration) + for _, namespace := range namespaces { + volumeVMMap, err := getRunningVmVolumeMap(t.sourceClient, namespace) + if err != nil { + return nil, err + } + for volumeName, vmName := range volumeVMMap { + vmimList := &virtv1.VirtualMachineInstanceMigrationList{} + if err := t.sourceClient.List(context.TODO(), vmimList, k8sclient.InNamespace(namespace)); err != nil { + return nil, err + } + for _, vmim := range vmimList.Items { + if vmim.Spec.VMIName == vmName { + vmimMap[fmt.Sprintf("%s/%s", namespace, volumeName)] = &vmim + } + } + } + } + return vmimMap, nil +} + +func getVMIMElapsedTime(vmim *virtv1.VirtualMachineInstanceMigration) metav1.Duration { + if vmim == nil || vmim.Status.MigrationState == nil { + return metav1.Duration{ + Duration: 0, + } + } + if vmim.Status.MigrationState.StartTimestamp == nil { + for _, timestamps := range vmim.Status.PhaseTransitionTimestamps { + if timestamps.Phase == virtv1.MigrationRunning { + vmim.Status.MigrationState.StartTimestamp = ×tamps.PhaseTransitionTimestamp + } + } + } + if vmim.Status.MigrationState.StartTimestamp == nil { + return metav1.Duration{ + Duration: 0, + } + } + if vmim.Status.MigrationState.EndTimestamp != nil { + return metav1.Duration{ + Duration: vmim.Status.MigrationState.EndTimestamp.Sub(vmim.Status.MigrationState.StartTimestamp.Time), + } + } + return metav1.Duration{ + Duration: time.Since(vmim.Status.MigrationState.StartTimestamp.Time), + } +} + +func (t *Task) getLastObservedProgressPercent(vmName, namespace string, currentProgress map[string]*migapi.LiveMigrationProgress) (string, error) { + if err := t.buildPrometheusAPI(); err != nil { + return "", err + } + + result, warnings, err := t.PromQuery(context.TODO(), fmt.Sprintf(progressQuery, vmName, vmName, vmName), time.Now()) + if err != nil { + t.Log.Error(err, "Failed to query prometheus, returning previous progress") + if progress, found := currentProgress[fmt.Sprintf("%s/%s", namespace, vmName)]; found { + return progress.LastObservedProgressPercent, nil + } + return "", nil + } + if len(warnings) > 0 { + t.Log.Info("Warnings", "warnings", warnings) + } + t.Log.V(5).Info("Prometheus query result", "type", result.Type(), "value", result.String()) + progress := parseProgress(result.String()) + if progress != "" { + return progress + "%", nil + } + return "", nil +} + +func (t *Task) buildPrometheusAPI() error { + if t.PrometheusAPI != nil { + return nil + } + if t.restConfig == nil { + var err error + t.restConfig, err = t.PlanResources.SrcMigCluster.BuildRestConfig(t.Client) + if err != nil { + return err + } + } + url, err := t.buildSourcePrometheusEndPointURL() + if err != nil { + return err + } + + // Prometheus URL not found, return blank progress + if url == "" { + return nil + } + httpClient, err := rest.HTTPClientFor(t.restConfig) + if err != nil { + return err + } + client, err := prometheusapi.NewClient(prometheusapi.Config{ + Address: url, + Client: httpClient, + }) + if err != nil { + return err + } + t.PrometheusAPI = prometheusv1.NewAPI(client) + t.PromQuery = t.PrometheusAPI.Query + return nil +} + +func parseProgress(progress string) string { + regExp := regexp.MustCompile(`\=\> (\d{1,3})\.\d* @`) + + if regExp.MatchString(progress) { + return regExp.FindStringSubmatch(progress)[1] + } + return "" +} + +// Find the URL that contains the prometheus metrics on the source cluster. +func (t *Task) buildSourcePrometheusEndPointURL() (string, error) { + urlString, err := t.getPrometheusURLFromConfig() + if err != nil { + return "", err + } + if urlString == "" { + // URL not found in config map, attempt to get the open shift prometheus route. + routes := &routev1.RouteList{} + req, err := labels.NewRequirement("app.kubernetes.io/part-of", selection.Equals, []string{"openshift-monitoring"}) + if err != nil { + return "", err + } + if err := t.sourceClient.List(context.TODO(), routes, k8sclient.InNamespace("openshift-monitoring"), &k8sclient.ListOptions{ + LabelSelector: labels.NewSelector().Add(*req), + }); err != nil { + return "", err + } + for _, r := range routes.Items { + if r.Spec.To.Name == prometheusRoute { + urlString = r.Spec.Host + break + } + } + } + if urlString == "" { + // Don't return error just return empty and skip the progress report + return "", nil + } + parsedUrl, err := url.Parse(urlString) + if err != nil { + return "", err + } + parsedUrl.Scheme = "https" + urlString = parsedUrl.String() + t.Log.V(3).Info("Prometheus route URL", "url", urlString) + return urlString, nil +} + +// The key in the config map should be in format _PROMETHEUS_URL +// For instance if the cluster name is "cluster1" the key should be "cluster1_PROMETHEUS_URL" +func (t *Task) getPrometheusURLFromConfig() (string, error) { + migControllerConfigMap := &corev1.ConfigMap{} + if err := t.sourceClient.Get(context.TODO(), k8sclient.ObjectKey{Namespace: migapi.OpenshiftMigrationNamespace, Name: "migration-controller"}, migControllerConfigMap); err != nil { + return "", err + } + key := fmt.Sprintf("%s_%s", t.PlanResources.SrcMigCluster.Name, prometheusURLKey) + if prometheusURL, found := migControllerConfigMap.Data[key]; found { + return prometheusURL, nil + } + return "", nil +} + +func (t *Task) deleteStaleVirtualMachineInstanceMigrations() error { + pvcMap, err := t.getNamespacedPVCPairs() + if err != nil { + return err + } + + for namespacePair := range pvcMap { + namespace, err := getNamespace(namespacePair) + if err != nil { + return err + } + vmMap, err := getVMNamesInNamespace(t.sourceClient, namespace) + if err != nil { + return err + } + + vmimList := &virtv1.VirtualMachineInstanceMigrationList{} + if err := t.sourceClient.List(context.TODO(), vmimList, k8sclient.InNamespace(namespace)); err != nil { + if meta.IsNoMatchError(err) { + return nil + } + return err + } + for _, vmim := range vmimList.Items { + if _, ok := vmMap[vmim.Spec.VMIName]; ok { + if vmim.Status.Phase == virtv1.MigrationSucceeded && + vmim.Status.MigrationState != nil && + vmim.Status.MigrationState.EndTimestamp.Before(&t.Owner.CreationTimestamp) { + // Old VMIM that has completed and succeeded before the migration was created, delete the VMIM + if err := t.sourceClient.Delete(context.TODO(), &vmim); err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + } + } + } + return nil +} diff --git a/pkg/controller/directvolumemigration/vm_test.go b/pkg/controller/directvolumemigration/vm_test.go new file mode 100644 index 000000000..2d3c6246d --- /dev/null +++ b/pkg/controller/directvolumemigration/vm_test.go @@ -0,0 +1,1800 @@ +package directvolumemigration + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "testing" + "time" + + transfer "github.com/konveyor/crane-lib/state_transfer/transfer" + "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + "github.com/konveyor/mig-controller/pkg/compat" + fakecompat "github.com/konveyor/mig-controller/pkg/compat/fake" + routev1 "github.com/openshift/api/route/v1" + prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + sourcePVC = "source-pvc" + sourceNs = "source-ns" + targetPVC = "target-pvc" + targetNs = "target-ns" + targetDv = "target-dv" +) + +func TestTask_startLiveMigrations(t *testing.T) { + tests := []struct { + name string + task *Task + client compat.Client + pairMap map[string][]transfer.PVCPair + expectedReasons []string + wantErr bool + }{ + { + name: "same namespace, no volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachine("vm", testNamespace)), + pairMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + }, + { + name: "different namespace, no volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachine("vm", testNamespace)), + pairMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace2: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace2)), + }, + }, + wantErr: true, + expectedReasons: []string{"source and target namespaces must match: test-namespace:test-namespace2"}, + }, + { + name: "running VM, no volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachine("vm", testNamespace), createVirtlauncherPod("vm", testNamespace, []string{"dv"})), + pairMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("pvc1", testNamespace), createPvc("pvc2", testNamespace)), + }, + }, + }, + { + name: "running VM, matching volume", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv", + }, + }, + }, + }), createVirtlauncherPod("vm", testNamespace, []string{"dv"}), + createDataVolume("dv", testNamespace)), + pairMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("dv", testNamespace), createPvc("dv2", testNamespace)), + }, + }, + }, + { + name: "running VM, no matching volume", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv-nomatch", + }, + }, + }, + }), createVirtlauncherPod("vm", testNamespace, []string{"dv"}), + createDataVolume("dv", testNamespace)), + pairMap: map[string][]transfer.PVCPair{ + testNamespace + ":" + testNamespace: { + transfer.NewPVCPair(createPvc("dv", testNamespace), createPvc("dv2", testNamespace)), + }, + }, + wantErr: false, + expectedReasons: []string{"source and target volumes do not match for VM vm"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.task.Owner.Spec.MigrationType == nil || tt.task.Owner.Spec.MigrationType == ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback) { + tt.task.sourceClient = tt.client + } else { + tt.task.destinationClient = tt.client + } + reasons, err := tt.task.startLiveMigrations(tt.pairMap) + if err != nil && !tt.wantErr { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } else if err == nil && tt.wantErr { + t.Errorf("expected error, got nil") + t.FailNow() + } + if len(reasons) != len(tt.expectedReasons) { + t.Errorf("expected %v, got %v", tt.expectedReasons, reasons) + t.FailNow() + } + for i, r := range reasons { + if r != tt.expectedReasons[i] { + t.Errorf("expected %v, got %v", tt.expectedReasons, reasons) + t.FailNow() + } + } + }) + } +} + +func TestGetNamespace(t *testing.T) { + _, err := getNamespace("nocolon") + if err == nil { + t.Errorf("expected error, got nil") + t.FailNow() + } +} + +func TestTaskGetRunningVMVolumes(t *testing.T) { + tests := []struct { + name string + task *Task + client compat.Client + expectedVolumes []string + }{ + { + name: "no VMs", + task: &Task{}, + client: getFakeClientWithObjs(), + expectedVolumes: []string{}, + }, + { + name: "no running vms", + task: &Task{}, + client: getFakeClientWithObjs(createVirtualMachine("vm", "ns1"), createVirtualMachine("vm2", "ns2")), + expectedVolumes: []string{}, + }, + { + name: "running all vms", + task: &Task{}, + client: getFakeClientWithObjs( + createVirtualMachine("vm", "ns1"), + createVirtualMachine("vm2", "ns2"), + createVirtlauncherPod("vm", "ns1", []string{"dv"}), + createVirtlauncherPod("vm2", "ns2", []string{"dv2"})), + expectedVolumes: []string{"ns1/dv", "ns2/dv2"}, + }, + { + name: "running single vms", + task: &Task{}, + client: getFakeClientWithObjs( + createVirtualMachine("vm", "ns1"), + createVirtualMachine("vm2", "ns2"), + createVirtlauncherPod("vm2", "ns2", []string{"dv2"})), + expectedVolumes: []string{"ns2/dv2"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + volumes, err := tt.task.getRunningVMVolumes([]string{"ns1", "ns2"}) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if len(volumes) != len(tt.expectedVolumes) { + t.Errorf("expected %v, got %v", tt.expectedVolumes, volumes) + t.FailNow() + } + for i, v := range volumes { + if v != tt.expectedVolumes[i] { + t.Errorf("expected %v, got %v", tt.expectedVolumes, volumes) + t.FailNow() + } + } + }) + } +} + +func TestTaskStorageLiveMigrateVM(t *testing.T) { + tests := []struct { + name string + task *Task + client compat.Client + volumes *vmVolumes + wantErr bool + }{ + { + name: "no vm, no volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(), + volumes: &vmVolumes{}, + wantErr: true, + }, + { + name: "vm, no volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachine("vm", testNamespace)), + volumes: &vmVolumes{}, + wantErr: false, + }, + { + name: "vm, matching volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv", + }, + }, + }, + }), createDataVolume("dv", testNamespace)), + volumes: &vmVolumes{ + sourceVolumes: []string{"dv"}, + targetVolumes: []string{targetDv}, + }, + wantErr: false, + }, + { + name: "vm, matching volumes, already modified", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: targetDv, + }, + }, + }, + }), createDataVolume("dv", testNamespace)), + volumes: &vmVolumes{ + sourceVolumes: []string{"dv"}, + targetVolumes: []string{targetDv}, + }, + wantErr: false, + }, + { + name: "vm, non matching volumes", + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + MigrationType: ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback), + }, + }, + }, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "unmatched", + }, + }, + }, + }), createDataVolume("dv", testNamespace)), + volumes: &vmVolumes{ + sourceVolumes: []string{"dv"}, + targetVolumes: []string{targetDv}, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.task.Owner.Spec.MigrationType == nil || tt.task.Owner.Spec.MigrationType == ptr.To[v1alpha1.DirectVolumeMigrationType](v1alpha1.MigrationTypeRollback) { + tt.task.sourceClient = tt.client + } else { + tt.task.destinationClient = tt.client + } + err := tt.task.storageLiveMigrateVM("vm", testNamespace, tt.volumes) + if err != nil && !tt.wantErr { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } else if err == nil && tt.wantErr { + t.Errorf("expected error, got nil") + t.FailNow() + } + }) + } +} + +func TestVirtualMachineMigrationStatus(t *testing.T) { + tests := []struct { + name string + client k8sclient.Client + expectedStatus string + }{ + { + name: "In progress VMIM", + client: getFakeClientWithObjs(createInProgressVirtualMachineMigration("vmim", testNamespace, "vm")), + expectedStatus: fmt.Sprintf("VMI %s not found in namespace %s", "vm", testNamespace), + }, + { + name: "No VMIM or VMI", + client: getFakeClientWithObjs(), + expectedStatus: fmt.Sprintf("VMI %s not found in namespace %s", "vm", testNamespace), + }, + { + name: "Failed VMIM with message", + client: getFakeClientWithObjs(createFailedVirtualMachineMigration("vmim", testNamespace, "vm", "failed")), + expectedStatus: "failed", + }, + { + name: "Canceled VMIM", + client: getFakeClientWithObjs(createCanceledVirtualMachineMigration("vmim", testNamespace, "vm", virtv1.MigrationAbortSucceeded)), + expectedStatus: "Migration canceled", + }, + { + name: "Canceled VMIM inprogress", + client: getFakeClientWithObjs(createCanceledVirtualMachineMigration("vmim", testNamespace, "vm", virtv1.MigrationAbortInProgress)), + expectedStatus: "Migration cancel in progress", + }, + { + name: "Canceled VMIM failed", + client: getFakeClientWithObjs(createCanceledVirtualMachineMigration("vmim", testNamespace, "vm", virtv1.MigrationAbortFailed)), + expectedStatus: "Migration canceled failed", + }, + { + name: "Completed VMIM", + client: getFakeClientWithObjs(createCompletedVirtualMachineMigration("vmim", testNamespace, "vm")), + expectedStatus: "", + }, + { + name: "VMI without conditions", + client: getFakeClientWithObjs(createVirtualMachineInstance("vm", testNamespace, virtv1.Running)), + expectedStatus: "", + }, + { + name: "VMI with update strategy conditions, and live migrate possible", + client: getFakeClientWithObjs(createVirtualMachineInstanceWithConditions("vm", testNamespace, []virtv1.VirtualMachineInstanceCondition{ + { + Type: virtv1.VirtualMachineInstanceVolumesChange, + Status: corev1.ConditionTrue, + }, + { + Type: virtv1.VirtualMachineInstanceIsMigratable, + Status: corev1.ConditionTrue, + }, + })), + expectedStatus: "", + }, + { + name: "VMI with update strategy conditions, and live migrate not possible", + client: getFakeClientWithObjs(createVirtualMachineInstanceWithConditions("vm", testNamespace, []virtv1.VirtualMachineInstanceCondition{ + { + Type: virtv1.VirtualMachineInstanceVolumesChange, + Status: corev1.ConditionTrue, + }, + { + Type: virtv1.VirtualMachineInstanceIsMigratable, + Status: corev1.ConditionFalse, + Message: "Migration not possible", + }, + })), + expectedStatus: "Migration not possible", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status, err := virtualMachineMigrationStatus(tt.client, "vm", testNamespace, log.WithName(tt.name)) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if status != tt.expectedStatus { + t.Errorf("expected %s, got %s", tt.expectedStatus, status) + t.FailNow() + } + }) + } +} + +func TestCancelLiveMigration(t *testing.T) { + tests := []struct { + name string + client k8sclient.Client + vmVolumes *vmVolumes + expectErr bool + }{ + { + name: "no changed volumes", + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{})), + vmVolumes: &vmVolumes{}, + }, + { + name: "no virtual machine", + client: getFakeClientWithObjs(), + vmVolumes: &vmVolumes{}, + expectErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := cancelLiveMigration(tt.client, "vm", testNamespace, tt.vmVolumes, log.WithName(tt.name)) + if err != nil && !tt.expectErr { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + }) + } +} + +func TestLiveMigrationsCompleted(t *testing.T) { + tests := []struct { + name string + client k8sclient.Client + vmNames []string + expectComplete bool + }{ + { + name: "no VMIMs", + client: getFakeClientWithObjs(), + vmNames: []string{"vm1", "vm2"}, + expectComplete: true, + }, + { + name: "all VMIMs completed, but no matching VMs", + client: getFakeClientWithObjs(createCompletedVirtualMachineMigration("vmim1", testNamespace, "vm")), + vmNames: []string{"vm1", "vm2"}, + expectComplete: true, + }, + { + name: "all VMIMs completed, and one matching VM", + client: getFakeClientWithObjs( + createCompletedVirtualMachineMigration("vmim1", testNamespace, "vm1"), + createCompletedVirtualMachineMigration("vmim2", testNamespace, "vm2")), + vmNames: []string{"vm1"}, + expectComplete: true, + }, + { + name: "not all VMIMs completed, and matching VM", + client: getFakeClientWithObjs( + createCompletedVirtualMachineMigration("vmim1", testNamespace, "vm1"), + createInProgressVirtualMachineMigration("vmim2", testNamespace, "vm2")), + vmNames: []string{"vm1"}, + expectComplete: true, + }, + { + name: "not all VMIMs completed, and matching VM", + client: getFakeClientWithObjs( + createCompletedVirtualMachineMigration("vmim1", testNamespace, "vm1"), + createInProgressVirtualMachineMigration("vmim2", testNamespace, "vm2")), + vmNames: []string{"vm1", "vm2"}, + expectComplete: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + complete, err := liveMigrationsCompleted(tt.client, testNamespace, tt.vmNames) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if complete != tt.expectComplete { + t.Errorf("expected %t, got %t", tt.expectComplete, complete) + t.FailNow() + } + }) + } +} + +func TestUpdateVM(t *testing.T) { + tests := []struct { + name string + client k8sclient.Client + vmName string + sourceVolumes []string + targetVolumes []string + expectedVM *virtv1.VirtualMachine + expectedUpdate bool + }{ + { + name: "name nil vm", + client: getFakeClientWithObjs(), + vmName: "", + }, + { + name: "vm without volumes", + client: getFakeClientWithObjs(createVirtualMachine("vm", testNamespace)), + vmName: "vm", + expectedVM: createVirtualMachine("vm", testNamespace), + expectedUpdate: true, + }, + { + name: "already migrated vm, no update", + client: getFakeClientWithObjs(createVirtualMachineWithUpdateStrategy("vm", testNamespace, []virtv1.Volume{})), + vmName: "vm", + expectedVM: createVirtualMachineWithUpdateStrategy("vm", testNamespace, []virtv1.Volume{}), + expectedUpdate: false, + }, + { + name: "update volumes in VM, no datavolume template", + client: getFakeClientWithObjs( + createDataVolume("volume-source", testNamespace), + createDataVolume("volume-source2", testNamespace), + createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "volume-source", + }, + }, + }, + { + Name: "dv2", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "volume-source2", + }, + }, + }, + })), + vmName: "vm", + expectedVM: createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "volume-target", + }, + }, + }, + { + Name: "dv2", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "volume-target2", + }, + }, + }, + }), + expectedUpdate: true, + sourceVolumes: []string{"volume-source", "volume-source2"}, + targetVolumes: []string{"volume-target", "volume-target2"}, + }, + { + name: "update volume in VM, with datavolume template", + client: getFakeClientWithObjs( + createDataVolume("volume-source", testNamespace), + createVirtualMachineWithTemplateAndVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "volume-source", + }, + }, + }, + })), + vmName: "vm", + expectedVM: createVirtualMachineWithTemplateAndVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "dv", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "volume-target", + }, + }, + }, + }), + expectedUpdate: true, + sourceVolumes: []string{"volume-source"}, + targetVolumes: []string{"volume-target"}, + }, + { + name: "update persisten volumes in VM, no datavolume template", + client: getFakeClientWithObjs( + createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "pvc", + VolumeSource: virtv1.VolumeSource{ + PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "volume-source", + }, + }, + }, + }, + })), + vmName: "vm", + expectedVM: createVirtualMachineWithVolumes("vm", testNamespace, []virtv1.Volume{ + { + Name: "pvc", + VolumeSource: virtv1.VolumeSource{ + PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "volume-target", + }, + }, + }, + }, + }), + expectedUpdate: true, + sourceVolumes: []string{"volume-source"}, + targetVolumes: []string{"volume-target"}, + }} + for _, tt := range tests { + sourceVM := &virtv1.VirtualMachine{} + err := tt.client.Get(context.Background(), k8sclient.ObjectKey{Name: "vm", Namespace: testNamespace}, sourceVM) + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + t.Run(tt.name, func(t *testing.T) { + err := updateVM(tt.client, sourceVM, tt.sourceVolumes, tt.targetVolumes, log.WithName(tt.name)) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if tt.expectedVM != nil { + vm := &virtv1.VirtualMachine{} + err = tt.client.Get(context.Background(), k8sclient.ObjectKey{Name: "vm", Namespace: testNamespace}, vm) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if vm.Spec.Template != nil && len(tt.expectedVM.Spec.Template.Spec.Volumes) != len(vm.Spec.Template.Spec.Volumes) { + t.Errorf("expected volumes to be equal") + t.FailNow() + } else if vm.Spec.Template != nil { + for i, v := range vm.Spec.Template.Spec.Volumes { + if v.VolumeSource.DataVolume != nil { + if v.VolumeSource.DataVolume.Name != tt.expectedVM.Spec.Template.Spec.Volumes[i].VolumeSource.DataVolume.Name { + t.Errorf("expected volumes to be equal") + t.FailNow() + } + } + if v.VolumeSource.PersistentVolumeClaim != nil { + if v.VolumeSource.PersistentVolumeClaim.ClaimName != tt.expectedVM.Spec.Template.Spec.Volumes[i].VolumeSource.PersistentVolumeClaim.ClaimName { + t.Errorf("expected volumes to be equal") + t.FailNow() + } + } + } + for i, tp := range vm.Spec.DataVolumeTemplates { + if tp.Name != tt.expectedVM.Spec.DataVolumeTemplates[i].Name { + t.Errorf("expected data volume templates to be equal") + t.FailNow() + } + } + } + if vm.Spec.UpdateVolumesStrategy == nil || *vm.Spec.UpdateVolumesStrategy != virtv1.UpdateVolumesStrategyMigration { + t.Errorf("expected update volumes strategy to be migration") + t.FailNow() + } + if tt.expectedUpdate { + newVersion, _ := strconv.Atoi(vm.GetResourceVersion()) + oldVersion, _ := strconv.Atoi(sourceVM.GetResourceVersion()) + if newVersion <= oldVersion { + t.Errorf("expected resource version to be updated, originalVersion: %s, updatedVersion: %s", sourceVM.GetResourceVersion(), vm.GetResourceVersion()) + t.FailNow() + } + } else { + if vm.GetResourceVersion() != sourceVM.GetResourceVersion() { + t.Errorf("expected resource version to be the same, originalVersion: %s, updatedVersion: %s", sourceVM.GetResourceVersion(), vm.GetResourceVersion()) + t.FailNow() + } + } + } + }) + } +} + +func TestCreateNewDataVolume(t *testing.T) { + tests := []struct { + name string + client k8sclient.Client + sourceDv *cdiv1.DataVolume + expectedDv *cdiv1.DataVolume + expectedNewDv bool + }{ + { + name: "create new data volume", + client: getFakeClientWithObjs(createDataVolume("source-dv", testNamespace)), + sourceDv: &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-dv", + }, + }, + expectedNewDv: true, + expectedDv: createDataVolume("source-dv", testNamespace), + }, + { + name: "don't update existing new data volume", + client: getFakeClientWithObjs(createDataVolume("source-dv", testNamespace), createDataVolume(targetDv, testNamespace)), + sourceDv: &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-dv", + }, + }, + expectedNewDv: false, + expectedDv: createDataVolume("source-dv", testNamespace), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := CreateNewDataVolume(tt.client, tt.sourceDv.Name, targetDv, testNamespace, log.WithName(tt.name)) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + dv := &cdiv1.DataVolume{} + err = tt.client.Get(context.Background(), k8sclient.ObjectKey{Name: targetDv, Namespace: testNamespace}, dv) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if tt.expectedNewDv { + if dv.GetAnnotations()["cdi.kubevirt.io/allowClaimAdoption"] != "true" { + t.Errorf("expected allowClaimAdoption annotation to be true") + t.FailNow() + } + if dv.Spec.Source == nil { + t.Errorf("expected source to be set") + t.FailNow() + } + if dv.Spec.Source.Blank == nil { + t.Errorf("expected source blank to be set") + t.FailNow() + } + } else { + if _, ok := dv.GetAnnotations()["cdi.kubevirt.io/allowClaimAdoption"]; ok { + t.Errorf("expected allowClaimAdoption annotation to not be set") + t.FailNow() + } + } + }) + } + +} + +func TestTask_getVolumeVMIMInNamespaces(t *testing.T) { + tests := []struct { + name string + sourceNamespace string + task *Task + client compat.Client + expectedVMIM map[string]*virtv1.VirtualMachineInstanceMigration + }{ + { + name: "empty volume name, due to no VMs", + task: &Task{}, + sourceNamespace: sourceNs, + client: getFakeClientWithObjs(), + }, + { + name: "empty volume name, due to no running VMs", + task: &Task{}, + sourceNamespace: sourceNs, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", sourceNs, []virtv1.Volume{ + { + Name: "data", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv", + }, + }, + }, + })), + }, + { + name: "empty volume name, due to no migrations", + task: &Task{}, + sourceNamespace: sourceNs, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", sourceNs, []virtv1.Volume{ + { + Name: "data", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv", + }, + }, + }, + }), createVirtlauncherPod("vm", sourceNs, []string{"dv"})), + }, + { + name: "running VMIM", + task: &Task{}, + sourceNamespace: sourceNs, + client: getFakeClientWithObjs(createVirtualMachineWithVolumes("vm", sourceNs, []virtv1.Volume{ + { + Name: "data", + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv", + }, + }, + }, + }), createVirtlauncherPod("vm", sourceNs, []string{"dv"}), + createCompletedVirtualMachineMigration("vmim", sourceNs, "vm")), + expectedVMIM: map[string]*virtv1.VirtualMachineInstanceMigration{ + "source-ns/dv": createCompletedVirtualMachineMigration("vmim", sourceNs, "vm"), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + out, err := tt.task.getVolumeVMIMInNamespaces([]string{tt.sourceNamespace}) + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + for k, v := range out { + if tt.expectedVMIM[k] == nil { + t.Errorf("unexpected VMIM %s", k) + t.FailNow() + } + if tt.expectedVMIM[k] == nil { + t.Errorf("got unexpected VMIM %s", k) + t.FailNow() + } + if v.Name != tt.expectedVMIM[k].Name { + t.Errorf("expected %s, got %s", tt.expectedVMIM[k].Name, v.Name) + t.FailNow() + } + } + }) + } +} + +func TestGetVMIMElapsedTime(t *testing.T) { + now := metav1.Now() + tests := []struct { + name string + vmim *virtv1.VirtualMachineInstanceMigration + expectedValue metav1.Duration + }{ + { + name: "nil VMIM", + vmim: nil, + expectedValue: metav1.Duration{ + Duration: 0, + }, + }, + { + name: "nil VMIM.Status.MigrationState", + vmim: &virtv1.VirtualMachineInstanceMigration{ + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: nil, + }, + }, + expectedValue: metav1.Duration{ + Duration: 0, + }, + }, + { + name: "VMIM.Status.MigrationState.StartTimestamp nil, and no replacement from PhaseTransition", + vmim: &virtv1.VirtualMachineInstanceMigration{ + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + StartTimestamp: nil, + }, + }, + }, + expectedValue: metav1.Duration{ + Duration: 0, + }, + }, + { + name: "VMIM.Status.MigrationState.StartTimestamp nil, and replacement from PhaseTransition", + vmim: &virtv1.VirtualMachineInstanceMigration{ + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + StartTimestamp: nil, + }, + PhaseTransitionTimestamps: []virtv1.VirtualMachineInstanceMigrationPhaseTransitionTimestamp{ + { + Phase: virtv1.MigrationRunning, + PhaseTransitionTimestamp: metav1.Time{ + Time: now.Add(-1 * time.Hour), + }, + }, + }, + }, + }, + expectedValue: metav1.Duration{ + Duration: time.Hour, + }, + }, + { + name: "VMIM.Status.MigrationState.StartTimestamp set, and end timestamp set", + vmim: &virtv1.VirtualMachineInstanceMigration{ + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + StartTimestamp: &metav1.Time{ + Time: now.Add(-1 * time.Hour), + }, + EndTimestamp: &now, + }, + }, + }, + expectedValue: metav1.Duration{ + Duration: time.Hour, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := getVMIMElapsedTime(tt.vmim) + // Round to nearest to avoid issues with time.Duration precision + // we could still get really unlucky and just be on the edge of a minute + // but it is unlikely + if out.Round(time.Minute) != tt.expectedValue.Round(time.Minute) { + t.Errorf("expected %s, got %s", tt.expectedValue, out) + } + }) + } +} + +func TestTaskGetLastObservedProgressPercent(t *testing.T) { + tests := []struct { + name string + vmName string + sourceNamespace string + currentProgress map[string]*migapi.LiveMigrationProgress + task *Task + client compat.Client + expectedPercent string + }{ + { + name: "valid result from query", + task: &Task{ + PrometheusAPI: prometheusv1.NewAPI(&mockPrometheusClient{ + fakeUrl: "https://fakeurl", + expectedQuery: "query=kubevirt_vmi_migration_data_processed_bytes", + responseBody: "[0,'59.3']", + }), + PromQuery: func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) { + return &model.String{ + Value: "=> 59.3 @", + }, nil, nil + }, + }, + expectedPercent: "59%", + }, + { + name: "invalid result from query", + task: &Task{ + PrometheusAPI: prometheusv1.NewAPI(&mockPrometheusClient{ + fakeUrl: "https://fakeurl", + expectedQuery: "query=kubevirt_vmi_migration_data_processed_bytes", + responseBody: "[0,'59.3']", + }), + PromQuery: func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) { + return &model.String{ + Value: "=> x59.3 @", + }, nil, nil + }, + }, + expectedPercent: "", + }, + { + name: "error result from query", + task: &Task{ + PrometheusAPI: prometheusv1.NewAPI(&mockPrometheusClient{ + fakeUrl: "https://fakeurl", + expectedQuery: "query=kubevirt_vmi_migration_data_processed_bytes", + responseBody: "[0,'59.3']", + }), + PromQuery: func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) { + return &model.String{ + Value: "=> x59.3 @", + }, nil, fmt.Errorf("error") + }, + }, + currentProgress: map[string]*migapi.LiveMigrationProgress{ + "source-ns/vm": &migapi.LiveMigrationProgress{ + LastObservedProgressPercent: "43%", + }, + }, + vmName: "vm", + sourceNamespace: sourceNs, + expectedPercent: "43%", + }, + { + name: "error result from query, no existing progress", + task: &Task{ + PrometheusAPI: prometheusv1.NewAPI(&mockPrometheusClient{ + fakeUrl: "https://fakeurl", + expectedQuery: "query=kubevirt_vmi_migration_data_processed_bytes", + responseBody: "[0,'59.3']", + }), + PromQuery: func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) { + return &model.String{ + Value: "=> x59.3 @", + }, nil, fmt.Errorf("error") + }, + }, + vmName: "vm", + sourceNamespace: sourceNs, + expectedPercent: "", + }, + { + name: "warning result from query, no existing progress", + task: &Task{ + PrometheusAPI: prometheusv1.NewAPI(&mockPrometheusClient{ + fakeUrl: "https://fakeurl", + expectedQuery: "query=kubevirt_vmi_migration_data_processed_bytes", + responseBody: "[0,'59.3']", + }), + PromQuery: func(ctx context.Context, query string, ts time.Time, opts ...prometheusv1.Option) (model.Value, prometheusv1.Warnings, error) { + return &model.String{ + Value: "=> 21.7 @", + }, prometheusv1.Warnings{"warning"}, nil + }, + }, + vmName: "vm", + sourceNamespace: sourceNs, + expectedPercent: "21%", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + percent, err := tt.task.getLastObservedProgressPercent(tt.vmName, tt.sourceNamespace, tt.currentProgress) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if percent != tt.expectedPercent { + t.Errorf("expected %s, got %s", tt.expectedPercent, percent) + } + }) + } +} + +func TestTaskBuildPrometheusAPI(t *testing.T) { + tests := []struct { + name string + task *Task + client compat.Client + apiNil bool + }{ + { + name: "API already built", + task: &Task{ + PrometheusAPI: prometheusv1.NewAPI(&mockPrometheusClient{}), + }, + apiNil: false, + }, + { + name: "API not built, should build", + client: getFakeClientWithObjs( + createControllerConfigMap( + "migration-controller", + migapi.OpenshiftMigrationNamespace, + map[string]string{ + "source-cluster_PROMETHEUS_URL": "http://prometheus", + }, + ), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + Spec: migapi.MigClusterSpec{ + IsHostCluster: true, + }, + }, + }, + restConfig: &rest.Config{}, + }, + apiNil: false, + }, + { + name: "API not built, should build, no URL", + client: getFakeClientWithObjs( + createControllerConfigMap( + "migration-controller", + migapi.OpenshiftMigrationNamespace, + map[string]string{ + "source-cluster_PROMETHEUS_URL": "", + }, + ), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + Spec: migapi.MigClusterSpec{ + IsHostCluster: true, + }, + }, + }, + restConfig: &rest.Config{}, + }, + apiNil: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + err := tt.task.buildPrometheusAPI() + if err != nil { + t.Errorf("unexpected error: %v", err) + t.FailNow() + } + if tt.apiNil && tt.task.PrometheusAPI != nil { + t.Errorf("expected API to be nil") + t.FailNow() + } + }) + } +} + +func TestParseProgress(t *testing.T) { + tests := []struct { + name string + intput string + expectedValue string + }{ + { + name: "Empty string", + intput: "", + expectedValue: "", + }, + { + name: "Valid progress", + intput: "=> 59.3 @", + expectedValue: "59", + }, + { + name: "Invalid progress", + intput: "=> x59.3 @", + expectedValue: "", + }, + { + name: "Invalid progress over 100", + intput: "=> 1159.3 @", + expectedValue: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := parseProgress(tt.intput) + if out != tt.expectedValue { + t.Errorf("expected %s, got %s", tt.expectedValue, out) + } + }) + } +} + +func TestTaskBuildSourcePrometheusEndPointURL(t *testing.T) { + tests := []struct { + name string + client compat.Client + task *Task + expectedError bool + expectedErrorMessage string + expectedValue string + }{ + { + name: "No prometheus config map, should return not found error", + client: getFakeClientWithObjs(), + task: &Task{}, + expectedError: true, + expectedErrorMessage: "not found", + }, + { + name: "Prometheus config map exists, but no prometheus url, should return empty url", + client: getFakeClientWithObjs( + createControllerConfigMap("migration-controller", migapi.OpenshiftMigrationNamespace, nil), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + }, + }, + }, + expectedValue: "", + }, + { + name: "Prometheus config map exists, with prometheus url, should return correct url", + client: getFakeClientWithObjs( + createControllerConfigMap( + "migration-controller", + migapi.OpenshiftMigrationNamespace, + map[string]string{ + "source-cluster_PROMETHEUS_URL": "http://prometheus", + }, + ), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + }, + }, + }, + expectedValue: "https://prometheus", + }, + { + name: "Prometheus config map exists, but no prometheus url, but route, should return route url", + client: getFakeClientWithObjs( + createControllerConfigMap("migration-controller", migapi.OpenshiftMigrationNamespace, nil), + createRoute("prometheus-route", "openshift-monitoring", "https://route.prometheus"), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + }, + }, + }, + expectedValue: "https://route.prometheus", + }, + { + name: "Prometheus config map exists, but no prometheus url, but route, should return route url", + client: getFakeClientWithObjs( + createControllerConfigMap("migration-controller", migapi.OpenshiftMigrationNamespace, nil), + createRoute("prometheus-route", "openshift-monitoring", "http://route.prometheus"), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + }, + }, + }, + expectedValue: "https://route.prometheus", + }, + { + name: "Prometheus config map exists, with invalid prometheus url, should return blank", + client: getFakeClientWithObjs( + createControllerConfigMap("migration-controller", migapi.OpenshiftMigrationNamespace, + map[string]string{ + "source-cluster_PROMETHEUS_URL": "%#$invalid", + }, + ), + ), + task: &Task{ + PlanResources: &migapi.PlanResources{ + SrcMigCluster: &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "source-cluster", + }, + }, + }, + }, + expectedValue: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + value, err := tt.task.buildSourcePrometheusEndPointURL() + if tt.expectedError { + if err == nil { + t.Errorf("expected error but got nil") + t.FailNow() + } + if !strings.Contains(err.Error(), tt.expectedErrorMessage) { + t.Errorf("expected error message to contain %s, got %s", tt.expectedErrorMessage, err.Error()) + } + } + if value != tt.expectedValue { + t.Errorf("expected %s, got %s", tt.expectedValue, value) + } + }) + } +} + +func TestTaskDeleteStaleVirtualMachineInstanceMigrations(t *testing.T) { + tests := []struct { + name string + client compat.Client + task *Task + expectedError bool + expectedMsg string + expectedVMIMs []*virtv1.VirtualMachineInstanceMigration + }{ + { + name: "No pvcs in either namespace, but persistent volume claims in DVM, should error on missing PVCs", + client: getFakeClientWithObjs(), + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []v1alpha1.PVCToMigrate{ + { + ObjectReference: &corev1.ObjectReference{ + Name: sourcePVC, + Namespace: sourceNs, + }, + TargetName: targetPVC, + TargetNamespace: targetNs, + }, + }, + }, + }, + }, + expectedError: true, + expectedMsg: "persistentvolumeclaims \"source-pvc\" not found", + }, + { + name: "PVCs in different namespaces, and persistent volume claims in DVM, should error on mismatched namespaces", + client: getFakeClientWithObjs(createPvc(sourcePVC, sourceNs), createPvc(targetPVC, targetNs)), + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []v1alpha1.PVCToMigrate{ + { + ObjectReference: &corev1.ObjectReference{ + Name: sourcePVC, + Namespace: sourceNs, + }, + TargetName: targetPVC, + TargetNamespace: targetNs, + }, + }, + }, + }, + }, + expectedError: true, + expectedMsg: "source and target namespaces must match", + }, + { + name: "PVCs in same namespace, and persistent volume claims in DVM, but no VMs or VMIMs, should not return error", + client: getFakeClientWithObjs(createPvc(sourcePVC, sourceNs), createPvc(targetPVC, sourceNs)), + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + Spec: v1alpha1.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []v1alpha1.PVCToMigrate{ + { + ObjectReference: &corev1.ObjectReference{ + Name: sourcePVC, + Namespace: sourceNs, + }, + TargetName: targetPVC, + TargetNamespace: sourceNs, + }, + }, + }, + }, + }, + }, + { + name: "PVCs in same namespace, and persistent volume claims in DVM, VMs and VMIMs created before the DVM, should not return error, should delete VMIM", + client: getFakeClientWithObjs( + createPvc(sourcePVC, sourceNs), + createPvc(targetPVC, sourceNs), + createVirtualMachine("vm", sourceNs), + createCompletedVirtualMachineMigration("vmim", sourceNs, "vm"), + ), + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + CreationTimestamp: metav1.Now(), + }, + Spec: v1alpha1.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []v1alpha1.PVCToMigrate{ + { + ObjectReference: &corev1.ObjectReference{ + Name: sourcePVC, + Namespace: sourceNs, + }, + TargetName: targetPVC, + TargetNamespace: sourceNs, + }, + }, + }, + }, + }, + expectedVMIMs: []*virtv1.VirtualMachineInstanceMigration{}, + }, + { + name: "PVCs in same namespace, and persistent volume claims in DVM, VMs and VMIMs created after the DVM, should not return error, should NOT delete VMIM", + client: getFakeClientWithObjs( + createPvc(sourcePVC, sourceNs), + createPvc(targetPVC, sourceNs), + createVirtualMachine("vm", sourceNs), + createCompletedVirtualMachineMigration("vmim", sourceNs, "vm"), + ), + task: &Task{ + Owner: &v1alpha1.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + CreationTimestamp: metav1.Time{ + Time: metav1.Now().Add(-2 * time.Hour), + }, + }, + Spec: v1alpha1.DirectVolumeMigrationSpec{ + PersistentVolumeClaims: []v1alpha1.PVCToMigrate{ + { + ObjectReference: &corev1.ObjectReference{ + Name: sourcePVC, + Namespace: sourceNs, + }, + TargetName: targetPVC, + TargetNamespace: sourceNs, + }, + }, + }, + }, + }, + expectedVMIMs: []*virtv1.VirtualMachineInstanceMigration{ + createCompletedVirtualMachineMigration("vmim", sourceNs, "vm"), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + tt.task.destinationClient = tt.client + err := tt.task.deleteStaleVirtualMachineInstanceMigrations() + if tt.expectedError { + if err == nil { + t.Errorf("expected error but got nil") + t.FailNow() + } else if !strings.Contains(err.Error(), tt.expectedMsg) { + t.Errorf("expected error message to contain %s, got %s", tt.expectedMsg, err.Error()) + } + } + remainingVMIMs := &virtv1.VirtualMachineInstanceMigrationList{} + err = tt.client.List(context.Background(), remainingVMIMs, k8sclient.InNamespace(sourceNs), &k8sclient.ListOptions{}) + if err != nil { + t.Errorf("error listing VMIMs: %v", err) + t.FailNow() + } + if len(remainingVMIMs.Items) != len(tt.expectedVMIMs) { + t.Errorf("expected %d VMIMs, got %d", len(tt.expectedVMIMs), len(remainingVMIMs.Items)) + t.FailNow() + } + for _, remainingVMIM := range remainingVMIMs.Items { + found := false + for _, expectedVMIM := range tt.expectedVMIMs { + if remainingVMIM.Name == expectedVMIM.Name { + found = true + break + } + } + if !found { + t.Errorf("unexpected VMIM %s", remainingVMIM.Name) + t.FailNow() + } + } + }) + } +} + +func getFakeClientWithObjs(obj ...k8sclient.Object) compat.Client { + client, _ := fakecompat.NewFakeClient(obj...) + return client +} + +func createVirtualMachine(name, namespace string) *virtv1.VirtualMachine { + return &virtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } +} + +func createCompletedVirtualMachineMigration(name, namespace, vmName string) *virtv1.VirtualMachineInstanceMigration { + return &virtv1.VirtualMachineInstanceMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: virtv1.VirtualMachineInstanceMigrationSpec{ + VMIName: vmName, + }, + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + Phase: virtv1.MigrationSucceeded, + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + EndTimestamp: &metav1.Time{ + Time: metav1.Now().Add(-1 * time.Hour), + }, + Completed: true, + }, + }, + } +} + +func createFailedVirtualMachineMigration(name, namespace, vmName, failedMessage string) *virtv1.VirtualMachineInstanceMigration { + return &virtv1.VirtualMachineInstanceMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: virtv1.VirtualMachineInstanceMigrationSpec{ + VMIName: vmName, + }, + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + Phase: virtv1.MigrationFailed, + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + EndTimestamp: &metav1.Time{ + Time: metav1.Now().Add(-1 * time.Hour), + }, + FailureReason: failedMessage, + Failed: true, + }, + }, + } +} + +func createCanceledVirtualMachineMigration(name, namespace, vmName string, reason virtv1.MigrationAbortStatus) *virtv1.VirtualMachineInstanceMigration { + return &virtv1.VirtualMachineInstanceMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: virtv1.VirtualMachineInstanceMigrationSpec{ + VMIName: vmName, + }, + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + Phase: virtv1.MigrationFailed, + MigrationState: &virtv1.VirtualMachineInstanceMigrationState{ + EndTimestamp: &metav1.Time{ + Time: metav1.Now().Add(-1 * time.Hour), + }, + AbortStatus: reason, + }, + }, + } +} + +func createInProgressVirtualMachineMigration(name, namespace, vmName string) *virtv1.VirtualMachineInstanceMigration { + return &virtv1.VirtualMachineInstanceMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: virtv1.VirtualMachineInstanceMigrationSpec{ + VMIName: vmName, + }, + Status: virtv1.VirtualMachineInstanceMigrationStatus{ + Phase: virtv1.MigrationRunning, + }, + } +} + +func createVirtualMachineWithVolumes(name, namespace string, volumes []virtv1.Volume) *virtv1.VirtualMachine { + vm := createVirtualMachine(name, namespace) + vm.Spec = virtv1.VirtualMachineSpec{ + Template: &virtv1.VirtualMachineInstanceTemplateSpec{ + Spec: virtv1.VirtualMachineInstanceSpec{ + Volumes: volumes, + }, + }, + } + return vm +} + +func createVirtualMachineWithUpdateStrategy(name, namespace string, volumes []virtv1.Volume) *virtv1.VirtualMachine { + vm := createVirtualMachineWithVolumes(name, namespace, volumes) + vm.Spec.UpdateVolumesStrategy = ptr.To[virtv1.UpdateVolumesStrategy](virtv1.UpdateVolumesStrategyMigration) + return vm +} + +func createVirtualMachineWithTemplateAndVolumes(name, namespace string, volumes []virtv1.Volume) *virtv1.VirtualMachine { + vm := createVirtualMachineWithVolumes(name, namespace, volumes) + for _, volume := range volumes { + vm.Spec.DataVolumeTemplates = append(vm.Spec.DataVolumeTemplates, virtv1.DataVolumeTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: volume.DataVolume.Name, + }, + }) + } + return vm +} + +func createControllerConfigMap(name, namespace string, data map[string]string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: data, + } +} + +func createRoute(name, namespace, url string) *routev1.Route { + return &routev1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/part-of": "openshift-monitoring", + }, + }, + Spec: routev1.RouteSpec{ + To: routev1.RouteTargetReference{ + Name: prometheusRoute, + }, + Host: url, + }, + } +} + +type mockPrometheusClient struct { + fakeUrl string + responseBody string + expectedQuery string +} + +func (m *mockPrometheusClient) URL(ep string, args map[string]string) *url.URL { + url, err := url.Parse(m.fakeUrl) + if err != nil { + panic(err) + } + return url +} + +func (m *mockPrometheusClient) Do(ctx context.Context, req *http.Request) (*http.Response, []byte, error) { + if req.Body != nil { + defer req.Body.Close() + } + b, err := io.ReadAll(req.Body) + queryBody := string(b) + if !strings.Contains(queryBody, m.expectedQuery) { + return nil, nil, fmt.Errorf("expected query %s, got %s", m.expectedQuery, queryBody) + } + if err != nil { + return nil, nil, err + } + + out := []byte(m.responseBody) + + t := &http.Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Body: io.NopCloser(bytes.NewBuffer(out)), + ContentLength: int64(len(out)), + Request: req, + Header: make(http.Header, 0), + } + return t, out, nil +} + +func createVirtlauncherPod(vmName, namespace string, dataVolumes []string) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("virt-launcher-%s", vmName), + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{ + { + Name: vmName, + Kind: "VirtualMachineInstance", + }, + }, + }, + Spec: corev1.PodSpec{}, + } + for _, dv := range dataVolumes { + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: dv, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: dv, + }, + }, + }) + } + return pod +} + +func createDataVolume(name, namespace string) *cdiv1.DataVolume { + return &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } +} + +func createVirtualMachineInstance(name, namespace string, phase virtv1.VirtualMachineInstancePhase) *virtv1.VirtualMachineInstance { + return &virtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Status: virtv1.VirtualMachineInstanceStatus{ + Phase: phase, + }, + } +} + +func createVirtualMachineInstanceWithConditions(name, namespace string, conditions []virtv1.VirtualMachineInstanceCondition) *virtv1.VirtualMachineInstance { + vm := createVirtualMachineInstance(name, namespace, virtv1.Running) + vm.Status.Conditions = append(vm.Status.Conditions, conditions...) + return vm +} diff --git a/pkg/controller/migmigration/description.go b/pkg/controller/migmigration/description.go index c2e67db6f..15fc9a165 100644 --- a/pkg/controller/migmigration/description.go +++ b/pkg/controller/migmigration/description.go @@ -52,8 +52,10 @@ var PhaseDescriptions = map[string]string{ WaitForResticReady: "Waiting for Restic Pods to restart, ensuring latest PVC mounts are available for PVC backups.", RestartVelero: "Restarting Velero Pods, ensuring work queue is empty.", WaitForVeleroReady: "Waiting for Velero Pods to restart, ensuring work queue is empty.", - QuiesceApplications: "Quiescing (Scaling to 0 replicas): Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines.", - EnsureQuiesced: "Waiting for Quiesce (Scaling to 0 replicas) to finish for Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines.", + QuiesceSourceApplications: "Quiescing (Scaling to 0 replicas): Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines in source namespace.", + QuiesceDestinationApplications: "Quiescing (Scaling to 0 replicas): Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines in target namespace.", + EnsureSrcQuiesced: "Waiting for Quiesce (Scaling to 0 replicas) to finish for Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines in source namespace.", + EnsureDestQuiesced: "Waiting for Quiesce (Scaling to 0 replicas) to finish for Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines in target namespace.", UnQuiesceSrcApplications: "UnQuiescing (Scaling to N replicas) source cluster Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines.", UnQuiesceDestApplications: "UnQuiescing (Scaling to N replicas) target cluster Deployments, DeploymentConfigs, StatefulSets, ReplicaSets, DaemonSets, CronJobs, Jobs and Virtual Machines.", EnsureStageBackup: "Creating a stage backup.", @@ -70,7 +72,9 @@ var PhaseDescriptions = map[string]string{ Verification: "Verifying health of migrated Pods.", Rollback: "Starting rollback", CreateDirectImageMigration: "Creating Direct Image Migration", - CreateDirectVolumeMigration: "Creating Direct Volume Migration", + CreateDirectVolumeMigrationStage: "Creating Direct Volume Migration for staging", + CreateDirectVolumeMigrationFinal: "Creating Direct Volume Migration for cutover", + CreateDirectVolumeMigrationRollback: "Creating Direct Volume Migration for rollback", WaitForDirectImageMigrationToComplete: "Waiting for Direct Image Migration to complete.", WaitForDirectVolumeMigrationToComplete: "Waiting for Direct Volume Migration to complete.", EnsureStagePodsDeleted: "Deleting any leftover stage Pods.", diff --git a/pkg/controller/migmigration/dvm.go b/pkg/controller/migmigration/dvm.go index 3ccd70a75..7cbc1aca0 100644 --- a/pkg/controller/migmigration/dvm.go +++ b/pkg/controller/migmigration/dvm.go @@ -17,7 +17,7 @@ import ( k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) -func (t *Task) createDirectVolumeMigration() error { +func (t *Task) createDirectVolumeMigration(migType *migapi.DirectVolumeMigrationType) error { existingDvm, err := t.getDirectVolumeMigration() if err != nil { return err @@ -31,11 +31,13 @@ func (t *Task) createDirectVolumeMigration() error { if dvm == nil { return errors.New("failed to build directvolumeclaim list") } + if migType != nil { + dvm.Spec.MigrationType = migType + } t.Log.Info("Creating DirectVolumeMigration on host cluster", "directVolumeMigration", path.Join(dvm.Namespace, dvm.Name)) err = t.Client.Create(context.TODO(), dvm) return err - } func (t *Task) buildDirectVolumeMigration() *migapi.DirectVolumeMigration { @@ -46,6 +48,10 @@ func (t *Task) buildDirectVolumeMigration() *migapi.DirectVolumeMigration { if pvcList == nil { return nil } + migrationType := migapi.MigrationTypeStage + if t.Owner.Spec.QuiescePods { + migrationType = migapi.MigrationTypeFinal + } dvm := &migapi.DirectVolumeMigration{ ObjectMeta: metav1.ObjectMeta{ Labels: labels, @@ -57,6 +63,8 @@ func (t *Task) buildDirectVolumeMigration() *migapi.DirectVolumeMigration { DestMigClusterRef: t.PlanResources.DestMigCluster.GetObjectReference(), PersistentVolumeClaims: pvcList, CreateDestinationNamespaces: true, + LiveMigrate: t.PlanResources.MigPlan.Spec.LiveMigrate, + MigrationType: &migrationType, }, } migapi.SetOwnerReference(t.Owner, t.Owner, dvm) @@ -86,29 +94,15 @@ func (t *Task) getDirectVolumeMigration() (*migapi.DirectVolumeMigration, error) // Check if the DVM has completed. // Returns if it has completed, why it failed, and it's progress results func (t *Task) hasDirectVolumeMigrationCompleted(dvm *migapi.DirectVolumeMigration) (completed bool, failureReasons, progress []string) { - totalVolumes := len(dvm.Spec.PersistentVolumeClaims) - successfulPods := 0 - failedPods := 0 - runningPods := 0 - if dvm.Status.SuccessfulPods != nil { - successfulPods = len(dvm.Status.SuccessfulPods) - } - if dvm.Status.FailedPods != nil { - failedPods = len(dvm.Status.FailedPods) - } - if dvm.Status.RunningPods != nil { - runningPods = len(dvm.Status.RunningPods) - } - volumeProgress := fmt.Sprintf("%v total volumes; %v successful; %v running; %v failed", - totalVolumes, - successfulPods, - runningPods, - failedPods) + len(dvm.Spec.PersistentVolumeClaims), + len(dvm.Status.SuccessfulPods)+len(dvm.Status.SuccessfulLiveMigrations), + len(dvm.Status.RunningPods)+len(dvm.Status.FailedLiveMigrations), + len(dvm.Status.FailedPods)+len(dvm.Status.RunningLiveMigrations)) switch { - //case dvm.Status.Phase != "" && dvm.Status.Phase != dvmc.Completed: - // // TODO: Update this to check on the associated dvmp resources and build up a progress indicator back to - case dvm.Status.Phase == dvmc.Completed && dvm.Status.Itinerary == "VolumeMigration" && dvm.Status.HasCondition(dvmc.Succeeded): + // case dvm.Status.Phase != "" && dvm.Status.Phase != dvmc.Completed: + // TODO: Update this to check on the associated dvmp resources and build up a progress indicator back to + case dvm.Status.Phase == dvmc.Completed && (dvm.Status.Itinerary == dvmc.VolumeMigrationItinerary || dvm.Status.Itinerary == dvmc.VolumeMigrationRollbackItinerary) && dvm.Status.HasCondition(dvmc.Succeeded): // completed successfully completed = true case (dvm.Status.Phase == dvmc.MigrationFailed || dvm.Status.Phase == dvmc.Completed) && dvm.Status.HasCondition(dvmc.Failed): @@ -117,14 +111,14 @@ func (t *Task) hasDirectVolumeMigrationCompleted(dvm *migapi.DirectVolumeMigrati default: progress = append(progress, volumeProgress) } - progress = append(progress, t.getDVMPodProgress(*dvm)...) + progress = append(progress, t.getDVMProgress(dvm)...) // sort the progress report so we dont have flapping for the same progress info sort.Strings(progress) return completed, failureReasons, progress } -func (t *Task) getWarningForDVM(dvm *migapi.DirectVolumeMigration) (*migapi.Condition, error) { +func (t *Task) getWarningForDVM(dvm *migapi.DirectVolumeMigration) *migapi.Condition { conditions := dvm.Status.Conditions.FindConditionByCategory(dvmc.Warn) if len(conditions) > 0 { return &migapi.Condition{ @@ -133,10 +127,10 @@ func (t *Task) getWarningForDVM(dvm *migapi.DirectVolumeMigration) (*migapi.Cond Reason: migapi.NotReady, Category: migapi.Warn, Message: joinConditionMessages(conditions), - }, nil + } } - return nil, nil + return nil } func joinConditionMessages(conditions []*migapi.Condition) string { @@ -160,7 +154,12 @@ func (t *Task) setDirectVolumeMigrationFailureWarning(dvm *migapi.DirectVolumeMi }) } -func (t *Task) getDVMPodProgress(dvm migapi.DirectVolumeMigration) []string { +func (t *Task) getDVMProgress(dvm *migapi.DirectVolumeMigration) []string { + progress := getDVMPodProgress(dvm) + return append(progress, getDVMLiveMigrationProgress(dvm)...) +} + +func getDVMPodProgress(dvm *migapi.DirectVolumeMigration) []string { progress := []string{} podProgressIterator := map[string][]*migapi.PodProgress{ "Pending": dvm.Status.PendingPods, @@ -202,6 +201,44 @@ func (t *Task) getDVMPodProgress(dvm migapi.DirectVolumeMigration) []string { return progress } +// Live migration progress +func getDVMLiveMigrationProgress(dvm *migapi.DirectVolumeMigration) []string { + progress := []string{} + if dvm == nil { + return progress + } + liveMigrationProgressIterator := map[string][]*migapi.LiveMigrationProgress{ + "Running": dvm.Status.RunningLiveMigrations, + "Completed": dvm.Status.SuccessfulLiveMigrations, + "Failed": dvm.Status.FailedLiveMigrations, + "Pending": dvm.Status.PendingLiveMigrations, + } + for state, liveMigrations := range liveMigrationProgressIterator { + for _, liveMigration := range liveMigrations { + p := fmt.Sprintf( + "[%s] Live Migration %s: %s", + liveMigration.PVCReference.Name, + path.Join(liveMigration.VMNamespace, liveMigration.VMName), + state) + if liveMigration.Message != "" { + p += fmt.Sprintf(" %s", liveMigration.Message) + } + if liveMigration.LastObservedProgressPercent != "" { + p += fmt.Sprintf(" %s", liveMigration.LastObservedProgressPercent) + } + if liveMigration.LastObservedTransferRate != "" { + p += fmt.Sprintf(" (Transfer rate %s)", liveMigration.LastObservedTransferRate) + } + if liveMigration.TotalElapsedTime != nil { + p += fmt.Sprintf(" (%s)", liveMigration.TotalElapsedTime.Duration.Round(time.Second)) + } + progress = append(progress, p) + } + + } + return progress +} + func (t *Task) getDirectVolumeClaimList() []migapi.PVCToMigrate { nsMapping := t.PlanResources.MigPlan.GetNamespaceMapping() var pvcList []migapi.PVCToMigrate diff --git a/pkg/controller/migmigration/dvm_test.go b/pkg/controller/migmigration/dvm_test.go index 15f1c5a19..0489c8e6c 100644 --- a/pkg/controller/migmigration/dvm_test.go +++ b/pkg/controller/migmigration/dvm_test.go @@ -2,11 +2,13 @@ package migmigration import ( "reflect" + "slices" "testing" migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" dvmc "github.com/konveyor/mig-controller/pkg/controller/directvolumemigration" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestTask_hasDirectVolumeMigrationCompleted(t1 *testing.T) { @@ -160,3 +162,134 @@ func TestTask_hasDirectVolumeMigrationCompleted(t1 *testing.T) { }) } } + +func TestTask_getDVMLiveMigrationProgress(t *testing.T) { + tests := []struct { + name string + dvm *migapi.DirectVolumeMigration + expectedProgress []string + }{ + { + name: "no dvm", + }, + { + name: "dvm with running live migrations, no message, no progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + RunningLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "vm-0", + VMNamespace: "ns", + PVCReference: &v1.ObjectReference{ + Name: "pvc-0", + Kind: "PersistentVolumeClaim", + APIVersion: "", + }, + }, + }, + }, + }, + expectedProgress: []string{"[pvc-0] Live Migration ns/vm-0: Running"}, + }, + { + name: "dvm with failed live migrations, message, no progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + FailedLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "vm-0", + VMNamespace: "ns", + PVCReference: &v1.ObjectReference{ + Name: "pvc-0", + Kind: "PersistentVolumeClaim", + APIVersion: "", + }, + Message: "Failed because of test", + }, + }, + }, + }, + expectedProgress: []string{"[pvc-0] Live Migration ns/vm-0: Failed Failed because of test"}, + }, + { + name: "dvm with completed live migrations, message, progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + SuccessfulLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "vm-0", + VMNamespace: "ns", + PVCReference: &v1.ObjectReference{ + Name: "pvc-0", + Kind: "PersistentVolumeClaim", + APIVersion: "", + }, + Message: "Successfully completed", + LastObservedProgressPercent: "100%", + }, + }, + }, + }, + expectedProgress: []string{"[pvc-0] Live Migration ns/vm-0: Completed Successfully completed 100%"}, + }, + { + name: "dvm with pending live migrations, message, blank progress", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + PendingLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "vm-0", + VMNamespace: "ns", + PVCReference: &v1.ObjectReference{ + Name: "pvc-0", + Kind: "PersistentVolumeClaim", + APIVersion: "", + }, + Message: "Pending", + LastObservedProgressPercent: "", + }, + }, + }, + }, + expectedProgress: []string{"[pvc-0] Live Migration ns/vm-0: Pending Pending"}, + }, + { + name: "dvm with running live migrations, message, progress, transferrate, and elapsed time", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + RunningLiveMigrations: []*migapi.LiveMigrationProgress{ + { + VMName: "vm-0", + VMNamespace: "ns", + PVCReference: &v1.ObjectReference{ + Name: "pvc-0", + Kind: "PersistentVolumeClaim", + APIVersion: "", + }, + Message: "Running", + LastObservedProgressPercent: "50%", + LastObservedTransferRate: "10MB/s", + TotalElapsedTime: &metav1.Duration{ + Duration: 1000, + }, + }, + }, + }, + }, + expectedProgress: []string{"[pvc-0] Live Migration ns/vm-0: Running Running 50% (Transfer rate 10MB/s) (0s)"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + res := getDVMLiveMigrationProgress(tt.dvm) + if len(res) != len(tt.expectedProgress) { + t.Errorf("getDVMLiveMigrationProgress() = %v, want %v", res, tt.expectedProgress) + } + for _, p := range tt.expectedProgress { + if !slices.Contains(res, p) { + t.Errorf("getDVMLiveMigrationProgress() = %v, want %v", res, tt.expectedProgress) + } + } + }) + } +} diff --git a/pkg/controller/migmigration/migmigration_controller_test.go b/pkg/controller/migmigration/migmigration_controller_test.go index 71bc1b618..53f21d37f 100644 --- a/pkg/controller/migmigration/migmigration_controller_test.go +++ b/pkg/controller/migmigration/migmigration_controller_test.go @@ -17,7 +17,6 @@ limitations under the License. package migmigration import ( - "reflect" "testing" "time" @@ -103,5 +102,5 @@ func Test_Itineraries(t *testing.T) { } } - g.Expect(reflect.DeepEqual(stage.Phases, common.Phases)).To(gomega.BeTrue()) + g.Expect(stage.Phases).To(gomega.ContainElements(common.Phases)) } diff --git a/pkg/controller/migmigration/quiesce.go b/pkg/controller/migmigration/quiesce.go index 5a36e934c..c8579109a 100644 --- a/pkg/controller/migmigration/quiesce.go +++ b/pkg/controller/migmigration/quiesce.go @@ -29,8 +29,7 @@ const ( vmSubresourceURLFmt = "/apis/subresources.kubevirt.io/%s/namespaces/%s/virtualmachines/%s/%s" ) -// Quiesce applications on source cluster -func (t *Task) quiesceApplications() error { +func (t *Task) quiesceSourceApplications() error { client, err := t.getSourceClient() if err != nil { return liberr.Wrap(err) @@ -39,7 +38,24 @@ func (t *Task) quiesceApplications() error { if err != nil { return liberr.Wrap(err) } - err = t.quiesceCronJobs(client) + return t.quiesceApplications(client, restConfig) +} + +func (t *Task) quiesceDestinationApplications() error { + client, err := t.getDestinationClient() + if err != nil { + return liberr.Wrap(err) + } + restConfig, err := t.getDestinationRestConfig() + if err != nil { + return liberr.Wrap(err) + } + return t.quiesceApplications(client, restConfig) +} + +// Quiesce applications on source cluster +func (t *Task) quiesceApplications(client compat.Client, restConfig *rest.Config) error { + err := t.quiesceCronJobs(client) if err != nil { return liberr.Wrap(err) } @@ -67,15 +83,16 @@ func (t *Task) quiesceApplications() error { if err != nil { return liberr.Wrap(err) } - restClient, err := t.createRestClient(restConfig) - if err != nil { - return liberr.Wrap(err) - } - err = t.quiesceVirtualMachines(client, restClient) - if err != nil { - return liberr.Wrap(err) + if !t.PlanResources.MigPlan.LiveMigrationChecked() { + restClient, err := t.createRestClient(restConfig) + if err != nil { + return liberr.Wrap(err) + } + err = t.quiesceVirtualMachines(client, restClient) + if err != nil { + return liberr.Wrap(err) + } } - return nil } @@ -88,7 +105,7 @@ func (t *Task) unQuiesceSrcApplications() error { if err != nil { return liberr.Wrap(err) } - t.Log.Info("Unquiescing applications on source cluster.") + t.Log.V(3).Info("Unquiescing applications on source cluster.") err = t.unQuiesceApplications(srcClient, restConfig, t.sourceNamespaces()) if err != nil { return liberr.Wrap(err) @@ -105,7 +122,7 @@ func (t *Task) unQuiesceDestApplications() error { if err != nil { return liberr.Wrap(err) } - t.Log.Info("Unquiescing applications on destination cluster.") + t.Log.V(3).Info("Unquiescing applications on destination cluster.") err = t.unQuiesceApplications(destClient, restConfig, t.destinationNamespaces()) if err != nil { return liberr.Wrap(err) @@ -143,15 +160,16 @@ func (t *Task) unQuiesceApplications(client compat.Client, restConfig *rest.Conf if err != nil { return liberr.Wrap(err) } - restClient, err := t.createRestClient(restConfig) - if err != nil { - return liberr.Wrap(err) - } - err = t.unQuiesceVirtualMachines(client, restClient, namespaces) - if err != nil { - return liberr.Wrap(err) + if !t.PlanResources.MigPlan.LiveMigrationChecked() { + restClient, err := t.createRestClient(restConfig) + if err != nil { + return liberr.Wrap(err) + } + err = t.unQuiesceVirtualMachines(client, restClient, namespaces) + if err != nil { + return liberr.Wrap(err) + } } - return nil } @@ -971,27 +989,41 @@ func (t *Task) startVM(vm *virtv1.VirtualMachine, client k8sclient.Client, restC return nil } +func (t *Task) ensureSourceQuiescedPodsTerminated() (bool, error) { + client, err := t.getSourceClient() + if err != nil { + return false, liberr.Wrap(err) + } + return t.ensureQuiescedPodsTerminated(client, t.sourceNamespaces()) +} + +func (t *Task) ensureDestinationQuiescedPodsTerminated() (bool, error) { + client, err := t.getDestinationClient() + if err != nil { + return false, liberr.Wrap(err) + } + return t.ensureQuiescedPodsTerminated(client, t.destinationNamespaces()) +} + // Ensure scaled down pods have terminated. // Returns: `true` when all pods terminated. -func (t *Task) ensureQuiescedPodsTerminated() (bool, error) { +func (t *Task) ensureQuiescedPodsTerminated(client compat.Client, namespaces []string) (bool, error) { kinds := map[string]bool{ "ReplicationController": true, "StatefulSet": true, "ReplicaSet": true, "DaemonSet": true, "Job": true, - "VirtualMachine": true, + } + if !t.PlanResources.MigPlan.LiveMigrationChecked() { + kinds["VirtualMachineInstance"] = true } skippedPhases := map[v1.PodPhase]bool{ v1.PodSucceeded: true, v1.PodFailed: true, v1.PodUnknown: true, } - client, err := t.getSourceClient() - if err != nil { - return false, liberr.Wrap(err) - } - for _, ns := range t.sourceNamespaces() { + for _, ns := range namespaces { list := v1.PodList{} options := k8sclient.InNamespace(ns) err := client.List( diff --git a/pkg/controller/migmigration/quiesce_test.go b/pkg/controller/migmigration/quiesce_test.go index 6c9e15931..654789d33 100644 --- a/pkg/controller/migmigration/quiesce_test.go +++ b/pkg/controller/migmigration/quiesce_test.go @@ -362,6 +362,162 @@ func TestUnQuiesceVirtualMachine(t *testing.T) { } } +func TestEnsureDestinationQuiescedPodsTerminated(t *testing.T) { + tests := []struct { + name string + client compat.Client + task *Task + allterminated bool + }{ + { + name: "no pods", + client: getFakeClientWithObjs(), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: true, + }, + { + name: "no pods with owner ref", + client: getFakeClientWithObjs(createPodWithOwner("pod", "tgt-namespace", "", "", "")), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: true, + }, + { + name: "pods with deployment owner ref", + client: getFakeClientWithObjs(createPodWithOwner("pod", "tgt-namespace", "v1", "ReplicaSet", "deployment")), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: false, + }, + { + name: "skipped pods with vm owner ref", + client: getFakeClientWithObjs(createPodWithOwnerAndPhase("pod", "tgt-namespace", "v1", "VirtualMachineInstance", "virt-launcher", corev1.PodSucceeded)), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.destinationClient = tt.client + allTerminated, err := tt.task.ensureDestinationQuiescedPodsTerminated() + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if allTerminated != tt.allterminated { + t.Errorf("ensureDestinationQuiescedPodsTerminated() allTerminated = %v, want %v", allTerminated, tt.allterminated) + } + }) + } +} + +func TestEnsureSourceQuiescedPodsTerminated(t *testing.T) { + tests := []struct { + name string + client compat.Client + task *Task + allterminated bool + }{ + { + name: "no pods", + client: getFakeClientWithObjs(), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: true, + }, + { + name: "no pods with owner ref", + client: getFakeClientWithObjs(createPodWithOwner("pod", "src-namespace", "", "", "")), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: true, + }, + { + name: "pods with deployment owner ref", + client: getFakeClientWithObjs(createPodWithOwner("pod", "src-namespace", "v1", "ReplicationController", "controller")), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: false, + }, + { + name: "skipped pods with vm owner ref", + client: getFakeClientWithObjs(createPodWithOwnerAndPhase("pod", "src-namespace", "v1", "VirtualMachineInstance", "virt-launcher", corev1.PodFailed)), + task: &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"src-namespace:tgt-namespace"}, + }, + }, + }, + }, + allterminated: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.task.sourceClient = tt.client + allTerminated, err := tt.task.ensureSourceQuiescedPodsTerminated() + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + if allTerminated != tt.allterminated { + t.Errorf("ensureDestinationQuiescedPodsTerminated() allTerminated = %v, want %v", allTerminated, tt.allterminated) + } + }) + } +} + func getFakeClientWithObjs(obj ...k8sclient.Object) compat.Client { client, _ := fakecompat.NewFakeClient(obj...) return client @@ -401,17 +557,37 @@ func createVMWithAnnotation(name, namespace string, ann map[string]string) *virt } func createVirtlauncherPod(vmName, namespace string) *corev1.Pod { - return &corev1.Pod{ + pod := createPodWithOwner(vmName+"-virt-launcher", namespace, "kubevirt.io/v1", "VirtualMachineInstance", vmName) + pod.Labels = map[string]string{ + "kubevirt.io": "virt-launcher", + } + return pod +} + +func createPodWithOwner(name, namespace, apiversion, kind, ownerName string) *corev1.Pod { + pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: vmName + "-virt-launcher", + Name: name, Namespace: namespace, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "kubevirt.io/v1", - Kind: "VirtualMachineInstance", - Name: vmName, - }, - }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, }, } + if apiversion != "" && kind != "" && ownerName != "" { + pod.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: apiversion, + Kind: kind, + Name: ownerName, + }, + } + } + return pod +} + +func createPodWithOwnerAndPhase(name, namespace, apiversion, kind, ownerName string, phase corev1.PodPhase) *corev1.Pod { + pod := createPodWithOwner(name, namespace, apiversion, kind, ownerName) + pod.Status.Phase = phase + return pod } diff --git a/pkg/controller/migmigration/rollback.go b/pkg/controller/migmigration/rollback.go index b82b41c0b..e439c569a 100644 --- a/pkg/controller/migmigration/rollback.go +++ b/pkg/controller/migmigration/rollback.go @@ -18,18 +18,19 @@ import ( // Delete namespace and cluster-scoped resources on dest cluster func (t *Task) deleteMigrated() error { // Delete 'deployer' and 'hooks' Pods that DeploymentConfig leaves behind upon DC deletion. - err := t.deleteDeploymentConfigLeftoverPods() - if err != nil { + if err := t.deleteDeploymentConfigLeftoverPods(); err != nil { return liberr.Wrap(err) } - err = t.deleteMigratedNamespaceScopedResources() - if err != nil { + if err := t.deleteMigratedNamespaceScopedResources(); err != nil { return liberr.Wrap(err) } - err = t.deleteMovedNfsPVs() - if err != nil { + if err := t.deleteMovedNfsPVs(); err != nil { + return liberr.Wrap(err) + } + + if err := t.deleteLiveMigrationCompletedPods(); err != nil { return liberr.Wrap(err) } @@ -227,6 +228,66 @@ func (t *Task) deleteMovedNfsPVs() error { return nil } +// Completed virt-launcher pods remain after migration is complete. These pods reference +// the migrated PVCs and prevent the PVCs from being deleted. This function deletes +// the completed virt-launcher pods. +func (t *Task) deleteLiveMigrationCompletedPods() error { + if t.PlanResources.MigPlan.LiveMigrationChecked() { + // Possible live migrations were performed, check for completed virt-launcher pods. + destClient, err := t.getDestinationClient() + if err != nil { + return liberr.Wrap(err) + } + namespaceVolumeNamesMap := t.getDestinationVolumeNames() + for _, namespace := range t.destinationNamespaces() { + pods := corev1.PodList{} + destClient.List(context.TODO(), &pods, k8sclient.InNamespace(namespace), k8sclient.MatchingLabels(map[string]string{ + "kubevirt.io": "virt-launcher", + })) + for _, pod := range pods.Items { + if !t.isLiveMigrationCompletedPod(&pod, namespaceVolumeNamesMap[namespace]) { + continue + } + t.Log.V(3).Info("Deleting virt-launcher pod that has completed live migration", "namespace", pod.Namespace, "name", pod.Name) + err := destClient.Delete(context.TODO(), &pod) + if err != nil && !k8serror.IsNotFound(err) { + return err + } + } + } + } + return nil +} + +func (t *Task) getDestinationVolumeNames() map[string][]string { + namespaceVolumeNamesMap := make(map[string][]string) + pvcs := t.getDirectVolumeClaimList() + for _, pvc := range pvcs { + namespaceVolumeNamesMap[pvc.Namespace] = append(namespaceVolumeNamesMap[pvc.Namespace], pvc.TargetName) + } + return namespaceVolumeNamesMap +} + +func (t *Task) isLiveMigrationCompletedPod(pod *corev1.Pod, volumeNames []string) bool { + if len(pod.Spec.Volumes) == 0 { + return false + } + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + found := false + for _, volumeName := range volumeNames { + if volumeName == volume.PersistentVolumeClaim.ClaimName { + found = true + } + } + if !found { + return false + } + } + } + return pod.Status.Phase == corev1.PodSucceeded +} + func (t *Task) ensureMigratedResourcesDeleted() (bool, error) { t.Log.Info("Scanning all GVKs in all migrated namespaces to ensure " + "resources have finished deleting.") diff --git a/pkg/controller/migmigration/rollback_test.go b/pkg/controller/migmigration/rollback_test.go new file mode 100644 index 000000000..940bbeae6 --- /dev/null +++ b/pkg/controller/migmigration/rollback_test.go @@ -0,0 +1,166 @@ +// FILEPATH: /home/awels/go/src/github.com/awels/mig-controller/pkg/controller/migmigration/rollback_test.go + +package migmigration + +import ( + "context" + "testing" + + "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestTask_DeleteLiveMigrationCompletedPods(t *testing.T) { + tests := []struct { + name string + objects []client.Object + liveMigrate bool + expectedPods []*corev1.Pod + deletedPods []*corev1.Pod + }{ + { + name: "live migrate is not checked", + liveMigrate: false, + }, + { + name: "live migrate is checked, no running pods", + liveMigrate: true, + }, + { + name: "live migrate is checked, running and completed pods, should delete completed pods", + liveMigrate: true, + objects: []client.Object{ + createVirtlauncherPodWithStatus("pod1", "ns1", corev1.PodRunning), + createVirtlauncherPodWithStatus("pod1", "ns2", corev1.PodSucceeded), + createVirtlauncherPod("pod2", "ns1"), + }, + expectedPods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", + Name: "pod1-virt-launcher", + }, + }, + }, + deletedPods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns2", + Name: "pod1-virt-launcher", + }, + }, + }, + }, + { + name: "live migrate is checked, running and completed pods, but non matching volumes, should delete completed pods", + liveMigrate: true, + objects: []client.Object{ + createVirtlauncherPodWithStatus("pod1", "ns1", corev1.PodRunning), + createVirtlauncherPodWithExtraVolume("pod1", "ns2", corev1.PodSucceeded), + createVirtlauncherPod("pod2", "ns1"), + }, + expectedPods: []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", + Name: "pod1-virt-launcher", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := getFakeClientWithObjs(tt.objects...) + task := &Task{ + PlanResources: &v1alpha1.PlanResources{ + MigPlan: &v1alpha1.MigPlan{ + Spec: v1alpha1.MigPlanSpec{ + Namespaces: []string{"ns1:ns1", "ns2"}, + LiveMigrate: ptr.To[bool](tt.liveMigrate), + PersistentVolumes: v1alpha1.PersistentVolumes{ + List: []v1alpha1.PV{ + { + PVC: v1alpha1.PVC{ + Namespace: "ns1", + Name: "pvc1", + }, + Selection: v1alpha1.Selection{ + Action: v1alpha1.PvCopyAction, + CopyMethod: v1alpha1.PvBlockCopyMethod, + StorageClass: "sc2", + AccessMode: "ReadWriteOnce", + }, + StorageClass: "sc1", + }, + { + PVC: v1alpha1.PVC{ + Namespace: "ns2", + Name: "pvc1", + }, + Selection: v1alpha1.Selection{ + Action: v1alpha1.PvCopyAction, + CopyMethod: v1alpha1.PvFilesystemCopyMethod, + StorageClass: "sc2", + AccessMode: "ReadWriteOnce", + }, + StorageClass: "sc1", + }, + }, + }, + }, + }, + }, + destinationClient: c, + } + err := task.deleteLiveMigrationCompletedPods() + if err != nil { + t.Errorf("Task.deleteLiveMigrationCompletedPods() error = %v", err) + } + for _, pod := range tt.expectedPods { + res := &corev1.Pod{} + err := c.Get(context.TODO(), client.ObjectKeyFromObject(pod), res) + if err != nil { + t.Errorf("Task.deleteLiveMigrationCompletedPods() pod not found, while it should remain: %s/%s, %v", pod.Namespace, pod.Name, err) + } + } + for _, pod := range tt.deletedPods { + res := &corev1.Pod{} + err := c.Get(context.TODO(), client.ObjectKeyFromObject(pod), res) + if err == nil { + t.Errorf("Task.deleteLiveMigrationCompletedPods() pod %s/%s found, while it should be deleted", pod.Namespace, pod.Name) + } + } + }) + } +} + +func createVirtlauncherPodWithStatus(name, namespace string, phase corev1.PodPhase) *corev1.Pod { + pod := createVirtlauncherPod(name, namespace) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "volume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc1", + }, + }, + }) + pod.Status.Phase = phase + return pod +} + +func createVirtlauncherPodWithExtraVolume(name, namespace string, phase corev1.PodPhase) *corev1.Pod { + pod := createVirtlauncherPodWithStatus(name, namespace, phase) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "extra-volume", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc2", + }, + }, + }) + return pod +} diff --git a/pkg/controller/migmigration/storage.go b/pkg/controller/migmigration/storage.go index bb7427d5a..0413381cc 100644 --- a/pkg/controller/migmigration/storage.go +++ b/pkg/controller/migmigration/storage.go @@ -11,12 +11,14 @@ import ( "github.com/go-logr/logr" liberr "github.com/konveyor/controller/pkg/error" migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + dvmc "github.com/konveyor/mig-controller/pkg/controller/directvolumemigration" ocappsv1 "github.com/openshift/api/apps/v1" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" batchv1beta "k8s.io/api/batch/v1beta1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" k8smeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/client-go/rest" "k8s.io/utils/ptr" @@ -127,7 +129,11 @@ func (t *Task) swapPVCReferences() (reasons []string, err error) { reasons = append(reasons, fmt.Sprintf("Failed updating PVC references on StatefulSets [%s]", strings.Join(failedStatefulSetNames, ","))) } - failedVirtualMachineSwaps := t.swapVirtualMachinePVCRefs(client, restConfig, mapping) + restClient, err := t.createRestClient(restConfig) + if err != nil { + t.Log.Error(err, "failed creating rest client") + } + failedVirtualMachineSwaps := t.swapVirtualMachinePVCRefs(client, restClient, mapping) if len(failedVirtualMachineSwaps) > 0 { reasons = append(reasons, fmt.Sprintf("Failed updating PVC references on VirtualMachines [%s]", strings.Join(failedVirtualMachineSwaps, ","))) @@ -636,11 +642,10 @@ func (t *Task) swapCronJobsPVCRefs(client k8sclient.Client, mapping pvcNameMappi return } -func (t *Task) swapVirtualMachinePVCRefs(client k8sclient.Client, restConfig *rest.Config, mapping pvcNameMapping) (failedVirtualMachines []string) { +func (t *Task) swapVirtualMachinePVCRefs(client k8sclient.Client, restClient rest.Interface, mapping pvcNameMapping) (failedVirtualMachines []string) { for _, ns := range t.destinationNamespaces() { list := &virtv1.VirtualMachineList{} - options := k8sclient.InNamespace(ns) - if err := client.List(context.TODO(), list, options); err != nil { + if err := client.List(context.TODO(), list, k8sclient.InNamespace(ns)); err != nil { if k8smeta.IsNoMatchError(err) { continue } @@ -648,59 +653,77 @@ func (t *Task) swapVirtualMachinePVCRefs(client k8sclient.Client, restConfig *re continue } for _, vm := range list.Items { - for i, volume := range vm.Spec.Template.Spec.Volumes { - if volume.PersistentVolumeClaim != nil { - if isFailed := updatePVCRef(&vm.Spec.Template.Spec.Volumes[i].PersistentVolumeClaim.PersistentVolumeClaimVolumeSource, vm.Namespace, mapping); isFailed { - failedVirtualMachines = append(failedVirtualMachines, fmt.Sprintf("%s/%s", vm.Namespace, vm.Name)) + retryCount := 1 + retry := true + for retry && retryCount <= 3 { + message, err := t.swapVirtualMachinePVCRef(client, restClient, &vm, mapping) + if err != nil && !k8serrors.IsConflict(err) { + failedVirtualMachines = append(failedVirtualMachines, message) + return + } else if k8serrors.IsConflict(err) { + t.Log.Info("Conflict updating VM, retrying after reloading VM resource") + // Conflict, reload VM and try again + if err := client.Get(context.TODO(), k8sclient.ObjectKey{Namespace: ns, Name: vm.Name}, &vm); err != nil { + failedVirtualMachines = append(failedVirtualMachines, fmt.Sprintf("failed reloading %s/%s", ns, vm.Name)) } - } - if volume.DataVolume != nil { - isFailed, err := updateDataVolumeRef(client, vm.Spec.Template.Spec.Volumes[i].DataVolume, vm.Namespace, mapping, t.Log) - if err != nil || isFailed { - failedVirtualMachines = append(failedVirtualMachines, fmt.Sprintf("%s/%s", vm.Namespace, vm.Name)) - } else { - // Update datavolume template if it exists. - for i, dvt := range vm.Spec.DataVolumeTemplates { - if destinationDVName, exists := mapping.Get(ns, dvt.Name); exists { - vm.Spec.DataVolumeTemplates[i].Name = destinationDVName - } - } + retryCount++ + } else { + retry = false + if message != "" { + failedVirtualMachines = append(failedVirtualMachines, message) } } } - for _, dvt := range vm.Spec.DataVolumeTemplates { - t.Log.Info("DataVolumeTemplate", "dv", dvt) + } + } + return +} + +func (t *Task) swapVirtualMachinePVCRef(client k8sclient.Client, restClient rest.Interface, vm *virtv1.VirtualMachine, mapping pvcNameMapping) (string, error) { + if vm.Spec.Template == nil { + return "", nil + } + for i, volume := range vm.Spec.Template.Spec.Volumes { + if volume.PersistentVolumeClaim != nil { + if isFailed := updatePVCRef(&vm.Spec.Template.Spec.Volumes[i].PersistentVolumeClaim.PersistentVolumeClaimVolumeSource, vm.Namespace, mapping); isFailed { + return fmt.Sprintf("%s/%s", vm.Namespace, vm.Name), nil } - for _, volume := range vm.Spec.Template.Spec.Volumes { - if volume.DataVolume != nil { - t.Log.Info("datavolume", "dv", volume.DataVolume) + } + if volume.DataVolume != nil { + isFailed, err := updateDataVolumeRef(client, vm.Spec.Template.Spec.Volumes[i].DataVolume, vm.Namespace, mapping, t.Log) + if err != nil || isFailed { + return fmt.Sprintf("%s/%s", vm.Namespace, vm.Name), err + } + // Update datavolume template if it exists. + for i, dvt := range vm.Spec.DataVolumeTemplates { + if destinationDVName, exists := mapping.Get(vm.Namespace, dvt.Name); exists { + vm.Spec.DataVolumeTemplates[i].Name = destinationDVName } } - if shouldStartVM(&vm) { - if !isVMActive(&vm, client) { - restClient, err := t.createRestClient(restConfig) - if err != nil { - t.Log.Error(err, "failed creating rest client", "namespace", vm.Namespace, "name", vm.Name) - } - if err := t.startVM(&vm, client, restClient); err != nil { - t.Log.Error(err, "failed starting VM", "namespace", vm.Namespace, "name", vm.Name) - failedVirtualMachines = append(failedVirtualMachines, fmt.Sprintf("%s/%s", vm.Namespace, vm.Name)) - } - } - } else { - if err := client.Update(context.Background(), &vm); err != nil { - t.Log.Error(err, "failed updating VM", "namespace", vm.Namespace, "name", vm.Name) - failedVirtualMachines = append(failedVirtualMachines, fmt.Sprintf("%s/%s", vm.Namespace, vm.Name)) - } + } + } + if !isVMActive(vm, client) { + if err := client.Update(context.Background(), vm); err != nil { + t.Log.Error(err, "failed updating VM", "namespace", vm.Namespace, "name", vm.Name) + return fmt.Sprintf("%s/%s", vm.Namespace, vm.Name), err + } + if err := client.Get(context.Background(), k8sclient.ObjectKey{Namespace: vm.Namespace, Name: vm.Name}, vm); err != nil { + t.Log.Error(err, "failed getting VM", "namespace", vm.Namespace, "name", vm.Name) + return fmt.Sprintf("%s/%s", vm.Namespace, vm.Name), err + } + if shouldStartVM(vm) { + if err := t.startVM(vm, client, restClient); err != nil { + t.Log.Error(err, "failed starting VM", "namespace", vm.Namespace, "name", vm.Name) + return fmt.Sprintf("%s/%s", vm.Namespace, vm.Name), err } } } - return + return "", nil } // updatePVCRef given a PVCSource, namespace and a mapping of pvc names, swaps the claim // present in the pvc source with the mapped pvc name found in the mapping -// returns whether the swap was successful or not +// returns whether the swap was successful or not, true is failure, false is success func updatePVCRef(pvcSource *v1.PersistentVolumeClaimVolumeSource, ns string, mapping pvcNameMapping) bool { if pvcSource != nil { originalName := pvcSource.ClaimName @@ -729,26 +752,13 @@ func updateDataVolumeRef(client k8sclient.Client, dv *virtv1.DataVolumeSource, n } if destinationDVName, exists := mapping.Get(ns, originalName); exists { - log.Info("Found DataVolume mapping", "namespace", ns, "name", originalName, "destination", destinationDVName) dv.Name = destinationDVName - // Create adopting datavolume. - adoptingDV := originalDv.DeepCopy() - adoptingDV.Name = destinationDVName - if adoptingDV.Annotations == nil { - adoptingDV.Annotations = make(map[string]string) - } - adoptingDV.Annotations["cdi.kubevirt.io/allowClaimAdoption"] = "true" - adoptingDV.ResourceVersion = "" - adoptingDV.ManagedFields = nil - adoptingDV.UID = "" - - err := client.Create(context.Background(), adoptingDV) + err := dvmc.CreateNewDataVolume(client, originalDv.Name, destinationDVName, ns, log) if err != nil && !errors.IsAlreadyExists(err) { log.Error(err, "failed creating DataVolume", "namespace", ns, "name", destinationDVName) return true, err } } else { - log.Info("DataVolume reference already updated", "namespace", ns, "name", originalName) // attempt to figure out whether the current DV reference // already points to the new migrated PVC. This is needed to // guarantee idempotency of the operation diff --git a/pkg/controller/migmigration/storage_test.go b/pkg/controller/migmigration/storage_test.go new file mode 100644 index 000000000..57afaef6e --- /dev/null +++ b/pkg/controller/migmigration/storage_test.go @@ -0,0 +1,435 @@ +package migmigration + +import ( + "context" + "slices" + "testing" + + "github.com/go-logr/logr" + migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + virtv1 "kubevirt.io/api/core/v1" + cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestTask_updateDataVolumeRef(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + dvSource *virtv1.DataVolumeSource + ns string + mapping pvcNameMapping + log logr.Logger + expectedFailure bool + wantErr bool + newDV *cdiv1.DataVolume + }{ + { + name: "no dv source", + expectedFailure: false, + wantErr: false, + }, + { + name: "dv source set to unknown dv", + dvSource: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + expectedFailure: true, + wantErr: true, + }, + { + name: "dv source set to known dv, but missing in mapping", + objects: []runtime.Object{ + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + dvSource: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + ns: "ns-0", + expectedFailure: true, + wantErr: false, + }, + { + name: "dv source set to known dv, but mapping as value only", + objects: []runtime.Object{ + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + dvSource: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + mapping: pvcNameMapping{"ns-0/src-0": "dv-0"}, + ns: "ns-0", + expectedFailure: false, + wantErr: false, + }, + { + name: "dv source set to known dv, but mapping as key", + objects: []runtime.Object{ + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + dvSource: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + mapping: pvcNameMapping{"ns-0/dv-0": "tgt-0"}, + ns: "ns-0", + expectedFailure: false, + wantErr: false, + newDV: &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tgt-0", + Namespace: "ns-0", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := scheme.Scheme + err := cdiv1.AddToScheme(s) + if err != nil { + panic(err) + } + c := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(tt.objects...).Build() + res, err := updateDataVolumeRef(c, tt.dvSource, tt.ns, tt.mapping, tt.log) + if (err != nil) != tt.wantErr { + t.Errorf("updateDataVolumeRef() error = %v, wantErr %v", err, tt.wantErr) + t.FailNow() + } + if res != tt.expectedFailure { + t.Errorf("updateDataVolumeRef() expected failure = %v, want %v", res, tt.expectedFailure) + t.FailNow() + } + if tt.newDV != nil { + err := c.Get(context.TODO(), client.ObjectKeyFromObject(tt.newDV), tt.newDV) + if err != nil { + t.Errorf("updateDataVolumeRef() failed to create new DV: %v", err) + t.FailNow() + } + } else { + dvs := &cdiv1.DataVolumeList{} + err := c.List(context.TODO(), dvs, client.InNamespace(tt.ns)) + if err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("Error reading datavolumes: %v", err) + t.FailNow() + } else if err == nil && len(dvs.Items) > 0 { + for _, dv := range dvs.Items { + if dv.Name != tt.dvSource.Name { + t.Errorf("updateDataVolumeRef() created new DV when it shouldn't have, %v", dvs.Items) + t.FailNow() + } + } + } + } + }) + } +} + +func TestTask_swapVirtualMachinePVCRefs(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + restConfig rest.Interface + pvcMapping pvcNameMapping + expectedFailures []string + expectedNewName string + shouldStartVM bool + }{ + { + name: "no VMs, should return no failed VMs", + restConfig: getFakeRestClient(), + }, + { + name: "VMs without volumes, should return no failed VMs", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + &virtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vm-0", + Namespace: "ns-0", + }, + }, + }, + }, + { + name: "VMs with DVs, no mapping, should return all VMs", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + createVirtualMachine("vm-0", "ns-0", []virtv1.Volume{ + { + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + }, + }, + }), + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + expectedFailures: []string{"ns-0/vm-0"}, + }, + { + name: "VMs with PVCs, no mapping, should return all VMs", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + createVirtualMachine("vm-0", "ns-0", []virtv1.Volume{ + { + VolumeSource: virtv1.VolumeSource{ + PersistentVolumeClaim: &virtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "pvc-0", + }, + }, + }, + }, + }), + }, + expectedFailures: []string{"ns-0/vm-0"}, + }, + { + name: "VMs with DVs, mapping to new name, should return no failures", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + createVirtualMachine("vm-0", "ns-0", []virtv1.Volume{ + { + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + }, + }, + }), + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + pvcMapping: pvcNameMapping{"ns-0/dv-0": "dv-1"}, + expectedNewName: "dv-1", + }, + { + name: "VMs with DVTemplatess, mapping to new name, should return no failures", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + createVirtualMachineWithDVTemplate("vm-0", "ns-0", []virtv1.Volume{ + { + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + }, + }, + }), + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + pvcMapping: pvcNameMapping{"ns-0/dv-0": "dv-1"}, + expectedNewName: "dv-1", + }, + { + name: "VMs with DVs, mapping to new name, but running VM, should return no failures, and no updates", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + createVirtualMachine("vm-0", "ns-0", []virtv1.Volume{ + { + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + }, + }, + }), + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + createVirtlauncherPod("vm-0", "ns-0"), + }, + pvcMapping: pvcNameMapping{"ns-0/dv-0": "dv-1"}, + }, + { + name: "VMs with DVs, mapping to new name, should return no failures", + restConfig: getFakeRestClient(), + objects: []runtime.Object{ + createVirtualMachineWithAnnotation("vm-0", "ns-0", migapi.StartVMAnnotation, "true", []virtv1.Volume{ + { + VolumeSource: virtv1.VolumeSource{ + DataVolume: &virtv1.DataVolumeSource{ + Name: "dv-0", + }, + }, + }, + }), + &cdiv1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dv-0", + Namespace: "ns-0", + }, + }, + }, + pvcMapping: pvcNameMapping{"ns-0/dv-0": "dv-1"}, + expectedNewName: "dv-1", + shouldStartVM: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{ + PlanResources: &migapi.PlanResources{ + MigPlan: &migapi.MigPlan{ + Spec: migapi.MigPlanSpec{ + Namespaces: []string{"ns-0:ns-0", "ns-1:ns-1"}, + }, + }, + }, + } + s := scheme.Scheme + err := cdiv1.AddToScheme(s) + if err != nil { + panic(err) + } + err = virtv1.AddToScheme(s) + if err != nil { + panic(err) + } + c := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(tt.objects...).Build() + failedVMs := task.swapVirtualMachinePVCRefs(c, tt.restConfig, tt.pvcMapping) + if len(failedVMs) != len(tt.expectedFailures) { + t.Errorf("swapVirtualMachinePVCRefs() failed to swap PVC refs for VMs: %v, expected failures: %v", failedVMs, tt.expectedFailures) + t.FailNow() + } + for _, failedVM := range failedVMs { + if !slices.Contains(tt.expectedFailures, failedVM) { + t.Errorf("unexpected failed VM: %s, expected failures: %v", failedVM, tt.expectedFailures) + t.FailNow() + } + } + vm := &virtv1.VirtualMachine{} + if err := c.Get(context.TODO(), client.ObjectKey{Namespace: "ns-0", Name: "vm-0"}, vm); err != nil && !k8serrors.IsNotFound(err) { + t.Errorf("failed to get VM: %v", err) + t.FailNow() + } + if vm.Spec.Template != nil { + found := false + for _, volume := range vm.Spec.Template.Spec.Volumes { + if volume.VolumeSource.DataVolume != nil && volume.VolumeSource.DataVolume.Name == tt.expectedNewName { + found = true + } + if volume.VolumeSource.PersistentVolumeClaim != nil && volume.VolumeSource.PersistentVolumeClaim.ClaimName == tt.expectedNewName { + found = true + } + } + if !found && tt.expectedNewName != "" { + t.Errorf("Didn't find new volume name %s", tt.expectedNewName) + t.FailNow() + } else if tt.expectedNewName == "" && vm.ObjectMeta.ResourceVersion == "1000" { + t.Errorf("VM updated when it shouldn't have") + t.FailNow() + } + // Check DVTemplates + if len(vm.Spec.DataVolumeTemplates) > 0 { + found = false + for _, dvTemplate := range vm.Spec.DataVolumeTemplates { + if dvTemplate.Name == tt.expectedNewName { + found = true + } + } + if !found && tt.expectedNewName != "" { + t.Errorf("Didn't find new volume name %s in DVTemplate", tt.expectedNewName) + t.FailNow() + } else if found && tt.expectedNewName == "" { + t.Errorf("Found new volume name %s in DVTemplate when it shouldn't have", tt.expectedNewName) + t.FailNow() + } + } + if tt.shouldStartVM { + if _, ok := vm.GetAnnotations()[migapi.StartVMAnnotation]; ok { + t.Errorf("VM should have started") + t.FailNow() + } + } + } + }) + } +} + +func createVirtualMachine(name, namespace string, volumes []virtv1.Volume) *virtv1.VirtualMachine { + return &virtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: virtv1.VirtualMachineSpec{ + Template: &virtv1.VirtualMachineInstanceTemplateSpec{ + Spec: virtv1.VirtualMachineInstanceSpec{ + Volumes: volumes, + }, + }, + }, + } +} + +func createVirtualMachineWithDVTemplate(name, namespace string, volumes []virtv1.Volume) *virtv1.VirtualMachine { + vm := createVirtualMachine(name, namespace, volumes) + for _, volume := range volumes { + if volume.VolumeSource.DataVolume != nil { + vm.Spec.DataVolumeTemplates = append(vm.Spec.DataVolumeTemplates, virtv1.DataVolumeTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: volume.VolumeSource.DataVolume.Name, + Namespace: namespace, + }, + Spec: cdiv1.DataVolumeSpec{ + Storage: &cdiv1.StorageSpec{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }) + } + } + return vm +} + +func createVirtualMachineWithAnnotation(name, namespace, key, value string, volumes []virtv1.Volume) *virtv1.VirtualMachine { + vm := createVirtualMachine(name, namespace, volumes) + vm.Annotations = map[string]string{key: value} + return vm +} diff --git a/pkg/controller/migmigration/task.go b/pkg/controller/migmigration/task.go index 67d60062e..88f5b5cad 100644 --- a/pkg/controller/migmigration/task.go +++ b/pkg/controller/migmigration/task.go @@ -27,78 +27,83 @@ var NoReQ = time.Duration(0) // Phases const ( - Created = "" - Started = "Started" - CleanStaleAnnotations = "CleanStaleAnnotations" - CleanStaleVeleroCRs = "CleanStaleVeleroCRs" - CleanStaleResticCRs = "CleanStaleResticCRs" - CleanStaleStagePods = "CleanStaleStagePods" - WaitForStaleStagePodsTerminated = "WaitForStaleStagePodsTerminated" - StartRefresh = "StartRefresh" - WaitForRefresh = "WaitForRefresh" - CreateRegistries = "CreateRegistries" - CreateDirectImageMigration = "CreateDirectImageMigration" - WaitForDirectImageMigrationToComplete = "WaitForDirectImageMigrationToComplete" - EnsureCloudSecretPropagated = "EnsureCloudSecretPropagated" - PreBackupHooks = "PreBackupHooks" - PostBackupHooks = "PostBackupHooks" - PreRestoreHooks = "PreRestoreHooks" - PostRestoreHooks = "PostRestoreHooks" - PreBackupHooksFailed = "PreBackupHooksFailed" - PostBackupHooksFailed = "PostBackupHooksFailed" - PreRestoreHooksFailed = "PreRestoreHooksFailed" - PostRestoreHooksFailed = "PostRestoreHooksFailed" - EnsureInitialBackup = "EnsureInitialBackup" - InitialBackupCreated = "InitialBackupCreated" - InitialBackupFailed = "InitialBackupFailed" - AnnotateResources = "AnnotateResources" - EnsureStagePodsFromRunning = "EnsureStagePodsFromRunning" - EnsureStagePodsFromTemplates = "EnsureStagePodsFromTemplates" - EnsureStagePodsFromOrphanedPVCs = "EnsureStagePodsFromOrphanedPVCs" - StagePodsCreated = "StagePodsCreated" - StagePodsFailed = "StagePodsFailed" - SourceStagePodsFailed = "SourceStagePodsFailed" - RestartVelero = "RestartVelero" - WaitForVeleroReady = "WaitForVeleroReady" - RestartRestic = "RestartRestic" - WaitForResticReady = "WaitForResticReady" - QuiesceApplications = "QuiesceApplications" - EnsureQuiesced = "EnsureQuiesced" - UnQuiesceSrcApplications = "UnQuiesceSrcApplications" - UnQuiesceDestApplications = "UnQuiesceDestApplications" - SwapPVCReferences = "SwapPVCReferences" - WaitForRegistriesReady = "WaitForRegistriesReady" - EnsureStageBackup = "EnsureStageBackup" - StageBackupCreated = "StageBackupCreated" - StageBackupFailed = "StageBackupFailed" - EnsureInitialBackupReplicated = "EnsureInitialBackupReplicated" - EnsureStageBackupReplicated = "EnsureStageBackupReplicated" - EnsureStageRestore = "EnsureStageRestore" - StageRestoreCreated = "StageRestoreCreated" - StageRestoreFailed = "StageRestoreFailed" - CreateDirectVolumeMigration = "CreateDirectVolumeMigration" - WaitForDirectVolumeMigrationToComplete = "WaitForDirectVolumeMigrationToComplete" - DirectVolumeMigrationFailed = "DirectVolumeMigrationFailed" - EnsureFinalRestore = "EnsureFinalRestore" - FinalRestoreCreated = "FinalRestoreCreated" - FinalRestoreFailed = "FinalRestoreFailed" - Verification = "Verification" - EnsureStagePodsDeleted = "EnsureStagePodsDeleted" - EnsureStagePodsTerminated = "EnsureStagePodsTerminated" - EnsureAnnotationsDeleted = "EnsureAnnotationsDeleted" - EnsureMigratedDeleted = "EnsureMigratedDeleted" - DeleteRegistries = "DeleteRegistries" - DeleteMigrated = "DeleteMigrated" - DeleteBackups = "DeleteBackups" - DeleteRestores = "DeleteRestores" - DeleteHookJobs = "DeleteHookJobs" - DeleteDirectVolumeMigrationResources = "DeleteDirectVolumeMigrationResources" - DeleteDirectImageMigrationResources = "DeleteDirectImageMigrationResources" - MigrationFailed = "MigrationFailed" - Canceling = "Canceling" - Canceled = "Canceled" - Rollback = "Rollback" - Completed = "Completed" + Created = "" + Started = "Started" + CleanStaleAnnotations = "CleanStaleAnnotations" + CleanStaleVeleroCRs = "CleanStaleVeleroCRs" + CleanStaleResticCRs = "CleanStaleResticCRs" + CleanStaleStagePods = "CleanStaleStagePods" + WaitForStaleStagePodsTerminated = "WaitForStaleStagePodsTerminated" + StartRefresh = "StartRefresh" + WaitForRefresh = "WaitForRefresh" + CreateRegistries = "CreateRegistries" + CreateDirectImageMigration = "CreateDirectImageMigration" + WaitForDirectImageMigrationToComplete = "WaitForDirectImageMigrationToComplete" + EnsureCloudSecretPropagated = "EnsureCloudSecretPropagated" + PreBackupHooks = "PreBackupHooks" + PostBackupHooks = "PostBackupHooks" + PreRestoreHooks = "PreRestoreHooks" + PostRestoreHooks = "PostRestoreHooks" + PreBackupHooksFailed = "PreBackupHooksFailed" + PostBackupHooksFailed = "PostBackupHooksFailed" + PreRestoreHooksFailed = "PreRestoreHooksFailed" + PostRestoreHooksFailed = "PostRestoreHooksFailed" + EnsureInitialBackup = "EnsureInitialBackup" + InitialBackupCreated = "InitialBackupCreated" + InitialBackupFailed = "InitialBackupFailed" + AnnotateResources = "AnnotateResources" + EnsureStagePodsFromRunning = "EnsureStagePodsFromRunning" + EnsureStagePodsFromTemplates = "EnsureStagePodsFromTemplates" + EnsureStagePodsFromOrphanedPVCs = "EnsureStagePodsFromOrphanedPVCs" + StagePodsCreated = "StagePodsCreated" + StagePodsFailed = "StagePodsFailed" + SourceStagePodsFailed = "SourceStagePodsFailed" + RestartVelero = "RestartVelero" + WaitForVeleroReady = "WaitForVeleroReady" + RestartRestic = "RestartRestic" + WaitForResticReady = "WaitForResticReady" + QuiesceSourceApplications = "QuiesceSourceApplications" + QuiesceDestinationApplications = "QuiesceDestinationApplications" + EnsureSrcQuiesced = "EnsureSrcQuiesced" + EnsureDestQuiesced = "EnsureDestQuiesced" + UnQuiesceSrcApplications = "UnQuiesceSrcApplications" + UnQuiesceDestApplications = "UnQuiesceDestApplications" + SwapPVCReferences = "SwapPVCReferences" + WaitForRegistriesReady = "WaitForRegistriesReady" + EnsureStageBackup = "EnsureStageBackup" + StageBackupCreated = "StageBackupCreated" + StageBackupFailed = "StageBackupFailed" + EnsureInitialBackupReplicated = "EnsureInitialBackupReplicated" + EnsureStageBackupReplicated = "EnsureStageBackupReplicated" + EnsureStageRestore = "EnsureStageRestore" + StageRestoreCreated = "StageRestoreCreated" + StageRestoreFailed = "StageRestoreFailed" + CreateDirectVolumeMigrationStage = "CreateDirectVolumeMigrationStage" + CreateDirectVolumeMigrationFinal = "CreateDirectVolumeMigrationFinal" + CreateDirectVolumeMigrationRollback = "CreateDirectVolumeMigrationRollback" + WaitForDirectVolumeMigrationToComplete = "WaitForDirectVolumeMigrationToComplete" + WaitForDirectVolumeMigrationRollbackToComplete = "WaitForDirectVolumeMigrationToRollbackComplete" + DirectVolumeMigrationFailed = "DirectVolumeMigrationFailed" + EnsureFinalRestore = "EnsureFinalRestore" + FinalRestoreCreated = "FinalRestoreCreated" + FinalRestoreFailed = "FinalRestoreFailed" + Verification = "Verification" + EnsureStagePodsDeleted = "EnsureStagePodsDeleted" + EnsureStagePodsTerminated = "EnsureStagePodsTerminated" + EnsureAnnotationsDeleted = "EnsureAnnotationsDeleted" + EnsureMigratedDeleted = "EnsureMigratedDeleted" + DeleteRegistries = "DeleteRegistries" + DeleteMigrated = "DeleteMigrated" + DeleteBackups = "DeleteBackups" + DeleteRestores = "DeleteRestores" + DeleteHookJobs = "DeleteHookJobs" + DeleteDirectVolumeMigrationResources = "DeleteDirectVolumeMigrationResources" + DeleteDirectImageMigrationResources = "DeleteDirectImageMigrationResources" + MigrationFailed = "MigrationFailed" + Canceling = "Canceling" + Canceled = "Canceled" + Rollback = "Rollback" + Completed = "Completed" ) // Flags @@ -124,18 +129,28 @@ const ( // Migration steps const ( - StepPrepare = "Prepare" - StepDirectImage = "DirectImage" - StepDirectVolume = "DirectVolume" - StepBackup = "Backup" - StepStageBackup = "StageBackup" - StepStageRestore = "StageRestore" - StepRestore = "Restore" - StepCleanup = "Cleanup" - StepCleanupVelero = "CleanupVelero" - StepCleanupHelpers = "CleanupHelpers" - StepCleanupMigrated = "CleanupMigrated" - StepCleanupUnquiesce = "CleanupUnquiesce" + StepPrepare = "Prepare" + StepDirectImage = "DirectImage" + StepDirectVolume = "DirectVolume" + StepBackup = "Backup" + StepStageBackup = "StageBackup" + StepStageRestore = "StageRestore" + StepRestore = "Restore" + StepCleanup = "Cleanup" + StepCleanupVelero = "CleanupVelero" + StepCleanupHelpers = "CleanupHelpers" + StepCleanupMigrated = "CleanupMigrated" + StepCleanupUnquiesce = "CleanupUnquiesce" + StepRollbackLiveMigration = "RollbackLiveMigration" +) + +// Itinerary names +const ( + StageItineraryName = "Stage" + FinalItineraryName = "Final" + CancelItineraryName = "Cancel" + FailedItineraryName = "Failed" + RollbackItineraryName = "Rollback" ) // Itinerary defines itinerary @@ -145,7 +160,7 @@ type Itinerary struct { } var StageItinerary = Itinerary{ - Name: "Stage", + Name: StageItineraryName, Phases: []Phase{ {Name: Created, Step: StepPrepare}, {Name: Started, Step: StepPrepare}, @@ -159,9 +174,9 @@ var StageItinerary = Itinerary{ {Name: WaitForStaleStagePodsTerminated, Step: StepPrepare}, {Name: CreateRegistries, Step: StepPrepare, all: IndirectImage | EnableImage | HasISs}, {Name: CreateDirectImageMigration, Step: StepStageBackup, all: DirectImage | EnableImage}, - {Name: QuiesceApplications, Step: StepStageBackup, all: Quiesce}, - {Name: EnsureQuiesced, Step: StepStageBackup, all: Quiesce}, - {Name: CreateDirectVolumeMigration, Step: StepStageBackup, all: DirectVolume | EnableVolume}, + {Name: QuiesceSourceApplications, Step: StepStageBackup, all: Quiesce}, + {Name: EnsureSrcQuiesced, Step: StepStageBackup, all: Quiesce}, + {Name: CreateDirectVolumeMigrationStage, Step: StepStageBackup, all: DirectVolume | EnableVolume}, {Name: EnsureStagePodsFromRunning, Step: StepStageBackup, all: HasPVs | IndirectVolume}, {Name: EnsureStagePodsFromTemplates, Step: StepStageBackup, all: HasPVs | IndirectVolume}, {Name: EnsureStagePodsFromOrphanedPVCs, Step: StepStageBackup, all: HasPVs | IndirectVolume}, @@ -189,7 +204,7 @@ var StageItinerary = Itinerary{ } var FinalItinerary = Itinerary{ - Name: "Final", + Name: FinalItineraryName, Phases: []Phase{ {Name: Created, Step: StepPrepare}, {Name: Started, Step: StepPrepare}, @@ -207,10 +222,11 @@ var FinalItinerary = Itinerary{ {Name: EnsureCloudSecretPropagated, Step: StepPrepare}, {Name: PreBackupHooks, Step: PreBackupHooks, all: HasPreBackupHooks}, {Name: CreateDirectImageMigration, Step: StepBackup, all: DirectImage | EnableImage}, + {Name: CreateDirectVolumeMigrationStage, Step: StepStageBackup, all: DirectVolume | EnableVolume}, {Name: EnsureInitialBackup, Step: StepBackup}, {Name: InitialBackupCreated, Step: StepBackup}, - {Name: QuiesceApplications, Step: StepStageBackup, all: Quiesce}, - {Name: EnsureQuiesced, Step: StepStageBackup, all: Quiesce}, + {Name: QuiesceSourceApplications, Step: StepStageBackup, all: Quiesce}, + {Name: EnsureSrcQuiesced, Step: StepStageBackup, all: Quiesce}, {Name: EnsureStagePodsFromRunning, Step: StepStageBackup, all: HasPVs | IndirectVolume}, {Name: EnsureStagePodsFromTemplates, Step: StepStageBackup, all: HasPVs | IndirectVolume}, {Name: EnsureStagePodsFromOrphanedPVCs, Step: StepStageBackup, all: HasPVs | IndirectVolume}, @@ -218,7 +234,7 @@ var FinalItinerary = Itinerary{ {Name: RestartRestic, Step: StepStageBackup, all: HasStagePods}, {Name: AnnotateResources, Step: StepStageBackup, all: HasStageBackup}, {Name: WaitForResticReady, Step: StepStageBackup, any: HasPVs | HasStagePods}, - {Name: CreateDirectVolumeMigration, Step: StepStageBackup, all: DirectVolume | EnableVolume}, + {Name: CreateDirectVolumeMigrationFinal, Step: StepStageBackup, all: DirectVolume | EnableVolume}, {Name: EnsureStageBackup, Step: StepStageBackup, all: HasStageBackup}, {Name: StageBackupCreated, Step: StepStageBackup, all: HasStageBackup}, {Name: EnsureStageBackupReplicated, Step: StepStageBackup, all: HasStageBackup}, @@ -244,7 +260,7 @@ var FinalItinerary = Itinerary{ } var CancelItinerary = Itinerary{ - Name: "Cancel", + Name: CancelItineraryName, Phases: []Phase{ {Name: Canceling, Step: StepCleanupVelero}, {Name: DeleteBackups, Step: StepCleanupVelero}, @@ -255,13 +271,14 @@ var CancelItinerary = Itinerary{ {Name: DeleteDirectImageMigrationResources, Step: StepCleanupHelpers, all: DirectImage}, {Name: EnsureStagePodsDeleted, Step: StepCleanupHelpers, all: HasStagePods}, {Name: EnsureAnnotationsDeleted, Step: StepCleanupHelpers, all: HasStageBackup}, + {Name: UnQuiesceSrcApplications, Step: StepCleanupUnquiesce}, {Name: Canceled, Step: StepCleanup}, {Name: Completed, Step: StepCleanup}, }, } var FailedItinerary = Itinerary{ - Name: "Failed", + Name: FailedItineraryName, Phases: []Phase{ {Name: MigrationFailed, Step: StepCleanupHelpers}, {Name: DeleteRegistries, Step: StepCleanupHelpers}, @@ -271,7 +288,7 @@ var FailedItinerary = Itinerary{ } var RollbackItinerary = Itinerary{ - Name: "Rollback", + Name: RollbackItineraryName, Phases: []Phase{ {Name: Rollback, Step: StepCleanupVelero}, {Name: DeleteBackups, Step: StepCleanupVelero}, @@ -279,7 +296,11 @@ var RollbackItinerary = Itinerary{ {Name: DeleteRegistries, Step: StepCleanupHelpers}, {Name: EnsureStagePodsDeleted, Step: StepCleanupHelpers}, {Name: EnsureAnnotationsDeleted, Step: StepCleanupHelpers, any: HasPVs | HasISs}, + {Name: QuiesceDestinationApplications, Step: StepCleanupMigrated, any: DirectVolume}, + {Name: EnsureDestQuiesced, Step: StepCleanupMigrated}, {Name: SwapPVCReferences, Step: StepCleanupMigrated, all: StorageConversion}, + {Name: CreateDirectVolumeMigrationRollback, Step: StepRollbackLiveMigration, all: DirectVolume | EnableVolume}, + {Name: WaitForDirectVolumeMigrationRollbackToComplete, Step: StepRollbackLiveMigration, all: DirectVolume | EnableVolume}, {Name: DeleteMigrated, Step: StepCleanupMigrated}, {Name: EnsureMigratedDeleted, Step: StepCleanupMigrated}, {Name: UnQuiesceSrcApplications, Step: StepCleanupUnquiesce}, @@ -327,18 +348,20 @@ func (r Itinerary) progressReport(phaseName string) (string, int, int) { // Errors - Migration errors. // Failed - Task phase has failed. type Task struct { - Scheme *runtime.Scheme - Log logr.Logger - Client k8sclient.Client - Owner *migapi.MigMigration - PlanResources *migapi.PlanResources - Annotations map[string]string - BackupResources mapset.Set - Phase string - Requeue time.Duration - Itinerary Itinerary - Errors []string - Step string + Scheme *runtime.Scheme + Log logr.Logger + Client k8sclient.Client + destinationClient compat.Client + sourceClient compat.Client + Owner *migapi.MigMigration + PlanResources *migapi.PlanResources + Annotations map[string]string + BackupResources mapset.Set + Phase string + Requeue time.Duration + Itinerary Itinerary + Errors []string + Step string Tracer opentracing.Tracer ReconcileSpan opentracing.Span @@ -671,16 +694,38 @@ func (t *Task) Run(ctx context.Context) error { t.Log.Info("Velero Pod(s) are unready on the source or target cluster. Waiting.") t.Requeue = PollReQ } - case QuiesceApplications: - err := t.quiesceApplications() + case QuiesceSourceApplications: + err := t.quiesceSourceApplications() + if err != nil { + return liberr.Wrap(err) + } + if err = t.next(); err != nil { + return liberr.Wrap(err) + } + case QuiesceDestinationApplications: + err := t.quiesceDestinationApplications() if err != nil { return liberr.Wrap(err) } if err = t.next(); err != nil { return liberr.Wrap(err) } - case EnsureQuiesced: - quiesced, err := t.ensureQuiescedPodsTerminated() + case EnsureSrcQuiesced: + quiesced, err := t.ensureSourceQuiescedPodsTerminated() + if err != nil { + return liberr.Wrap(err) + } + if quiesced { + if err = t.next(); err != nil { + return liberr.Wrap(err) + } + } else { + t.Log.Info("Quiescing on source cluster is incomplete. " + + "Pods are not yet terminated, waiting.") + t.Requeue = PollReQ + } + case EnsureDestQuiesced: + quiesced, err := t.ensureDestinationQuiescedPodsTerminated() if err != nil { return liberr.Wrap(err) } @@ -721,9 +766,9 @@ func (t *Task) Run(ctx context.Context) error { if err = t.next(); err != nil { return liberr.Wrap(err) } - case CreateDirectVolumeMigration: + case CreateDirectVolumeMigrationStage, CreateDirectVolumeMigrationFinal: if t.hasDirectVolumes() { - err := t.createDirectVolumeMigration() + err := t.createDirectVolumeMigration(nil) if err != nil { return liberr.Wrap(err) } @@ -731,7 +776,16 @@ func (t *Task) Run(ctx context.Context) error { if err := t.next(); err != nil { return liberr.Wrap(err) } - case WaitForDirectVolumeMigrationToComplete: + case CreateDirectVolumeMigrationRollback: + rollback := migapi.MigrationTypeRollback + err := t.createDirectVolumeMigration(&rollback) + if err != nil { + return liberr.Wrap(err) + } + if err = t.next(); err != nil { + return liberr.Wrap(err) + } + case WaitForDirectVolumeMigrationToComplete, WaitForDirectVolumeMigrationRollbackToComplete: dvm, err := t.getDirectVolumeMigration() if err != nil { return liberr.Wrap(err) @@ -743,30 +797,8 @@ func (t *Task) Run(ctx context.Context) error { } break } - // Check if DVM is complete and report progress - completed, reasons, progress := t.hasDirectVolumeMigrationCompleted(dvm) - PhaseDescriptions[t.Phase] = dvm.Status.PhaseDescription - t.setProgress(progress) - if completed { - step := t.Owner.Status.FindStep(t.Step) - step.MarkCompleted() - if len(reasons) > 0 { - t.setDirectVolumeMigrationFailureWarning(dvm) - } - if err = t.next(); err != nil { - return liberr.Wrap(err) - } - } else { - t.Requeue = PollReQ - criticalWarning, err := t.getWarningForDVM(dvm) - if err != nil { - return liberr.Wrap(err) - } - if criticalWarning != nil { - t.Owner.Status.SetCondition(*criticalWarning) - return nil - } - t.Owner.Status.DeleteCondition(DirectVolumeMigrationBlocked) + if err := t.waitForDVMToComplete(dvm); err != nil { + return liberr.Wrap(err) } case EnsureStageBackup: _, err := t.ensureStageBackup() @@ -1516,9 +1548,7 @@ func (t *Task) failCurrentStep() { // Add errors. func (t *Task) addErrors(errors []string) { - for _, e := range errors { - t.Errors = append(t.Errors, e) - } + t.Errors = append(t.Errors, errors...) } // Migration UID. @@ -1604,7 +1634,14 @@ func (t *Task) keepAnnotations() bool { // Get a client for the source cluster. func (t *Task) getSourceClient() (compat.Client, error) { - return t.PlanResources.SrcMigCluster.GetClient(t.Client) + if t.sourceClient == nil { + c, err := t.PlanResources.SrcMigCluster.GetClient(t.Client) + if err != nil { + return nil, err + } + t.sourceClient = c + } + return t.sourceClient, nil } // Get a client for the source cluster. @@ -1614,7 +1651,14 @@ func (t *Task) getSourceRestConfig() (*rest.Config, error) { // Get a client for the destination cluster. func (t *Task) getDestinationClient() (compat.Client, error) { - return t.PlanResources.DestMigCluster.GetClient(t.Client) + if t.destinationClient == nil { + c, err := t.PlanResources.DestMigCluster.GetClient(t.Client) + if err != nil { + return nil, err + } + t.destinationClient = c + } + return t.destinationClient, nil } // Get a client for the source cluster. @@ -1852,3 +1896,33 @@ func (t *Task) hasPostRestoreHooks() bool { } return anyPostRestoreHooks } + +func (t *Task) waitForDVMToComplete(dvm *migapi.DirectVolumeMigration) error { + // Check if DVM is complete and report progress + completed, reasons, progress := t.hasDirectVolumeMigrationCompleted(dvm) + t.Log.V(3).Info("DVM status", "completed", completed, "reasons", reasons, "progress", progress) + PhaseDescriptions[t.Phase] = dvm.Status.PhaseDescription + t.setProgress(progress) + if completed { + step := t.Owner.Status.FindStep(t.Step) + if step == nil { + return fmt.Errorf("step %s not found in pipeline", t.Step) + } + step.MarkCompleted() + if len(reasons) > 0 { + t.setDirectVolumeMigrationFailureWarning(dvm) + } + if err := t.next(); err != nil { + return liberr.Wrap(err) + } + } else { + t.Requeue = PollReQ + criticalWarning := t.getWarningForDVM(dvm) + if criticalWarning != nil { + t.Owner.Status.SetCondition(*criticalWarning) + return nil + } + t.Owner.Status.DeleteCondition(DirectVolumeMigrationBlocked) + } + return nil +} diff --git a/pkg/controller/migmigration/task_test.go b/pkg/controller/migmigration/task_test.go index e539b2318..f4c4f3e14 100644 --- a/pkg/controller/migmigration/task_test.go +++ b/pkg/controller/migmigration/task_test.go @@ -1,13 +1,16 @@ package migmigration import ( - "github.com/go-logr/logr" - migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" "reflect" "testing" + + "github.com/go-logr/logr" + migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + dvmc "github.com/konveyor/mig-controller/pkg/controller/directvolumemigration" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestTask_getStagePVs(t1 *testing.T) { +func TestTask_getStagePVs(t *testing.T) { type fields struct { Log logr.Logger PlanResources *migapi.PlanResources @@ -176,15 +179,189 @@ func TestTask_getStagePVs(t1 *testing.T) { }, } for _, tt := range tests { - t1.Run(tt.name, func(t1 *testing.T) { - t := &Task{ + t.Run(tt.name, func(t *testing.T) { + task := &Task{ Log: tt.fields.Log, PlanResources: tt.fields.PlanResources, Phase: tt.fields.Phase, Step: tt.fields.Step, } - if got := t.getStagePVs(); !reflect.DeepEqual(got, tt.want) { - t1.Errorf("getStagePVs() = %v, want %v", got, tt.want) + if got := task.getStagePVs(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getStagePVs() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestTask_waitForDMVToComplete(t *testing.T) { + tests := []struct { + name string + step string + dvm *migapi.DirectVolumeMigration + initialConditions []migapi.Condition + expectedConditions []migapi.Condition + wantErr bool + }{ + { + name: "dvm uncompleted, no warnings", + dvm: &migapi.DirectVolumeMigration{}, + initialConditions: []migapi.Condition{ + { + Type: DirectVolumeMigrationBlocked, + }, + }, + expectedConditions: []migapi.Condition{}, + wantErr: false, + }, + { + name: "dvm uncompleted, warnings", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + Conditions: migapi.Conditions{ + List: []migapi.Condition{ + { + Category: migapi.Warn, + Message: "warning", + }, + }, + }, + }, + }, + initialConditions: []migapi.Condition{ + { + Type: DirectVolumeMigrationBlocked, + }, + }, + expectedConditions: []migapi.Condition{ + { + Type: DirectVolumeMigrationBlocked, + Status: True, + Reason: migapi.NotReady, + Category: migapi.Warn, + Message: "warning", + }, + }, + wantErr: false, + }, + { + name: "dvm completed, no warnings", + step: "test", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + Phase: dvmc.Completed, + Itinerary: dvmc.VolumeMigrationItinerary, + Conditions: migapi.Conditions{ + List: []migapi.Condition{ + { + Type: dvmc.Succeeded, + Status: True, + }, + }, + }, + }, + }, + initialConditions: []migapi.Condition{}, + expectedConditions: []migapi.Condition{}, + wantErr: false, + }, + { + name: "dvm completed, invalid next step", + step: "invalid", + dvm: &migapi.DirectVolumeMigration{ + Status: migapi.DirectVolumeMigrationStatus{ + Phase: dvmc.Completed, + Itinerary: dvmc.VolumeMigrationItinerary, + Conditions: migapi.Conditions{ + List: []migapi.Condition{ + { + Type: dvmc.Succeeded, + Status: True, + }, + }, + }, + }, + }, + initialConditions: []migapi.Condition{}, + expectedConditions: []migapi.Condition{}, + wantErr: true, + }, + { + name: "dvm completed, warnings", + step: "test", + dvm: &migapi.DirectVolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + Status: migapi.DirectVolumeMigrationStatus{ + Phase: dvmc.Completed, + Itinerary: dvmc.VolumeMigrationItinerary, + Conditions: migapi.Conditions{ + List: []migapi.Condition{ + { + Type: dvmc.Failed, + Reason: "test failure", + Status: True, + }, + }, + }, + }, + }, + initialConditions: []migapi.Condition{}, + expectedConditions: []migapi.Condition{ + { + Type: DirectVolumeMigrationFailed, + Status: True, + Category: migapi.Warn, + Message: "DirectVolumeMigration (dvm): test/test failed. See in dvm status.Errors", + Durable: true, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + task := &Task{ + Step: tt.step, + Owner: &migapi.MigMigration{ + Status: migapi.MigMigrationStatus{ + Pipeline: []*migapi.Step{ + { + Name: "test", + }, + }, + Conditions: migapi.Conditions{ + List: tt.initialConditions, + }, + }, + }, + } + err := task.waitForDVMToComplete(tt.dvm) + if (err != nil) != tt.wantErr { + t.Errorf("waitForDMVToComplete() error = %v, wantErr %v", err, tt.wantErr) + t.FailNow() + } + if len(task.Owner.Status.Conditions.List) != len(tt.expectedConditions) { + t.Errorf("waitForDMVToComplete() = %v, want %v", task.Owner.Status.Conditions.List, tt.expectedConditions) + t.FailNow() + } + for i, c := range task.Owner.Status.Conditions.List { + if c.Category != tt.expectedConditions[i].Category { + t.Errorf("category = %s, want %s", c.Category, tt.expectedConditions[i].Category) + } + if c.Type != tt.expectedConditions[i].Type { + t.Errorf("type = %s, want %s", c.Type, tt.expectedConditions[i].Type) + } + if c.Status != tt.expectedConditions[i].Status { + t.Errorf("status = %s, want %s", c.Status, tt.expectedConditions[i].Status) + } + if c.Reason != tt.expectedConditions[i].Reason { + t.Errorf("reason = %s, want %s", c.Reason, tt.expectedConditions[i].Reason) + } + if c.Message != tt.expectedConditions[i].Message { + t.Errorf("message = %s, want %s", c.Message, tt.expectedConditions[i].Message) + } } }) } diff --git a/pkg/controller/migplan/validation.go b/pkg/controller/migplan/validation.go index 7897fce49..0b8a48d41 100644 --- a/pkg/controller/migplan/validation.go +++ b/pkg/controller/migplan/validation.go @@ -6,11 +6,14 @@ import ( "fmt" "net" "path" + "slices" "sort" + "strconv" "strings" liberr "github.com/konveyor/controller/pkg/error" migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + "github.com/konveyor/mig-controller/pkg/compat" "github.com/konveyor/mig-controller/pkg/controller/migcluster" "github.com/konveyor/mig-controller/pkg/health" "github.com/konveyor/mig-controller/pkg/pods" @@ -27,6 +30,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/util/exec" + virtv1 "kubevirt.io/api/core/v1" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -88,6 +92,9 @@ const ( HookPhaseUnknown = "HookPhaseUnknown" HookPhaseDuplicate = "HookPhaseDuplicate" IntraClusterMigration = "IntraClusterMigration" + KubeVirtNotInstalledSourceCluster = "KubeVirtNotInstalledSourceCluster" + KubeVirtVersionNotSupported = "KubeVirtVersionNotSupported" + KubeVirtStorageLiveMigrationNotEnabled = "KubeVirtStorageLiveMigrationNotEnabled" ) // Categories @@ -115,6 +122,14 @@ const ( DuplicateNs = "DuplicateNamespaces" ConflictingNamespaces = "ConflictingNamespaces" ConflictingPermissions = "ConflictingPermissions" + NotSupported = "NotSupported" +) + +// Messages +const ( + KubeVirtNotInstalledSourceClusterMessage = "KubeVirt is not installed on the source cluster" + KubeVirtVersionNotSupportedMessage = "KubeVirt version does not support storage live migration, Virtual Machines will be stopped instead" + KubeVirtStorageLiveMigrationNotEnabledMessage = "KubeVirt storage live migration is not enabled, Virtual Machines will be stopped instead" ) // Statuses @@ -130,6 +145,13 @@ const ( openShiftUIDRangeAnnotation = "openshift.io/sa.scc.uid-range" ) +// Valid kubevirt feature gates +const ( + VolumesUpdateStrategy = "VolumesUpdateStrategy" + VolumeMigrationConfig = "VolumeMigration" + VMLiveUpdateFeatures = "VMLiveUpdateFeatures" +) + // Valid AccessMode values var validAccessModes = []kapi.PersistentVolumeAccessMode{kapi.ReadWriteOnce, kapi.ReadOnlyMany, kapi.ReadWriteMany} @@ -224,6 +246,13 @@ func (r ReconcileMigPlan) validate(ctx context.Context, plan *migapi.MigPlan) er if err != nil { return liberr.Wrap(err) } + + if plan.LiveMigrationChecked() { + // Live migration possible + if err := r.validateLiveMigrationPossible(ctx, plan); err != nil { + return liberr.Wrap(err) + } + } return nil } @@ -1523,7 +1552,7 @@ func (r ReconcileMigPlan) validatePodHealth(ctx context.Context, plan *migapi.Mi return nil } -func (r ReconcileMigPlan) validateHooks(ctx context.Context, plan *migapi.MigPlan) error { +func (r *ReconcileMigPlan) validateHooks(ctx context.Context, plan *migapi.MigPlan) error { if opentracing.SpanFromContext(ctx) != nil { span, _ := opentracing.StartSpanFromContextWithTracer(ctx, r.tracer, "validateHooks") defer span.Finish() @@ -1626,6 +1655,127 @@ func (r ReconcileMigPlan) validateHooks(ctx context.Context, plan *migapi.MigPla return nil } +func (r *ReconcileMigPlan) validateLiveMigrationPossible(ctx context.Context, plan *migapi.MigPlan) error { + // Check if kubevirt is installed, if not installed, return nil + srcCluster, err := plan.GetSourceCluster(r) + if err != nil { + return liberr.Wrap(err) + } + if err := r.validateCluster(ctx, srcCluster, plan); err != nil { + return liberr.Wrap(err) + } + dstCluster, err := plan.GetDestinationCluster(r) + if err != nil { + return liberr.Wrap(err) + } + return r.validateCluster(ctx, dstCluster, plan) +} + +func (r *ReconcileMigPlan) validateCluster(ctx context.Context, cluster *migapi.MigCluster, plan *migapi.MigPlan) error { + if cluster == nil || !cluster.Status.IsReady() { + return nil + } + srcClient, err := cluster.GetClient(r) + if err != nil { + return liberr.Wrap(err) + } + if err := r.validateKubeVirtInstalled(ctx, srcClient, plan); err != nil { + return err + } + return nil +} + +func (r *ReconcileMigPlan) validateKubeVirtInstalled(ctx context.Context, client compat.Client, plan *migapi.MigPlan) error { + if !plan.LiveMigrationChecked() { + return nil + } + kubevirtList := &virtv1.KubeVirtList{} + if err := client.List(ctx, kubevirtList); err != nil { + return liberr.Wrap(err) + } + if len(kubevirtList.Items) == 0 || len(kubevirtList.Items) > 1 { + plan.Status.SetCondition(migapi.Condition{ + Type: KubeVirtNotInstalledSourceCluster, + Status: True, + Reason: NotFound, + Category: Advisory, + Message: KubeVirtNotInstalledSourceClusterMessage, + }) + return nil + } + kubevirt := kubevirtList.Items[0] + operatorVersion := kubevirt.Status.OperatorVersion + major, minor, bugfix, err := parseKubeVirtOperatorSemver(operatorVersion) + if err != nil { + plan.Status.SetCondition(migapi.Condition{ + Type: KubeVirtVersionNotSupported, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtVersionNotSupportedMessage, + }) + return nil + } + log.V(3).Info("KubeVirt operator version", "major", major, "minor", minor, "bugfix", bugfix) + // Check if kubevirt operator version is at least 1.3.0 if live migration is enabled. + if major < 1 || (major == 1 && minor < 3) { + plan.Status.SetCondition(migapi.Condition{ + Type: KubeVirtVersionNotSupported, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtVersionNotSupportedMessage, + }) + return nil + } + // Check if the appropriate feature gates are enabled + if kubevirt.Spec.Configuration.VMRolloutStrategy == nil || + *kubevirt.Spec.Configuration.VMRolloutStrategy != virtv1.VMRolloutStrategyLiveUpdate || + kubevirt.Spec.Configuration.DeveloperConfiguration == nil || + !slices.Contains(kubevirt.Spec.Configuration.DeveloperConfiguration.FeatureGates, VolumesUpdateStrategy) || + !slices.Contains(kubevirt.Spec.Configuration.DeveloperConfiguration.FeatureGates, VolumeMigrationConfig) || + !slices.Contains(kubevirt.Spec.Configuration.DeveloperConfiguration.FeatureGates, VMLiveUpdateFeatures) { + plan.Status.SetCondition(migapi.Condition{ + Type: KubeVirtStorageLiveMigrationNotEnabled, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtStorageLiveMigrationNotEnabledMessage, + }) + return nil + } + return nil +} + +func parseKubeVirtOperatorSemver(operatorVersion string) (int, int, int, error) { + // example versions: v1.1.1-106-g0be1a2073, or: v1.3.0-beta.0.202+f8efa57713ba76-dirty + tokens := strings.Split(operatorVersion, ".") + if len(tokens) < 3 { + return -1, -1, -1, liberr.Wrap(fmt.Errorf("version string was not in semver format, != 3 tokens")) + } + + if tokens[0][0] == 'v' { + tokens[0] = tokens[0][1:] + } + major, err := strconv.Atoi(tokens[0]) + if err != nil { + return -1, -1, -1, liberr.Wrap(fmt.Errorf("major version could not be parsed as integer")) + } + + minor, err := strconv.Atoi(tokens[1]) + if err != nil { + return -1, -1, -1, liberr.Wrap(fmt.Errorf("minor version could not be parsed as integer")) + } + + bugfixTokens := strings.Split(tokens[2], "-") + bugfix, err := strconv.Atoi(bugfixTokens[0]) + if err != nil { + return -1, -1, -1, liberr.Wrap(fmt.Errorf("bugfix version could not be parsed as integer")) + } + + return major, minor, bugfix, nil +} + func containsAccessMode(modeList []kapi.PersistentVolumeAccessMode, accessMode kapi.PersistentVolumeAccessMode) bool { for _, mode := range modeList { if mode == accessMode { diff --git a/pkg/controller/migplan/validation_test.go b/pkg/controller/migplan/validation_test.go index ea86f6f03..091ebb4b5 100644 --- a/pkg/controller/migplan/validation_test.go +++ b/pkg/controller/migplan/validation_test.go @@ -11,43 +11,49 @@ import ( "github.com/opentracing/opentracing-go/mocktracer" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + virtv1 "kubevirt.io/api/core/v1" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" ) -func TestReconcileMigPlan_validatePossibleMigrationTypes(t *testing.T) { - getFakeClientWithObjs := func(obj ...k8sclient.Object) compat.Client { - client, _ := fakecompat.NewFakeClient(obj...) - return client - } - getTestMigCluster := func(name string, url string) *migapi.MigCluster { - return &migapi.MigCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: migapi.OpenshiftMigrationNamespace, - }, - Spec: migapi.MigClusterSpec{ - URL: url, - }, - } +func getFakeClientWithObjs(obj ...k8sclient.Object) compat.Client { + client, _ := fakecompat.NewFakeClient(obj...) + return client +} + +func getTestMigCluster(name string, url string) *migapi.MigCluster { + return &migapi.MigCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: migapi.OpenshiftMigrationNamespace, + }, + Spec: migapi.MigClusterSpec{ + URL: url, + }, } - getTestMigPlan := func(srcCluster string, destCluster string, ns []string, conds []migapi.Condition) *migapi.MigPlan { - return &migapi.MigPlan{ - ObjectMeta: metav1.ObjectMeta{ - Name: "migplan", - Namespace: migapi.OpenshiftMigrationNamespace, - }, - Spec: migapi.MigPlanSpec{ - SrcMigClusterRef: &v1.ObjectReference{Name: srcCluster, Namespace: migapi.OpenshiftMigrationNamespace}, - DestMigClusterRef: &v1.ObjectReference{Name: destCluster, Namespace: migapi.OpenshiftMigrationNamespace}, - Namespaces: ns, - }, - Status: migapi.MigPlanStatus{ - Conditions: migapi.Conditions{ - List: conds, - }, +} + +func getTestMigPlan(srcCluster string, destCluster string, ns []string, conds []migapi.Condition) *migapi.MigPlan { + return &migapi.MigPlan{ + ObjectMeta: metav1.ObjectMeta{ + Name: "migplan", + Namespace: migapi.OpenshiftMigrationNamespace, + }, + Spec: migapi.MigPlanSpec{ + SrcMigClusterRef: &v1.ObjectReference{Name: srcCluster, Namespace: migapi.OpenshiftMigrationNamespace}, + DestMigClusterRef: &v1.ObjectReference{Name: destCluster, Namespace: migapi.OpenshiftMigrationNamespace}, + Namespaces: ns, + LiveMigrate: ptr.To[bool](true), + }, + Status: migapi.MigPlanStatus{ + Conditions: migapi.Conditions{ + List: conds, }, - } + }, } +} + +func TestReconcileMigPlan_validatePossibleMigrationTypes(t *testing.T) { tests := []struct { name string client k8sclient.Client @@ -273,3 +279,346 @@ func TestReconcileMigPlan_validatePossibleMigrationTypes(t *testing.T) { }) } } + +func TestReconcileMigPlan_validateparseKubeVirtOperatorSemver(t *testing.T) { + tests := []struct { + name string + input string + wantErr bool + major int + minor int + bugfix int + }{ + { + name: "given a valid semver string, should not return an error", + input: "v0.0.0", + wantErr: false, + major: 0, + minor: 0, + bugfix: 0, + }, + { + name: "given a valid semver string with extra info, should not return an error", + input: "v1.2.3-rc1", + wantErr: false, + major: 1, + minor: 2, + bugfix: 3, + }, + { + name: "given a valid semver string with extra info, should not return an error", + input: "v1.2.3-rc1.debug.1", + wantErr: false, + major: 1, + minor: 2, + bugfix: 3, + }, + { + name: "given a semver string with two dots, should return an error", + input: "v0.0", + wantErr: true, + }, + { + name: "given a semver string without a v should not return an error", + input: "1.1.1", + wantErr: false, + major: 1, + minor: 1, + bugfix: 1, + }, + { + name: "given a semver with an invalid major version, should return an error", + input: "va.1.1", + wantErr: true, + }, + { + name: "given a semver with an invalid minor version, should return an error", + input: "v4.b.1", + wantErr: true, + }, + { + name: "given a semver with an invalid bugfix version, should return an error", + input: "v2.1.c", + wantErr: true, + }, + { + name: "given a semver with an invalid bugfix version with dash, should return an error", + input: "v2.1.-", + wantErr: true, + }, + { + name: "given a semver with a dot instead of a valid bugfix version, should return an error", + input: "v2.1.-", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + major, minor, patch, err := parseKubeVirtOperatorSemver(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("parseKubeVirtOperatorSemver() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err == nil { + if major != tt.major { + t.Errorf("parseKubeVirtOperatorSemver() major = %v, want %v", major, tt.major) + } + if minor != tt.minor { + t.Errorf("parseKubeVirtOperatorSemver() minor = %v, want %v", minor, tt.minor) + } + if patch != tt.bugfix { + t.Errorf("parseKubeVirtOperatorSemver() patch = %v, want %v", patch, tt.bugfix) + } + } + }) + } +} + +func TestReconcileMigPlan_validateKubeVirtInstalled(t *testing.T) { + plan := getTestMigPlan("test-cluster", "test-cluster", []string{ + "ns-00:ns-00", + "ns-01:ns-02", + }, []migapi.Condition{}) + noLiveMigratePlan := plan.DeepCopy() + noLiveMigratePlan.Spec.LiveMigrate = ptr.To[bool](false) + tests := []struct { + name string + client compat.Client + plan *migapi.MigPlan + wantErr bool + wantConditions []migapi.Condition + dontWantConditions []migapi.Condition + }{ + { + name: "given a cluster without kubevirt installed, should return a warning condition", + client: getFakeClientWithObjs(), + plan: plan.DeepCopy(), + wantErr: false, + wantConditions: []migapi.Condition{ + { + Type: KubeVirtNotInstalledSourceCluster, + Status: True, + Reason: NotFound, + Category: Advisory, + Message: KubeVirtNotInstalledSourceClusterMessage, + }, + }, + }, + { + name: "given a cluster with multiple kubevirt CRs, should return a warning condition", + client: getFakeClientWithObjs( + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt", + Namespace: "kubevirt", + }, + }, + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt-two", + Namespace: "openshift-cnv", + }, + }, + ), + plan: plan.DeepCopy(), + wantErr: false, + wantConditions: []migapi.Condition{ + { + Type: KubeVirtNotInstalledSourceCluster, + Status: True, + Reason: NotFound, + Category: Advisory, + Message: KubeVirtNotInstalledSourceClusterMessage, + }, + }, + }, + { + name: "given a cluster with kubevirt installed, but invalid version, should return a warning condition", + client: getFakeClientWithObjs( + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt", + Namespace: "kubevirt", + }, + Status: virtv1.KubeVirtStatus{ + OperatorVersion: "a.b.c", + }, + }, + ), + plan: plan.DeepCopy(), + wantErr: false, + wantConditions: []migapi.Condition{ + { + Type: KubeVirtVersionNotSupported, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtVersionNotSupportedMessage, + }, + }, + }, + { + name: "given a cluster with kubevirt installed, but older version, should return a warning condition", + client: getFakeClientWithObjs( + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt", + Namespace: "kubevirt", + }, + Status: virtv1.KubeVirtStatus{ + OperatorVersion: "0.4.3", + }, + }, + ), + plan: plan.DeepCopy(), + wantErr: false, + wantConditions: []migapi.Condition{ + { + Type: KubeVirtVersionNotSupported, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtVersionNotSupportedMessage, + }, + }, + }, + { + name: "given a cluster with kubevirt installed, but plan has no live migration, should not return a warning condition", + client: getFakeClientWithObjs( + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt", + Namespace: "kubevirt", + }, + Status: virtv1.KubeVirtStatus{ + OperatorVersion: "1.4.3", + }, + }, + ), + plan: noLiveMigratePlan, + wantErr: false, + dontWantConditions: []migapi.Condition{ + { + Type: KubeVirtStorageLiveMigrationNotEnabled, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtStorageLiveMigrationNotEnabledMessage, + }, + { + Type: KubeVirtVersionNotSupported, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtVersionNotSupportedMessage, + }, + { + Type: KubeVirtNotInstalledSourceCluster, + Status: True, + Reason: NotFound, + Category: Advisory, + Message: KubeVirtNotInstalledSourceClusterMessage, + }, + }, + }, + { + name: "given a cluster with new enough kubevirt installed, should not return a warning condition", + client: getFakeClientWithObjs( + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt", + Namespace: "kubevirt", + }, + Spec: virtv1.KubeVirtSpec{ + Configuration: virtv1.KubeVirtConfiguration{ + VMRolloutStrategy: ptr.To[virtv1.VMRolloutStrategy](virtv1.VMRolloutStrategyLiveUpdate), + DeveloperConfiguration: &virtv1.DeveloperConfiguration{ + FeatureGates: []string{ + VMLiveUpdateFeatures, + VolumeMigrationConfig, + VolumesUpdateStrategy, + }, + }, + }, + }, + Status: virtv1.KubeVirtStatus{ + OperatorVersion: "1.4.3", + }, + }, + ), + plan: plan.DeepCopy(), + wantErr: false, + dontWantConditions: []migapi.Condition{ + { + Type: KubeVirtStorageLiveMigrationNotEnabled, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtStorageLiveMigrationNotEnabledMessage, + }, + }, + }, + { + name: "given a cluster with new enough kubevirt installed, but not all featuregates should a warning condition", + client: getFakeClientWithObjs( + &virtv1.KubeVirt{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-kubevirt", + Namespace: "kubevirt", + }, + Spec: virtv1.KubeVirtSpec{ + Configuration: virtv1.KubeVirtConfiguration{ + VMRolloutStrategy: ptr.To[virtv1.VMRolloutStrategy](virtv1.VMRolloutStrategyLiveUpdate), + DeveloperConfiguration: &virtv1.DeveloperConfiguration{ + FeatureGates: []string{ + VolumeMigrationConfig, + VolumesUpdateStrategy, + }, + }, + }, + }, + Status: virtv1.KubeVirtStatus{ + OperatorVersion: "1.4.3", + }, + }, + ), + plan: plan.DeepCopy(), + wantErr: false, + wantConditions: []migapi.Condition{ + { + Type: KubeVirtStorageLiveMigrationNotEnabled, + Status: True, + Reason: NotSupported, + Category: Warn, + Message: KubeVirtStorageLiveMigrationNotEnabledMessage, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := ReconcileMigPlan{ + Client: tt.client, + tracer: mocktracer.New(), + } + if err := r.validateKubeVirtInstalled(context.TODO(), tt.client, tt.plan); (err != nil) != tt.wantErr { + t.Errorf("ReconcileMigPlan.validateKubeVirtInstalled() error = %v, wantErr %v", err, tt.wantErr) + } + for _, wantCond := range tt.wantConditions { + foundCond := tt.plan.Status.FindCondition(wantCond.Type) + if foundCond == nil { + t.Errorf("ReconcileMigPlan.validatePossibleMigrationTypes() wantCondition = %s, found nil", wantCond.Type) + } + if foundCond != nil && foundCond.Reason != wantCond.Reason { + t.Errorf("ReconcileMigPlan.validatePossibleMigrationTypes() want reason = %s, found %s", wantCond.Reason, foundCond.Reason) + } + } + for _, dontWantCond := range tt.dontWantConditions { + foundCond := tt.plan.Status.FindCondition(dontWantCond.Type) + if foundCond != nil { + t.Errorf("ReconcileMigPlan.validatePossibleMigrationTypes() dontWantCondition = %s, found = %s", dontWantCond.Type, foundCond.Type) + } + } + }) + } +}