From 2773cfd61314c086252bed617e00a1cd6c915926 Mon Sep 17 00:00:00 2001 From: Antonin Bas Date: Fri, 20 Dec 2024 16:14:15 -0800 Subject: [PATCH] Add e2e test for SupportBundleCollection for K8s Nodes (#6866) The SupportBundleCollection CRD can be used to collect support bundle files and upload them to a file server. It supports both K8s Nodes and VMs (External Nodes). Prior to this change, we would only test the VM case. The SupportBundleCollection e2e test for VMs cannot easily be run locally on a Kind cluster, which makes it impractical for development. The corresponding CI job (which runs the e2e test) also has to be triggered manually, which means it is not always run for all PRs, which can cause breakage. This change introduces a new SupportBundleCollection e2e test, which only collects bundles from K8s Nodes. The test can be run locally on a Kind cluster, as long as Antrea is installed with the SupportBundleCollection Feature Gate enabled. We also use a uniform mechanism to deploy an SFTP server for e2e tests. hack/externalnode/sftp-deployment.yml is no longer used for e2e tests, and the necessary Deployment / Service are created programmatically, using the same mechanism as for existing PacketCapture tests. Finally, we add a new e2e test case for the PacketCapture feature, to test the case where an unexpected public host key is provided in the PacketCapture CR (the key does not match any of the server's keys). Note that as part of this change, the toolbox image is updated to 1.5-0, as the curl command included with ubuntu 22.04 seems to have issues with SFTP (the 1.5 version of the image is based on ubuntu 24.04). Signed-off-by: Antonin Bas --- ci/jenkins/test-vm.sh | 1 - ci/jenkins/test-vmc.sh | 2 +- ci/kind/test-e2e-kind.sh | 2 +- ci/kind/test-secondary-network-kind.sh | 2 +- test/e2e/framework.go | 32 +- test/e2e/packetcapture_test.go | 587 ++++++++++--------------- test/e2e/sftp_util.go | 162 +++++++ test/e2e/supportbundle_test.go | 313 +++++++++++++ test/e2e/trafficcontrol_test.go | 13 +- test/e2e/vmagent_test.go | 83 +--- 10 files changed, 771 insertions(+), 426 deletions(-) create mode 100644 test/e2e/sftp_util.go diff --git a/ci/jenkins/test-vm.sh b/ci/jenkins/test-vm.sh index 5ccd4640c0a..32e94c451a7 100755 --- a/ci/jenkins/test-vm.sh +++ b/ci/jenkins/test-vm.sh @@ -179,7 +179,6 @@ function configure_vm_agent { cp ./build/yamls/externalnode/support-bundle-collection-rbac.yml ${WORKDIR}/support-bundle-collection-rbac.yml echo "Applying support-bundle-collection rbac yaml" kubectl apply -f ${WORKDIR}/support-bundle-collection-rbac.yml -n $TEST_NAMESPACE - cp ./hack/externalnode/sftp-deployment.yml ${WORKDIR}/sftp-deployment.yml cp ./hack/externalnode/install-vm.sh ${WORKDIR}/install-vm.sh cp ./hack/externalnode/install-vm.ps1 ${WORKDIR}/install-vm.ps1 create_kubeconfig_files diff --git a/ci/jenkins/test-vmc.sh b/ci/jenkins/test-vmc.sh index 73e6989630a..a8df4fc828f 100755 --- a/ci/jenkins/test-vmc.sh +++ b/ci/jenkins/test-vmc.sh @@ -447,7 +447,7 @@ function deliver_antrea { ${SCP_WITH_ANTREA_CI_KEY} $GIT_CHECKOUT_DIR/build/yamls/*.yml capv@${control_plane_ip}:~ IPs=($(kubectl get nodes -o wide --no-headers=true | awk '{print $6}' | xargs)) - antrea_images=("registry.k8s.io/e2e-test-images/agnhost:2.40" "antrea/nginx:1.21.6-alpine" "antrea/sonobuoy:v0.56.16" "antrea/toolbox:1.3-0" "antrea/systemd-logs:v0.4") + antrea_images=("registry.k8s.io/e2e-test-images/agnhost:2.40" "antrea/nginx:1.21.6-alpine" "antrea/sonobuoy:v0.56.16" "antrea/toolbox:1.5-1" "antrea/systemd-logs:v0.4") k8s_images=("registry.k8s.io/e2e-test-images/agnhost:2.45" "registry.k8s.io/e2e-test-images/jessie-dnsutils:1.5" "registry.k8s.io/e2e-test-images/nginx:1.14-2") e2e_images=("k8sprow.azurecr.io/kubernetes-e2e-test-images/agnhost:2.45" "k8sprow.azurecr.io/kubernetes-e2e-test-images/jessie-dnsutils:1.5" "k8sprow.azurecr.io/kubernetes-e2e-test-images/nginx:1.14-2") for image in "${antrea_images[@]}"; do diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh index 1b9030c76df..635191076e7 100755 --- a/ci/kind/test-e2e-kind.sh +++ b/ci/kind/test-e2e-kind.sh @@ -251,7 +251,7 @@ fi COMMON_IMAGES_LIST=("registry.k8s.io/e2e-test-images/agnhost:2.40" \ "antrea/nginx:1.21.6-alpine" \ - "antrea/toolbox:1.3-0") + "antrea/toolbox:1.5-1") FLOW_VISIBILITY_IMAGE_LIST=("antrea/ipfix-collector:v0.11.0" \ "antrea/clickhouse-operator:0.21.0" \ diff --git a/ci/kind/test-secondary-network-kind.sh b/ci/kind/test-secondary-network-kind.sh index 6d437ab2130..c2db3618bea 100755 --- a/ci/kind/test-secondary-network-kind.sh +++ b/ci/kind/test-secondary-network-kind.sh @@ -90,7 +90,7 @@ fi trap "quit" INT EXIT -IMAGE_LIST=("antrea/toolbox:1.3-0" \ +IMAGE_LIST=("antrea/toolbox:1.5-1" \ "antrea/antrea-agent-ubuntu:latest" \ "antrea/antrea-controller-ubuntu:latest") diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 387445e5adc..576f98db8c3 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -29,6 +29,7 @@ import ( "regexp" "strconv" "strings" + "testing" "time" "github.com/containernetworking/plugins/pkg/ip" @@ -40,6 +41,7 @@ import ( networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" @@ -121,7 +123,7 @@ const ( flowAggregatorConfName = "flow-aggregator.conf" agnhostImage = "registry.k8s.io/e2e-test-images/agnhost:2.40" - ToolboxImage = "antrea/toolbox:1.3-0" + ToolboxImage = "antrea/toolbox:1.5-1" mcjoinImage = "antrea/mcjoin:v2.9" nginxImage = "antrea/nginx:1.21.6-alpine" iisImage = "mcr.microsoft.com/windows/servercore/iis" @@ -3300,3 +3302,31 @@ func (data *TestData) setPodAnnotation(namespace, podName, annotationKey string, log.Infof("Successfully patched Pod %s in Namespace %s", podName, namespace) return nil } + +func (data *TestData) waitForDeploymentReady(t *testing.T, namespace string, name string, timeout time.Duration) error { + t.Logf("Waiting for Deployment '%s/%s' to be ready", namespace, name) + var labelSelector *metav1.LabelSelector + err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeout, false, func(ctx context.Context) (bool, error) { + dp, err := data.clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + labelSelector = dp.Spec.Selector + return dp.Status.ObservedGeneration == dp.Generation && dp.Status.ReadyReplicas == *dp.Spec.Replicas, nil + }) + if wait.Interrupted(err) { + labelMap, err := metav1.LabelSelectorAsMap(labelSelector) + var stdout string + if err != nil { + t.Logf("Cannot convert Selector for Deployment into kubectl label query: %v", err) + stdout = "" + } else { + labelQuery := labels.SelectorFromSet(labelMap).String() + _, stdout, _, _ = data.provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl -n %s describe pod -l %s", namespace, labelQuery)) + } + return fmt.Errorf("some replicas for Deployment '%s/%s' are not ready after %v:\n%s", namespace, name, timeout, stdout) + } else if err != nil { + return fmt.Errorf("error when waiting for Deployment '%s/%s' to be ready: %w", namespace, name, err) + } + return nil +} diff --git a/test/e2e/packetcapture_test.go b/test/e2e/packetcapture_test.go index 28e1f41f1a1..6ac6670f342 100644 --- a/test/e2e/packetcapture_test.go +++ b/test/e2e/packetcapture_test.go @@ -19,7 +19,9 @@ import ( "fmt" "io" "net" + "net/url" "os" + "path" "path/filepath" "sort" "strings" @@ -31,7 +33,6 @@ import ( "github.com/gopacket/gopacket/pcapgo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -60,106 +61,6 @@ type pcTestCase struct { ipVersion int } -func genSFTPService() *v1.Service { - selector := map[string]string{"app": "sftp"} - return &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sftp", - Labels: selector, - }, - Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeNodePort, - Selector: selector, - Ports: []v1.ServicePort{ - { - Port: 22, - TargetPort: intstr.FromInt32(22), - NodePort: 30010, - }, - }, - }, - } -} - -func genSSHKeysSecret(ed25519Key, rsaKey []byte) *v1.Secret { - return &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ssh-keys", - }, - Immutable: ptr.To(true), - Data: map[string][]byte{ - "ed25519": ed25519Key, - "rsa": rsaKey, - }, - } -} - -func genSFTPDeployment() *appsv1.Deployment { - replicas := int32(1) - selector := map[string]string{"app": "sftp"} - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sftp", - Labels: selector, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: selector, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sftp", - Labels: selector, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "sftp", - Image: "ghcr.io/atmoz/sftp/debian:latest", - ImagePullPolicy: v1.PullIfNotPresent, - Args: []string{"foo:pass:::upload"}, - ReadinessProbe: &v1.Probe{ - ProbeHandler: v1.ProbeHandler{ - TCPSocket: &v1.TCPSocketAction{ - Port: intstr.FromInt32(int32(22)), - }, - }, - PeriodSeconds: 3, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "ssh-keys", - ReadOnly: true, - MountPath: "/etc/ssh/ssh_host_ed25519_key", - SubPath: "ed25519", - }, - { - Name: "ssh-keys", - ReadOnly: true, - MountPath: "/etc/ssh/ssh_host_rsa_key", - SubPath: "rsa", - }, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "ssh-keys", - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: "ssh-keys", - DefaultMode: ptr.To[int32](0400), - }, - }, - }, - }, - }, - }, - }, - } -} - func createUDPServerPod(name string, ns string, portNum int32, serverNode string) error { port := v1.ContainerPort{Name: fmt.Sprintf("port-%d", portNum), ContainerPort: portNum} return NewPodBuilder(name, ns, agnhostImage). @@ -181,18 +82,12 @@ func TestPacketCapture(t *testing.T) { } defer teardownTest(t, data) - ed25519PubKey, ed25519PrivateKey, err := sftptesting.GenerateEd25519Key() - require.NoError(t, err) - rsaPubKey, rsaPrivateKey, err := sftptesting.GenerateRSAKey(4096) - require.NoError(t, err) - - _, err = data.clientset.CoreV1().Secrets(data.testNamespace).Create(context.TODO(), genSSHKeysSecret(ed25519PrivateKey, rsaPrivateKey), metav1.CreateOptions{}) - require.NoError(t, err) - deployment, err := data.clientset.AppsV1().Deployments(data.testNamespace).Create(context.TODO(), genSFTPDeployment(), metav1.CreateOptions{}) - require.NoError(t, err) - _, err = data.clientset.CoreV1().Services(data.testNamespace).Create(context.TODO(), genSFTPService(), metav1.CreateOptions{}) - require.NoError(t, err) - failOnError(data.waitForDeploymentReady(t, deployment.Namespace, deployment.Name, defaultTimeout), t) + deployment, svc, pubKeys, err := data.deploySFTPServer(context.TODO(), 0) + require.NoError(t, err, "failed to deploy SFTP server") + require.Len(t, pubKeys, 2) + pubKey1, pubKey2 := pubKeys[0], pubKeys[1] + require.NoError(t, data.waitForDeploymentReady(t, deployment.Namespace, deployment.Name, defaultTimeout)) + require.NotEmpty(t, svc.Spec.ClusterIP) sec := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -201,8 +96,8 @@ func TestPacketCapture(t *testing.T) { Namespace: "kube-system", }, Data: map[string][]byte{ - "username": []byte("foo"), - "password": []byte("pass"), + "username": []byte(sftpUser), + "password": []byte(sftpPassword), }, } _, err = data.clientset.CoreV1().Secrets(sec.Namespace).Create(context.TODO(), sec, metav1.CreateOptions{}) @@ -210,19 +105,50 @@ func TestPacketCapture(t *testing.T) { defer data.clientset.CoreV1().Secrets(sec.Namespace).Delete(context.TODO(), sec.Name, metav1.DeleteOptions{}) t.Run("testPacketCaptureBasic", func(t *testing.T) { - testPacketCaptureBasic(t, data, ed25519PubKey.Marshal(), rsaPubKey.Marshal()) + testPacketCaptureBasic(t, data, svc.Spec.ClusterIP, pubKey1.Marshal(), pubKey2.Marshal()) }) } +// getLocalPcapFilepath returns the path of the local pcap file present inside the Pod, for the +// Antrea Agent which ran the packet capture. +func getLocalPcapFilepath(pcName string) string { + return path.Join("/tmp", "antrea", "packetcapture", "packets", pcName+".pcapng") +} + +type packetCaptureOption func(pc *crdv1alpha1.PacketCapture) + +func packetCaptureTimeout(timeout *int32) packetCaptureOption { + return func(pc *crdv1alpha1.PacketCapture) { + pc.Spec.Timeout = timeout + } +} + +func packetCaptureFirstN(firstN int32) packetCaptureOption { + return func(pc *crdv1alpha1.PacketCapture) { + pc.Spec.CaptureConfig.FirstN = &crdv1alpha1.PacketCaptureFirstNConfig{ + Number: firstN, + } + } +} + +func packetCaptureHostPublicKey(pubKey []byte) packetCaptureOption { + return func(pc *crdv1alpha1.PacketCapture) { + pc.Spec.FileServer.HostPublicKey = pubKey + } +} + // testPacketCaptureTCP verifies if PacketCapture can capture tcp packets. this function only contains basic // cases with pod-to-pod. -func testPacketCaptureBasic(t *testing.T, data *TestData, ed25519PubKey, rsaPubKey []byte) { +func testPacketCaptureBasic(t *testing.T, data *TestData, sftpServerIP string, pubKey1, pubKey2 []byte) { node1 := nodeName(0) clientPodName := "client" tcpServerPodName := "tcp-server" udpServerPodName := "udp-server" nonExistingPodName := "non-existing-pod" + sftpURL := fmt.Sprintf("sftp://%s:22/%s", sftpServerIP, sftpUploadDir) + invalidPubKey, _, err := sftptesting.GenerateEd25519Key() + require.NoError(t, err) require.NoError(t, data.createToolboxPodOnNode(clientPodName, data.testNamespace, node1, false)) defer data.DeletePodAndWait(defaultTimeout, clientPodName, data.testNamespace) @@ -237,64 +163,85 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, ed25519PubKey, rsaPubK {Name: udpServerPodName}, }) - testcases := []pcTestCase{ - { - name: "ipv4-icmp-timeout", - ipVersion: 4, - pc: &crdv1alpha1.PacketCapture{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ipv4-icmp-timeout", - }, - Spec: crdv1alpha1.PacketCaptureSpec{ - Timeout: ptr.To[int32](15), - Source: crdv1alpha1.Source{ - Pod: &crdv1alpha1.PodReference{ - Namespace: data.testNamespace, - Name: clientPodName, - }, - }, - Destination: crdv1alpha1.Destination{ - Pod: &crdv1alpha1.PodReference{ - Namespace: data.testNamespace, - Name: udpServerPodName, - }, - }, - CaptureConfig: crdv1alpha1.CaptureConfig{ - FirstN: &crdv1alpha1.PacketCaptureFirstNConfig{ - Number: 500, - }, + // This is the name of the Antrea Pod which performs the capture. The capture is performed + // on the Node where the source Pod (clientPodName) is running, which is node1. + antreaPodName, err := data.getAntreaPodOnNode(node1) + require.NoError(t, err) + + getPcapURL := func(name string) string { + p, err := url.JoinPath(sftpURL, name+".pcapng") + require.NoError(t, err) + return p + } + + getPacketCaptureCR := func(name string, destinationPodName string, packet *crdv1alpha1.Packet, options ...packetCaptureOption) *crdv1alpha1.PacketCapture { + pc := &crdv1alpha1.PacketCapture{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: crdv1alpha1.PacketCaptureSpec{ + Source: crdv1alpha1.Source{ + Pod: &crdv1alpha1.PodReference{ + Namespace: data.testNamespace, + Name: clientPodName, }, - FileServer: &crdv1alpha1.PacketCaptureFileServer{ - URL: fmt.Sprintf("sftp://%s:30010/upload", controlPlaneNodeIPv4()), + }, + Destination: crdv1alpha1.Destination{ + Pod: &crdv1alpha1.PodReference{ + Namespace: data.testNamespace, + Name: destinationPodName, }, - Packet: &crdv1alpha1.Packet{ - Protocol: &icmpProto, - IPFamily: v1.IPv4Protocol, + }, + CaptureConfig: crdv1alpha1.CaptureConfig{ + FirstN: &crdv1alpha1.PacketCaptureFirstNConfig{ + Number: 5, }, }, + FileServer: &crdv1alpha1.PacketCaptureFileServer{ + URL: sftpURL, + }, + Packet: packet, }, + } + for _, option := range options { + option(pc) + } + return pc + } + + testcases := []pcTestCase{ + { + name: "ipv4-icmp-timeout", + ipVersion: 4, + pc: getPacketCaptureCR( + "ipv4-icmp-timeout", + udpServerPodName, + &crdv1alpha1.Packet{ + Protocol: &icmpProto, + IPFamily: v1.IPv4Protocol, + }, + packetCaptureTimeout(ptr.To[int32](15)), + packetCaptureFirstN(500), + ), expectedStatus: crdv1alpha1.PacketCaptureStatus{ NumberCaptured: 10, - FilePath: fmt.Sprintf("sftp://%s:30010/upload/ipv4-icmp-timeout.pcapng", controlPlaneNodeIPv4()), + FilePath: getPcapURL("ipv4-icmp-timeout"), Conditions: []crdv1alpha1.PacketCaptureCondition{ { - Type: crdv1alpha1.PacketCaptureStarted, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Started", + Type: crdv1alpha1.PacketCaptureStarted, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Started", }, { - Type: crdv1alpha1.PacketCaptureComplete, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Timeout", - Message: "context deadline exceeded", + Type: crdv1alpha1.PacketCaptureComplete, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Timeout", + Message: "context deadline exceeded", }, { - Type: crdv1alpha1.PacketCaptureFileUploaded, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Succeed", + Type: crdv1alpha1.PacketCaptureFileUploaded, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Succeed", }, }, }, @@ -302,47 +249,23 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, ed25519PubKey, rsaPubK { name: nonExistingPodName, ipVersion: 4, - pc: &crdv1alpha1.PacketCapture{ - ObjectMeta: metav1.ObjectMeta{ - Name: nonExistingPodName, - }, - Spec: crdv1alpha1.PacketCaptureSpec{ - Source: crdv1alpha1.Source{ - Pod: &crdv1alpha1.PodReference{ - Namespace: data.testNamespace, - Name: clientPodName, - }, - }, - Destination: crdv1alpha1.Destination{ - Pod: &crdv1alpha1.PodReference{ - Namespace: data.testNamespace, - Name: nonExistingPodName, - }, - }, - CaptureConfig: crdv1alpha1.CaptureConfig{ - FirstN: &crdv1alpha1.PacketCaptureFirstNConfig{ - Number: 5, - }, - }, - FileServer: &crdv1alpha1.PacketCaptureFileServer{ - URL: fmt.Sprintf("sftp://%s:30010/upload", controlPlaneNodeIPv4()), - }, - }, - }, + pc: getPacketCaptureCR( + nonExistingPodName, + nonExistingPodName, + nil, + ), expectedStatus: crdv1alpha1.PacketCaptureStatus{ Conditions: []crdv1alpha1.PacketCaptureCondition{ { - Type: crdv1alpha1.PacketCaptureStarted, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Started", + Type: crdv1alpha1.PacketCaptureStarted, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Started", }, { - Type: crdv1alpha1.PacketCaptureComplete, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Failed", - Message: fmt.Sprintf("failed to get Pod %s/%s: pods \"%s\" not found", data.testNamespace, nonExistingPodName, nonExistingPodName), + Type: crdv1alpha1.PacketCaptureComplete, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Failed", + Message: fmt.Sprintf("failed to get Pod %s/%s: pods \"%s\" not found", data.testNamespace, nonExistingPodName, nonExistingPodName), }, }, }, @@ -350,64 +273,38 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, ed25519PubKey, rsaPubK { name: "ipv4-tcp", ipVersion: 4, - pc: &crdv1alpha1.PacketCapture{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ipv4-tcp", - }, - Spec: crdv1alpha1.PacketCaptureSpec{ - Source: crdv1alpha1.Source{ - Pod: &crdv1alpha1.PodReference{ - Namespace: data.testNamespace, - Name: clientPodName, - }, - }, - Destination: crdv1alpha1.Destination{ - Pod: &crdv1alpha1.PodReference{ - Namespace: data.testNamespace, - Name: tcpServerPodName, - }, - }, - CaptureConfig: crdv1alpha1.CaptureConfig{ - FirstN: &crdv1alpha1.PacketCaptureFirstNConfig{ - Number: 5, - }, - }, - FileServer: &crdv1alpha1.PacketCaptureFileServer{ - URL: fmt.Sprintf("sftp://%s:30010/upload", controlPlaneNodeIPv4()), - HostPublicKey: ed25519PubKey, - }, - Packet: &crdv1alpha1.Packet{ - Protocol: &tcpProto, - IPFamily: v1.IPv4Protocol, - TransportHeader: crdv1alpha1.TransportHeader{ - TCP: &crdv1alpha1.TCPHeader{ - DstPort: ptr.To(serverPodPort), - }, + pc: getPacketCaptureCR( + "ipv4-tcp", + tcpServerPodName, + &crdv1alpha1.Packet{ + Protocol: &tcpProto, + IPFamily: v1.IPv4Protocol, + TransportHeader: crdv1alpha1.TransportHeader{ + TCP: &crdv1alpha1.TCPHeader{ + DstPort: ptr.To(serverPodPort), }, }, }, - }, + packetCaptureHostPublicKey(pubKey1), + ), expectedStatus: crdv1alpha1.PacketCaptureStatus{ NumberCaptured: 5, - FilePath: fmt.Sprintf("sftp://%s:30010/upload/ipv4-tcp.pcapng", controlPlaneNodeIPv4()), + FilePath: getPcapURL("ipv4-tcp"), Conditions: []crdv1alpha1.PacketCaptureCondition{ { - Type: crdv1alpha1.PacketCaptureStarted, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Started", + Type: crdv1alpha1.PacketCaptureStarted, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Started", }, { - Type: crdv1alpha1.PacketCaptureComplete, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Succeed", + Type: crdv1alpha1.PacketCaptureComplete, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Succeed", }, { - Type: crdv1alpha1.PacketCaptureFileUploaded, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Succeed", + Type: crdv1alpha1.PacketCaptureFileUploaded, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Succeed", }, }, }, @@ -415,64 +312,38 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, ed25519PubKey, rsaPubK { name: "ipv4-udp", ipVersion: 4, - pc: &crdv1alpha1.PacketCapture{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ipv4-udp", - }, - Spec: crdv1alpha1.PacketCaptureSpec{ - Source: crdv1alpha1.Source{ - Pod: &crdv1alpha1.PodReference{ - Namespace: data.testNamespace, - Name: clientPodName, - }, - }, - Destination: crdv1alpha1.Destination{ - Pod: &crdv1alpha1.PodReference{ - Namespace: data.testNamespace, - Name: udpServerPodName, - }, - }, - CaptureConfig: crdv1alpha1.CaptureConfig{ - FirstN: &crdv1alpha1.PacketCaptureFirstNConfig{ - Number: 5, - }, - }, - FileServer: &crdv1alpha1.PacketCaptureFileServer{ - URL: fmt.Sprintf("sftp://%s:30010/upload", controlPlaneNodeIPv4()), - HostPublicKey: rsaPubKey, - }, - Packet: &crdv1alpha1.Packet{ - Protocol: &udpProto, - IPFamily: v1.IPv4Protocol, - TransportHeader: crdv1alpha1.TransportHeader{ - UDP: &crdv1alpha1.UDPHeader{ - DstPort: ptr.To(serverPodPort), - }, + pc: getPacketCaptureCR( + "ipv4-udp", + udpServerPodName, + &crdv1alpha1.Packet{ + Protocol: &udpProto, + IPFamily: v1.IPv4Protocol, + TransportHeader: crdv1alpha1.TransportHeader{ + UDP: &crdv1alpha1.UDPHeader{ + DstPort: ptr.To(serverPodPort), }, }, }, - }, + packetCaptureHostPublicKey(pubKey2), + ), expectedStatus: crdv1alpha1.PacketCaptureStatus{ NumberCaptured: 5, - FilePath: fmt.Sprintf("sftp://%s:30010/upload/ipv4-udp.pcapng", controlPlaneNodeIPv4()), + FilePath: getPcapURL("ipv4-udp"), Conditions: []crdv1alpha1.PacketCaptureCondition{ { - Type: crdv1alpha1.PacketCaptureStarted, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Started", + Type: crdv1alpha1.PacketCaptureStarted, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Started", }, { - Type: crdv1alpha1.PacketCaptureComplete, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Succeed", + Type: crdv1alpha1.PacketCaptureComplete, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Succeed", }, { - Type: crdv1alpha1.PacketCaptureFileUploaded, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Succeed", + Type: crdv1alpha1.PacketCaptureFileUploaded, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Succeed", }, }, }, @@ -480,58 +351,68 @@ func testPacketCaptureBasic(t *testing.T, data *TestData, ed25519PubKey, rsaPubK { name: "ipv4-icmp", ipVersion: 4, - pc: &crdv1alpha1.PacketCapture{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ipv4-icmp", + pc: getPacketCaptureCR( + "ipv4-icmp", + tcpServerPodName, + &crdv1alpha1.Packet{ + Protocol: &icmpProto, + IPFamily: v1.IPv4Protocol, }, - Spec: crdv1alpha1.PacketCaptureSpec{ - Source: crdv1alpha1.Source{ - Pod: &crdv1alpha1.PodReference{ - Namespace: data.testNamespace, - Name: clientPodName, - }, - }, - Destination: crdv1alpha1.Destination{ - Pod: &crdv1alpha1.PodReference{ - Namespace: data.testNamespace, - Name: tcpServerPodName, - }, - }, - CaptureConfig: crdv1alpha1.CaptureConfig{ - FirstN: &crdv1alpha1.PacketCaptureFirstNConfig{ - Number: 5, - }, + ), + expectedStatus: crdv1alpha1.PacketCaptureStatus{ + NumberCaptured: 5, + FilePath: getPcapURL("ipv4-icmp"), + Conditions: []crdv1alpha1.PacketCaptureCondition{ + { + Type: crdv1alpha1.PacketCaptureStarted, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Started", }, - FileServer: &crdv1alpha1.PacketCaptureFileServer{ - URL: fmt.Sprintf("sftp://%s:30010/upload", controlPlaneNodeIPv4()), + { + Type: crdv1alpha1.PacketCaptureComplete, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Succeed", }, - Packet: &crdv1alpha1.Packet{ - Protocol: &icmpProto, - IPFamily: v1.IPv4Protocol, + { + Type: crdv1alpha1.PacketCaptureFileUploaded, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Succeed", }, }, }, + }, + { + // The key is correctly formatted but does not match the server's keys. + name: "invalid-host-public-key", + ipVersion: 4, + pc: getPacketCaptureCR( + "invalid-host-public-key", + tcpServerPodName, + &crdv1alpha1.Packet{ + Protocol: &icmpProto, + IPFamily: v1.IPv4Protocol, + }, + packetCaptureHostPublicKey(invalidPubKey.Marshal()), + ), expectedStatus: crdv1alpha1.PacketCaptureStatus{ NumberCaptured: 5, - FilePath: fmt.Sprintf("sftp://%s:30010/upload/ipv4-icmp.pcapng", controlPlaneNodeIPv4()), + FilePath: antreaPodName + ":" + getLocalPcapFilepath("invalid-host-public-key"), Conditions: []crdv1alpha1.PacketCaptureCondition{ { - Type: crdv1alpha1.PacketCaptureStarted, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Started", + Type: crdv1alpha1.PacketCaptureStarted, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Started", }, { - Type: crdv1alpha1.PacketCaptureComplete, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Succeed", + Type: crdv1alpha1.PacketCaptureComplete, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "Succeed", }, { - Type: crdv1alpha1.PacketCaptureFileUploaded, - Status: metav1.ConditionStatus(v1.ConditionTrue), - LastTransitionTime: metav1.Now(), - Reason: "Succeed", + Type: crdv1alpha1.PacketCaptureFileUploaded, + Status: metav1.ConditionStatus(v1.ConditionFalse), + Reason: "Failed", + Message: "failed to upload file after 5 attempts", }, }, }, @@ -659,10 +540,9 @@ func runPacketCaptureTest(t *testing.T, data *TestData, tc pcTestCase) { // verify packets. antreaPodName, err := data.getAntreaPodOnNode(nodeName(0)) require.NoError(t, err) - fileName := fmt.Sprintf("%s.pcapng", tc.pc.Name) tmpDir := t.TempDir() - dstFileName := filepath.Join(tmpDir, fileName) - packetFile := filepath.Join("/tmp", "antrea", "packetcapture", "packets", fileName) + dstFileName := filepath.Join(tmpDir, tc.pc.Name+".pcapng") + packetFile := getLocalPcapFilepath(tc.pc.Name) require.NoError(t, data.copyPodFile(antreaPodName, "antrea-agent", "kube-system", packetFile, tmpDir)) defer os.Remove(dstFileName) file, err := os.Open(dstFileName) @@ -679,10 +559,11 @@ func (data *TestData) waitForPacketCapture(t *testing.T, name string, specTimeou timeout = time.Duration(specTimeout) * time.Second } if err = wait.PollUntilContextTimeout(context.Background(), defaultInterval, timeout, true, func(ctx context.Context) (bool, error) { - pc, err = data.crdClient.CrdV1alpha1().PacketCaptures().Get(ctx, name, metav1.GetOptions{}) + c, err := data.crdClient.CrdV1alpha1().PacketCaptures().Get(ctx, name, metav1.GetOptions{}) if err != nil { return false, nil } + pc = c if fn(pc) { return true, nil } @@ -717,37 +598,35 @@ func isPacketCaptureRunning(pc *crdv1alpha1.PacketCapture) bool { } -func conditionEqualsIgnoreLastTransitionTime(a, b crdv1alpha1.PacketCaptureCondition) bool { - a1 := a - a1.LastTransitionTime = metav1.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC) - b1 := b - b1.LastTransitionTime = metav1.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC) - return a1 == b1 +func packetCaptureConditionEqual(c1, c2 crdv1alpha1.PacketCaptureCondition) bool { + c1.LastTransitionTime = metav1.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC) + c2.LastTransitionTime = metav1.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC) + return c1 == c2 } -var semanticIgnoreLastTransitionTime = conversion.EqualitiesOrDie( - conditionSliceEqualsIgnoreLastTransitionTime, +var packetCaptureStatusSemanticEquality = conversion.EqualitiesOrDie( + packetCaptureConditionSliceEqual, ) -func packetCaptureStatusEqual(oldStatus, newStatus crdv1alpha1.PacketCaptureStatus) bool { - return semanticIgnoreLastTransitionTime.DeepEqual(oldStatus, newStatus) +func packetCaptureStatusEqual(status1, status2 crdv1alpha1.PacketCaptureStatus) bool { + return packetCaptureStatusSemanticEquality.DeepEqual(status1, status2) } -func conditionSliceEqualsIgnoreLastTransitionTime(as, bs []crdv1alpha1.PacketCaptureCondition) bool { - sort.Slice(as, func(i, j int) bool { - return as[i].Type < as[j].Type +func packetCaptureConditionSliceEqual(s1, s2 []crdv1alpha1.PacketCaptureCondition) bool { + sort.Slice(s1, func(i, j int) bool { + return s1[i].Type < s1[j].Type }) - sort.Slice(bs, func(i, j int) bool { - return bs[i].Type < bs[j].Type + sort.Slice(s2, func(i, j int) bool { + return s2[i].Type < s2[j].Type }) - if len(as) != len(bs) { + if len(s1) != len(s2) { return false } - for i := range as { - a := as[i] - b := bs[i] - if !conditionEqualsIgnoreLastTransitionTime(a, b) { + for i := range s1 { + a := s1[i] + b := s2[i] + if !packetCaptureConditionEqual(a, b) { return false } } diff --git a/test/e2e/sftp_util.go b/test/e2e/sftp_util.go new file mode 100644 index 00000000000..945bb42a601 --- /dev/null +++ b/test/e2e/sftp_util.go @@ -0,0 +1,162 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "fmt" + + "golang.org/x/crypto/ssh" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + + sftptesting "antrea.io/antrea/pkg/util/sftp/testing" +) + +var sftpLabels = map[string]string{"app": "sftp"} + +const ( + sftpUser = "foo" + sftpPassword = "pass" + sftpUploadDir = "upload" +) + +func genSFTPService(nodePort int32) *v1.Service { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sftp", + Labels: sftpLabels, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeNodePort, + Selector: sftpLabels, + Ports: []v1.ServicePort{ + { + Port: 22, + TargetPort: intstr.FromInt32(22), + NodePort: nodePort, + }, + }, + }, + } +} + +func genSSHKeysSecret(ed25519Key, rsaKey []byte) *v1.Secret { + return &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ssh-keys", + }, + Immutable: ptr.To(true), + Data: map[string][]byte{ + "ed25519": ed25519Key, + "rsa": rsaKey, + }, + } +} + +func genSFTPDeployment() *appsv1.Deployment { + replicas := int32(1) + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sftp", + Labels: sftpLabels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: sftpLabels, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sftp", + Labels: sftpLabels, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "sftp", + Image: "ghcr.io/atmoz/sftp/debian:latest", + ImagePullPolicy: v1.PullIfNotPresent, + Args: []string{fmt.Sprintf("%s:%s:::%s", sftpUser, sftpPassword, sftpUploadDir)}, + ReadinessProbe: &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.FromInt32(int32(22)), + }, + }, + PeriodSeconds: 3, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "ssh-keys", + ReadOnly: true, + MountPath: "/etc/ssh/ssh_host_ed25519_key", + SubPath: "ed25519", + }, + { + Name: "ssh-keys", + ReadOnly: true, + MountPath: "/etc/ssh/ssh_host_rsa_key", + SubPath: "rsa", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "ssh-keys", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "ssh-keys", + DefaultMode: ptr.To[int32](0400), + }, + }, + }, + }, + }, + }, + }, + } +} + +func (data *TestData) deploySFTPServer(ctx context.Context, nodePort int32) (*appsv1.Deployment, *v1.Service, []ssh.PublicKey, error) { + ed25519PubKey, ed25519PrivateKey, err := sftptesting.GenerateEd25519Key() + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to generate Ed25519 key: %w", err) + } + rsaPubKey, rsaPrivateKey, err := sftptesting.GenerateRSAKey(4096) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to generate RSA key: %w", err) + } + pubKeys := []ssh.PublicKey{ed25519PubKey, rsaPubKey} + + _, err = data.clientset.CoreV1().Secrets(data.testNamespace).Create(ctx, genSSHKeysSecret(ed25519PrivateKey, rsaPrivateKey), metav1.CreateOptions{}) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to create Secret for SSH private keys: %w", err) + } + deployment, err := data.clientset.AppsV1().Deployments(data.testNamespace).Create(ctx, genSFTPDeployment(), metav1.CreateOptions{}) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to create SFTP Deployment: %w", err) + } + svc, err := data.clientset.CoreV1().Services(data.testNamespace).Create(ctx, genSFTPService(nodePort), metav1.CreateOptions{}) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to create SFTP Service: %w", err) + } + + return deployment, svc, pubKeys, nil +} diff --git a/test/e2e/supportbundle_test.go b/test/e2e/supportbundle_test.go index 9cb644321a8..d865e2bd41e 100644 --- a/test/e2e/supportbundle_test.go +++ b/test/e2e/supportbundle_test.go @@ -20,18 +20,27 @@ import ( "fmt" "io" "net" + "slices" + "sort" + "strings" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/conversion" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" "antrea.io/antrea/pkg/apis" + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" systemv1beta1 "antrea.io/antrea/pkg/apis/system/v1beta1" clientset "antrea.io/antrea/pkg/client/clientset/versioned" + "antrea.io/antrea/pkg/features" + sftptesting "antrea.io/antrea/pkg/util/sftp/testing" "antrea.io/antrea/test/e2e/utils/portforwarder" ) @@ -153,3 +162,307 @@ func TestSupportBundleController(t *testing.T) { func TestSupportBundleAgent(t *testing.T) { testSupportBundle("agent", t) } + +func TestSupportBundleCollection(t *testing.T) { + skipIfFeatureDisabled(t, features.SupportBundleCollection, true, true) + skipIfHasWindowsNodes(t) + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + + deployment, svc, pubKeys, err := data.deploySFTPServer(context.TODO(), 0) + require.NoError(t, err, "failed to deploy SFTP server") + require.NotEmpty(t, pubKeys) + require.NoError(t, data.waitForDeploymentReady(t, deployment.Namespace, deployment.Name, defaultTimeout)) + require.NotEmpty(t, svc.Spec.ClusterIP) + + secretName := "support-bundle-secret" + sec := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: antreaNamespace, + }, + Data: map[string][]byte{ + "username": []byte(sftpUser), + "password": []byte(sftpPassword), + }, + } + _, err = data.clientset.CoreV1().Secrets(sec.Namespace).Create(context.TODO(), sec, metav1.CreateOptions{}) + require.NoError(t, err) + defer data.clientset.CoreV1().Secrets(sec.Namespace).Delete(context.TODO(), sec.Name, metav1.DeleteOptions{}) + + grantAntreaAccessToSecret(t, data, secretName) + + clientPod := "client" + require.NoError(t, data.createToolboxPodOnNode(clientPod, data.testNamespace, "", false)) + require.NoError(t, data.podWaitForRunning(defaultTimeout, clientPod, data.testNamespace)) + + invalidPubKey, _, err := sftptesting.GenerateEd25519Key() + require.NoError(t, err) + + // If cluster has more than 3 Nodes, only consider the first 3. + const maxNodes = 3 + var nodeNames []string + for idx := 0; idx < min(maxNodes, clusterInfo.numNodes); idx++ { + nodeNames = append(nodeNames, nodeName(idx)) + } + sortedNodeNames := slices.Sorted(slices.Values(nodeNames)) + + expectedStatusSuccess := crdv1alpha1.SupportBundleCollectionStatus{ + CollectedNodes: int32(len(nodeNames)), + DesiredNodes: int32(len(nodeNames)), + Conditions: []crdv1alpha1.SupportBundleCollectionCondition{ + { + Type: crdv1alpha1.CollectionStarted, + Status: metav1.ConditionStatus(v1.ConditionTrue), + }, + { + Type: crdv1alpha1.BundleCollected, + Status: metav1.ConditionStatus(v1.ConditionTrue), + }, + { + Type: crdv1alpha1.CollectionFailure, + Status: metav1.ConditionStatus(v1.ConditionFalse), + }, + { + Type: crdv1alpha1.CollectionCompleted, + Status: metav1.ConditionStatus(v1.ConditionTrue), + }, + }, + } + + t.Run("with ssh host key", func(t *testing.T) { + testSupportBundleCollection(t, data, "sbc-0", nodeNames, clientPod, svc.Spec.ClusterIP, pubKeys[0].Marshal(), expectedStatusSuccess) + }) + t.Run("without ssh host key", func(t *testing.T) { + testSupportBundleCollection(t, data, "sbc-1", nodeNames, clientPod, svc.Spec.ClusterIP, nil, expectedStatusSuccess) + }) + t.Run("with invalid ssh host key", func(t *testing.T) { + expectedStatus := crdv1alpha1.SupportBundleCollectionStatus{ + CollectedNodes: 0, + DesiredNodes: int32(len(nodeNames)), + Conditions: []crdv1alpha1.SupportBundleCollectionCondition{ + { + Type: crdv1alpha1.CollectionStarted, + Status: metav1.ConditionStatus(v1.ConditionTrue), + }, + { + Type: crdv1alpha1.BundleCollected, + Status: metav1.ConditionStatus(v1.ConditionFalse), + }, + { + Type: crdv1alpha1.CollectionFailure, + Status: metav1.ConditionStatus(v1.ConditionTrue), + Reason: "InternalError", + Message: fmt.Sprintf("Failed Agent count: %d, \"failed to upload file after 5 attempts\":[%s]", len(nodeNames), strings.Join(sortedNodeNames, ", ")), + }, + { + Type: crdv1alpha1.CollectionCompleted, + Status: metav1.ConditionStatus(v1.ConditionTrue), + }, + }, + } + // The key is correctly formatted but does not match the server's keys. + testSupportBundleCollection(t, data, "sbc-2", nodeNames, clientPod, svc.Spec.ClusterIP, invalidPubKey.Marshal(), expectedStatus) + }) +} + +func testSupportBundleCollection( + t *testing.T, + data *TestData, + bundleName string, + nodeNames []string, + clientPod string, + sftpServerIP string, + pubKey []byte, + expectedStatus crdv1alpha1.SupportBundleCollectionStatus, +) { + sftpURL := fmt.Sprintf("sftp://%s/%s", sftpServerIP, sftpUploadDir) + + // First, create a dedicated upload directory for this test case. + cmd := []string{"curl", "--insecure", "--user", fmt.Sprintf("%s:%s", sftpUser, sftpPassword), "-Q", fmt.Sprintf("mkdir %s/%s", sftpUploadDir, bundleName), sftpURL + "/"} + stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, clientPod, toolboxContainerName, cmd) + require.NoErrorf(t, err, "failed to create upload directory with sftp, stdout: %s, stderr: %s", stdout, stderr) + + sbc := &crdv1alpha1.SupportBundleCollection{ + ObjectMeta: metav1.ObjectMeta{ + Name: bundleName, + }, + Spec: crdv1alpha1.SupportBundleCollectionSpec{ + Nodes: &crdv1alpha1.BundleNodes{ + NodeNames: nodeNames, + }, + ExpirationMinutes: 300, + FileServer: crdv1alpha1.BundleFileServer{ + URL: fmt.Sprintf("%s:22/%s/%s", sftpServerIP, sftpUploadDir, bundleName), + HostPublicKey: pubKey, + }, + Authentication: crdv1alpha1.BundleServerAuthConfiguration{ + AuthType: "BasicAuthentication", + AuthSecret: &v1.SecretReference{ + Name: "support-bundle-secret", + Namespace: antreaNamespace, + }, + }, + }, + } + _, err = data.crdClient.CrdV1alpha1().SupportBundleCollections().Create(context.TODO(), sbc, metav1.CreateOptions{}) + require.NoError(t, err) + defer data.crdClient.CrdV1alpha1().SupportBundleCollections().Delete(context.TODO(), bundleName, metav1.DeleteOptions{}) + sbc, err = data.waitForSupportBundleCollectionCompleted(t, bundleName, 30*time.Second) + require.NoError(t, err) + + require.True(t, supportBundleCollectionStatusEqual(sbc.Status, expectedStatus)) + + condFailure := findSupportBundleCollectionCondition(sbc.Status.Conditions, crdv1alpha1.CollectionFailure) + if condFailure != nil && condFailure.Status == metav1.ConditionTrue || sbc.Status.CollectedNodes != int32(len(nodeNames)) { + // don't check for uploaded files in case of failure + return + } + + // Finally, we check that the expected files have been uploaded the server, but we do not + // check their contents. + // --list-only is to ensure that the output only includes file names, with no additional metadata + cmd = []string{"curl", "--insecure", "--list-only", "--user", fmt.Sprintf("%s:%s", sftpUser, sftpPassword), fmt.Sprintf("%s/%s/", sftpURL, bundleName)} + stdout, stderr, err = data.RunCommandFromPod(data.testNamespace, clientPod, toolboxContainerName, cmd) + require.NoErrorf(t, err, "failed to list upload directory with sftp, stdout: %s, stderr: %s", stdout, stderr) + files := slices.DeleteFunc(strings.Fields(stdout), func(fileName string) bool { + // Remove symbolic links "." and ".." + return strings.HasPrefix(fileName, ".") + }) + expectedFiles := make([]string, len(nodeNames)) + for idx := range nodeNames { + expectedFiles[idx] = fmt.Sprintf("%s_%s.tar.gz", nodeNames[idx], bundleName) + } + assert.ElementsMatch(t, expectedFiles, files, "files uploaded by Antrea to sftp server do not match expectations") +} + +func (data *TestData) waitForSupportBundleCollection( + t *testing.T, + name string, + timeout time.Duration, + condition func(*crdv1alpha1.SupportBundleCollection) bool, +) (*crdv1alpha1.SupportBundleCollection, error) { + var sbc *crdv1alpha1.SupportBundleCollection + if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, timeout, false, func(ctx context.Context) (bool, error) { + c, err := data.crdClient.CrdV1alpha1().SupportBundleCollections().Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + sbc = c + if condition(sbc) { + return true, nil + } + return false, nil + }); err != nil { + if sbc != nil { + t.Logf("Status for SupportBundleCollection: %+v", sbc.Status) + } + return nil, err + } + return sbc, nil +} + +func findSupportBundleCollectionCondition(conditions []crdv1alpha1.SupportBundleCollectionCondition, t crdv1alpha1.SupportBundleCollectionConditionType) *crdv1alpha1.SupportBundleCollectionCondition { + for idx := range conditions { + cond := &conditions[idx] + if cond.Type == t { + return cond + } + } + return nil +} + +func (data *TestData) waitForSupportBundleCollectionCompleted(t *testing.T, name string, timeout time.Duration) (*crdv1alpha1.SupportBundleCollection, error) { + t.Logf("Waiting for SupportBundleCollection '%s' to be completed", name) + return data.waitForSupportBundleCollection(t, name, timeout, func(sbc *crdv1alpha1.SupportBundleCollection) bool { + cond := findSupportBundleCollectionCondition(sbc.Status.Conditions, crdv1alpha1.CollectionCompleted) + return cond != nil && cond.Status == metav1.ConditionTrue + }) +} + +func supportBundleCollectionConditionEqual(c1, c2 crdv1alpha1.SupportBundleCollectionCondition) bool { + c1.LastTransitionTime = metav1.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC) + c2.LastTransitionTime = metav1.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC) + return c1 == c2 +} + +var supportBundleCollectionStatusSemanticEquality = conversion.EqualitiesOrDie( + supportBundleCollectionConditionSliceEqual, +) + +func supportBundleCollectionStatusEqual(status1, status2 crdv1alpha1.SupportBundleCollectionStatus) bool { + return supportBundleCollectionStatusSemanticEquality.DeepEqual(status1, status2) +} + +func supportBundleCollectionConditionSliceEqual(s1, s2 []crdv1alpha1.SupportBundleCollectionCondition) bool { + sort.Slice(s1, func(i, j int) bool { + return s1[i].Type < s1[j].Type + }) + sort.Slice(s2, func(i, j int) bool { + return s2[i].Type < s2[j].Type + }) + + if len(s1) != len(s2) { + return false + } + for i := range s1 { + a := s1[i] + b := s2[i] + if !supportBundleCollectionConditionEqual(a, b) { + return false + } + } + return true +} + +func grantAntreaAccessToSecret(t *testing.T, data *TestData, secretName string) { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + ResourceNames: []string{secretName}, + }, + }, + } + _, err := data.clientset.RbacV1().Roles(antreaNamespace).Create(context.TODO(), role, metav1.CreateOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + err := data.clientset.RbacV1().Roles(antreaNamespace).Delete(context.TODO(), role.Name, metav1.DeleteOptions{}) + assert.NoError(t, err) + }) + + for _, serviceAccount := range []string{"antrea-controller", "antrea-agent"} { + name := fmt.Sprintf("%s-%s", serviceAccount, secretName) + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: serviceAccount, + Namespace: antreaNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: secretName, + }, + } + _, err := data.clientset.RbacV1().RoleBindings(antreaNamespace).Create(context.TODO(), roleBinding, metav1.CreateOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + err := data.clientset.RbacV1().RoleBindings(antreaNamespace).Delete(context.TODO(), roleBinding.Name, metav1.DeleteOptions{}) + assert.NoError(t, err) + }) + } +} diff --git a/test/e2e/trafficcontrol_test.go b/test/e2e/trafficcontrol_test.go index db5a4c83c42..81a69cef8a7 100644 --- a/test/e2e/trafficcontrol_test.go +++ b/test/e2e/trafficcontrol_test.go @@ -33,6 +33,7 @@ import ( type trafficControlTestConfig struct { nodeName string + podLabels map[string]string podName string podIPs map[corev1.IPFamily]string collectorPodName string @@ -42,9 +43,9 @@ type trafficControlTestConfig struct { var ( vni = int32(1) dstVXLANPort = int32(1111) - labels = map[string]string{"tc-e2e": "agnhost"} tcTestConfig = trafficControlTestConfig{ + podLabels: map[string]string{"tc-e2e": "agnhost"}, podName: "test-tc-pod", podIPs: map[corev1.IPFamily]string{}, collectorPodName: "test-packets-collector-pod", @@ -64,7 +65,7 @@ func TestTrafficControl(t *testing.T) { tcTestConfig.nodeName = controlPlaneNodeName() - createTrafficControlTestPod(t, data, tcTestConfig.podName) + createTrafficControlTestPod(t, data, tcTestConfig.podName, tcTestConfig.podLabels) createTrafficControlPacketsCollectorPod(t, data, tcTestConfig.collectorPodName) t.Run("TestMirrorToRemote", func(t *testing.T) { testMirrorToRemote(t, data) }) @@ -72,7 +73,7 @@ func TestTrafficControl(t *testing.T) { t.Run("TestRedirectToLocal", func(t *testing.T) { testRedirectToLocal(t, data) }) } -func createTrafficControlTestPod(t *testing.T, data *TestData, podName string) { +func createTrafficControlTestPod(t *testing.T, data *TestData, podName string, labels map[string]string) { args := []string{"netexec", "--http-port=8080"} ports := []corev1.ContainerPort{ { @@ -229,7 +230,7 @@ ip link set %[3]s up`, vni, dstVXLANPort, tunnelPeer) // Create a TrafficControl whose target port is VXLAN. targetPort := &v1alpha2.UDPTunnel{RemoteIP: tcTestConfig.collectorPodIPs[corev1.IPv4Protocol], VNI: &vni, DestinationPort: &dstVXLANPort} - tc := data.createTrafficControl(t, "tc-", nil, labels, v1alpha2.DirectionBoth, v1alpha2.ActionMirror, targetPort, true, nil) + tc := data.createTrafficControl(t, "tc-", nil, tcTestConfig.podLabels, v1alpha2.DirectionBoth, v1alpha2.ActionMirror, targetPort, true, nil) defer data.crdClient.CrdV1alpha2().TrafficControls().Delete(context.TODO(), tc.Name, metav1.DeleteOptions{}) // Wait flows of the TrafficControl to be realized. time.Sleep(time.Second) @@ -242,7 +243,7 @@ func testMirrorToLocal(t *testing.T, data *TestData) { // Create a TrafficControl whose target port is OVS internal port. portName := "test-port" targetPort := &v1alpha2.OVSInternalPort{Name: portName} - tc := data.createTrafficControl(t, "tc-", nil, labels, v1alpha2.DirectionBoth, v1alpha2.ActionMirror, targetPort, false, nil) + tc := data.createTrafficControl(t, "tc-", nil, tcTestConfig.podLabels, v1alpha2.DirectionBoth, v1alpha2.ActionMirror, targetPort, false, nil) defer data.crdClient.CrdV1alpha2().TrafficControls().Delete(context.TODO(), tc.Name, metav1.DeleteOptions{}) // Wait flows of the TrafficControl to be realized. time.Sleep(time.Second) @@ -310,7 +311,7 @@ ip link set dev %[2]s up`, targetPortName, returnPortName) targetPort := &v1alpha2.NetworkDevice{Name: targetPortName} returnPort := &v1alpha2.NetworkDevice{Name: returnPortName} - tc := data.createTrafficControl(t, "tc-", nil, labels, v1alpha2.DirectionBoth, v1alpha2.ActionRedirect, targetPort, false, returnPort) + tc := data.createTrafficControl(t, "tc-", nil, tcTestConfig.podLabels, v1alpha2.DirectionBoth, v1alpha2.ActionRedirect, targetPort, false, returnPort) defer data.crdClient.CrdV1alpha2().TrafficControls().Delete(context.TODO(), tc.Name, metav1.DeleteOptions{}) // Wait flows of TrafficControl to be realized. time.Sleep(time.Second) diff --git a/test/e2e/vmagent_test.go b/test/e2e/vmagent_test.go index 2e1d75682b0..a83181ed17d 100644 --- a/test/e2e/vmagent_test.go +++ b/test/e2e/vmagent_test.go @@ -82,83 +82,43 @@ func TestVMAgent(t *testing.T) { t.Run("testExternalNodeSupportBundleCollection", func(t *testing.T) { testExternalNodeSupportBundleCollection(t, data, vmList) }) } -func (data *TestData) waitForDeploymentReady(t *testing.T, namespace string, name string, timeout time.Duration) error { - t.Logf("Waiting for Deployment '%s/%s' to be ready", namespace, name) - err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, timeout, false, func(ctx context.Context) (bool, error) { - dp, err := data.clientset.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) - if err != nil { - return false, err - } - return dp.Status.ObservedGeneration == dp.Generation && dp.Status.ReadyReplicas == *dp.Spec.Replicas, nil - }) - if wait.Interrupted(err) { - _, stdout, _, _ := data.provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl -n %s describe pod -l app=sftp", namespace)) - return fmt.Errorf("some replicas for Deployment '%s/%s' are not ready after %v:\n%v", namespace, name, timeout, stdout) - } else if err != nil { - return fmt.Errorf("error when waiting for Deployment '%s/%s' to be ready: %w", namespace, name, err) - } - return nil -} - func (data *TestData) waitForSupportBundleCollectionRealized(t *testing.T, name string, timeout time.Duration) error { t.Logf("Waiting for SupportBundleCollection '%s' to be realized", name) - var sbc *crdv1alpha1.SupportBundleCollection - if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, timeout, false, func(ctx context.Context) (bool, error) { - var getErr error - sbc, getErr = data.crdClient.CrdV1alpha1().SupportBundleCollections().Get(context.TODO(), name, metav1.GetOptions{}) - if getErr != nil { - return false, getErr - } - for _, cond := range sbc.Status.Conditions { - if cond.Status == metav1.ConditionTrue && cond.Type == crdv1alpha1.CollectionCompleted { - return sbc.Status.DesiredNodes == sbc.Status.CollectedNodes, nil - } - } - return false, nil - }); err != nil { - if sbc != nil { - t.Logf("The conditions of SupportBundleCollection for the vms are %v", sbc.Status.Conditions) - } - return fmt.Errorf("error when waiting for SupportBundleCollection '%s' to be realized: %v", name, err) + _, err := data.waitForSupportBundleCollection(t, name, timeout, func(sbc *crdv1alpha1.SupportBundleCollection) bool { + cond := findSupportBundleCollectionCondition(sbc.Status.Conditions, crdv1alpha1.CollectionCompleted) + return cond != nil && cond.Status == metav1.ConditionTrue && sbc.Status.DesiredNodes == sbc.Status.CollectedNodes + }) + if err != nil { + return fmt.Errorf("error when waiting for SupportBundleCollection '%s' to be realized: %w", name, err) } return nil } func testExternalNodeSupportBundleCollection(t *testing.T, data *TestData, vmList []vmInfo) { - sftpServiceYAML := "sftp-deployment.yml" - secretUserName := "foo" - secretPassword := "pass" - uploadFolder := "upload" - uploadPath := path.Join("/home", secretUserName, uploadFolder) + const sftpNodePort = 30010 + deployment, _, _, err := data.deploySFTPServer(context.TODO(), 30010) + require.NoError(t, err, "failed to deploy SFTP server") + require.NoError(t, data.waitForDeploymentReady(t, deployment.Namespace, deployment.Name, defaultTimeout)) + secretName := "support-bundle-secret" - vmNames := make([]string, 0, len(vmList)) - for _, vm := range vmList { - vmNames = append(vmNames, vm.nodeName) - } - applySFTPYamlCommand := fmt.Sprintf("kubectl apply -f %s -n %s", sftpServiceYAML, data.testNamespace) - code, stdout, stderr, err := data.RunCommandOnNode(controlPlaneNodeName(), applySFTPYamlCommand) - require.NoError(t, err) - defer func() { - deleteSFTPYamlCommand := fmt.Sprintf("kubectl delete -f %s -n %s", sftpServiceYAML, data.testNamespace) - data.RunCommandOnNode(controlPlaneNodeName(), deleteSFTPYamlCommand) - }() - t.Logf("Stdout of the command '%s': %s", applySFTPYamlCommand, stdout) - if code != 0 { - t.Errorf("Error when applying %s: %v", sftpServiceYAML, stderr) - } - failOnError(data.waitForDeploymentReady(t, data.testNamespace, "sftp", defaultTimeout), t) sec := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, }, Data: map[string][]byte{ - "username": []byte(secretUserName), - "password": []byte(secretPassword), + "username": []byte(sftpUser), + "password": []byte(sftpPassword), }, } _, err = data.clientset.CoreV1().Secrets(namespace).Create(context.TODO(), sec, metav1.CreateOptions{}) require.NoError(t, err) - defer data.clientset.CoreV1().Secrets(namespace).Delete(context.TODO(), secretName, metav1.DeleteOptions{}) + defer data.clientset.CoreV1().Secrets(namespace).Delete(context.TODO(), sec.Name, metav1.DeleteOptions{}) + + vmNames := make([]string, 0, len(vmList)) + for _, vm := range vmList { + vmNames = append(vmNames, vm.nodeName) + } + bundleName := "support-bundle-collection-external-node" sbc := &crdv1alpha1.SupportBundleCollection{ ObjectMeta: metav1.ObjectMeta{ @@ -172,7 +132,7 @@ func testExternalNodeSupportBundleCollection(t *testing.T, data *TestData, vmLis }, ExpirationMinutes: 300, FileServer: crdv1alpha1.BundleFileServer{ - URL: fmt.Sprintf("%s:30010/upload", controlPlaneNodeIPv4()), + URL: fmt.Sprintf("%s:%d/%s", controlPlaneNodeIPv4(), sftpNodePort, sftpUploadDir), }, Authentication: crdv1alpha1.BundleServerAuthConfiguration{ AuthType: "BasicAuthentication", @@ -191,6 +151,7 @@ func testExternalNodeSupportBundleCollection(t *testing.T, data *TestData, vmLis require.NoError(t, err) require.Len(t, pods.Items, 1) pod := pods.Items[0] + uploadPath := path.Join("/home", sftpUser, sftpUploadDir) for _, vm := range vmList { extractPath := path.Join(uploadPath, vm.nodeName) mkdirCommand := fmt.Sprintf("mkdir %s", extractPath)