From 051c9726c6db679ab8470856b1f3b6f40bf64c9a Mon Sep 17 00:00:00 2001 From: Daman Arora Date: Tue, 25 Apr 2023 23:07:46 +0530 Subject: [PATCH] setting the fullstate ipvs as main Signed-off-by: Daman --- .github/workflows/backend-build.yml | 17 +- .github/workflows/e2e_tests.yaml | 2 +- Makefile | 12 - backends/ipvs-as-sink/baseportinfo.go | 104 --- backends/ipvs-as-sink/clusteripsvc.go | 143 ---- backends/ipvs-as-sink/common.go | 480 ----------- backends/ipvs-as-sink/exec/exec.go | 252 ------ backends/ipvs-as-sink/externalIP.go | 74 -- backends/ipvs-as-sink/flags.go | 58 -- backends/ipvs-as-sink/go.mod | 49 -- backends/ipvs-as-sink/go.sum | 48 -- backends/ipvs-as-sink/ipset.go | 163 ---- backends/ipvs-as-sink/iptables.go | 414 --------- backends/ipvs-as-sink/iptables_test.go | 272 ------ backends/ipvs-as-sink/ipvs.go | 491 ----------- backends/ipvs-as-sink/ipvslb.go | 92 -- backends/ipvs-as-sink/loadbalancersvc.go | 281 ------- backends/ipvs-as-sink/nodeportsvc.go | 163 ---- backends/ipvs-as-sink/proxier.go | 115 --- backends/ipvs-as-sink/util/ipset.go | 537 ------------ backends/ipvs-as-sink/util/iptables.go | 792 ------------------ backends/ipvs-as-sink/util/iptables_linux.go | 101 --- backends/ipvs-as-sink/util/save_restore.go | 122 --- backends/ipvs-as-sink/util/types.go | 59 -- backends/ipvs-as-sink/util/utils.go | 194 ----- backends/{ipvsfullstate => ipvs}/README.md | 4 +- backends/{ipvsfullstate => ipvs}/common.go | 4 +- .../{ipvsfullstate => ipvs}/controller.go | 8 +- backends/{ipvsfullstate => ipvs}/flags.go | 2 +- backends/{ipvsfullstate => ipvs}/go.mod | 9 +- backends/{ipvsfullstate => ipvs}/go.sum | 5 + .../handle_clusterip_service.go | 4 +- .../handle_loadbalancer_service.go | 8 +- .../handle_nodeport_service.go | 4 +- .../internal/ipsets/entry.go | 0 .../internal/ipsets/ipset.go | 0 .../internal/ipsets/ipset_util.go | 0 .../internal/ipsets/manager.go | 0 .../internal/ipsets/runner.go | 0 .../internal/ipsets/setup.go | 0 .../internal/ipsets/types.go | 0 .../internal/iptables/manager.go | 0 .../internal/iptables/templates.go | 0 .../internal/iptables/types.go | 0 .../internal/ipvs/destination.go | 0 .../internal/ipvs/manager.go | 0 .../internal/ipvs/setup.go | 0 .../internal/ipvs/types.go | 0 .../internal/ipvs/util.go | 2 +- .../internal/ipvs/virtual_server.go | 0 .../ipset_definitions.go | 4 +- .../ipset_entry_builder.go | 4 +- .../{ipvsfullstate => ipvs}/iptable_rules.go | 4 +- backends/{ipvsfullstate => ipvs}/register.go | 4 +- backends/{ipvsfullstate => ipvs}/setup.go | 6 +- backends/{ipvsfullstate => ipvs}/types.go | 2 +- cmd/kpng/storecmds/storecmds_linux.go | 3 +- go.work | 3 +- go.work.sum | 381 +++++++++ hack/test_backend_build.sh | 3 +- hack/test_e2e.sh | 2 +- hack/test_skip_list.sh | 79 +- hack/tilt/deploy.sh | 2 +- hack/tilt/setup.sh | 4 +- 64 files changed, 436 insertions(+), 5150 deletions(-) delete mode 100644 backends/ipvs-as-sink/baseportinfo.go delete mode 100644 backends/ipvs-as-sink/clusteripsvc.go delete mode 100644 backends/ipvs-as-sink/common.go delete mode 100644 backends/ipvs-as-sink/exec/exec.go delete mode 100644 backends/ipvs-as-sink/externalIP.go delete mode 100644 backends/ipvs-as-sink/flags.go delete mode 100644 backends/ipvs-as-sink/go.mod delete mode 100644 backends/ipvs-as-sink/go.sum delete mode 100644 backends/ipvs-as-sink/ipset.go delete mode 100644 backends/ipvs-as-sink/iptables.go delete mode 100644 backends/ipvs-as-sink/iptables_test.go delete mode 100644 backends/ipvs-as-sink/ipvs.go delete mode 100644 backends/ipvs-as-sink/ipvslb.go delete mode 100644 backends/ipvs-as-sink/loadbalancersvc.go delete mode 100644 backends/ipvs-as-sink/nodeportsvc.go delete mode 100644 backends/ipvs-as-sink/proxier.go delete mode 100644 backends/ipvs-as-sink/util/ipset.go delete mode 100644 backends/ipvs-as-sink/util/iptables.go delete mode 100644 backends/ipvs-as-sink/util/iptables_linux.go delete mode 100644 backends/ipvs-as-sink/util/save_restore.go delete mode 100644 backends/ipvs-as-sink/util/types.go delete mode 100644 backends/ipvs-as-sink/util/utils.go rename backends/{ipvsfullstate => ipvs}/README.md (97%) rename backends/{ipvsfullstate => ipvs}/common.go (98%) rename backends/{ipvsfullstate => ipvs}/controller.go (93%) rename backends/{ipvsfullstate => ipvs}/flags.go (98%) rename backends/{ipvsfullstate => ipvs}/go.mod (75%) rename backends/{ipvsfullstate => ipvs}/go.sum (89%) rename backends/{ipvsfullstate => ipvs}/handle_clusterip_service.go (97%) rename backends/{ipvsfullstate => ipvs}/handle_loadbalancer_service.go (95%) rename backends/{ipvsfullstate => ipvs}/handle_nodeport_service.go (97%) rename backends/{ipvsfullstate => ipvs}/internal/ipsets/entry.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/ipsets/ipset.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/ipsets/ipset_util.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/ipsets/manager.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/ipsets/runner.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/ipsets/setup.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/ipsets/types.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/iptables/manager.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/iptables/templates.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/iptables/types.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/ipvs/destination.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/ipvs/manager.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/ipvs/setup.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/ipvs/types.go (100%) rename backends/{ipvsfullstate => ipvs}/internal/ipvs/util.go (99%) rename backends/{ipvsfullstate => ipvs}/internal/ipvs/virtual_server.go (100%) rename backends/{ipvsfullstate => ipvs}/ipset_definitions.go (98%) rename backends/{ipvsfullstate => ipvs}/ipset_entry_builder.go (97%) rename backends/{ipvsfullstate => ipvs}/iptable_rules.go (99%) rename backends/{ipvsfullstate => ipvs}/register.go (95%) rename backends/{ipvsfullstate => ipvs}/setup.go (96%) rename backends/{ipvsfullstate => ipvs}/types.go (98%) diff --git a/.github/workflows/backend-build.yml b/.github/workflows/backend-build.yml index f9563966a..8a9035591 100644 --- a/.github/workflows/backend-build.yml +++ b/.github/workflows/backend-build.yml @@ -32,28 +32,17 @@ jobs: - name: build backends/iptables run: ./hack/test_backend_build.sh iptables - ipvs-as-sink: - name: build backend package ipvs-as-sink + ipvs: + name: build backend package ipvs needs: setup runs-on: ubuntu-latest steps: - name: checkout uses: actions/checkout@v3 - - name: build backends/ipvs-as-sink + - name: build backends/ipvs run: ./hack/test_backend_build.sh ipvs - ipvsfullstate: - name: build backend package ipvsfullstate - needs: setup - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@v3 - - - name: build backends/ipvsfullstate - run: ./hack/test_backend_build.sh ipvsfullstate - nft: name: build backend package nft needs: setup diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index a82e3ce0d..edf949e34 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -33,7 +33,7 @@ jobs: fail-fast: false matrix: ipFamily: ["ipv4", "ipv6", "dual"] - backend: ["iptables", "nft", "ipvs", "ipvsfullstate", "ebpf", "userspacelin"] + backend: ["iptables", "nft", "ipvs", "ebpf", "userspacelin"] exclude: - ipFamily: "ipv6" backend: "ebpf" diff --git a/Makefile b/Makefile index deeef2c99..cacfcd323 100644 --- a/Makefile +++ b/Makefile @@ -94,18 +94,6 @@ e2e-ipv6-ipvs: go_mod_tests_requirement e2e-dual-ipvs: go_mod_tests_requirement ./hack/test_e2e.sh -i dual -b ipvs -## E2E with IPV4 and IPVS -e2e-ipv4-ipvsfullstate: go_mod_tests_requirement - ./hack/test_e2e.sh -i ipv4 -b ipvsfullstate - -## E2E with IPV6 and IPVS -e2e-ipv6-ipvsfullstate: go_mod_tests_requirement - ./hack/test_e2e.sh -i ipv6 -b ipvsfullstate - -## E2E with DUAL and IPVS -e2e-dual-ipvsfullstate: go_mod_tests_requirement - ./hack/test_e2e.sh -i dual -b ipvsfullstate - ## E2E with IPV4 and NFT e2e-ipv4-nft: go_mod_tests_requirement ./hack/test_e2e.sh -i ipv4 -b nft diff --git a/backends/ipvs-as-sink/baseportinfo.go b/backends/ipvs-as-sink/baseportinfo.go deleted file mode 100644 index cb418202f..000000000 --- a/backends/ipvs-as-sink/baseportinfo.go +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - v1 "k8s.io/api/core/v1" - "sigs.k8s.io/kpng/api/localv1" - "sigs.k8s.io/kpng/client/serviceevents" -) - -type BaseServicePortInfo struct { - serviceIP string - serviceType string - port int32 - targetPort int32 - targetPortName string - nodePort int32 - protocol localv1.Protocol - schedulingMethod string - weight int32 - sessionAffinity serviceevents.SessionAffinity - stickyMaxAgeSeconds int - healthCheckNodePort int - nodeLocalExternal bool - nodeLocalInternal bool - internalTrafficPolicy *v1.ServiceInternalTrafficPolicyType - hintsAnnotation string -} - -func NewBaseServicePortInfo(svc *localv1.Service, port *localv1.PortMapping, - serviceIP, serviceType, - schedulingMethod string, - weight int32) *BaseServicePortInfo { - return &BaseServicePortInfo{ - serviceIP: serviceIP, - port: port.Port, - targetPort: port.TargetPort, - targetPortName: port.Name, - nodePort: port.NodePort, - protocol: port.Protocol, - schedulingMethod: schedulingMethod, - weight: weight, - serviceType: serviceType, - sessionAffinity: serviceevents.GetSessionAffinity(svc.SessionAffinity), - } -} - -func (b *BaseServicePortInfo) ServiceIP() string { - return b.serviceIP -} - -func (b *BaseServicePortInfo) Port() int32 { - return b.port -} - -func (b *BaseServicePortInfo) TargetPort() int32 { - return b.targetPort -} - -func (b *BaseServicePortInfo) TargetPortName() string { - return b.targetPortName -} - -func (b *BaseServicePortInfo) Protocol() localv1.Protocol { - return b.protocol -} - -func (b *BaseServicePortInfo) GetVirtualServer() ipvsLB { - vs := ipvsLB{IP: b.serviceIP, - SchedulingMethod: b.schedulingMethod, - ServiceType: b.serviceType, - Port: uint16(b.port), - Protocol: b.protocol, - NodePort: uint16(b.nodePort), - } - - if b.sessionAffinity.ClientIP != nil { - vs.Flags |= FlagPersistent - vs.Timeout = uint32(b.sessionAffinity.ClientIP.ClientIP.TimeoutSeconds) - } - return vs -} - -func (b *BaseServicePortInfo) SetSessionAffinity(sa serviceevents.SessionAffinity) { - b.sessionAffinity = sa -} - -func (b *BaseServicePortInfo) ResetSessionAffinity() { - b.sessionAffinity = serviceevents.SessionAffinity{} -} diff --git a/backends/ipvs-as-sink/clusteripsvc.go b/backends/ipvs-as-sink/clusteripsvc.go deleted file mode 100644 index 9b3a2d9f2..000000000 --- a/backends/ipvs-as-sink/clusteripsvc.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - "k8s.io/klog/v2" - "sigs.k8s.io/kpng/api/localv1" - "sigs.k8s.io/kpng/client/serviceevents" -) - -func (s *Backend) handleClusterIPService(svc *localv1.Service, serviceIP string, IPKind serviceevents.IPKind, port *localv1.PortMapping) { - serviceKey := getServiceKey(svc) - ipFamily := getIPFamily(serviceIP) - - // Handle Cluster-IP for the service - if IPKind == serviceevents.ClusterIP { - isServiceUpdated := s.isServiceUpdated(serviceKey) - if !isServiceUpdated { - s.proxiers[ipFamily].handleNewClusterIPService(serviceKey, serviceIP, svc, port) - } else { - s.proxiers[ipFamily].handleUpdatedClusterIPService(serviceKey, serviceIP, svc, port) - } - } - - // Handle External-IP for the service - if IPKind == serviceevents.ExternalIP { - isServiceUpdated := s.isServiceUpdated(serviceKey) - if !isServiceUpdated { - s.proxiers[ipFamily].handleNewExternalIP(serviceKey, serviceIP, ClusterIPService, svc, port) - } else { - s.proxiers[ipFamily].handleUpdatedExternalIP(serviceKey, serviceIP, ClusterIPService, svc, port) - } - } -} - -func (s *Backend) deleteClusterIPService(svc *localv1.Service, serviceIP string, IPKind serviceevents.IPKind, port *localv1.PortMapping) { - serviceKey := getServiceKey(svc) - s.svcs[serviceKey] = svc - ipFamily := getIPFamily(serviceIP) - p := s.proxiers[ipFamily] - - spKey := getServicePortKey(serviceKey, serviceIP, port) - kv := p.servicePorts.GetByPrefix([]byte(spKey)) - portInfo := kv[0].Value.(BaseServicePortInfo) - - epList := p.deleteRealServerForPort(serviceKey, []*BaseServicePortInfo{&portInfo}) - for _, ep := range epList { - p.AddOrDelEndPointInIPSet(ep.endPointIP, port.Protocol.String(), port.TargetPort, ep.isLocalEndPoint, AddEndPoint) - } - - p.deleteVirtualServer(&portInfo) - - // Delete Cluster-IP for the service - if IPKind == serviceevents.ClusterIP { - p.AddOrDelClusterIPInIPSet(&portInfo, DeleteService) - } - - // Delete External-IP for the service - if IPKind == serviceevents.ExternalIP { - p.AddOrDelExternalIPInIPSet(serviceIP, &portInfo, DeleteService) - } - - p.servicePorts.DeleteByPrefix([]byte(spKey)) - - portMapKey := getPortKey(serviceKey, port) - p.deletePortFromPortMap(serviceKey, portMapKey) -} - -func (p *proxier) handleNewClusterIPService(serviceKey, clusterIP string, svc *localv1.Service, port *localv1.PortMapping) { - if _, ok := p.portMap[serviceKey]; !ok { - p.portMap[serviceKey] = make(map[string]localv1.PortMapping) - } - - spKey := getServicePortKey(serviceKey, clusterIP, port) - portInfo := NewBaseServicePortInfo(svc, port, clusterIP, ClusterIPService, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - - portMapKey := getPortKey(serviceKey, port) - p.portMap[serviceKey][portMapKey] = *port - - p.addVirtualServer(portInfo) - - //Cluster service IP needs to be programmed in ipset. - p.AddOrDelClusterIPInIPSet(portInfo, AddService) -} - -func (p *proxier) handleUpdatedClusterIPService(serviceKey, clusterIP string, svc *localv1.Service, port *localv1.PortMapping) { - if _, ok := p.portMap[serviceKey]; !ok { - klog.Errorf("can't update port into non-existent service") - return - } - - spKey := getServicePortKey(serviceKey, clusterIP, port) - portInfo := NewBaseServicePortInfo(svc, port, clusterIP, ClusterIPService, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - - portMapKey := getPortKey(serviceKey, port) - p.portMap[serviceKey][portMapKey] = *port - - //Update the service with added ports into LB tree - p.addVirtualServer(portInfo) - //Cluster service IP needs to be programmed in ipset with added ports. - p.AddOrDelClusterIPInIPSet(portInfo, AddService) - - epList := p.addRealServerForPort(serviceKey, []*BaseServicePortInfo{portInfo}) - - for _, ep := range epList { - p.AddOrDelEndPointInIPSet(ep.endPointIP, port.Protocol.String(), port.TargetPort, ep.isLocalEndPoint, AddEndPoint) - } -} - -func (s *Backend) handleEndPointForClusterIP(svcKey, key string, endpoint *localv1.Endpoint, op Operation) { - prefix := svcKey + "/" + key + "/" - - if op == AddEndPoint { - // endpoint will have only one family IP, either v4/6. - endPointIPs := endpoint.IPs.All() - for _, ip := range endPointIPs { - ipFamily := getIPFamily(ip) - s.proxiers[ipFamily].addRealServer(svcKey, prefix, ip, endpoint) - } - } - - if op == DeleteEndPoint { - for _, proxier := range s.proxiers { - proxier.deleteRealServer(svcKey, prefix) - } - } -} diff --git a/backends/ipvs-as-sink/common.go b/backends/ipvs-as-sink/common.go deleted file mode 100644 index 874486863..000000000 --- a/backends/ipvs-as-sink/common.go +++ /dev/null @@ -1,480 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - "errors" - "strconv" - "strings" - - "github.com/google/seesaw/ipvs" - "sigs.k8s.io/kpng/client/lightdiffstore" - "sigs.k8s.io/kpng/client/serviceevents" - - v1 "k8s.io/api/core/v1" - "k8s.io/klog/v2" - netutils "k8s.io/utils/net" - - "sigs.k8s.io/kpng/api/localv1" - ipsetutil "sigs.k8s.io/kpng/backends/ipvs-as-sink/util" -) - -const ( - ClusterIPService = "ClusterIP" - NodePortService = "NodePort" - LoadBalancerService = "LoadBalancer" -) - -const ( - // FlagPersistent specify IPVS service session affinity - FlagPersistent = 0x1 - // FlagHashed specify IPVS service hash flag - FlagHashed = 0x2 -) - -var protocolIPSetMap = map[string]string{ - ipsetutil.ProtocolTCP: kubeNodePortSetTCP, - ipsetutil.ProtocolUDP: kubeNodePortSetUDP, - ipsetutil.ProtocolSCTP: kubeNodePortSetSCTP, -} - -type Operation int32 - -const ( - AddService Operation = 0 - DeleteService Operation = 1 - AddEndPoint Operation = 2 - DeleteEndPoint Operation = 3 -) - -type endPointInfo struct { - endPointIP string - isLocalEndPoint bool - portMap map[string]int32 -} - -func asDummyIPs(ip string, ipFamily v1.IPFamily) string { - if ipFamily == v1.IPv4Protocol { - return ip + "/32" - } - - if ipFamily == v1.IPv6Protocol { - return ip + "/128" - } - return ip + "/32" -} - -func epPortSuffix(port *localv1.PortMapping) string { - return port.Protocol.String() + ":" + strconv.Itoa(int(port.Port)) -} - -func getServiceKey(svc *localv1.Service) string { - return svc.Namespace + "/" + svc.Name -} - -func getServicePortKey(serviceKey, serviceIP string, port *localv1.PortMapping) string { - return serviceKey + "/" + serviceIP + "/" + epPortSuffix(port) -} - -func getPortKey(serviceKey string, port *localv1.PortMapping) string { - return serviceKey + "/" + epPortSuffix(port) -} - -// Any service event (ip or port update) received after -// EP creation is considered as service update. Else -// its a new service creation. -func (s *Backend) isServiceUpdated(serviceKey string) bool { - if s.svcEPMap[serviceKey] >= 1 { - return true - } - return false -} - -func (p *proxier) AddOrDelEndPointInIPSet(endPointIP, protocol string, targetPort int32, isLocalEndPoint bool, op Operation) { - if !isLocalEndPoint { - return - } - - entry := getEndPointEntry(endPointIP, protocol, targetPort) - set := p.ipsetList[kubeLoopBackIPSet] - if valid := set.validateEntry(entry); !valid { - klog.Errorf("error adding entry to ipset. entry:%s, ipset:%s", entry.String(), p.ipsetList[kubeLoopBackIPSet].Name) - return - } - if op == AddEndPoint { - if err := set.handle.AddEntry(entry.String(), &set.IPSet, true); err != nil { - klog.Errorf("Failed to add entry %v into ip set: %s, error: %v", entry, set.Name, err) - } else { - klog.V(3).Infof("Successfully add entry: %v into ip set: %s", entry, set.Name) - } - p.updateRefCountForIPSet(kubeLoopBackIPSet, op) - } - if op == DeleteEndPoint { - if err := set.handle.DelEntry(entry.String(), set.Name); err != nil { - klog.Errorf("Failed to delete entry: %v from ip set: %s, error: %v", entry, set.Name, err) - } else { - klog.V(3).Infof("Successfully deleted entry: %v to ip set: %s", entry, set.Name) - } - p.updateRefCountForIPSet(kubeLoopBackIPSet, op) - } -} - -func getIPFamily(ipAddr string) v1.IPFamily { - var ipAddrFamily v1.IPFamily - if netutils.IsIPv4String(ipAddr) { - ipAddrFamily = v1.IPv4Protocol - } - - if netutils.IsIPv6String(ipAddr) { - ipAddrFamily = v1.IPv6Protocol - } - return ipAddrFamily -} - -func getEndPointEntry(endPointIP, protocol string, targetPort int32) *ipsetutil.Entry { - return &ipsetutil.Entry{ - IP: endPointIP, - Port: int(targetPort), - Protocol: strings.ToLower(protocol), - IP2: endPointIP, - SetType: ipsetutil.HashIPPortIP, - } -} - -func (p *proxier) AddOrDelClusterIPInIPSet(port *BaseServicePortInfo, op Operation) { - // Capture the clusterIP. - entry := getIPSetEntry("", port) - // add service Cluster IP:Port to kubeServiceAccess ip set for the purpose of solving hairpin. - if valid := p.ipsetList[kubeClusterIPSet].validateEntry(entry); !valid { - klog.Errorf("error adding entry :%s, to ipset:%s", entry.String(), p.ipsetList[kubeClusterIPSet].Name) - return - } - set := p.ipsetList[kubeClusterIPSet] - if op == AddService { - if err := set.handle.AddEntry(entry.String(), &set.IPSet, true); err != nil { - klog.Errorf("Failed to add entry %v into ip set: %s, error: %v", entry, set.Name, err) - } else { - klog.V(3).Infof("Successfully add entry: %v into ip set: %s", entry, set.Name) - } - //Increment ref count for clusterIP - p.updateRefCountForIPSet(kubeClusterIPSet, op) - } - if op == DeleteService { - if err := set.handle.DelEntry(entry.String(), set.Name); err != nil { - klog.Errorf("Failed to delete entry: %v from ip set: %s, error: %v", entry, set.Name, err) - } else { - klog.V(3).Infof("Successfully deleted entry: %v to ip set: %s", entry, set.Name) - } - //Increment ref count for clusterIP - p.updateRefCountForIPSet(kubeClusterIPSet, op) - } -} - -func (p *proxier) addRealServerForPort(serviceKey string, portList []*BaseServicePortInfo) []endPointInfo { - var epList []endPointInfo - for _, port := range portList { - klog.V(2).Infof("addRealServerForPort port (%v)", port) - for _, epKV := range p.endpoints.GetByPrefix([]byte(serviceKey)) { - epInfo := epKV.Value.(endPointInfo) - epList = append(epList, epInfo) - destination := ipvsSvcDst{ - Svc: port.GetVirtualServer().ToService(), - Dst: ipvsDestination(epInfo, port), - } - klog.V(2).Infof("adding destination ep (%v)", epInfo.endPointIP) - if err := ipvs.AddDestination(destination.Svc, destination.Dst); err != nil && !strings.HasSuffix(err.Error(), "object exists") { - klog.Error("failed to add destination ", serviceKey, ": ", err) - } - } - } - return epList -} - -func (p *proxier) deleteRealServerForPort(serviceKey string, portList []*BaseServicePortInfo) []endPointInfo { - var epList []endPointInfo - for _, port := range portList { - klog.V(2).Infof("deleteRealServerForPort port (%v)", port) - for _, epKV := range p.endpoints.GetByPrefix([]byte(serviceKey)) { - epInfo := epKV.Value.(endPointInfo) - epList = append(epList, epInfo) - destination := ipvsSvcDst{ - Svc: port.GetVirtualServer().ToService(), - Dst: ipvsDestination(epInfo, port), - } - klog.V(2).Infof("deleting destination ep (%v)", epInfo.endPointIP) - if err := ipvs.DeleteDestination(destination.Svc, destination.Dst); err != nil && !strings.HasSuffix(err.Error(), "object exists") { - klog.Error("failed to delete destination ", serviceKey, ": ", err) - } - } - } - return epList -} - -func getIPSetEntry(srcAddr string, port *BaseServicePortInfo) *ipsetutil.Entry { - if srcAddr != "" { - return &ipsetutil.Entry{ - IP: port.ServiceIP(), - Port: int(port.Port()), - Protocol: strings.ToLower(port.Protocol().String()), - SetType: ipsetutil.HashIPPort, - Net: srcAddr, - } - } - return &ipsetutil.Entry{ - IP: port.ServiceIP(), - Port: int(port.Port()), - Protocol: strings.ToLower(port.Protocol().String()), - SetType: ipsetutil.HashIPPort, - } -} - -func getIPFamiliesOfService(svc *localv1.Service) []v1.IPFamily { - var ipFamilies []v1.IPFamily - if len(svc.IPs.ClusterIPs.V4) != 0 { - ipFamilies = append(ipFamilies, v1.IPv4Protocol) - } - if len(svc.IPs.ClusterIPs.V6) != 0 { - ipFamilies = append(ipFamilies, v1.IPv6Protocol) - } - return ipFamilies -} - -func (p *proxier) getLbIPForIPFamily(svc *localv1.Service) (error, string) { - var svcIP string - if svc.IPs.LoadBalancerIPs == nil { - return errors.New("LB IPs are not configured"), svcIP - } - - if p.ipFamily == v1.IPv4Protocol && len(svc.IPs.LoadBalancerIPs.V4) > 0 { - svcIP = svc.IPs.LoadBalancerIPs.V4[0] - } - if p.ipFamily == v1.IPv6Protocol && len(svc.IPs.LoadBalancerIPs.V6) > 0 { - svcIP = svc.IPs.LoadBalancerIPs.V6[0] - } - return nil, svcIP -} - -func (p *proxier) addVirtualServer(portInfo *BaseServicePortInfo) { - vs := portInfo.GetVirtualServer() - - klog.V(2).Infof("adding AddVirtualServer: port: %v", portInfo) - // Programme virtual-server directly - ipvsSvc := vs.ToService() - err := ipvs.AddService(ipvsSvc) - if err != nil && !strings.HasSuffix(err.Error(), "object exists") { - klog.Error("failed to add service in IPVS", ": ", err) - } -} - -func (p *proxier) deleteVirtualServer(portInfo *BaseServicePortInfo) { - klog.V(2).Infof("deleting service , serviceIP (%v) , port (%v)", portInfo.serviceIP, portInfo.Port()) - err := ipvs.DeleteService(portInfo.GetVirtualServer().ToService()) - if err != nil { - klog.Error("failed to delete service from IPVS", portInfo.serviceIP, ": ", err) - } -} - -func (p *proxier) AddOrDelNodePortInIPSet(port *localv1.PortMapping, op Operation) { - var entries []*ipsetutil.Entry - protocol := strings.ToLower(port.Protocol.String()) - ipSetName := protocolIPSetMap[protocol] - p.updateRefCountForIPSet(ipSetName, op) - nodePortSet := p.ipsetList[ipSetName] - switch protocol { - case ipsetutil.ProtocolTCP, ipsetutil.ProtocolUDP: - entries = []*ipsetutil.Entry{getNodePortIPSetEntry(int(port.NodePort), protocol, ipsetutil.BitmapPort)} - - case ipsetutil.ProtocolSCTP: - // Since hash ip:port is used for SCTP, all the nodeIPs to be used in the SCTP ipset entries. - entries = []*ipsetutil.Entry{} - for _, nodeIP := range p.nodeAddresses { - entry := getNodePortIPSetEntry(int(port.NodePort), protocol, ipsetutil.HashIPPort) - entry.IP = nodeIP - entries = append(entries, entry) - } - default: - // It should never hit - klog.Errorf("Unsupported protocol type %v protocol ", protocol) - } - if nodePortSet != nil { - for _, entry := range entries { - if valid := nodePortSet.validateEntry(entry); !valid { - klog.Errorf("error adding entry (%v) to ipset (%v)", entry.String(), nodePortSet.Name) - } - set := p.ipsetList[ipSetName] - if op == AddService { - if err := set.handle.AddEntry(entry.String(), &set.IPSet, true); err != nil { - klog.Errorf("Failed to add entry %v into ip set: %s, error: %v", entry, set.Name, err) - } else { - klog.V(3).Infof("Successfully add entry: %v into ip set: %s", entry, set.Name) - } - } - if op == DeleteService { - if err := set.handle.DelEntry(entry.String(), set.Name); err != nil { - klog.Errorf("Failed to delete entry: %v from ip set: %s, error: %v", entry, set.Name, err) - } else { - klog.V(3).Infof("Successfully deleted entry: %v to ip set: %s", entry, set.Name) - } - } - } - } -} - -func getNodePortIPSetEntry(port int, protocol string, ipSetType ipsetutil.Type) *ipsetutil.Entry { - return &ipsetutil.Entry{ - // No need to provide ip info - Port: port, - Protocol: protocol, - SetType: ipSetType, - } -} - -func (p *proxier) sync() { - // sync iptable rules - p.syncIPTableRules() - - // signal diffstores we've finished - p.endpoints.Reset(lightdiffstore.ItemUnchanged) -} - -func (p *proxier) updateRefCountForIPSet(setName string, op Operation) { - if op == AddService || op == AddEndPoint { - p.ipsetList[setName].refCountOfSvc++ - } - if op == DeleteService || op == DeleteEndPoint { - p.ipsetList[setName].refCountOfSvc-- - } -} - -func (s *Backend) enableSessionAffinityForServiceIPs(svc *localv1.Service, sessionAffinity serviceevents.SessionAffinity) { - serviceKey := getServiceKey(svc) - ipFamilies := getIPFamiliesOfService(svc) - for _, ipFamily := range ipFamilies { - s.proxiers[ipFamily].enableSessionAffinityForServiceIP(serviceKey, sessionAffinity) - } -} - -func (s *Backend) disableSessionAffinityForServiceIPs(svc *localv1.Service) { - serviceKey := getServiceKey(svc) - ipFamilies := getIPFamiliesOfService(svc) - for _, ipFamily := range ipFamilies { - s.proxiers[ipFamily].disableSessionAffinityForServiceIP(serviceKey) - } -} - -func (p *proxier) enableSessionAffinityForServiceIP(serviceKey string, sa serviceevents.SessionAffinity) { - for _, sp := range p.servicePorts.GetByPrefix([]byte(serviceKey)) { - portInfo := sp.Value.(BaseServicePortInfo) - // Set session affinity for port - portInfo.SetSessionAffinity(sa) - - vs := portInfo.GetVirtualServer() - // Programme virtual-server directly - ipvsSvc := vs.ToService() - err := ipvs.UpdateService(ipvsSvc) - if err != nil && !strings.HasSuffix(err.Error(), "object exists") { - klog.Error("failed to add service in IPVS", serviceKey, ": ", err) - } - klog.V(2).Infof("enable sess-aff ipvsSvc: %v", ipvsSvc) - p.servicePorts.Set([]byte(serviceKey), 0, portInfo) - } -} - -func (p *proxier) disableSessionAffinityForServiceIP(serviceKey string) { - for _, sp := range p.servicePorts.GetByPrefix([]byte(serviceKey)) { - portInfo := sp.Value.(BaseServicePortInfo) - portInfo.ResetSessionAffinity() - vs := portInfo.GetVirtualServer() - - // Programme virtual-server directly - ipvsSvc := vs.ToService() - err := ipvs.UpdateService(ipvsSvc) - if err != nil && !strings.HasSuffix(err.Error(), "object exists") { - klog.Error("failed to add service in IPVS", serviceKey, ": ", err) - } - klog.V(2).Infof("disable sess-aff : %v", ipvsSvc) - p.servicePorts.Set([]byte(serviceKey), 0, portInfo) - } -} - -func (p *proxier) addRealServer(serviceKey, prefix, endPointIP string, endpoint *localv1.Endpoint) { - epInfo := endPointInfo{ - endPointIP: endPointIP, - isLocalEndPoint: endpoint.Local, - portMap: make(map[string]int32), - } - - for _, port := range endpoint.PortOverrides { - epInfo.portMap[port.Name] = port.Port - } - p.endpoints.Set([]byte(prefix), 0, epInfo) - for _, sp := range p.servicePorts.GetByPrefix([]byte(serviceKey)) { - portInfo := sp.Value.(BaseServicePortInfo) - klog.V(2).Infof("addRealServer, portInfo : %v", portInfo) - vs := portInfo.GetVirtualServer() - dest := ipvsSvcDst{ - Svc: vs.ToService(), - Dst: ipvsDestination(epInfo, &portInfo), - } - klog.V(2).Infof("adding destination ep (%v)", endPointIP) - if err := ipvs.AddDestination(dest.Svc, dest.Dst); err != nil && !strings.HasSuffix(err.Error(), "object exists") { - klog.Error("failed to add destination ", dest, ": ", err) - } - } - portList := p.portMap[serviceKey] - klog.V(2).Infof("addRealServer, portList : %v", portList) - for _, port := range portList { - p.AddOrDelEndPointInIPSet(endPointIP, port.Protocol.String(), port.TargetPort, endpoint.Local, AddEndPoint) - } -} - -func (p *proxier) deleteRealServer(serviceKey, prefix string) { - for _, kv := range p.endpoints.GetByPrefix([]byte(prefix)) { - epInfo := kv.Value.(endPointInfo) - for _, sp := range p.servicePorts.GetByPrefix([]byte(serviceKey)) { - portInfo := sp.Value.(BaseServicePortInfo) - vs := portInfo.GetVirtualServer() - klog.V(2).Infof("deleteRealServer, portInfo : %v", portInfo) - dest := ipvsSvcDst{ - Svc: vs.ToService(), - Dst: ipvsDestination(epInfo, &portInfo), - } - - klog.V(2).Infof("deleting destination : %v", dest) - if err := ipvs.DeleteDestination(dest.Svc, dest.Dst); err != nil { - klog.Error("failed to delete destination ", dest, ": ", err) - } - } - portList := p.portMap[serviceKey] - klog.V(2).Infof("deleteRealServer, portList : %v", portList) - for _, port := range portList { - p.AddOrDelEndPointInIPSet(epInfo.endPointIP, port.Protocol.String(), port.TargetPort, epInfo.isLocalEndPoint, DeleteEndPoint) - } - } - - // remove this endpoint from the endpoints - p.endpoints.DeleteByPrefix([]byte(prefix)) -} - -func (p *proxier) deletePortFromPortMap(serviceKey, portMapKey string) { - klog.V(2).Infof("deletePortFromPortMap, portMapKey= %v, portMap=%+v", portMapKey, p.portMap[serviceKey]) - delete(p.portMap[serviceKey], portMapKey) - if len(p.portMap[serviceKey]) == 0 { - delete(p.portMap, serviceKey) - klog.V(2).Infof("deletePortFromPortMap, svcKey=%v", serviceKey) - } -} diff --git a/backends/ipvs-as-sink/exec/exec.go b/backends/ipvs-as-sink/exec/exec.go deleted file mode 100644 index 6d8b9c3d7..000000000 --- a/backends/ipvs-as-sink/exec/exec.go +++ /dev/null @@ -1,252 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package exec - -import ( - "context" - "io" - osexec "os/exec" - "syscall" - "time" -) - -// ErrExecutableNotFound is returned if the executable is not found. -var ErrExecutableNotFound = osexec.ErrNotFound - -// Interface is an interface that presents a subset of the os/exec API. Use this -// when you want to inject fakeable/mockable exec behavior. -type Interface interface { - // Command returns a Cmd instance which can be used to run a single command. - // This follows the pattern of package os/exec. - Command(cmd string, args ...string) Cmd - - // CommandContext returns a Cmd instance which can be used to run a single command. - // - // The provided context is used to kill the process if the context becomes done - // before the command completes on its own. For example, a timeout can be set in - // the context. - CommandContext(ctx context.Context, cmd string, args ...string) Cmd - - // LookPath wraps os/exec.LookPath - LookPath(file string) (string, error) -} - -// Cmd is an interface that presents an API that is very similar to Cmd from os/exec. -// As more functionality is needed, this can grow. Since Cmd is a struct, we will have -// to replace fields with get/set method pairs. -type Cmd interface { - // Run runs the command to the completion. - Run() error - // CombinedOutput runs the command and returns its combined standard output - // and standard error. This follows the pattern of package os/exec. - CombinedOutput() ([]byte, error) - // Output runs the command and returns standard output, but not standard err - Output() ([]byte, error) - SetDir(dir string) - SetStdin(in io.Reader) - SetStdout(out io.Writer) - SetStderr(out io.Writer) - SetEnv(env []string) - - // StdoutPipe and StderrPipe for getting the process' Stdout and Stderr as - // Readers - StdoutPipe() (io.ReadCloser, error) - StderrPipe() (io.ReadCloser, error) - - // Start and Wait are for running a process non-blocking - Start() error - Wait() error - - // Stops the command by sending SIGTERM. It is not guaranteed the - // process will stop before this function returns. If the process is not - // responding, an internal timer function will send a SIGKILL to force - // terminate after 10 seconds. - Stop() -} - -// ExitError is an interface that presents an API similar to os.ProcessState, which is -// what ExitError from os/exec is. This is designed to make testing a bit easier and -// probably loses some of the cross-platform properties of the underlying library. -type ExitError interface { - String() string - Error() string - Exited() bool - ExitStatus() int -} - -// Implements Interface in terms of really exec()ing. -type executor struct{} - -// New returns a new Interface which will os/exec to run commands. -func New() Interface { - return &executor{} -} - -// Command is part of the Interface interface. -func (executor *executor) Command(cmd string, args ...string) Cmd { - return (*cmdWrapper)(osexec.Command(cmd, args...)) -} - -// CommandContext is part of the Interface interface. -func (executor *executor) CommandContext(ctx context.Context, cmd string, args ...string) Cmd { - return (*cmdWrapper)(osexec.CommandContext(ctx, cmd, args...)) -} - -// LookPath is part of the Interface interface -func (executor *executor) LookPath(file string) (string, error) { - return osexec.LookPath(file) -} - -// Wraps exec.Cmd so we can capture errors. -type cmdWrapper osexec.Cmd - -var _ Cmd = &cmdWrapper{} - -func (cmd *cmdWrapper) SetDir(dir string) { - cmd.Dir = dir -} - -func (cmd *cmdWrapper) SetStdin(in io.Reader) { - cmd.Stdin = in -} - -func (cmd *cmdWrapper) SetStdout(out io.Writer) { - cmd.Stdout = out -} - -func (cmd *cmdWrapper) SetStderr(out io.Writer) { - cmd.Stderr = out -} - -func (cmd *cmdWrapper) SetEnv(env []string) { - cmd.Env = env -} - -func (cmd *cmdWrapper) StdoutPipe() (io.ReadCloser, error) { - r, err := (*osexec.Cmd)(cmd).StdoutPipe() - return r, handleError(err) -} - -func (cmd *cmdWrapper) StderrPipe() (io.ReadCloser, error) { - r, err := (*osexec.Cmd)(cmd).StderrPipe() - return r, handleError(err) -} - -func (cmd *cmdWrapper) Start() error { - err := (*osexec.Cmd)(cmd).Start() - return handleError(err) -} - -func (cmd *cmdWrapper) Wait() error { - err := (*osexec.Cmd)(cmd).Wait() - return handleError(err) -} - -// Run is part of the Cmd interface. -func (cmd *cmdWrapper) Run() error { - err := (*osexec.Cmd)(cmd).Run() - return handleError(err) -} - -// CombinedOutput is part of the Cmd interface. -func (cmd *cmdWrapper) CombinedOutput() ([]byte, error) { - out, err := (*osexec.Cmd)(cmd).CombinedOutput() - return out, handleError(err) -} - -func (cmd *cmdWrapper) Output() ([]byte, error) { - out, err := (*osexec.Cmd)(cmd).Output() - return out, handleError(err) -} - -// Stop is part of the Cmd interface. -func (cmd *cmdWrapper) Stop() { - c := (*osexec.Cmd)(cmd) - - if c.Process == nil { - return - } - - c.Process.Signal(syscall.SIGTERM) - - time.AfterFunc(10*time.Second, func() { - if !c.ProcessState.Exited() { - c.Process.Signal(syscall.SIGKILL) - } - }) -} - -func handleError(err error) error { - if err == nil { - return nil - } - - switch e := err.(type) { - case *osexec.ExitError: - return &ExitErrorWrapper{e} - case *osexec.Error: - if e.Err == osexec.ErrNotFound { - return ErrExecutableNotFound - } - } - - return err -} - -// ExitErrorWrapper is an implementation of ExitError in terms of os/exec ExitError. -// Note: standard exec.ExitError is type *os.ProcessState, which already implements Exited(). -type ExitErrorWrapper struct { - *osexec.ExitError -} - -var _ ExitError = &ExitErrorWrapper{} - -// ExitStatus is part of the ExitError interface. -func (eew ExitErrorWrapper) ExitStatus() int { - ws, ok := eew.Sys().(syscall.WaitStatus) - if !ok { - panic("can't call ExitStatus() on a non-WaitStatus exitErrorWrapper") - } - return ws.ExitStatus() -} - -// CodeExitError is an implementation of ExitError consisting of an error object -// and an exit code (the upper bits of os.exec.ExitStatus). -type CodeExitError struct { - Err error - Code int -} - -var _ ExitError = CodeExitError{} - -func (e CodeExitError) Error() string { - return e.Err.Error() -} - -func (e CodeExitError) String() string { - return e.Err.Error() -} - -// Exited is to check if the process has finished -func (e CodeExitError) Exited() bool { - return true -} - -// ExitStatus is for checking the error code -func (e CodeExitError) ExitStatus() int { - return e.Code -} diff --git a/backends/ipvs-as-sink/externalIP.go b/backends/ipvs-as-sink/externalIP.go deleted file mode 100644 index b065e3b14..000000000 --- a/backends/ipvs-as-sink/externalIP.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - "k8s.io/klog/v2" - "sigs.k8s.io/kpng/api/localv1" -) - -func (p *proxier) handleNewExternalIP(serviceKey, externalIP, svcType string, svc *localv1.Service, port *localv1.PortMapping) { - spKey := getServicePortKey(serviceKey, externalIP, port) - portInfo := NewBaseServicePortInfo(svc, port, externalIP, svcType, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - - p.addVirtualServer(portInfo) - - //Cluster service IP needs to be programmed in ipset. - p.AddOrDelExternalIPInIPSet(externalIP, portInfo, AddService) -} - -func (p *proxier) handleUpdatedExternalIP(serviceKey, externalIP, svcType string, svc *localv1.Service, port *localv1.PortMapping) { - spKey := getServicePortKey(serviceKey, externalIP, port) - portInfo := NewBaseServicePortInfo(svc, port, externalIP, svcType, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - - //Update the service with added ports into LB tree - p.addVirtualServer(portInfo) - //Cluster service IP needs to be programmed in ipset with added ports. - p.AddOrDelExternalIPInIPSet(externalIP, portInfo, AddService) - - p.addRealServerForPort(serviceKey, []*BaseServicePortInfo{portInfo}) -} - -func (p *proxier) AddOrDelExternalIPInIPSet(externalIP string, port *BaseServicePortInfo, op Operation) { - entry := getIPSetEntry(externalIP, port) - // We have to SNAT packets to external IPs. - if valid := p.ipsetList[kubeExternalIPSet].validateEntry(entry); !valid { - klog.Errorf("error adding entry :%s, to ipset:%s", entry.String(), p.ipsetList[kubeClusterIPSet].Name) - return - } - set := p.ipsetList[kubeExternalIPSet] - if op == AddService { - if err := set.handle.AddEntry(entry.String(), &set.IPSet, true); err != nil { - klog.Errorf("Failed to add entry %v into ip set: %s, error: %v", entry, set.Name, err) - } else { - klog.V(3).Infof("Successfully add entry: %v into ip set: %s", entry, set.Name) - } - //Increment ref count - p.updateRefCountForIPSet(kubeExternalIPSet, op) - } - if op == DeleteService { - if err := set.handle.DelEntry(entry.String(), set.Name); err != nil { - klog.Errorf("Failed to delete entry: %v from ip set: %s, error: %v", entry, set.Name, err) - } else { - klog.V(3).Infof("Successfully deleted entry: %v to ip set: %s", entry, set.Name) - } - //Decrement ref count - p.updateRefCountForIPSet(kubeExternalIPSet, op) - } -} diff --git a/backends/ipvs-as-sink/flags.go b/backends/ipvs-as-sink/flags.go deleted file mode 100644 index 4112f23ba..000000000 --- a/backends/ipvs-as-sink/flags.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - "net" - - "github.com/spf13/pflag" -) - -func (s *Backend) BindFlags(flags *pflag.FlagSet) { - s.Config.BindFlags(flags) - - // real ipvs sink flags - flags.BoolVar(&s.dryRun, "dry-run", false, "dry run (print instead of applying)") - flags.StringSliceVar(&s.nodeAddresses, "node-address", interfaceAddresses(), "A comma-separated list of IPs to associate when using NodePort type. Defaults to all the Node addresses") - flags.StringVar(&s.schedulingMethod, "scheduling-method", "rr", "Algorithm for allocating TCP conn & UDP datagrams to real servers. Values: rr,wrr,lc,wlc,lblc,lblcr,dh,sh,seq,nq") - flags.Int32Var(&s.weight, "weight", 1, "An integer specifying the capacity of server relative to others in the pool") - //flags.Int32Var(s.masqueradeBit, "iptables-masquerade-bit", Int32PtrDerefOr(s.masqueradeBit, 14), "If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with. Must be within the range [0, 31].") - flags.BoolVar(&s.masqueradeAll, "masquerade-all", s.masqueradeAll, "If using the pure iptables proxy, SNAT all traffic sent via Service cluster IPs (this not commonly needed)") -} - -func interfaceAddresses() []string { - ifacesAddress, err := net.InterfaceAddrs() - if err != nil { - panic(err) - } - - var addresses []string - for _, addr := range ifacesAddress { - // TODO: Ignore interfaces in PodCIDR or ClusterCIDR - ip, _, err := net.ParseCIDR(addr.String()) - if err != nil { - panic(err) - } - // I want to deal only with IPv4 right now... - if ipv4 := ip.To4(); ipv4 == nil { - continue - } - - addresses = append(addresses, ip.String()) - } - return addresses -} diff --git a/backends/ipvs-as-sink/go.mod b/backends/ipvs-as-sink/go.mod deleted file mode 100644 index 3154b1a66..000000000 --- a/backends/ipvs-as-sink/go.mod +++ /dev/null @@ -1,49 +0,0 @@ -module sigs.k8s.io/kpng/backends/ipvs-as-sink - -go 1.19 - -require ( - github.com/google/seesaw v0.0.0-20220321203705-0e93b4c33bc6 - github.com/lithammer/dedent v1.1.0 - github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.0 - github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 - golang.org/x/sys v0.0.0-20221010170243-090e33056c14 - k8s.io/api v0.25.2 - k8s.io/apimachinery v0.25.2 - k8s.io/apiserver v0.25.2 - k8s.io/klog/v2 v2.80.1 - k8s.io/utils v0.0.0-20221011040102-427025108f67 - sigs.k8s.io/kpng/backends/iptables v0.0.0-20220824013548-88b8a1d9bc62 - sigs.k8s.io/kpng/client v0.0.0-20221011133104-469299451522 -) - -require ( - github.com/cespare/xxhash v1.1.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/btree v1.1.2 // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/kr/pretty v0.3.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect - golang.org/x/net v0.0.0-20221004154528-8021a29435af // indirect - golang.org/x/text v0.3.7 // indirect - google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e // indirect - google.golang.org/grpc v1.50.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/kpng/api v0.0.0-20220824013548-88b8a1d9bc62 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect -) diff --git a/backends/ipvs-as-sink/go.sum b/backends/ipvs-as-sink/go.sum deleted file mode 100644 index 2680f8473..000000000 --- a/backends/ipvs-as-sink/go.sum +++ /dev/null @@ -1,48 +0,0 @@ -github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/seesaw v0.0.0-20220321203705-0e93b4c33bc6 h1:fcKp0spMrrD9kA8kPisEjUE3iyn1ncauEOmCYofxpH4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= -github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= -golang.org/x/net v0.0.0-20221004154528-8021a29435af h1:wv66FM3rLZGPdxpYL+ApnDe2HzHcTFta3z5nsc13wI4= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e h1:halCgTFuLWDRD61piiNSxPsARANGD3Xl16hPrLgLiIg= -google.golang.org/grpc v1.50.0 h1:fPVVDxY9w++VjTZsYvXWqEf9Rqar/e+9zYfxKK+W+YU= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -k8s.io/api v0.25.2 h1:v6G8RyFcwf0HR5jQGIAYlvtRNrxMJQG1xJzaSeVnIS8= -k8s.io/apimachinery v0.25.2 h1:WbxfAjCx+AeN8Ilp9joWnyJ6xu9OMeS/fsfjK/5zaQs= -k8s.io/apiserver v0.25.2 h1:YePimobk187IMIdnmsMxsfIbC5p4eX3WSOrS9x6FEYw= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/utils v0.0.0-20221011040102-427025108f67 h1:ZmUY7x0cwj9e7pGyCTIalBi5jpNfigO5sU46/xFoF/w= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/kpng/api v0.0.0-20220824013548-88b8a1d9bc62 h1:yCjRx4awGZF5+7nt1PDz9b514W/v/oeEOLLZ63Q9HQY= -sigs.k8s.io/kpng/backends/iptables v0.0.0-20220824013548-88b8a1d9bc62 h1:R5o2nCK3JZ6RbgWk6C6HVxw2IW1QrOql8Ak6R952TcA= -sigs.k8s.io/kpng/client v0.0.0-20221011133104-469299451522 h1:uexG5zX/+RMBitJ/J4586YHxV2866nO3/pfNl0vPDQQ= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= diff --git a/backends/ipvs-as-sink/ipset.go b/backends/ipvs-as-sink/ipset.go deleted file mode 100644 index 6ddb77ae3..000000000 --- a/backends/ipvs-as-sink/ipset.go +++ /dev/null @@ -1,163 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - "fmt" - "strings" - - v1 "k8s.io/api/core/v1" - - "k8s.io/klog/v2" - - ipsetutil "sigs.k8s.io/kpng/backends/ipvs-as-sink/util" -) - -const ( - kubeLoopBackIPSetComment = "Kubernetes endpoints dst ip:port, source ip for solving hairpin purpose" - kubeLoopBackIPSet = "KUBE-LOOP-BACK" - - kubeClusterIPSetComment = "Kubernetes service cluster ip + port for masquerade purpose" - kubeClusterIPSet = "KUBE-CLUSTER-IP" - - kubeExternalIPSetComment = "Kubernetes service external ip + port for masquerade and filter purpose" - kubeExternalIPSet = "KUBE-EXTERNAL-IP" - - kubeExternalIPLocalSetComment = "Kubernetes service external ip + port with externalTrafficPolicy=local" - kubeExternalIPLocalSet = "KUBE-EXTERNAL-IP-LOCAL" - - kubeLoadBalancerSetComment = "Kubernetes service lb portal" - kubeLoadBalancerSet = "KUBE-LOAD-BALANCER" - - kubeLoadBalancerLocalSetComment = "Kubernetes service load balancer ip + port with externalTrafficPolicy=local" - kubeLoadBalancerLocalSet = "KUBE-LOAD-BALANCER-LOCAL" - - kubeLoadbalancerFWSetComment = "Kubernetes service load balancer ip + port for load balancer with sourceRange" - kubeLoadbalancerFWSet = "KUBE-LOAD-BALANCER-FW" - - kubeLoadBalancerSourceIPSetComment = "Kubernetes service load balancer ip + port + source IP for packet filter purpose" - kubeLoadBalancerSourceIPSet = "KUBE-LOAD-BALANCER-SOURCE-IP" - - kubeLoadBalancerSourceCIDRSetComment = "Kubernetes service load balancer ip + port + source cidr for packet filter purpose" - kubeLoadBalancerSourceCIDRSet = "KUBE-LOAD-BALANCER-SOURCE-CIDR" - - kubeNodePortSetTCPComment = "Kubernetes nodeport TCP port for masquerade purpose" - kubeNodePortSetTCP = "KUBE-NODE-PORT-TCP" - - kubeNodePortLocalSetTCPComment = "Kubernetes nodeport TCP port with externalTrafficPolicy=local" - kubeNodePortLocalSetTCP = "KUBE-NODE-PORT-LOCAL-TCP" - - kubeNodePortSetUDPComment = "Kubernetes nodeport UDP port for masquerade purpose" - kubeNodePortSetUDP = "KUBE-NODE-PORT-UDP" - - kubeNodePortLocalSetUDPComment = "Kubernetes nodeport UDP port with externalTrafficPolicy=local" - kubeNodePortLocalSetUDP = "KUBE-NODE-PORT-LOCAL-UDP" - - kubeNodePortSetSCTPComment = "Kubernetes nodeport SCTP port for masquerade purpose with type 'hash ip:port'" - kubeNodePortSetSCTP = "KUBE-NODE-PORT-SCTP-HASH" - - kubeNodePortLocalSetSCTPComment = "Kubernetes nodeport SCTP port with externalTrafficPolicy=local with type 'hash ip:port'" - kubeNodePortLocalSetSCTP = "KUBE-NODE-PORT-LOCAL-SCTP-HASH" - - kubeHealthCheckNodePortSetComment = "Kubernetes health check node port" - kubeHealthCheckNodePortSet = "KUBE-HEALTH-CHECK-NODE-PORT" -) - -// ipsetInfo is all ipset we needed in ipvs proxier -var ipsetInfo = []struct { - name string - setType ipsetutil.Type - comment string -}{ - {kubeLoopBackIPSet, ipsetutil.HashIPPortIP, kubeLoopBackIPSetComment}, - {kubeClusterIPSet, ipsetutil.HashIPPort, kubeClusterIPSetComment}, - {kubeExternalIPSet, ipsetutil.HashIPPort, kubeExternalIPSetComment}, - {kubeExternalIPLocalSet, ipsetutil.HashIPPort, kubeExternalIPLocalSetComment}, - {kubeLoadBalancerSet, ipsetutil.HashIPPort, kubeLoadBalancerSetComment}, - {kubeLoadbalancerFWSet, ipsetutil.HashIPPort, kubeLoadbalancerFWSetComment}, - {kubeLoadBalancerLocalSet, ipsetutil.HashIPPort, kubeLoadBalancerLocalSetComment}, - {kubeLoadBalancerSourceIPSet, ipsetutil.HashIPPortIP, kubeLoadBalancerSourceIPSetComment}, - {kubeLoadBalancerSourceCIDRSet, ipsetutil.HashIPPortNet, kubeLoadBalancerSourceCIDRSetComment}, - {kubeNodePortSetTCP, ipsetutil.BitmapPort, kubeNodePortSetTCPComment}, - {kubeNodePortLocalSetTCP, ipsetutil.BitmapPort, kubeNodePortLocalSetTCPComment}, - {kubeNodePortSetUDP, ipsetutil.BitmapPort, kubeNodePortSetUDPComment}, - {kubeNodePortLocalSetUDP, ipsetutil.BitmapPort, kubeNodePortLocalSetUDPComment}, - {kubeNodePortSetSCTP, ipsetutil.HashIPPort, kubeNodePortSetSCTPComment}, - {kubeNodePortLocalSetSCTP, ipsetutil.HashIPPort, kubeNodePortLocalSetSCTPComment}, - {kubeHealthCheckNodePortSet, ipsetutil.BitmapPort, kubeHealthCheckNodePortSetComment}, -} - -type IPSet struct { - ipsetutil.IPSet - // handle is the util ipset interface handle. - handle ipsetutil.Interface - - refCountOfSvc int -} - -// NewIPSet initialize a new IPSet struct -func newIPSet(handle ipsetutil.Interface, name string, setType ipsetutil.Type, ipFamily v1.IPFamily, comment string) *IPSet { - hashFamily := ipsetutil.ProtocolFamilyIPV4 - if ipFamily == v1.IPv6Protocol { - hashFamily = ipsetutil.ProtocolFamilyIPV6 - // In dual-stack both ipv4 and ipv6 ipset's can co-exist. To - // ensure unique names the prefix for ipv6 is changed from - // "KUBE-" to "KUBE-6-". The "KUBE-" prefix is kept for - // backward compatibility. The maximum name length of an ipset - // is 31 characters which must be taken into account. The - // ipv4 names are not altered to minimize the risk for - // problems on upgrades. - if strings.HasPrefix(name, "KUBE-") { - name = strings.Replace(name, "KUBE-", "KUBE-6-", 1) - if len(name) > 31 { - klog.Info("Ipset name truncated", "ipSetName", name, "truncatedName", name[:31]) - name = name[:31] - } - } - } - set := &IPSet{ - IPSet: ipsetutil.IPSet{ - Name: name, - SetType: setType, - HashFamily: hashFamily, - Comment: comment, - }, - handle: handle, - } - return set -} - -func (set *IPSet) isRefCountZero() bool { - klog.V(2).Infof("isRefCountZero, set=%v, count=%v", set.Name, set.refCountOfSvc) - return set.refCountOfSvc == 0 -} - -func (set *IPSet) validateEntry(entry *ipsetutil.Entry) bool { - return entry.Validate(&set.IPSet) -} - -func (set *IPSet) getComment() string { - return fmt.Sprintf("\"%s\"", set.Comment) -} - -func ensureIPSet(set *IPSet) error { - if err := set.handle.CreateSet(&set.IPSet, true); err != nil { - klog.Errorf("Failed to ensure ip set %v exist, error: %v", set, err) - return err - } - return nil -} diff --git a/backends/ipvs-as-sink/iptables.go b/backends/ipvs-as-sink/iptables.go deleted file mode 100644 index 32fd3789e..000000000 --- a/backends/ipvs-as-sink/iptables.go +++ /dev/null @@ -1,414 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - "bytes" - "fmt" - - "k8s.io/klog/v2" - - "sigs.k8s.io/kpng/backends/ipvs-as-sink/util" -) - -const ( - // kubeServicesChain is the services portal chain - kubeServicesChain util.Chain = "KUBE-SERVICES" - - // KubeFireWallChain is the kubernetes firewall chain. - KubeFireWallChain util.Chain = "KUBE-FIREWALL" - - // kubePostroutingChain is the kubernetes postrouting chain - kubePostroutingChain util.Chain = "KUBE-POSTROUTING" - - // KubeMarkMasqChain is the mark-for-masquerade chain - KubeMarkMasqChain util.Chain = "KUBE-MARK-MASQ" - - // KubeNodePortChain is the kubernetes node port chain - KubeNodePortChain util.Chain = "KUBE-NODE-PORT" - - // KubeMarkDropChain is the mark-for-drop chain - KubeMarkDropChain util.Chain = "KUBE-MARK-DROP" - - // KubeForwardChain is the kubernetes forward chain - KubeForwardChain util.Chain = "KUBE-FORWARD" - - // KubeLoadBalancerChain is the kubernetes chain for loadbalancer type service - KubeLoadBalancerChain util.Chain = "KUBE-LOAD-BALANCER" -) - -var iptablesEnsureChains = []struct { - table util.Table - chain util.Chain -}{ - {util.TableNAT, KubeMarkDropChain}, -} - -var iptablesChains = []struct { - table util.Table - chain util.Chain -}{ - {util.TableNAT, kubeServicesChain}, - {util.TableNAT, kubePostroutingChain}, - {util.TableNAT, KubeFireWallChain}, - {util.TableNAT, KubeNodePortChain}, - {util.TableNAT, KubeLoadBalancerChain}, - {util.TableNAT, KubeMarkMasqChain}, - {util.TableFilter, KubeForwardChain}, - {util.TableFilter, KubeNodePortChain}, -} - -// iptablesJumpChain is tables of iptables chains that ipvs proxier used to install iptables or cleanup iptables. -// `to` is the iptables chain we want to operate. -// `from` is the source iptables chain -var iptablesJumpChain = []struct { - table util.Table - from util.Chain - to util.Chain - comment string -}{ - {util.TableNAT, util.ChainOutput, kubeServicesChain, "kubernetes service portals"}, - {util.TableNAT, util.ChainPrerouting, kubeServicesChain, "kubernetes service portals"}, - {util.TableNAT, util.ChainPostrouting, kubePostroutingChain, "kubernetes postrouting rules"}, - {util.TableFilter, util.ChainForward, KubeForwardChain, "kubernetes forwarding rules"}, - {util.TableFilter, util.ChainInput, KubeNodePortChain, "kubernetes health check rules"}, -} - -// ipsetWithIptablesChain is the ipsets list with iptables source chain and the chain jump to -// `iptables -t nat -A -m set --match-set -j ` -// example: iptables -t nat -A KUBE-SERVICES -m set --match-set KUBE-NODE-PORT-TCP dst -j KUBE-NODE-PORT -// ipsets with other match rules will be created Individually. -// Note: kubeNodePortLocalSetTCP must be prior to kubeNodePortSetTCP, the same for UDP. -var ipsetWithIptablesChain = []struct { - name string - from string - to string - matchType string - protocolMatch string -}{ - {kubeLoopBackIPSet, string(kubePostroutingChain), "MASQUERADE", "dst,dst,src", ""}, - {kubeLoadBalancerSet, string(kubeServicesChain), string(KubeLoadBalancerChain), "dst,dst", ""}, - {kubeLoadbalancerFWSet, string(KubeLoadBalancerChain), string(KubeFireWallChain), "dst,dst", ""}, - {kubeLoadBalancerSourceCIDRSet, string(KubeFireWallChain), "RETURN", "dst,dst,src", ""}, - {kubeLoadBalancerSourceIPSet, string(KubeFireWallChain), "RETURN", "dst,dst,src", ""}, - {kubeLoadBalancerLocalSet, string(KubeLoadBalancerChain), "RETURN", "dst,dst", ""}, - {kubeNodePortLocalSetTCP, string(KubeNodePortChain), "RETURN", "dst", util.ProtocolTCP}, - {kubeNodePortSetTCP, string(KubeNodePortChain), string(KubeMarkMasqChain), "dst", util.ProtocolTCP}, - {kubeNodePortLocalSetUDP, string(KubeNodePortChain), "RETURN", "dst", util.ProtocolUDP}, - {kubeNodePortSetUDP, string(KubeNodePortChain), string(KubeMarkMasqChain), "dst", util.ProtocolUDP}, - {kubeNodePortLocalSetSCTP, string(KubeNodePortChain), "RETURN", "dst,dst", util.ProtocolSCTP}, - {kubeNodePortSetSCTP, string(KubeNodePortChain), string(KubeMarkMasqChain), "dst,dst", util.ProtocolSCTP}, -} - -// createAndLinkKubeChain create all kube chains that ipvs proxier need and write basic link. -func (p *proxier) createAndLinkKubeChain() { - existingFilterChains := p.getExistingChains(p.filterChainsData, util.TableFilter) - existingNATChains := p.getExistingChains(p.iptablesData, util.TableNAT) - - // ensure KUBE-MARK-DROP chain exist but do not change any rules - for _, ch := range iptablesEnsureChains { - if _, err := p.iptables.EnsureChain(ch.table, ch.chain); err != nil { - klog.Error(err, "Failed to ensure chain exists", "table", ch.table, "chain", ch.chain) - return - } - } - - // Make sure we keep stats for the top-level chains - for _, ch := range iptablesChains { - if _, err := p.iptables.EnsureChain(ch.table, ch.chain); err != nil { - klog.Error(err, "Failed to ensure chain exists", "table", ch.table, "chain", ch.chain) - return - } - if ch.table == util.TableNAT { - if chain, ok := existingNATChains[ch.chain]; ok { - p.natChains.WriteBytes(chain) - } else { - p.natChains.Write(util.MakeChainLine(ch.chain)) - } - } else { - if chain, ok := existingFilterChains[ch.chain]; ok { - p.filterChains.WriteBytes(chain) - } else { - p.filterChains.Write(util.MakeChainLine(ch.chain)) - } - } - } - - for _, jc := range iptablesJumpChain { - args := []string{"-m", "comment", "--comment", jc.comment, "-j", string(jc.to)} - if _, err := p.iptables.EnsureRule(util.Prepend, jc.table, jc.from, args...); err != nil { - klog.Error(err, "Failed to ensure chain jumps", "table", jc.table, "srcChain", jc.from, "dstChain", jc.to) - } - } -} - -// getExistingChains get iptables-save output so we can check for existing chains and rules. -// This will be a map of chain name to chain with rules as stored in iptables-save/iptables-restore -// Result may SHARE memory with contents of buffer. -func (p *proxier) getExistingChains(buffer *bytes.Buffer, table util.Table) map[util.Chain][]byte { - buffer.Reset() - err := p.iptables.SaveInto(table, buffer) - if err != nil { // if we failed to get any rules - klog.Error(err, "Failed to execute iptables-save, syncing all rules") - } else { // otherwise parse the output - return util.GetChainLines(table, buffer.Bytes()) - } - return nil -} - -// writeIptablesRules write all iptables rules to proxier.natRules or proxier.FilterRules that ipvs proxier needed -// according to proxier.ipsetList information and the ipset match relationship that `ipsetWithIptablesChain` specified. -// some ipset(kubeClusterIPSet for example) have particular match rules and iptables jump relation should be sync separately. -func (p *proxier) writeIptablesRules() { - // We are creating those slices ones here to avoid memory reallocations - // in every loop. Note that reuse the memory, instead of doing: - // slice = - // you should always do one of the below: - // slice = slice[:0] // and then append to it - // slice = append(slice[:0], ...) - // To avoid growing this slice, we arbitrarily set its size to 64, - // there is never more than that many arguments for a single line. - // Note that even if we go over 64, it will still be correct - it - // is just for efficiency, not correctness. - args := make([]string, 64) - - for _, set := range ipsetWithIptablesChain { - if !p.ipsetList[set.name].isRefCountZero() { - args = append(args[:0], "-A", set.from) - if set.protocolMatch != "" { - args = append(args, "-p", set.protocolMatch) - } - args = append(args, - "-m", "comment", "--comment", p.ipsetList[set.name].getComment(), - "-m", "set", "--match-set", p.ipsetList[set.name].Name, - set.matchType, - ) - p.natRules.Write(args, "-j", set.to) - } - } - - if !p.ipsetList[kubeClusterIPSet].isRefCountZero() { - args = append(args[:0], - "-A", string(kubeServicesChain), - "-m", "comment", "--comment", p.ipsetList[kubeClusterIPSet].getComment(), - "-m", "set", "--match-set", p.ipsetList[kubeClusterIPSet].Name, - ) - if p.masqueradeAll { - p.natRules.Write(args, "dst,dst", "-j", string(KubeMarkMasqChain)) - //TODO: localDetector code needs to be added later - //} else if p.localDetector.IsImplemented() { - // // This masquerades off-cluster traffic to a service VIP. The idea - // // is that you can establish a static route for your Service range, - // // routing to any node, and that node will bridge into the Service - // // for you. Since that might bounce off-node, we masquerade here. - // // If/when we support "Local" policy for VIPs, we should update this. - // p.natRules.Write(p.localDetector.JumpIfNotLocal(append(args, "dst,dst"), string(KubeMarkMasqChain))) - //} else { - } else { - // Masquerade all OUTPUT traffic coming from a service ip. - // The kube dummy interface has all service VIPs assigned which - // results in the service VIP being picked as the source IP to reach - // a VIP. This leads to a connection from VIP: to - // VIP:. - // Always masquerading OUTPUT (node-originating) traffic with a VIP - // source ip and service port destination fixes the outgoing connections. - p.natRules.Write(args, "src,dst", "-j", string(KubeMarkMasqChain)) - } - } - - // externalIPRules adds iptables rules applies to Service ExternalIPs - externalIPRules := func(args []string) { - // Allow traffic for external IPs that does not come from a bridge (i.e. not from a container) - // nor from a local process to be forwarded to the service. - // This rule roughly translates to "all traffic from off-machine". - // This is imperfect in the face of network plugins that might not use a bridge, but we can revisit that later. - externalTrafficOnlyArgs := append(args, - "-m", "physdev", "!", "--physdev-is-in", - "-m", "addrtype", "!", "--src-type", "LOCAL") - p.natRules.Write(externalTrafficOnlyArgs, "-j", "ACCEPT") - dstLocalOnlyArgs := append(args, "-m", "addrtype", "--dst-type", "LOCAL") - // Allow traffic bound for external IPs that happen to be recognized as local IPs to stay local. - // This covers cases like GCE load-balancers which get added to the local routing table. - p.natRules.Write(dstLocalOnlyArgs, "-j", "ACCEPT") - } - - if !p.ipsetList[kubeExternalIPSet].isRefCountZero() { - // Build masquerade rules for packets to external IPs. - args = append(args[:0], - "-A", string(kubeServicesChain), - "-m", "comment", "--comment", p.ipsetList[kubeExternalIPSet].getComment(), - "-m", "set", "--match-set", p.ipsetList[kubeExternalIPSet].Name, - "dst,dst", - ) - p.natRules.Write(args, "-j", string(KubeMarkMasqChain)) - externalIPRules(args) - } - - if !p.ipsetList[kubeExternalIPLocalSet].isRefCountZero() { - args = append(args[:0], - "-A", string(kubeServicesChain), - "-m", "comment", "--comment", p.ipsetList[kubeExternalIPLocalSet].getComment(), - "-m", "set", "--match-set", p.ipsetList[kubeExternalIPLocalSet].Name, - "dst,dst", - ) - externalIPRules(args) - } - - // -A KUBE-SERVICES -m addrtype --dst-type LOCAL -j KUBE-NODE-PORT - args = append(args[:0], - "-A", string(kubeServicesChain), - "-m", "addrtype", "--dst-type", "LOCAL", - ) - p.natRules.Write(args, "-j", string(KubeNodePortChain)) - - // mark drop for KUBE-LOAD-BALANCER - p.natRules.Write( - "-A", string(KubeLoadBalancerChain), - "-j", string(KubeMarkMasqChain), - ) - // mark drop for KUBE-FIRE-WALL - p.natRules.Write( - "-A", string(KubeFireWallChain), - "-j", string(KubeMarkDropChain), - ) - - // Accept all traffic with destination of ipvs virtual service, in case other iptables rules - // block the traffic, that may result in ipvs rules invalid. - // Those rules must be in the end of KUBE-SERVICE chain - p.acceptIPVSTraffic() - - // If the masqueradeMark has been added then we want to forward that same - // traffic, this allows NodePort traffic to be forwarded even if the default - // FORWARD policy is not accept. - p.filterRules.Write( - "-A", string(KubeForwardChain), - "-m", "comment", "--comment", `"kubernetes forwarding rules"`, - "-m", "mark", "--mark", fmt.Sprintf("%s/%s", p.masqueradeMark, p.masqueradeMark), - "-j", "ACCEPT", - ) - - // The following rule ensures the traffic after the initial packet accepted - // by the "kubernetes forwarding rules" rule above will be accepted. - p.filterRules.Write( - "-A", string(KubeForwardChain), - "-m", "comment", "--comment", `"kubernetes forwarding conntrack rule"`, - "-m", "conntrack", - "--ctstate", "RELATED,ESTABLISHED", - "-j", "ACCEPT", - ) - - // Add rule to accept traffic towards health check node port - p.filterRules.Write( - "-A", string(KubeNodePortChain), - "-m", "comment", "--comment", p.ipsetList[kubeHealthCheckNodePortSet].getComment(), - "-m", "set", "--match-set", p.ipsetList[kubeHealthCheckNodePortSet].Name, "dst", - "-j", "ACCEPT", - ) - - // Install the kubernetes-specific postrouting rules. We use a whole chain for - // this so that it is easier to flush and change, for example if the mark - // value should ever change. - // NB: THIS MUST MATCH the corresponding code in the kubelet - p.natRules.Write( - "-A", string(kubePostroutingChain), - "-m", "mark", "!", "--mark", fmt.Sprintf("%s/%s", p.masqueradeMark, p.masqueradeMark), - "-j", "RETURN", - ) - - // Clear the mark to avoid re-masquerading if the packet re-traverses the network stack. - p.natRules.Write( - "-A", string(kubePostroutingChain), - // XOR proxier.masqueradeMark to unset it - "-j", "MARK", "--xor-mark", p.masqueradeMark, - ) - - masqRule := []string{ - "-A", string(kubePostroutingChain), - "-m", "comment", "--comment", `"kubernetes service traffic requiring SNAT"`, - "-j", "MASQUERADE", - } - if p.iptables.HasRandomFully() { - masqRule = append(masqRule, "--random-fully") - } - p.natRules.Write(masqRule) - - // Install the kubernetes-specific masquerade mark rule. We use a whole chain for - // this so that it is easier to flush and change, for example if the mark - // value should ever change. - p.natRules.Write( - "-A", string(KubeMarkMasqChain), - "-j", "MARK", "--or-mark", p.masqueradeMark, - ) - - // Write the end-of-table markers. - p.filterRules.Write("COMMIT") - p.natRules.Write("COMMIT") -} - -func (p *proxier) acceptIPVSTraffic() { - sets := []string{kubeClusterIPSet, kubeLoadBalancerSet} - for _, set := range sets { - var matchType string - if !p.ipsetList[set].isRefCountZero() { - switch p.ipsetList[set].SetType { - case util.BitmapPort: - matchType = "dst" - default: - matchType = "dst,dst" - } - p.natRules.Write( - "-A", string(kubeServicesChain), - "-m", "set", "--match-set", p.ipsetList[set].Name, matchType, - "-j", "ACCEPT", - ) - } - } -} - -func (p *proxier) syncIPTableRules() { - // Reset all buffers used later. - // This is to avoid memory reallocations and thus improve performance. - p.natChains.Reset() - p.natRules.Reset() - p.filterChains.Reset() - p.filterRules.Reset() - - // Write table headers. - p.filterChains.Write("*filter") - p.natChains.Write("*nat") - - p.createAndLinkKubeChain() - - // Tail call iptables rules for ipset, make sure only call iptables once - // in a single loop per ip set. - p.writeIptablesRules() - - // Sync iptables rules. - // NOTE: NoFlushTables is used so we don't flush non-kubernetes chains in the table. - p.iptablesData.Reset() - p.iptablesData.Write(p.natChains.Bytes()) - p.iptablesData.Write(p.natRules.Bytes()) - p.iptablesData.Write(p.filterChains.Bytes()) - p.iptablesData.Write(p.filterRules.Bytes()) - - klog.V(5).Info("Restoring iptables", "rules", string(p.iptablesData.Bytes())) - err := p.iptables.RestoreAll(p.iptablesData.Bytes(), util.NoFlushTables, util.RestoreCounters) - if err != nil { - klog.Error(err, "Failed to execute iptables-restore", "rules", string(p.iptablesData.Bytes())) - return - } -} diff --git a/backends/ipvs-as-sink/iptables_test.go b/backends/ipvs-as-sink/iptables_test.go deleted file mode 100644 index 146d962a1..000000000 --- a/backends/ipvs-as-sink/iptables_test.go +++ /dev/null @@ -1,272 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - "fmt" - "strings" - "testing" - - "github.com/lithammer/dedent" - "github.com/stretchr/testify/assert" -) - -// parseIPTablesData takes iptables-save output and returns a map of table name to array of lines. -func parseIPTablesData(ruleData string) (map[string][]string, error) { - // Split ruleData at the "COMMIT" lines; given valid input, this will result in - // one element for each table plus an extra empty element (since the ruleData - // should end with a "COMMIT" line). - rawTables := strings.Split(strings.TrimPrefix(ruleData, "\n"), "COMMIT\n") - nTables := len(rawTables) - 1 - if nTables < 2 || rawTables[nTables] != "" { - return nil, fmt.Errorf("bad ruleData (%d tables)\n%s", nTables, ruleData) - } - - tables := make(map[string][]string, nTables) - for i, table := range rawTables[:nTables] { - lines := strings.Split(strings.Trim(table, "\n"), "\n") - // The first line should be, eg, "*nat" or "*filter" - if lines[0][0] != '*' { - return nil, fmt.Errorf("bad ruleData (table %d starts with %q)", i+1, lines[0]) - } - // add back the "COMMIT" line that got eaten by the strings.Split above - lines = append(lines, "COMMIT") - tables[lines[0][1:]] = lines - } - - if tables["nat"] == nil { - return nil, fmt.Errorf("bad ruleData (no %q table)", "nat") - } - if tables["filter"] == nil { - return nil, fmt.Errorf("bad ruleData (no %q table)", "filter") - } - return tables, nil -} - -func TestParseIPTablesData(t *testing.T) { - for _, tc := range []struct { - name string - input string - output map[string][]string - error string - }{ - { - name: "basic test", - input: dedent.Dedent(` - *filter - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-NODEPORTS - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-SERVICES - [0:0] - :KUBE-NODEPORTS - [0:0] - :KUBE-POSTROUTING - [0:0] - :KUBE-MARK-MASQ - [0:0] - :KUBE-SVC-XPGD46QRK7WJZT7O - [0:0] - :KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0] - -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN - -A KUBE-POSTROUTING -j MARK --xor-mark 0x4000 - -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE - -A KUBE-MARK-MASQ -j MARK --or-mark 0x4000 - -A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ - -A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ - -A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80 - -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS - COMMIT - `), - output: map[string][]string{ - "filter": { - `*filter`, - `:KUBE-SERVICES - [0:0]`, - `:KUBE-EXTERNAL-SERVICES - [0:0]`, - `:KUBE-FORWARD - [0:0]`, - `:KUBE-NODEPORTS - [0:0]`, - `-A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT`, - `-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP`, - `-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT`, - `-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT`, - `COMMIT`, - }, - "nat": { - `*nat`, - `:KUBE-SERVICES - [0:0]`, - `:KUBE-NODEPORTS - [0:0]`, - `:KUBE-POSTROUTING - [0:0]`, - `:KUBE-MARK-MASQ - [0:0]`, - `:KUBE-SVC-XPGD46QRK7WJZT7O - [0:0]`, - `:KUBE-SEP-SXIVWICOYRO3J4NJ - [0:0]`, - `-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN`, - `-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000`, - `-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE`, - `-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000`, - `-A KUBE-SERVICES -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 -j KUBE-SVC-XPGD46QRK7WJZT7O`, - `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment "ns1/svc1:p80 cluster IP" -m tcp -p tcp -d 10.20.30.41 --dport 80 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ`, - `-A KUBE-SVC-XPGD46QRK7WJZT7O -m comment --comment ns1/svc1:p80 -j KUBE-SEP-SXIVWICOYRO3J4NJ`, - `-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -s 10.180.0.1 -j KUBE-MARK-MASQ`, - `-A KUBE-SEP-SXIVWICOYRO3J4NJ -m comment --comment ns1/svc1:p80 -m tcp -p tcp -j DNAT --to-destination 10.180.0.1:80`, - `-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS`, - `COMMIT`, - }, - }, - }, - { - name: "not enough tables", - input: dedent.Dedent(` - *filter - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-NODEPORTS - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - `), - error: "bad ruleData (1 tables)", - }, - { - name: "trailing junk", - input: dedent.Dedent(` - *filter - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-NODEPORTS - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-NODEPORTS - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - junk - `), - error: "bad ruleData (2 tables)", - }, - { - name: "bad start line", - input: dedent.Dedent(` - *filter - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-NODEPORTS - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-NODEPORTS - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - `), - error: `bad ruleData (table 2 starts with ":KUBE-SERVICES - [0:0]")`, - }, - { - name: "no nat", - input: dedent.Dedent(` - *filter - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-NODEPORTS - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *mangle - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-NODEPORTS - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - `), - error: `bad ruleData (no "nat" table)`, - }, - { - name: "no filter", - input: dedent.Dedent(` - *mangle - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-NODEPORTS - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - *nat - :KUBE-SERVICES - [0:0] - :KUBE-EXTERNAL-SERVICES - [0:0] - :KUBE-FORWARD - [0:0] - :KUBE-NODEPORTS - [0:0] - -A KUBE-NODEPORTS -m comment --comment "ns2/svc2:p80 health check node port" -m tcp -p tcp --dport 30000 -j ACCEPT - -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT - -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - COMMIT - `), - error: `bad ruleData (no "filter" table)`, - }, - } { - t.Run(tc.name, func(t *testing.T) { - out, err := parseIPTablesData(tc.input) - if err == nil { - if tc.error != "" { - t.Errorf("unexpectedly did not get error") - } else { - assert.Equal(t, tc.output, out) - } - } else { - if tc.error == "" { - t.Errorf("got unexpected error: %v", err) - } else if !strings.HasPrefix(err.Error(), tc.error) { - t.Errorf("got wrong error: %v (expected %q)", err, tc.error) - } - } - }) - } -} diff --git a/backends/ipvs-as-sink/ipvs.go b/backends/ipvs-as-sink/ipvs.go deleted file mode 100644 index d0f37dd72..000000000 --- a/backends/ipvs-as-sink/ipvs.go +++ /dev/null @@ -1,491 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - "fmt" - "net" - "net/http" - "os" - "time" - - "github.com/google/seesaw/ipvs" - "github.com/vishvananda/netlink" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/server/mux" - "k8s.io/klog/v2" - - "sigs.k8s.io/kpng/api/localv1" - "sigs.k8s.io/kpng/client/backendcmd" - "sigs.k8s.io/kpng/client/localsink" - "sigs.k8s.io/kpng/client/localsink/decoder" - "sigs.k8s.io/kpng/client/localsink/filterreset" - "sigs.k8s.io/kpng/client/serviceevents" - - "sigs.k8s.io/kpng/backends/ipvs-as-sink/exec" - "sigs.k8s.io/kpng/backends/ipvs-as-sink/util" -) - -// In IPVS proxy mode, the following flags need to be set -const ( - sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables" - sysctlVSConnTrack = "net/ipv4/vs/conntrack" - sysctlConnReuse = "net/ipv4/vs/conn_reuse_mode" - sysctlExpireNoDestConn = "net/ipv4/vs/expire_nodest_conn" - sysctlExpireQuiescentTemplate = "net/ipv4/vs/expire_quiescent_template" - sysctlForward = "net/ipv4/ip_forward" - sysctlArpIgnore = "net/ipv4/conf/all/arp_ignore" - sysctlArpAnnounce = "net/ipv4/conf/all/arp_announce" - connReuseMinSupportedKernelVersion = "4.1" - // https://github.com/torvalds/linux/commit/35dfb013149f74c2be1ff9c78f14e6a3cd1539d1 - connReuseFixedKernelVersion = "5.9" - readHeaderTimeout = time.Second * 5 -) - -func init() { - backendcmd.Register("to-ipvs", func() backendcmd.Cmd { return New() }) -} - -type Backend struct { - localsink.Config - svcs map[string]*localv1.Service - proxiers map[v1.IPFamily]*proxier - svcEPMap map[string]int - - dryRun bool - nodeAddresses []string - schedulingMethod string - weight int32 - - dummy netlink.Link - - masqueradeAll bool -} - -var _ decoder.Interface = &Backend{} - -func New() *Backend { - return &Backend{ - proxiers: make(map[v1.IPFamily]*proxier), - svcs: map[string]*localv1.Service{}, - svcEPMap: map[string]int{}, - } -} - -func (s *Backend) Sink() localsink.Sink { - return filterreset.New(decoder.New(serviceevents.Wrap(s))) -} - -// ------------------------------------------------------------------------ -// (IP, port) listener interface -var _ serviceevents.IPPortsListener = &Backend{} - -func (s *Backend) AddIPPort(svc *localv1.Service, ip string, IPKind serviceevents.IPKind, port *localv1.PortMapping) { - klog.V(2).Infof("AddIPPort (svc: %v, svc-ip: %v, port: %v)", svc, ip, port) - serviceKey := getServiceKey(svc) - s.svcs[serviceKey] = svc - if svc.Type == ClusterIPService { - s.handleClusterIPService(svc, ip, IPKind, port) - } - - if svc.Type == NodePortService { - s.handleNodePortService(svc, ip, port) - } - - if svc.Type == LoadBalancerService { - s.handleLbService(svc, ip, IPKind, port) - } -} - -func (s *Backend) DeleteIPPort(svc *localv1.Service, ip string, IPKind serviceevents.IPKind, port *localv1.PortMapping) { - klog.V(2).Infof("DeleteIPPort (svc: %v, svc-ip: %v, port: %v)", svc, ip, port) - if svc.Type == ClusterIPService { - s.deleteClusterIPService(svc, ip, IPKind, port) - } - - if svc.Type == NodePortService { - s.deleteNodePortService(svc, ip, port) - } - - if svc.Type == LoadBalancerService { - s.deleteLbService(svc, ip, IPKind, port) - } -} - -// ------------------------------------------------------------------------ -// IP listener interface -var _ serviceevents.IPsListener = &Backend{} - -func (s *Backend) AddIP(svc *localv1.Service, ip string, ipKind serviceevents.IPKind) { - klog.V(2).Infof("AddIP (svc: %v, svc-ip: %v, type: %v)", svc, ip, ipKind) - s.addServiceIPToKubeIPVSIntf(ip) -} -func (s *Backend) DeleteIP(svc *localv1.Service, ip string, ipKind serviceevents.IPKind) { - klog.V(2).Infof("DeleteIP (svc: %v, svc-ip: %v, type: %v)", svc, ip, ipKind) - s.deleteServiceIPToKubeIPVSIntf(ip) -} - -// Handle session affinity -var _ serviceevents.SessionAffinityListener = &Backend{} - -func (s *Backend) EnableSessionAffinity(svc *localv1.Service, sessionAffinity serviceevents.SessionAffinity) { - klog.V(2).Infof("EnableSessionAffinity (svc: %v, sessionAffinity: %v)", svc, sessionAffinity) - s.enableSessionAffinityForServiceIPs(svc, sessionAffinity) -} - -func (s *Backend) DisableSessionAffinity(svc *localv1.Service) { - klog.V(2).Infof("DisableSessionAffinity (svc: %v,)", svc) - s.disableSessionAffinityForServiceIPs(svc) -} - -// Handle traffic policy -var _ serviceevents.TrafficPolicyListener = &Backend{} - -func (s *Backend) EnableTrafficPolicy(svc *localv1.Service, policyKind serviceevents.TrafficPolicyKind) { - klog.V(2).Infof("EnableTrafficPolicy (svc: %v, policyKind: %v)", svc, policyKind) -} - -func (s *Backend) DisableTrafficPolicy(svc *localv1.Service, policyKind serviceevents.TrafficPolicyKind) { - klog.V(2).Infof("DisableTrafficPolicy (svc: %v, policyKind: %v)", svc, policyKind) -} - -// SetService ------------------------------------------------------ -// Service -func (s *Backend) SetService(svc *localv1.Service) {} - -func (s *Backend) DeleteService(namespace, name string) {} - -func (s *Backend) SetEndpoint(namespace, serviceName, key string, endpoint *localv1.Endpoint) { - klog.V(2).Infof("SetEndpoint(%q, %q, %q, %v)", namespace, serviceName, key, endpoint) - svcKey := namespace + "/" + serviceName - //TODO Check whether IPVS handles headless service - if _, ok := s.svcs[svcKey]; !ok { - klog.Infof("service (%s) could be headless-service", serviceName) - return - } - service := s.svcs[svcKey] - s.svcEPMap[svcKey]++ - - if service.Type == ClusterIPService { - s.handleEndPointForClusterIP(svcKey, key, endpoint, AddEndPoint) - } - - if service.Type == NodePortService { - s.handleEndPointForNodePortService(svcKey, key, endpoint, AddEndPoint) - } - - if service.Type == LoadBalancerService { - s.handleEndPointForLBService(svcKey, key, endpoint, AddEndPoint) - } -} - -func (s *Backend) DeleteEndpoint(namespace, serviceName, key string) { - klog.V(2).Infof("DeleteEndpoint(%q, %q, %q)", namespace, serviceName, key) - svcKey := namespace + "/" + serviceName - //TODO Check whether IPVS handles headless service - if _, ok := s.svcs[svcKey]; !ok { - klog.Infof("service (%s) could be headless-service", serviceName) - return - } - service := s.svcs[svcKey] - s.svcEPMap[svcKey]-- - if service.Type == ClusterIPService { - s.handleEndPointForClusterIP(svcKey, key, nil, DeleteEndPoint) - } - - if service.Type == NodePortService { - s.handleEndPointForNodePortService(svcKey, key, nil, DeleteEndPoint) - } - - if service.Type == LoadBalancerService { - s.handleEndPointForLBService(svcKey, key, nil, DeleteEndPoint) - } -} - -func (s *Backend) Setup() { - kernelHandler := util.NewLinuxKernelHandler() - err := s.initializeKernelConfig(kernelHandler) - if err != nil { - klog.Info(err) - return - } - - ipvs.Init() - - s.createIPVSDummyInterface() - - // Generate the masquerade mark to use for SNAT rules. - //TODO fetch masqueradeBit from config - masqueradeBit := 14 - masqueradeValue := 1 << uint(masqueradeBit) - masqueradeMark := fmt.Sprintf("%#08x", masqueradeValue) - - // Create a ipset utils. - execer := exec.New() - ipsetInterface := util.New(execer) - - for _, ipFamily := range []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol} { - var nodeIPs []string - - for _, nodeIP := range s.nodeAddresses { - if ipFamily == getIPFamily(nodeIP) { - nodeIPs = append(nodeIPs, nodeIP) - } - } - - iptInterface := util.NewIPTableInterface(execer, util.Protocol(ipFamily)) - - s.proxiers[ipFamily] = NewProxier( - ipFamily, - s.dummy, - ipsetInterface, - iptInterface, - nodeIPs, - s.schedulingMethod, - masqueradeMark, - s.masqueradeAll, - s.weight, - ) - - s.proxiers[ipFamily].initializeIPSets() - } - - go func() { - err := s.SetUpHttpListen() - if err != nil { - return - } - }() -} - -func (s *Backend) SetUpHttpListen() error { - errCh := make(chan error) - s.ServeProxyMode(errCh) - return <-errCh -} - -func (s *Backend) ServeProxyMode(errCh chan error) { - //TODO Get Bind address config. Time being leave it empty, kernel will choose loopback adress 127.0.0.1 - bindAddress := "127.0.0.1:10249" - proxyMode := "ipvs" - proxyMux := mux.NewPathRecorderMux("kpng-ipvs") - proxyMux.HandleFunc("/proxyMode", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Header().Set("X-Content-Type-Options", "nosniff") - fmt.Fprintf(w, "%s", proxyMode) - }) - - fn := func() { - server := &http.Server{ - Addr: bindAddress, - Handler: proxyMux, - ReadHeaderTimeout: readHeaderTimeout, - } - err := server.ListenAndServe() - if err != nil { - klog.Errorf("starting http server for proxyMode failed: %v", err) - if errCh != nil { - errCh <- err - } - } - } - go wait.Until(fn, 5*time.Second, wait.NeverStop) -} - -func (s *Backend) createIPVSDummyInterface() { - // populate dummyIPs - const dummyName = "kube-ipvs0" - - dummy, err := netlink.LinkByName(dummyName) - if err != nil { - if _, ok := err.(netlink.LinkNotFoundError); !ok { - klog.Fatal("failed to get dummy interface: ", err) - } - - // not found => create the dummy - dummy = &netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{Name: dummyName}, - } - - klog.Info("creating dummy interface ", dummyName) - if err = netlink.LinkAdd(dummy); err != nil { - klog.Fatal("failed to create dummy interface: ", err) - } - - dummy, err = netlink.LinkByName(dummyName) - if err != nil { - klog.Fatal("failed to get link after create: ", err) - } - } - - if dummy.Attrs().Flags&net.FlagUp == 0 { - klog.Info("setting dummy interface ", dummyName, " up") - if err = netlink.LinkSetUp(dummy); err != nil { - klog.Fatal("failed to set dummy interface up: ", err) - } - } - - s.dummy = dummy - - dummyIface, err := net.InterfaceByName(dummyName) - if err != nil { - klog.Fatal("failed to get dummy interface: ", err) - } - - addrs, err := dummyIface.Addrs() - if err != nil { - klog.Fatal("failed to list dummy interface IPs: ", err) - } - - for _, ip := range addrs { - cidr := ip.String() - ip, _, err := net.ParseCIDR(cidr) - if err != nil { - klog.Fatalf("failed to parse ip/net %q: %v", ip, err) - } - if ip.IsLinkLocalUnicast() { - continue - } - } -} - -// WaitRequest see localsink.Sink#WaitRequest -func (s *Backend) WaitRequest() (nodeName string, err error) { - name, _ := os.Hostname() - return name, nil -} - -func (s *Backend) Reset() { /* noop, we're wrapped in filterreset */ } - -func (s *Backend) Sync() { - if log := klog.V(1); log.Enabled() { - klog.Info("Sync()") - - start := time.Now() - defer klog.Info("sync took ", time.Now().Sub(start)) - } - - for _, proxier := range s.proxiers { - proxier.sync() - } -} - -func (s *Backend) addServiceIPToKubeIPVSIntf(serviceIP string) { - ipFamily := getIPFamily(serviceIP) - - ip := asDummyIPs(serviceIP, ipFamily) - - _, ipNet, err := net.ParseCIDR(ip) - if err != nil { - klog.Fatalf("failed to parse ip/net %q: %v", ip, err) - } - - if s.dummy == nil { - klog.Fatalf("exit early while adding dummy IP ", ip, "; dummy link device not found") - return - } - klog.V(2).Info("adding dummy IP ", ip) - if err = netlink.AddrAdd(s.dummy, &netlink.Addr{IPNet: ipNet}); err != nil { - klog.Error("failed to add dummy IP ", ip, ": ", err) - } -} - -func (s *Backend) deleteServiceIPToKubeIPVSIntf(serviceIP string) { - ipFamily := getIPFamily(serviceIP) - - ip := asDummyIPs(serviceIP, ipFamily) - - _, ipNet, err := net.ParseCIDR(ip) - if err != nil { - klog.Fatalf("failed to parse ip/net %q: %v", ip, err) - } - - if s.dummy == nil { - klog.Fatalf("exit early while deleting dummy IP ", ip, "; dummy link device not found") - return - } - klog.V(2).Info("deleting dummy IP ", ip) - if err = netlink.AddrDel(s.dummy, &netlink.Addr{IPNet: ipNet}); err != nil { - klog.Error("failed to delete dummy IP ", ip, ": ", err) - } -} - -func (s *Backend) initializeKernelConfig(kernelHandler util.KernelHandler) error { - // Proxy needs br_netfilter and bridge-nf-call-iptables=1 when containers - // are connected to a Linux bridge (but not SDN bridges). Until most - // plugins handle this, log when config is missing - sysctl := util.NewSysInterface() - if val, err := sysctl.GetSysctl(sysctlBridgeCallIPTables); err == nil && val != 1 { - klog.Info("Missing br-netfilter module or unset sysctl br-nf-call-iptables, proxy may not work as intended") - } - - // Set the conntrack sysctl we need for - if err := util.EnsureSysctl(sysctl, sysctlVSConnTrack, 1); err != nil { - return err - } - - kernelVersionStr, err := kernelHandler.GetKernelVersion() - if err != nil { - return fmt.Errorf("error determining kernel version to find required kernel modules for ipvs support: %v", err) - } - kernelVersion, err := version.ParseGeneric(kernelVersionStr) - if err != nil { - return fmt.Errorf("error parsing kernel version %q: %v", kernelVersionStr, err) - } - if kernelVersion.LessThan(version.MustParseGeneric(connReuseMinSupportedKernelVersion)) { - klog.Error(nil, "Can't set sysctl, kernel version doesn't satisfy minimum version requirements", "sysctl", sysctlConnReuse, "minimumKernelVersion", connReuseMinSupportedKernelVersion) - } else if kernelVersion.AtLeast(version.MustParseGeneric(connReuseFixedKernelVersion)) { - // https://github.com/kubernetes/kubernetes/issues/93297 - klog.V(2).Info("Left as-is", "sysctl", sysctlConnReuse) - } else { - // Set the connection reuse mode - if err := util.EnsureSysctl(sysctl, sysctlConnReuse, 0); err != nil { - return err - } - } - - // Set the expire_nodest_conn sysctl we need for - if err := util.EnsureSysctl(sysctl, sysctlExpireNoDestConn, 1); err != nil { - return err - } - - // Set the expire_quiescent_template sysctl we need for - if err := util.EnsureSysctl(sysctl, sysctlExpireQuiescentTemplate, 1); err != nil { - return err - } - - // Set the ip_forward sysctl we need for - if err := util.EnsureSysctl(sysctl, sysctlForward, 1); err != nil { - return err - } - - //if strictARP { - // // Set the arp_ignore sysctl we need for - // if err := utilproxy.EnsureSysctl(sysctl, sysctlArpIgnore, 1); err != nil { - // return err - // } - // - // // Set the arp_announce sysctl we need for - // if err := utilproxy.EnsureSysctl(sysctl, sysctlArpAnnounce, 2); err != nil { - // return err - // } - //} - return nil -} diff --git a/backends/ipvs-as-sink/ipvslb.go b/backends/ipvs-as-sink/ipvslb.go deleted file mode 100644 index 3119caae8..000000000 --- a/backends/ipvs-as-sink/ipvslb.go +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - "encoding/json" - "net" - "syscall" - - "sigs.k8s.io/kpng/api/localv1" - - "github.com/google/seesaw/ipvs" -) - -type ipvsSvcDst struct { - Svc ipvs.Service - Dst ipvs.Destination -} - -type ipvsLB struct { - IP string - ServiceKey string - ServiceType string - SchedulingMethod string - Flags ipvs.ServiceFlags - Timeout uint32 - Port uint16 - NodePort uint16 - Protocol localv1.Protocol -} - -func (lb ipvsLB) String() string { - ba, _ := json.Marshal(lb) - return string(ba) -} - -func (lb ipvsLB) ToService() ipvs.Service { - var port uint16 - if lb.ServiceType == ClusterIPService { - port = lb.Port - } - if lb.ServiceType == NodePortService { - port = lb.NodePort - } - if lb.ServiceType == LoadBalancerService { - port = lb.Port - } - s := ipvs.Service{ - Address: net.ParseIP(lb.IP), - Port: port, - Scheduler: lb.SchedulingMethod, - Flags: lb.Flags, - Timeout: lb.Timeout, - } - - switch lb.Protocol { - case localv1.Protocol_TCP: - s.Protocol = syscall.IPPROTO_TCP - case localv1.Protocol_UDP: - s.Protocol = syscall.IPPROTO_UDP - case localv1.Protocol_SCTP: - s.Protocol = syscall.IPPROTO_SCTP - } - - return s -} - -func ipvsDestination(epInfo endPointInfo, port *BaseServicePortInfo) ipvs.Destination { - targetPort := port.TargetPort() - if port.targetPort == 0 { - targetPort = epInfo.portMap[port.TargetPortName()] - } - return ipvs.Destination{ - Address: net.ParseIP(epInfo.endPointIP), - Port: uint16(targetPort), - Weight: port.weight, - } -} diff --git a/backends/ipvs-as-sink/loadbalancersvc.go b/backends/ipvs-as-sink/loadbalancersvc.go deleted file mode 100644 index c8bdd0369..000000000 --- a/backends/ipvs-as-sink/loadbalancersvc.go +++ /dev/null @@ -1,281 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - "k8s.io/klog/v2" - "sigs.k8s.io/kpng/api/localv1" - ipsetutil "sigs.k8s.io/kpng/backends/ipvs-as-sink/util" - "sigs.k8s.io/kpng/client/serviceevents" -) - -func (s *Backend) handleLbService(svc *localv1.Service, serviceIP string, IPKind serviceevents.IPKind, port *localv1.PortMapping) { - serviceKey := getServiceKey(svc) - s.svcs[serviceKey] = svc - ipFamily := getIPFamily(serviceIP) - isServiceUpdated := s.isServiceUpdated(serviceKey) - - if !isServiceUpdated { - s.proxiers[ipFamily].handleNewLBService(serviceKey, serviceIP, IPKind, svc, port) - } else { - s.proxiers[ipFamily].handleUpdatedLBService(serviceKey, serviceIP, IPKind, svc, port) - } -} - -func (s *Backend) deleteLbService(svc *localv1.Service, serviceIP string, IPKind serviceevents.IPKind, port *localv1.PortMapping) { - serviceKey := getServiceKey(svc) - s.svcs[serviceKey] = svc - ipFamily := getIPFamily(serviceIP) - p := s.proxiers[ipFamily] - - var portList []*BaseServicePortInfo - if IPKind == serviceevents.ClusterIP { - // -------------------------------------------------------------------------- - // ClusterIP needs to be removed from IPVS - spKey := getServicePortKey(serviceKey, serviceIP, port) - kv := p.servicePorts.GetByPrefix([]byte(spKey)) - portInfo := kv[0].Value.(BaseServicePortInfo) - portList = append(portList, &portInfo) - p.servicePorts.DeleteByPrefix([]byte(spKey)) - - p.deleteVirtualServer(&portInfo) - - //Cluster service IP needs to be programmed in ipset. - p.AddOrDelClusterIPInIPSet(&portInfo, DeleteService) - //--------------------------------------------------------------------------- - - // -------------------------------------------------------------------------- - // NodeIPs needs to be removed from IPVS - for _, nodeIP := range p.nodeAddresses { - spKey = getServicePortKey(serviceKey, nodeIP, port) - kv := p.servicePorts.GetByPrefix([]byte(spKey)) - portInfo := kv[0].Value.(BaseServicePortInfo) - portList = append(portList, &portInfo) - p.servicePorts.DeleteByPrefix([]byte(spKey)) - - p.deleteVirtualServer(&portInfo) - } - p.AddOrDelNodePortInIPSet(port, DeleteService) - // -------------------------------------------------------------------------- - } - - if IPKind == serviceevents.LoadBalancerIP { - spKey := getServicePortKey(serviceKey, serviceIP, port) - kv := p.servicePorts.GetByPrefix([]byte(spKey)) - portInfo := kv[0].Value.(BaseServicePortInfo) - portList = append(portList, &portInfo) - - p.servicePorts.DeleteByPrefix([]byte(spKey)) - - p.deleteVirtualServer(&portInfo) - p.AddOrDelLbIPInIPSet(svc, &portInfo, DeleteService) - } - - epList := p.deleteRealServerForPort(serviceKey, portList) - for _, ep := range epList { - p.AddOrDelEndPointInIPSet(ep.endPointIP, port.Protocol.String(), port.TargetPort, ep.isLocalEndPoint, DeleteEndPoint) - } - - portMapKey := getPortKey(serviceKey, port) - p.deletePortFromPortMap(serviceKey, portMapKey) -} - -func (p *proxier) handleNewLBService(serviceKey, serviceIP string, - IPKind serviceevents.IPKind, - svc *localv1.Service, - port *localv1.PortMapping, -) { - if _, ok := p.portMap[serviceKey]; !ok { - p.portMap[serviceKey] = make(map[string]localv1.PortMapping) - } - - portMapKey := getPortKey(serviceKey, port) - p.portMap[serviceKey][portMapKey] = *port - - if IPKind == serviceevents.ClusterIP { - // -------------------------------------------------------------------------- - // ClusterIP needs to be programmed in IPVS - spKey := getServicePortKey(serviceKey, serviceIP, port) - portInfo := NewBaseServicePortInfo(svc, port, serviceIP, ClusterIPService, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - - p.addVirtualServer(portInfo) - - //Cluster service IP needs to be programmed in ipset. - p.AddOrDelClusterIPInIPSet(portInfo, AddService) - //--------------------------------------------------------------------------- - - // -------------------------------------------------------------------------- - // NodeIPs needs to be programmed in IPVS - for _, nodeIP := range p.nodeAddresses { - spKey := getServicePortKey(serviceKey, nodeIP, port) - portInfo = NewBaseServicePortInfo(svc, port, nodeIP, NodePortService, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - - p.addVirtualServer(portInfo) - } - p.AddOrDelNodePortInIPSet(port, AddService) - // -------------------------------------------------------------------------- - } - - if IPKind == serviceevents.LoadBalancerIP { - spKey := getServicePortKey(serviceKey, serviceIP, port) - portInfo := NewBaseServicePortInfo(svc, port, serviceIP, LoadBalancerService, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - - p.addVirtualServer(portInfo) - - p.AddOrDelLbIPInIPSet(svc, portInfo, AddService) - } -} - -func (p *proxier) handleUpdatedLBService(serviceKey, serviceIP string, - IPKind serviceevents.IPKind, - svc *localv1.Service, - port *localv1.PortMapping, -) { - err, lbIP := p.getLbIPForIPFamily(svc) - if err != nil { - klog.Info(err) - } - portMapKey := getPortKey(serviceKey, port) - p.portMap[serviceKey][portMapKey] = *port - - var portList []*BaseServicePortInfo - var spKey string - if IPKind == serviceevents.ClusterIP { - // -------------------------------------------------------------------------- - // ClusterIP needs to be programmed in IPVS - spKey = getServicePortKey(serviceKey, serviceIP, port) - portInfo := NewBaseServicePortInfo(svc, port, serviceIP, ClusterIPService, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - portList = append(portList, portInfo) - - p.addVirtualServer(portInfo) - - //Cluster service IP needs to be programmed in ipset. - p.AddOrDelClusterIPInIPSet(portInfo, AddService) - //--------------------------------------------------------------------------- - - // -------------------------------------------------------------------------- - // NodeIPs needs to be programmed in IPVS - for _, nodeIP := range p.nodeAddresses { - spKey := getServicePortKey(serviceKey, nodeIP, port) - portInfo = NewBaseServicePortInfo(svc, port, nodeIP, NodePortService, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - portList = append(portList, portInfo) - - p.addVirtualServer(portInfo) - } - p.AddOrDelNodePortInIPSet(port, AddService) - // -------------------------------------------------------------------------- - } - - if IPKind == serviceevents.LoadBalancerIP { - // LbIP needs to be programmed in IPVS - spKey = getServicePortKey(serviceKey, lbIP, port) - portInfo := NewBaseServicePortInfo(svc, port, lbIP, LoadBalancerService, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - portList = append(portList, portInfo) - - p.addVirtualServer(portInfo) - p.AddOrDelLbIPInIPSet(svc, portInfo, AddService) - } - endPointList := p.addRealServerForPort(serviceKey, portList) - - for _, ep := range endPointList { - p.AddOrDelEndPointInIPSet(ep.endPointIP, port.Protocol.String(), port.TargetPort, ep.isLocalEndPoint, AddEndPoint) - } -} - -func (s *Backend) handleEndPointForLBService(svcKey, key string, endpoint *localv1.Endpoint, op Operation) { - prefix := svcKey + "/" + key + "/" - - if op == AddEndPoint { - // endpoint will have only one family IP, either v4/6. - endPointIPs := endpoint.IPs.All() - for _, ip := range endPointIPs { - ipFamily := getIPFamily(ip) - s.proxiers[ipFamily].addRealServer(svcKey, prefix, ip, endpoint) - } - } - - if op == DeleteEndPoint { - for _, proxier := range s.proxiers { - proxier.deleteRealServer(svcKey, prefix) - } - } -} - -func (p *proxier) AddOrDelLbIPInIPSet(svc *localv1.Service, port *BaseServicePortInfo, op Operation) { - var entry *ipsetutil.Entry - entry = getIPSetEntry("", port) - // add service load balancer ingressIP:Port to kubeServiceAccess ip set for the purpose of solving hairpin. - // proxier.kubeServiceAccessSet.activeEntries.Insert(entry.String()) - // If we are proxying globally, we need to masquerade in case we cross nodes. - // If we are proxying only locally, we can retain the source IP. - p.setKubeLBIPSet(kubeLoadBalancerSet, entry, op) - - if svc.ExternalTrafficToLocal { - //insert loadbalancer entry to lbIngressLocalSet if service externaltrafficpolicy=local - p.setKubeLBIPSet(kubeLoadBalancerLocalSet, entry, op) - } - - var isSourceRangeConfigured bool = false - if len(svc.IPFilters) > 0 { - for _, ip := range svc.IPFilters { - if len(ip.SourceRanges) > 0 { - isSourceRangeConfigured = true - } - for _, srcIP := range ip.SourceRanges { - srcRangeEntry := getIPSetEntry(srcIP, port) - p.setKubeLBIPSet(kubeLoadBalancerSourceCIDRSet, srcRangeEntry, op) - } - } - - // The service firewall rules are created based on ServiceSpec.loadBalancerSourceRanges field. - // This currently works for loadbalancers that preserves source ips. - // For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply. - if isSourceRangeConfigured { - p.setKubeLBIPSet(kubeLoadbalancerFWSet, entry, op) - } - } -} - -func (p *proxier) setKubeLBIPSet(ipSetName string, entry *ipsetutil.Entry, op Operation) { - if valid := p.ipsetList[ipSetName].validateEntry(entry); !valid { - klog.Errorf("error adding entry :%s, to ipset:%s", entry.String(), p.ipsetList[ipSetName].Name) - return - } - set := p.ipsetList[ipSetName] - if op == AddService { - if err := set.handle.AddEntry(entry.String(), &set.IPSet, true); err != nil { - klog.Errorf("Failed to add entry %v into ip set: %s, error: %v", entry, set.Name, err) - } else { - klog.V(3).Infof("Successfully add entry: %v into ip set: %s", entry, set.Name) - } - p.updateRefCountForIPSet(ipSetName, op) - } - if op == DeleteService { - if err := set.handle.DelEntry(entry.String(), set.Name); err != nil { - klog.Errorf("Failed to delete entry: %v from ip set: %s, error: %v", entry, set.Name, err) - } else { - klog.V(3).Infof("Successfully deleted entry: %v to ip set: %s", entry, set.Name) - } - p.updateRefCountForIPSet(ipSetName, op) - } -} diff --git a/backends/ipvs-as-sink/nodeportsvc.go b/backends/ipvs-as-sink/nodeportsvc.go deleted file mode 100644 index bda5bc4fc..000000000 --- a/backends/ipvs-as-sink/nodeportsvc.go +++ /dev/null @@ -1,163 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - "k8s.io/klog/v2" - "sigs.k8s.io/kpng/api/localv1" -) - -func (s *Backend) handleNodePortService(svc *localv1.Service, serviceIP string, port *localv1.PortMapping) { - serviceKey := getServiceKey(svc) - s.svcs[serviceKey] = svc - ipFamily := getIPFamily(serviceIP) - - isServiceUpdated := s.isServiceUpdated(serviceKey) - if !isServiceUpdated { - s.proxiers[ipFamily].handleNewNodePortService(serviceKey, serviceIP, svc, port) - } else { - s.proxiers[ipFamily].handleUpdatedNodePortService(serviceKey, serviceIP, svc, port) - } -} - -func (s *Backend) deleteNodePortService(svc *localv1.Service, serviceIP string, port *localv1.PortMapping) { - serviceKey := getServiceKey(svc) - s.svcs[serviceKey] = svc - ipFamily := getIPFamily(serviceIP) - p := s.proxiers[ipFamily] - - // All Node Addresses need to be deleted for port in IPVS. - var portList []*BaseServicePortInfo - for _, nodeIP := range p.nodeAddresses { - spKey := getServicePortKey(serviceKey, nodeIP, port) - kv := p.servicePorts.GetByPrefix([]byte(spKey)) - portInfo := kv[0].Value.(BaseServicePortInfo) - portList = append(portList, &portInfo) - p.servicePorts.DeleteByPrefix([]byte(spKey)) - - p.deleteVirtualServer(&portInfo) - } - p.AddOrDelNodePortInIPSet(port, DeleteService) - - // ClusterIP of nodePort service needs to be deleted for port in IPVS. - spKey := getServicePortKey(serviceKey, serviceIP, port) - kv := p.servicePorts.GetByPrefix([]byte(spKey)) - clusterIPPortInfo := kv[0].Value.(BaseServicePortInfo) - portList = append(portList, &clusterIPPortInfo) - p.servicePorts.DeleteByPrefix([]byte(spKey)) - - p.deleteVirtualServer(&clusterIPPortInfo) - - p.AddOrDelClusterIPInIPSet(&clusterIPPortInfo, DeleteService) - - epList := p.deleteRealServerForPort(serviceKey, portList) - for _, ep := range epList { - p.AddOrDelEndPointInIPSet(ep.endPointIP, port.Protocol.String(), port.TargetPort, ep.isLocalEndPoint, DeleteEndPoint) - } - - portMapKey := getPortKey(serviceKey, port) - p.deletePortFromPortMap(serviceKey, portMapKey) -} - -func (p *proxier) handleNewNodePortService(serviceKey, clusterIP string, svc *localv1.Service, port *localv1.PortMapping) { - if _, ok := p.portMap[serviceKey]; !ok { - p.portMap[serviceKey] = make(map[string]localv1.PortMapping) - } - - portMapKey := getPortKey(serviceKey, port) - p.portMap[serviceKey][portMapKey] = *port - - // All Node Addresses need to be added as virtual servers in IPVS. - for _, nodeIP := range p.nodeAddresses { - spKey := getServicePortKey(serviceKey, nodeIP, port) - portInfo := NewBaseServicePortInfo(svc, port, nodeIP, NodePortService, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - - p.addVirtualServer(portInfo) - } - // Only here direct port obj is used instead of portInfo - p.AddOrDelNodePortInIPSet(port, AddService) - - // ClusterIP of nodePort service needs to be added as virtual servers in IPVS. - spKey := getServicePortKey(serviceKey, clusterIP, port) - portInfo := NewBaseServicePortInfo(svc, port, clusterIP, ClusterIPService, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - - p.addVirtualServer(portInfo) - - p.AddOrDelClusterIPInIPSet(portInfo, AddService) -} - -func (p *proxier) handleUpdatedNodePortService(serviceKey, clusterIP string, svc *localv1.Service, port *localv1.PortMapping) { - if _, ok := p.portMap[serviceKey]; !ok { - klog.Errorf("can't update port into non-existent service") - return - } - - portMapKey := getPortKey(serviceKey, port) - p.portMap[serviceKey][portMapKey] = *port - - // -------------------------------------------------------------------------- - // All Node Addresses need to be updated with new port as virtual servers in IPVS. - var portList []*BaseServicePortInfo - for _, nodeIP := range p.nodeAddresses { - spKey := getServicePortKey(serviceKey, nodeIP, port) - portInfo := NewBaseServicePortInfo(svc, port, nodeIP, NodePortService, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - portList = append(portList, portInfo) - - p.addVirtualServer(portInfo) - } - p.AddOrDelNodePortInIPSet(port, AddService) - // -------------------------------------------------------------------------- - - // -------------------------------------------------------------------------- - // ClusterIP of nodePort service needs to be updated with new port as virtual servers in IPVS. - spKey := getServicePortKey(serviceKey, clusterIP, port) - portInfo := NewBaseServicePortInfo(svc, port, clusterIP, ClusterIPService, p.schedulingMethod, p.weight) - p.servicePorts.Set([]byte(spKey), 0, *portInfo) - - portList = append(portList, portInfo) - p.addVirtualServer(portInfo) - - p.AddOrDelClusterIPInIPSet(portInfo, AddService) - // -------------------------------------------------------------------------- - - endPointList := p.addRealServerForPort(serviceKey, portList) - - for _, ep := range endPointList { - p.AddOrDelEndPointInIPSet(ep.endPointIP, port.Protocol.String(), port.TargetPort, ep.isLocalEndPoint, AddEndPoint) - } -} - -func (s *Backend) handleEndPointForNodePortService(svcKey, key string, endpoint *localv1.Endpoint, op Operation) { - prefix := svcKey + "/" + key + "/" - if op == AddEndPoint { - // endpoint will have only one family IP, either v4/6. - endPointIPs := endpoint.IPs.All() - for _, ip := range endPointIPs { - ipFamily := getIPFamily(ip) - s.proxiers[ipFamily].addRealServer(svcKey, prefix, ip, endpoint) - } - } - - if op == DeleteEndPoint { - for _, proxier := range s.proxiers { - proxier.deleteRealServer(svcKey, prefix) - } - } -} diff --git a/backends/ipvs-as-sink/proxier.go b/backends/ipvs-as-sink/proxier.go deleted file mode 100644 index 4cba7dee9..000000000 --- a/backends/ipvs-as-sink/proxier.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package ipvssink - -import ( - "bytes" - - "github.com/vishvananda/netlink" - v1 "k8s.io/api/core/v1" - - "sigs.k8s.io/kpng/api/localv1" - iptablesutil "sigs.k8s.io/kpng/backends/iptables/util" - "sigs.k8s.io/kpng/backends/ipvs-as-sink/exec" - util "sigs.k8s.io/kpng/backends/ipvs-as-sink/util" - "sigs.k8s.io/kpng/client/lightdiffstore" -) - -type proxier struct { - ipFamily v1.IPFamily - - dryRun bool - nodeAddresses []string - schedulingMethod string - weight int32 - masqueradeMark string - masqueradeAll bool - - dummy netlink.Link - - iptables util.IPTableInterface - ipset util.Interface - exec exec.Interface - //ipvs util.Interface - //localDetector proxyutiliptables.LocalTrafficDetector - //portMapper netutils.PortOpener - //recorder events.EventRecorder - //serviceHealthServer healthcheck.ServiceHealthServer - //healthzServer healthcheck.ProxierHealthUpdater - - // /// -> - endpoints *lightdiffstore.DiffStore - // ///: -> ipvsLB - servicePorts *lightdiffstore.DiffStore - - ipsetList map[string]*IPSet - //servicePortMap map[string]map[string]*BaseServicePortInfo - portMap map[string]map[string]localv1.PortMapping - // The following buffers are used to reuse memory and avoid allocations - // that are significantly impacting performance. - iptablesData *bytes.Buffer - filterChainsData *bytes.Buffer - natChains iptablesutil.LineBuffer - filterChains iptablesutil.LineBuffer - natRules iptablesutil.LineBuffer - filterRules iptablesutil.LineBuffer -} - -func NewProxier(ipFamily v1.IPFamily, - dummy netlink.Link, - ipsetInterface util.Interface, - iptInterface util.IPTableInterface, - nodeIPs []string, - schedulingMethod, masqueradeMark string, - masqueradeAll bool, - weight int32) *proxier { - return &proxier{ - ipFamily: ipFamily, - dummy: dummy, - nodeAddresses: nodeIPs, - schedulingMethod: schedulingMethod, - weight: weight, - ipset: ipsetInterface, - iptables: iptInterface, - masqueradeMark: masqueradeMark, - masqueradeAll: masqueradeAll, - ipsetList: make(map[string]*IPSet), - portMap: make(map[string]map[string]localv1.PortMapping), - endpoints: lightdiffstore.New(), - servicePorts: lightdiffstore.New(), - iptablesData: bytes.NewBuffer(nil), - filterChainsData: bytes.NewBuffer(nil), - natChains: iptablesutil.LineBuffer{}, - natRules: iptablesutil.LineBuffer{}, - filterChains: iptablesutil.LineBuffer{}, - filterRules: iptablesutil.LineBuffer{}, - } -} - -func (p *proxier) initializeIPSets() { - // initialize ipsetList with all sets we needed - for _, is := range ipsetInfo { - p.ipsetList[is.name] = newIPSet(p.ipset, is.name, is.setType, p.ipFamily, is.comment) - } - - // make sure ip sets exists in the system. - for _, set := range p.ipsetList { - if err := ensureIPSet(set); err != nil { - return - } - } -} diff --git a/backends/ipvs-as-sink/util/ipset.go b/backends/ipvs-as-sink/util/ipset.go deleted file mode 100644 index b7239523a..000000000 --- a/backends/ipvs-as-sink/util/ipset.go +++ /dev/null @@ -1,537 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "bytes" - "fmt" - "net" - "regexp" - "strconv" - "strings" - - "k8s.io/apimachinery/pkg/util/version" - "sigs.k8s.io/kpng/backends/ipvs-as-sink/exec" - - "k8s.io/klog/v2" -) - -// Interface is an injectable interface for running ipset commands. Implementations must be goroutine-safe. -type Interface interface { - // FlushSet deletes all entries from a named set. - FlushSet(set string) error - // DestroySet deletes a named set. - DestroySet(set string) error - // DestroyAllSets deletes all sets. - DestroyAllSets() error - // CreateSet creates a new set. It will ignore error when the set already exists if ignoreExistErr=true. - CreateSet(set *IPSet, ignoreExistErr bool) error - // AddEntry adds a new entry to the named set. It will ignore error when the entry already exists if ignoreExistErr=true. - AddEntry(entry string, set *IPSet, ignoreExistErr bool) error - // DelEntry deletes one entry from the named set - DelEntry(entry string, set string) error - // Test test if an entry exists in the named set - TestEntry(entry string, set string) (bool, error) - // ListEntries lists all the entries from a named set - ListEntries(set string) ([]string, error) - // ListSets list all set names from kernel - ListSets() ([]string, error) - // GetVersion returns the "X.Y" version string for ipset. - GetVersion() (string, error) -} - -// IPSetCmd represents the ipset util. We use ipset command for ipset execute. -const IPSetCmd = "ipset" - -// EntryMemberPattern is the regular expression pattern of ipset member list. -// The raw output of ipset command `ipset list {set}` is similar to, -// Name: foobar -// Type: hash:ip,port -// Revision: 2 -// Header: family inet hashsize 1024 maxelem 65536 -// Size in memory: 16592 -// References: 0 -// Members: -// 192.168.1.2,tcp:8080 -// 192.168.1.1,udp:53 -var EntryMemberPattern = "(?m)^(.*\n)*Members:\n" - -// VersionPattern is the regular expression pattern of ipset version string. -// ipset version output is similar to "v6.10". -var VersionPattern = "v[0-9]+\\.[0-9]+" - -// IPSet implements an Interface to a set. -type IPSet struct { - // Name is the set name. - Name string - // SetType specifies the ipset type. - SetType Type - // HashFamily specifies the protocol family of the IP addresses to be stored in the set. - // The default is inet, i.e IPv4. If users want to use IPv6, they should specify inet6. - HashFamily string - // HashSize specifies the hash table size of ipset. - HashSize int - // MaxElem specifies the max element number of ipset. - MaxElem int - // PortRange specifies the port range of bitmap:port type ipset. - PortRange string - // comment message for ipset - Comment string -} - -// Validate checks if a given ipset is valid or not. -func (set *IPSet) Validate() bool { - // Check if protocol is valid for `HashIPPort`, `HashIPPortIP` and `HashIPPortNet` type set. - if set.SetType == HashIPPort || set.SetType == HashIPPortIP || set.SetType == HashIPPortNet { - if valid := validateHashFamily(set.HashFamily); !valid { - return false - } - } - // check set type - if valid := validateIPSetType(set.SetType); !valid { - return false - } - // check port range for bitmap type set - if set.SetType == BitmapPort { - if valid := validatePortRange(set.PortRange); !valid { - return false - } - } - // check hash size value of ipset - if set.HashSize <= 0 { - klog.Errorf("Invalid hashsize value %d, should be >0", set.HashSize) - return false - } - // check max elem value of ipset - if set.MaxElem <= 0 { - klog.Errorf("Invalid maxelem value %d, should be >0", set.MaxElem) - return false - } - - return true -} - -// setIPSetDefaults sets some IPSet fields if not present to their default values. -func (set *IPSet) setIPSetDefaults() { - // Setting default values if not present - if set.HashSize == 0 { - set.HashSize = 1024 - } - if set.MaxElem == 0 { - set.MaxElem = 65536 - } - // Default protocol is IPv4 - if set.HashFamily == "" { - set.HashFamily = ProtocolFamilyIPV4 - } - // Default ipset type is "hash:ip,port" - if len(set.SetType) == 0 { - set.SetType = HashIPPort - } - if len(set.PortRange) == 0 { - set.PortRange = DefaultPortRange - } -} - -// Entry represents a ipset entry. -type Entry struct { - // IP is the entry's IP. The IP address protocol corresponds to the HashFamily of IPSet. - // All entries' IP addresses in the same ip set has same the protocol, IPv4 or IPv6. - IP string - // Port is the entry's Port. - Port int - // Protocol is the entry's Protocol. The protocols of entries in the same ip set are all - // the same. The accepted protocols are TCP, UDP and SCTP. - Protocol string - // Net is the entry's IP network address. Network address with zero prefix size can NOT - // be stored. - Net string - // IP2 is the entry's second IP. IP2 may not be empty for `hash:ip,port,ip` type ip set. - IP2 string - // SetType is the type of ipset where the entry exists. - SetType Type -} - -// Validate checks if a given ipset entry is valid or not. The set parameter is the ipset that entry belongs to. -func (e *Entry) Validate(set *IPSet) bool { - if e.Port < 0 { - klog.Errorf("Entry %v port number %d should be >=0 for ipset %v", e, e.Port, set) - return false - } - switch e.SetType { - case HashIPPort: - //check if IP and Protocol of Entry is valid. - if valid := e.checkIPandProtocol(set); !valid { - return false - } - case HashIPPortIP: - //check if IP and Protocol of Entry is valid. - if valid := e.checkIPandProtocol(set); !valid { - return false - } - - // IP2 can not be empty for `hash:ip,port,ip` type ip set - if net.ParseIP(e.IP2) == nil { - klog.Errorf("Error parsing entry %v second ip address %v for ipset %v", e, e.IP2, set) - return false - } - case HashIPPortNet: - //check if IP and Protocol of Entry is valid. - if valid := e.checkIPandProtocol(set); !valid { - return false - } - - // Net can not be empty for `hash:ip,port,net` type ip set - if _, ipNet, err := net.ParseCIDR(e.Net); ipNet == nil { - klog.Errorf("Error parsing entry %v ip net %v for ipset %v, error: %v", e, e.Net, set, err) - return false - } - case BitmapPort: - // check if port number satisfies its ipset's requirement of port range - if set == nil { - klog.Errorf("Unable to reference ip set where the entry %v exists", e) - return false - } - begin, end, err := parsePortRange(set.PortRange) - if err != nil { - klog.Errorf("Failed to parse set %v port range %s for ipset %v, error: %v", set, set.PortRange, set, err) - return false - } - if e.Port < begin || e.Port > end { - klog.Errorf("Entry %v port number %d is not in the port range %s of its ipset %v", e, e.Port, set.PortRange, set) - return false - } - } - - return true -} - -// String returns the string format for ipset entry. -func (e *Entry) String() string { - switch e.SetType { - case HashIPPort: - // Entry{192.168.1.1, udp, 53} -> 192.168.1.1,udp:53 - // Entry{192.168.1.2, tcp, 8080} -> 192.168.1.2,tcp:8080 - return fmt.Sprintf("%s,%s:%s", e.IP, e.Protocol, strconv.Itoa(e.Port)) - case HashIPPortIP: - // Entry{192.168.1.1, udp, 53, 10.0.0.1} -> 192.168.1.1,udp:53,10.0.0.1 - // Entry{192.168.1.2, tcp, 8080, 192.168.1.2} -> 192.168.1.2,tcp:8080,192.168.1.2 - return fmt.Sprintf("%s,%s:%s,%s", e.IP, e.Protocol, strconv.Itoa(e.Port), e.IP2) - case HashIPPortNet: - // Entry{192.168.1.2, udp, 80, 10.0.1.0/24} -> 192.168.1.2,udp:80,10.0.1.0/24 - // Entry{192.168.2,25, tcp, 8080, 10.1.0.0/16} -> 192.168.2,25,tcp:8080,10.1.0.0/16 - return fmt.Sprintf("%s,%s:%s,%s", e.IP, e.Protocol, strconv.Itoa(e.Port), e.Net) - case BitmapPort: - // Entry{53} -> 53 - // Entry{8080} -> 8080 - return strconv.Itoa(e.Port) - } - return "" -} - -// checkIPandProtocol checks if IP and Protocol of Entry is valid. -func (e *Entry) checkIPandProtocol(set *IPSet) bool { - // set default protocol to tcp if empty - if len(e.Protocol) == 0 { - e.Protocol = ProtocolTCP - } else if !validateProtocol(e.Protocol) { - return false - } - - if net.ParseIP(e.IP) == nil { - klog.Errorf("Error parsing entry %v ip address %v for ipset %v", e, e.IP, set) - return false - } - - return true -} - -type runner struct { - exec exec.Interface -} - -// New returns a new Interface which will exec ipset. -func New(exec exec.Interface) Interface { - return &runner{ - exec: exec, - } -} - -// CreateSet creates a new set, it will ignore error when the set already exists if ignoreExistErr=true. -func (runner *runner) CreateSet(set *IPSet, ignoreExistErr bool) error { - // sets some IPSet fields if not present to their default values. - set.setIPSetDefaults() - - // Validate ipset before creating - valid := set.Validate() - if !valid { - return fmt.Errorf("error creating ipset since it's invalid") - } - return runner.createSet(set, ignoreExistErr) -} - -// If ignoreExistErr is set to true, then the -exist option of ipset will be specified, ipset ignores the error -// otherwise raised when the same set (setname and create parameters are identical) already exists. -func (runner *runner) createSet(set *IPSet, ignoreExistErr bool) error { - args := []string{"create", set.Name, string(set.SetType)} - if set.SetType == HashIPPortIP || set.SetType == HashIPPort || set.SetType == HashIPPortNet { - args = append(args, - "family", set.HashFamily, - "hashsize", strconv.Itoa(set.HashSize), - "maxelem", strconv.Itoa(set.MaxElem), - ) - } - if set.SetType == BitmapPort { - args = append(args, "range", set.PortRange) - } - if ignoreExistErr { - args = append(args, "-exist") - } - if _, err := runner.exec.Command(IPSetCmd, args...).CombinedOutput(); err != nil { - return fmt.Errorf("error creating ipset %s, error: %v", set.Name, err) - } - return nil -} - -// AddEntry adds a new entry to the named set. -// If the -exist option is specified, ipset ignores the error otherwise raised when -// the same set (setname and create parameters are identical) already exists. -func (runner *runner) AddEntry(entry string, set *IPSet, ignoreExistErr bool) error { - args := []string{"add", set.Name, entry} - if ignoreExistErr { - args = append(args, "-exist") - } - if _, err := runner.exec.Command(IPSetCmd, args...).CombinedOutput(); err != nil { - return fmt.Errorf("error adding entry %s, error: %v", entry, err) - } - return nil -} - -// DelEntry is used to delete the specified entry from the set. -func (runner *runner) DelEntry(entry string, set string) error { - if _, err := runner.exec.Command(IPSetCmd, "del", set, entry).CombinedOutput(); err != nil { - return fmt.Errorf("error deleting entry %s: from set: %s, error: %v", entry, set, err) - } - return nil -} - -// TestEntry is used to check whether the specified entry is in the set or not. -func (runner *runner) TestEntry(entry string, set string) (bool, error) { - if out, err := runner.exec.Command(IPSetCmd, "test", set, entry).CombinedOutput(); err == nil { - reg, e := regexp.Compile("is NOT in set " + set) - if e == nil && reg.MatchString(string(out)) { - return false, nil - } else if e == nil { - return true, nil - } else { - return false, fmt.Errorf("error testing entry: %s, error: %v", entry, e) - } - } else { - return false, fmt.Errorf("error testing entry %s: %v (%s)", entry, err, out) - } -} - -// FlushSet deletes all entries from a named set. -func (runner *runner) FlushSet(set string) error { - if _, err := runner.exec.Command(IPSetCmd, "flush", set).CombinedOutput(); err != nil { - return fmt.Errorf("error flushing set: %s, error: %v", set, err) - } - return nil -} - -// DestroySet is used to destroy a named set. -func (runner *runner) DestroySet(set string) error { - if out, err := runner.exec.Command(IPSetCmd, "destroy", set).CombinedOutput(); err != nil { - return fmt.Errorf("error destroying set %s, error: %v(%s)", set, err, out) - } - return nil -} - -// DestroyAllSets is used to destroy all sets. -func (runner *runner) DestroyAllSets() error { - if _, err := runner.exec.Command(IPSetCmd, "destroy").CombinedOutput(); err != nil { - return fmt.Errorf("error destroying all sets, error: %v", err) - } - return nil -} - -// ListSets list all set names from kernel -func (runner *runner) ListSets() ([]string, error) { - out, err := runner.exec.Command(IPSetCmd, "list", "-n").CombinedOutput() - if err != nil { - return nil, fmt.Errorf("error listing all sets, error: %v", err) - } - return strings.Split(string(out), "\n"), nil -} - -// ListEntries lists all the entries from a named set. -func (runner *runner) ListEntries(set string) ([]string, error) { - if len(set) == 0 { - return nil, fmt.Errorf("set name can't be nil") - } - out, err := runner.exec.Command(IPSetCmd, "list", set).CombinedOutput() - if err != nil { - return nil, fmt.Errorf("error listing set: %s, error: %v", set, err) - } - memberMatcher := regexp.MustCompile(EntryMemberPattern) - list := memberMatcher.ReplaceAllString(string(out[:]), "") - strs := strings.Split(list, "\n") - results := make([]string, 0) - for i := range strs { - if len(strs[i]) > 0 { - results = append(results, strs[i]) - } - } - return results, nil -} - -// GetVersion returns the version string. -func (runner *runner) GetVersion() (string, error) { - return getIPSetVersionString(runner.exec) -} - -// getIPSetVersionString runs "ipset --version" to get the version string -// in the form of "X.Y", i.e "6.19" -func getIPSetVersionString(exec exec.Interface) (string, error) { - cmd := exec.Command(IPSetCmd, "--version") - cmd.SetStdin(bytes.NewReader([]byte{})) - bytes, err := cmd.CombinedOutput() - if err != nil { - return "", err - } - versionMatcher := regexp.MustCompile(VersionPattern) - match := versionMatcher.FindStringSubmatch(string(bytes)) - if match == nil { - return "", fmt.Errorf("no ipset version found in string: %s", bytes) - } - return match[0], nil -} - -// checks if port range is valid. The begin port number is not necessarily less than -// end port number - ipset util can accept it. It means both 1-100 and 100-1 are valid. -func validatePortRange(portRange string) bool { - strs := strings.Split(portRange, "-") - if len(strs) != 2 { - klog.Errorf("port range should be in the format of `a-b`") - return false - } - for i := range strs { - num, err := strconv.Atoi(strs[i]) - if err != nil { - klog.Errorf("Failed to parse %s, error: %v", strs[i], err) - return false - } - if num < 0 { - klog.Errorf("port number %d should be >=0", num) - return false - } - } - return true -} - -// checks if the given ipset type is valid. -func validateIPSetType(set Type) bool { - for _, valid := range ValidIPSetTypes { - if set == valid { - return true - } - } - klog.Errorf("Currently supported ipset types are: %v, %s is not supported", ValidIPSetTypes, set) - return false -} - -// checks if given hash family is supported in ipset -func validateHashFamily(family string) bool { - if family == ProtocolFamilyIPV4 || family == ProtocolFamilyIPV6 { - return true - } - klog.Errorf("Currently supported ip set hash families are: [%s, %s], %s is not supported", ProtocolFamilyIPV4, ProtocolFamilyIPV6, family) - return false -} - -// checks if given protocol is supported in entry -func validateProtocol(protocol string) bool { - if protocol == ProtocolTCP || protocol == ProtocolUDP || protocol == ProtocolSCTP { - return true - } - klog.Errorf("Invalid entry's protocol: %s, supported protocols are [%s, %s, %s]", protocol, ProtocolTCP, ProtocolUDP, ProtocolSCTP) - return false -} - -// parsePortRange parse the begin and end port from a raw string(format: a-b). beginPort <= endPort -// in the return value. -func parsePortRange(portRange string) (beginPort int, endPort int, err error) { - if len(portRange) == 0 { - portRange = DefaultPortRange - } - - strs := strings.Split(portRange, "-") - if len(strs) != 2 { - // port number -1 indicates invalid - return -1, -1, fmt.Errorf("port range should be in the format of `a-b`") - } - for i := range strs { - num, err := strconv.Atoi(strs[i]) - if err != nil { - // port number -1 indicates invalid - return -1, -1, err - } - if num < 0 { - // port number -1 indicates invalid - return -1, -1, fmt.Errorf("port number %d should be >=0", num) - } - if i == 0 { - beginPort = num - continue - } - endPort = num - // switch when first port number > second port number - if beginPort > endPort { - endPort = beginPort - beginPort = num - } - } - return beginPort, endPort, nil -} - -var _ = Interface(&runner{}) - -// IPVS required kernel modules. -const ( - // KernelModuleIPVS is the kernel module "ip_vs" - KernelModuleIPVS string = "ip_vs" - // KernelModuleIPVSRR is the kernel module "ip_vs_rr" - KernelModuleIPVSRR string = "ip_vs_rr" - // KernelModuleIPVSWRR is the kernel module "ip_vs_wrr" - KernelModuleIPVSWRR string = "ip_vs_wrr" - // KernelModuleIPVSSH is the kernel module "ip_vs_sh" - KernelModuleIPVSSH string = "ip_vs_sh" - // KernelModuleNfConntrackIPV4 is the module "nf_conntrack_ipv4" - KernelModuleNfConntrackIPV4 string = "nf_conntrack_ipv4" - // KernelModuleNfConntrack is the kernel module "nf_conntrack" - KernelModuleNfConntrack string = "nf_conntrack" -) - -// GetRequiredIPVSModules returns the required ipvs modules for the given linux kernel version. -func GetRequiredIPVSModules(kernelVersion *version.Version) []string { - // "nf_conntrack_ipv4" has been removed since v4.19 - // see https://github.com/torvalds/linux/commit/a0ae2562c6c4b2721d9fddba63b7286c13517d9f - if kernelVersion.LessThan(version.MustParseGeneric("4.19")) { - return []string{KernelModuleIPVS, KernelModuleIPVSRR, KernelModuleIPVSWRR, KernelModuleIPVSSH, KernelModuleNfConntrackIPV4} - } - return []string{KernelModuleIPVS, KernelModuleIPVSRR, KernelModuleIPVSWRR, KernelModuleIPVSSH, KernelModuleNfConntrack} -} diff --git a/backends/ipvs-as-sink/util/iptables.go b/backends/ipvs-as-sink/util/iptables.go deleted file mode 100644 index ec901e03c..000000000 --- a/backends/ipvs-as-sink/util/iptables.go +++ /dev/null @@ -1,792 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "bytes" - "context" - "fmt" - "regexp" - "strings" - "sync" - "time" - - "sigs.k8s.io/kpng/backends/ipvs-as-sink/exec" - - "k8s.io/apimachinery/pkg/util/sets" - utilversion "k8s.io/apimachinery/pkg/util/version" - utilwait "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog/v2" - utiltrace "k8s.io/utils/trace" -) - -// RulePosition holds the -I/-A flags for iptable -type RulePosition string - -const ( - // Prepend is the insert flag for iptable - Prepend RulePosition = "-I" - // Append is the append flag for iptable - Append RulePosition = "-A" -) - -// Interface is an injectable interface for running iptables commands. Implementations must be goroutine-safe. -type IPTableInterface interface { - // EnsureChain checks if the specified chain exists and, if not, creates it. If the chain existed, return true. - EnsureChain(table Table, chain Chain) (bool, error) - // FlushChain clears the specified chain. If the chain did not exist, return error. - FlushChain(table Table, chain Chain) error - // DeleteChain deletes the specified chain. If the chain did not exist, return error. - DeleteChain(table Table, chain Chain) error - // ChainExists tests whether the specified chain exists, returning an error if it - // does not, or if it is unable to check. - ChainExists(table Table, chain Chain) (bool, error) - // EnsureRule checks if the specified rule is present and, if not, creates it. If the rule existed, return true. - EnsureRule(position RulePosition, table Table, chain Chain, args ...string) (bool, error) - // DeleteRule checks if the specified rule is present and, if so, deletes it. - DeleteRule(table Table, chain Chain, args ...string) error - // IsIPv6 returns true if this is managing ipv6 tables. - IsIPv6() bool - // Protocol returns the IP family this instance is managing, - Protocol() Protocol - // SaveInto calls `iptables-save` for table and stores result in a given buffer. - SaveInto(table Table, buffer *bytes.Buffer) error - // Restore runs `iptables-restore` passing data through []byte. - // table is the Table to restore - // data should be formatted like the output of SaveInto() - // flush sets the presence of the "--noflush" flag. see: FlushFlag - // counters sets the "--counters" flag. see: RestoreCountersFlag - Restore(table Table, data []byte, flush FlushFlag, counters RestoreCountersFlag) error - // RestoreAll is the same as Restore except that no table is specified. - RestoreAll(data []byte, flush FlushFlag, counters RestoreCountersFlag) error - // Monitor detects when the given iptables tables have been flushed by an external - // tool (e.g. a firewall reload) by creating canary chains and polling to see if - // they have been deleted. (Specifically, it polls tables[0] every interval until - // the canary has been deleted from there, then waits a short additional time for - // the canaries to be deleted from the remaining tables as well. You can optimize - // the polling by listing a relatively empty table in tables[0]). When a flush is - // detected, this calls the reloadFunc so the caller can reload their own iptables - // rules. If it is unable to create the canary chains (either initially or after - // a reload) it will log an error and stop monitoring. - // (This function should be called from a goroutine.) - Monitor(canary Chain, tables []Table, reloadFunc func(), interval time.Duration, stopCh <-chan struct{}) - // HasRandomFully reveals whether `-j MASQUERADE` takes the - // `--random-fully` option. This is helpful to work around a - // Linux kernel bug that sometimes causes multiple flows to get - // mapped to the same IP:PORT and consequently some suffer packet - // drops. - HasRandomFully() bool - - // Present checks if the kernel supports the iptable interface - Present() bool -} - -// Protocol defines the ip protocol either ipv4 or ipv6 -type Protocol string - -const ( - // ProtocolIPv4 represents ipv4 protocol in iptables - ProtocolIPv4 Protocol = "IPv4" - // ProtocolIPv6 represents ipv6 protocol in iptables - ProtocolIPv6 Protocol = "IPv6" -) - -// Table represents different iptable like filter,nat, mangle and raw -type Table string - -const ( - // TableNAT represents the built-in nat table - TableNAT Table = "nat" - // TableFilter represents the built-in filter table - TableFilter Table = "filter" - // TableMangle represents the built-in mangle table - TableMangle Table = "mangle" -) - -// Chain represents the different rules -type Chain string - -const ( - // ChainPostrouting used for source NAT in nat table - ChainPostrouting Chain = "POSTROUTING" - // ChainPrerouting used for DNAT (destination NAT) in nat table - ChainPrerouting Chain = "PREROUTING" - // ChainOutput used for the packets going out from local - ChainOutput Chain = "OUTPUT" - // ChainInput used for incoming packets - ChainInput Chain = "INPUT" - // ChainForward used for the packets for another NIC - ChainForward Chain = "FORWARD" -) - -const ( - cmdIPTablesSave string = "iptables-save" - cmdIPTablesRestore string = "iptables-restore" - cmdIPTables string = "iptables" - cmdIP6TablesRestore string = "ip6tables-restore" - cmdIP6TablesSave string = "ip6tables-save" - cmdIP6Tables string = "ip6tables" -) - -// RestoreCountersFlag is an option flag for Restore -type RestoreCountersFlag bool - -// RestoreCounters a boolean true constant for the option flag RestoreCountersFlag -const RestoreCounters RestoreCountersFlag = true - -// NoRestoreCounters a boolean false constant for the option flag RestoreCountersFlag -const NoRestoreCounters RestoreCountersFlag = false - -// FlushFlag an option flag for Flush -type FlushFlag bool - -// FlushTables a boolean true constant for option flag FlushFlag -const FlushTables FlushFlag = true - -// NoFlushTables a boolean false constant for option flag FlushFlag -const NoFlushTables FlushFlag = false - -// MinCheckVersion minimum version to be checked -// Versions of iptables less than this do not support the -C / --check flag -// (test whether a rule exists). -var MinCheckVersion = utilversion.MustParseGeneric("1.4.11") - -// RandomFullyMinVersion is the minimum version from which the --random-fully flag is supported, -// used for port mapping to be fully randomized -var RandomFullyMinVersion = utilversion.MustParseGeneric("1.6.2") - -// WaitMinVersion a minimum iptables versions supporting the -w and -w flags -var WaitMinVersion = utilversion.MustParseGeneric("1.4.20") - -// WaitIntervalMinVersion a minimum iptables versions supporting the wait interval useconds -var WaitIntervalMinVersion = utilversion.MustParseGeneric("1.6.1") - -// WaitSecondsMinVersion a minimum iptables versions supporting the wait seconds -var WaitSecondsMinVersion = utilversion.MustParseGeneric("1.4.22") - -// WaitRestoreMinVersion a minimum iptables versions supporting the wait restore seconds -var WaitRestoreMinVersion = utilversion.MustParseGeneric("1.6.2") - -// WaitString a constant for specifying the wait flag -const WaitString = "-w" - -// WaitSecondsValue a constant for specifying the default wait seconds -const WaitSecondsValue = "5" - -// WaitIntervalString a constant for specifying the wait interval flag -const WaitIntervalString = "-W" - -// WaitIntervalUsecondsValue a constant for specifying the default wait interval useconds -const WaitIntervalUsecondsValue = "100000" - -// LockfilePath16x is the iptables 1.6.x lock file acquired by any process that's making any change in the iptable rule -const LockfilePath16x = "/run/xtables.lock" - -// LockfilePath14x is the iptables 1.4.x lock file acquired by any process that's making any change in the iptable rule -const LockfilePath14x = "@xtables" - -// runner implements Interface in terms of exec("iptables"). -type iptablerunner struct { - mu sync.Mutex - exec exec.Interface - protocol Protocol - hasCheck bool - hasRandomFully bool - waitFlag []string - restoreWaitFlag []string - lockfilePath14x string - lockfilePath16x string -} - -// newInternal returns a new Interface which will exec iptables, and allows the -// caller to change the iptables-restore lockfile path -func newInternal(exec exec.Interface, protocol Protocol, lockfilePath14x, lockfilePath16x string) IPTableInterface { - version, err := getIPTablesVersion(exec, protocol) - if err != nil { - klog.Warningf("Error checking iptables version, assuming version at least %s: %v", MinCheckVersion, err) - version = MinCheckVersion - } - - if lockfilePath16x == "" { - lockfilePath16x = LockfilePath16x - } - if lockfilePath14x == "" { - lockfilePath14x = LockfilePath14x - } - - runner := &iptablerunner{ - exec: exec, - protocol: protocol, - hasCheck: version.AtLeast(MinCheckVersion), - hasRandomFully: version.AtLeast(RandomFullyMinVersion), - waitFlag: getIPTablesWaitFlag(version), - restoreWaitFlag: getIPTablesRestoreWaitFlag(version, exec, protocol), - lockfilePath14x: lockfilePath14x, - lockfilePath16x: lockfilePath16x, - } - return runner -} - -// NewIPTableExec returns a new Interface which will exec iptables. -func NewIPTableExec(exec exec.Interface, protocol Protocol) IPTableInterface { - return newInternal(exec, protocol, "", "") -} - -// New returns a new Interface which will exec iptables. -func NewIPTableInterface(exec exec.Interface, protocol Protocol) IPTableInterface { - return newInternal(exec, protocol, "", "") -} - -// EnsureChain is part of Interface. -func (runner *iptablerunner) EnsureChain(table Table, chain Chain) (bool, error) { - fullArgs := makeFullArgs(table, chain) - - runner.mu.Lock() - defer runner.mu.Unlock() - - out, err := runner.run(opCreateChain, fullArgs) - if err != nil { - if ee, ok := err.(exec.ExitError); ok { - if ee.Exited() && ee.ExitStatus() == 1 { - return true, nil - } - } - return false, fmt.Errorf("error creating chain %q: %v: %s", chain, err, out) - } - return false, nil -} - -// FlushChain is part of Interface. -func (runner *iptablerunner) FlushChain(table Table, chain Chain) error { - fullArgs := makeFullArgs(table, chain) - - runner.mu.Lock() - defer runner.mu.Unlock() - - out, err := runner.run(opFlushChain, fullArgs) - if err != nil { - return fmt.Errorf("error flushing chain %q: %v: %s", chain, err, out) - } - return nil -} - -// DeleteChain is part of Interface. -func (runner *iptablerunner) DeleteChain(table Table, chain Chain) error { - fullArgs := makeFullArgs(table, chain) - - runner.mu.Lock() - defer runner.mu.Unlock() - - // TODO: we could call iptables -S first, ignore the output and check for non-zero return (more like DeleteRule) - out, err := runner.run(opDeleteChain, fullArgs) - if err != nil { - return fmt.Errorf("error deleting chain %q: %v: %s", chain, err, out) - } - return nil -} - -// EnsureRule is part of Interface. -func (runner *iptablerunner) EnsureRule(position RulePosition, table Table, chain Chain, args ...string) (bool, error) { - fullArgs := makeFullArgs(table, chain, args...) - - runner.mu.Lock() - defer runner.mu.Unlock() - - exists, err := runner.checkRule(table, chain, args...) - if err != nil { - return false, err - } - if exists { - return true, nil - } - out, err := runner.run(operation(position), fullArgs) - if err != nil { - return false, fmt.Errorf("error appending rule: %v: %s", err, out) - } - return false, nil -} - -// DeleteRule is part of Interface. -func (runner *iptablerunner) DeleteRule(table Table, chain Chain, args ...string) error { - fullArgs := makeFullArgs(table, chain, args...) - - runner.mu.Lock() - defer runner.mu.Unlock() - - exists, err := runner.checkRule(table, chain, args...) - if err != nil { - return err - } - if !exists { - return nil - } - out, err := runner.run(opDeleteRule, fullArgs) - if err != nil { - return fmt.Errorf("error deleting rule: %v: %s", err, out) - } - return nil -} - -func (runner *iptablerunner) IsIPv6() bool { - return runner.protocol == ProtocolIPv6 -} - -func (runner *iptablerunner) Protocol() Protocol { - return runner.protocol -} - -// SaveInto is part of Interface. -func (runner *iptablerunner) SaveInto(table Table, buffer *bytes.Buffer) error { - runner.mu.Lock() - defer runner.mu.Unlock() - - trace := utiltrace.New("iptables save") - defer trace.LogIfLong(2 * time.Second) - - // run and return - iptablesSaveCmd := iptablesSaveCommand(runner.protocol) - args := []string{"-t", string(table)} - klog.V(4).Infof("running %s %v", iptablesSaveCmd, args) - cmd := runner.exec.Command(iptablesSaveCmd, args...) - cmd.SetStdout(buffer) - stderrBuffer := bytes.NewBuffer(nil) - cmd.SetStderr(stderrBuffer) - - err := cmd.Run() - if err != nil { - stderrBuffer.WriteTo(buffer) // ignore error, since we need to return the original error - } - return err -} - -// Restore is part of Interface. -func (runner *iptablerunner) Restore(table Table, data []byte, flush FlushFlag, counters RestoreCountersFlag) error { - // setup args - args := []string{"-T", string(table)} - return runner.restoreInternal(args, data, flush, counters) -} - -// RestoreAll is part of Interface. -func (runner *iptablerunner) RestoreAll(data []byte, flush FlushFlag, counters RestoreCountersFlag) error { - // setup args - args := make([]string, 0) - return runner.restoreInternal(args, data, flush, counters) -} - -type iptablesLocker interface { - Close() error -} - -// restoreInternal is the shared part of Restore/RestoreAll -func (runner *iptablerunner) restoreInternal(args []string, data []byte, flush FlushFlag, counters RestoreCountersFlag) error { - runner.mu.Lock() - defer runner.mu.Unlock() - - trace := utiltrace.New("iptables restore") - defer trace.LogIfLong(2 * time.Second) - - if !flush { - args = append(args, "--noflush") - } - if counters { - args = append(args, "--counters") - } - - // Grab the iptables lock to prevent iptables-restore and iptables - // from stepping on each other. iptables-restore 1.6.2 will have - // a --wait option like iptables itself, but that's not widely deployed. - if len(runner.restoreWaitFlag) == 0 { - locker, err := grabIptablesLocks(runner.lockfilePath14x, runner.lockfilePath16x) - if err != nil { - return err - } - trace.Step("Locks grabbed") - defer func(locker iptablesLocker) { - if err := locker.Close(); err != nil { - klog.Errorf("Failed to close iptables locks: %v", err) - } - }(locker) - } - - // run the command and return the output or an error including the output and error - fullArgs := append(runner.restoreWaitFlag, args...) - iptablesRestoreCmd := iptablesRestoreCommand(runner.protocol) - klog.V(4).Infof("running %s %v", iptablesRestoreCmd, fullArgs) - cmd := runner.exec.Command(iptablesRestoreCmd, fullArgs...) - cmd.SetStdin(bytes.NewBuffer(data)) - b, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("%v (%s)", err, b) - } - return nil -} - -func iptablesSaveCommand(protocol Protocol) string { - if protocol == ProtocolIPv6 { - return cmdIP6TablesSave - } - return cmdIPTablesSave -} - -func iptablesRestoreCommand(protocol Protocol) string { - if protocol == ProtocolIPv6 { - return cmdIP6TablesRestore - } - return cmdIPTablesRestore - -} - -func iptablesCommand(protocol Protocol) string { - if protocol == ProtocolIPv6 { - return cmdIP6Tables - } - return cmdIPTables -} - -func (runner *iptablerunner) run(op operation, args []string) ([]byte, error) { - return runner.runContext(context.TODO(), op, args) -} - -func (runner *iptablerunner) runContext(ctx context.Context, op operation, args []string) ([]byte, error) { - iptablesCmd := iptablesCommand(runner.protocol) - fullArgs := append(runner.waitFlag, string(op)) - fullArgs = append(fullArgs, args...) - klog.V(5).Infof("running iptables: %s %v", iptablesCmd, fullArgs) - if ctx == nil { - return runner.exec.Command(iptablesCmd, fullArgs...).CombinedOutput() - } - return runner.exec.CommandContext(ctx, iptablesCmd, fullArgs...).CombinedOutput() - // Don't log err here - callers might not think it is an error. -} - -// Returns (bool, nil) if it was able to check the existence of the rule, or -// (, error) if the process of checking failed. -func (runner *iptablerunner) checkRule(table Table, chain Chain, args ...string) (bool, error) { - if runner.hasCheck { - return runner.checkRuleUsingCheck(makeFullArgs(table, chain, args...)) - } - return runner.checkRuleWithoutCheck(table, chain, args...) -} - -var hexnumRE = regexp.MustCompile("0x0+([0-9])") - -func trimhex(s string) string { - return hexnumRE.ReplaceAllString(s, "0x$1") -} - -// Executes the rule check without using the "-C" flag, instead parsing iptables-save. -// Present for compatibility with <1.4.11 versions of iptables. This is full -// of hack and half-measures. We should nix this ASAP. -func (runner *iptablerunner) checkRuleWithoutCheck(table Table, chain Chain, args ...string) (bool, error) { - iptablesSaveCmd := iptablesSaveCommand(runner.protocol) - klog.V(1).Infof("running %s -t %s", iptablesSaveCmd, string(table)) - out, err := runner.exec.Command(iptablesSaveCmd, "-t", string(table)).CombinedOutput() - if err != nil { - return false, fmt.Errorf("error checking rule: %v", err) - } - - // Sadly, iptables has inconsistent quoting rules for comments. Just remove all quotes. - // Also, quoted multi-word comments (which are counted as a single arg) - // will be unpacked into multiple args, - // in order to compare against iptables-save output (which will be split at whitespace boundary) - // e.g. a single arg('"this must be before the NodePort rules"') will be unquoted and unpacked into 7 args. - var argsCopy []string - for i := range args { - tmpField := strings.Trim(args[i], "\"") - tmpField = trimhex(tmpField) - argsCopy = append(argsCopy, strings.Fields(tmpField)...) - } - argset := sets.NewString(argsCopy...) - - for _, line := range strings.Split(string(out), "\n") { - var fields = strings.Fields(line) - - // Check that this is a rule for the correct chain, and that it has - // the correct number of argument (+2 for "-A ") - if !strings.HasPrefix(line, fmt.Sprintf("-A %s", string(chain))) || len(fields) != len(argsCopy)+2 { - continue - } - - // Sadly, iptables has inconsistent quoting rules for comments. - // Just remove all quotes. - for i := range fields { - fields[i] = strings.Trim(fields[i], "\"") - fields[i] = trimhex(fields[i]) - } - - // TODO: This misses reorderings e.g. "-x foo ! -y bar" will match "! -x foo -y bar" - if sets.NewString(fields...).IsSuperset(argset) { - return true, nil - } - klog.V(5).Infof("DBG: fields is not a superset of args: fields=%v args=%v", fields, args) - } - - return false, nil -} - -// Executes the rule check using the "-C" flag -func (runner *iptablerunner) checkRuleUsingCheck(args []string) (bool, error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - out, err := runner.runContext(ctx, opCheckRule, args) - if ctx.Err() == context.DeadlineExceeded { - return false, fmt.Errorf("timed out while checking rules") - } - if err == nil { - return true, nil - } - if ee, ok := err.(exec.ExitError); ok { - // iptables uses exit(1) to indicate a failure of the operation, - // as compared to a malformed commandline, for example. - if ee.Exited() && ee.ExitStatus() == 1 { - return false, nil - } - } - return false, fmt.Errorf("error checking rule: %v: %s", err, out) -} - -const ( - // Max time we wait for an iptables flush to complete after we notice it has started - iptablesFlushTimeout = 5 * time.Second - // How often we poll while waiting for an iptables flush to complete - iptablesFlushPollTime = 100 * time.Millisecond -) - -// Monitor is part of Interface -func (runner *iptablerunner) Monitor(canary Chain, tables []Table, reloadFunc func(), interval time.Duration, stopCh <-chan struct{}) { - for { - _ = utilwait.PollImmediateUntil(interval, func() (bool, error) { - for _, table := range tables { - if _, err := runner.EnsureChain(table, canary); err != nil { - klog.Warningf("Could not set up iptables canary %s/%s: %v", string(table), string(canary), err) - return false, nil - } - } - return true, nil - }, stopCh) - - // Poll until stopCh is closed or iptables is flushed - err := utilwait.PollUntil(interval, func() (bool, error) { - if exists, err := runner.ChainExists(tables[0], canary); exists { - return false, nil - } else if isResourceError(err) { - klog.Warningf("Could not check for iptables canary %s/%s: %v", string(tables[0]), string(canary), err) - return false, nil - } - klog.V(2).Infof("iptables canary %s/%s deleted", string(tables[0]), string(canary)) - - // Wait for the other canaries to be deleted too before returning - // so we don't start reloading too soon. - err := utilwait.PollImmediate(iptablesFlushPollTime, iptablesFlushTimeout, func() (bool, error) { - for i := 1; i < len(tables); i++ { - if exists, err := runner.ChainExists(tables[i], canary); exists || isResourceError(err) { - return false, nil - } - } - return true, nil - }) - if err != nil { - klog.Warning("Inconsistent iptables state detected.") - } - return true, nil - }, stopCh) - - if err != nil { - // stopCh was closed - for _, table := range tables { - _ = runner.DeleteChain(table, canary) - } - return - } - - klog.V(2).Infof("Reloading after iptables flush") - reloadFunc() - } -} - -// ChainExists is part of Interface -func (runner *iptablerunner) ChainExists(table Table, chain Chain) (bool, error) { - fullArgs := makeFullArgs(table, chain) - - runner.mu.Lock() - defer runner.mu.Unlock() - - trace := utiltrace.New("iptables ChainExists") - defer trace.LogIfLong(2 * time.Second) - - _, err := runner.run(opListChain, fullArgs) - return err == nil, err -} - -type operation string - -const ( - opCreateChain operation = "-N" - opFlushChain operation = "-F" - opDeleteChain operation = "-X" - opListChain operation = "-S" - opCheckRule operation = "-C" - opDeleteRule operation = "-D" -) - -func makeFullArgs(table Table, chain Chain, args ...string) []string { - return append([]string{string(chain), "-t", string(table)}, args...) -} - -const iptablesVersionPattern = `v([0-9]+(\.[0-9]+)+)` - -// getIPTablesVersion runs "iptables --version" and parses the returned version -func getIPTablesVersion(exec exec.Interface, protocol Protocol) (*utilversion.Version, error) { - // this doesn't access mutable state so we don't need to use the interface / runner - iptablesCmd := iptablesCommand(protocol) - bytes, err := exec.Command(iptablesCmd, "--version").CombinedOutput() - if err != nil { - return nil, err - } - versionMatcher := regexp.MustCompile(iptablesVersionPattern) - match := versionMatcher.FindStringSubmatch(string(bytes)) - if match == nil { - return nil, fmt.Errorf("no iptables version found in string: %s", bytes) - } - version, err := utilversion.ParseGeneric(match[1]) - if err != nil { - return nil, fmt.Errorf("iptables version %q is not a valid version string: %v", match[1], err) - } - - return version, nil -} - -// Checks if iptables version has a "wait" flag -func getIPTablesWaitFlag(version *utilversion.Version) []string { - switch { - case version.AtLeast(WaitIntervalMinVersion): - return []string{WaitString, WaitSecondsValue, WaitIntervalString, WaitIntervalUsecondsValue} - case version.AtLeast(WaitSecondsMinVersion): - return []string{WaitString, WaitSecondsValue} - case version.AtLeast(WaitMinVersion): - return []string{WaitString} - default: - return nil - } -} - -// Checks if iptables-restore has a "wait" flag -func getIPTablesRestoreWaitFlag(version *utilversion.Version, exec exec.Interface, protocol Protocol) []string { - if version.AtLeast(WaitRestoreMinVersion) { - return []string{WaitString, WaitSecondsValue, WaitIntervalString, WaitIntervalUsecondsValue} - } - - // Older versions may have backported features; if iptables-restore supports - // --version, assume it also supports --wait - vstring, err := getIPTablesRestoreVersionString(exec, protocol) - if err != nil || vstring == "" { - klog.V(3).Infof("couldn't get iptables-restore version; assuming it doesn't support --wait") - return nil - } - if _, err := utilversion.ParseGeneric(vstring); err != nil { - klog.V(3).Infof("couldn't parse iptables-restore version; assuming it doesn't support --wait") - return nil - } - return []string{WaitString} -} - -// getIPTablesRestoreVersionString runs "iptables-restore --version" to get the version string -// in the form "X.X.X" -func getIPTablesRestoreVersionString(exec exec.Interface, protocol Protocol) (string, error) { - // this doesn't access mutable state so we don't need to use the interface / runner - - // iptables-restore hasn't always had --version, and worse complains - // about unrecognized commands but doesn't exit when it gets them. - // Work around that by setting stdin to nothing so it exits immediately. - iptablesRestoreCmd := iptablesRestoreCommand(protocol) - cmd := exec.Command(iptablesRestoreCmd, "--version") - cmd.SetStdin(bytes.NewReader([]byte{})) - bytes, err := cmd.CombinedOutput() - if err != nil { - return "", err - } - versionMatcher := regexp.MustCompile(iptablesVersionPattern) - match := versionMatcher.FindStringSubmatch(string(bytes)) - if match == nil { - return "", fmt.Errorf("no iptables version found in string: %s", bytes) - } - return match[1], nil -} - -func (runner *iptablerunner) HasRandomFully() bool { - return runner.hasRandomFully -} - -// Present tests if iptable is supported on current kernel by checking the existence -// of default table and chain -func (runner *iptablerunner) Present() bool { - if _, err := runner.ChainExists(TableNAT, ChainPostrouting); err != nil { - return false - } - - return true -} - -var iptablesNotFoundStrings = []string{ - // iptables-legacy [-A|-I] BAD-CHAIN [...] - // iptables-legacy [-C|-D] GOOD-CHAIN [...non-matching rule...] - // iptables-legacy [-X|-F|-Z] BAD-CHAIN - // iptables-nft -X BAD-CHAIN - // NB: iptables-nft [-F|-Z] BAD-CHAIN exits with no error - "No chain/target/match by that name", - - // iptables-legacy [...] -j BAD-CHAIN - // iptables-nft-1.8.0 [-A|-I] BAD-CHAIN [...] - // iptables-nft-1.8.0 [-A|-I] GOOD-CHAIN -j BAD-CHAIN - // NB: also matches some other things like "-m BAD-MODULE" - "No such file or directory", - - // iptables-legacy [-C|-D] BAD-CHAIN [...] - // iptables-nft [-C|-D] GOOD-CHAIN [...non-matching rule...] - "does a matching rule exist", - - // iptables-nft-1.8.2 [-A|-C|-D|-I] BAD-CHAIN [...] - // iptables-nft-1.8.2 [...] -j BAD-CHAIN - "does not exist", -} - -// IsNotFoundError returns true if the error indicates "not found". It parses -// the error string looking for known values, which is imperfect; beware using -// this function for anything beyond deciding between logging or ignoring an -// error. -//func IsNotFoundError(err error) bool { -// es := err.Error() -// for _, str := range iptablesNotFoundStrings { -// if strings.Contains(es, str) { -// return true -// } -// } -// return false -//} - -const iptablesStatusResourceProblem = 4 - -// isResourceError returns true if the error indicates that iptables ran into a "resource -// problem" and was unable to attempt the request. In particular, this will be true if it -// times out trying to get the iptables lock. -func isResourceError(err error) bool { - if ee, isExitError := err.(exec.ExitError); isExitError { - return ee.ExitStatus() == iptablesStatusResourceProblem - } - return false -} diff --git a/backends/ipvs-as-sink/util/iptables_linux.go b/backends/ipvs-as-sink/util/iptables_linux.go deleted file mode 100644 index bed4a35b3..000000000 --- a/backends/ipvs-as-sink/util/iptables_linux.go +++ /dev/null @@ -1,101 +0,0 @@ -//go:build linux -// +build linux - -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "fmt" - "net" - "os" - "time" - - "golang.org/x/sys/unix" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/wait" -) - -type locker struct { - lock16 *os.File - lock14 *net.UnixListener -} - -func (l *locker) Close() error { - errList := []error{} - if l.lock16 != nil { - if err := l.lock16.Close(); err != nil { - errList = append(errList, err) - } - } - if l.lock14 != nil { - if err := l.lock14.Close(); err != nil { - errList = append(errList, err) - } - } - return utilerrors.NewAggregate(errList) -} - -func grabIptablesLocks(lockfilePath14x, lockfilePath16x string) (iptablesLocker, error) { - var err error - var success bool - - l := &locker{} - defer func(l *locker) { - // Clean up immediately on failure - if !success { - l.Close() - } - }(l) - - // Grab both 1.6.x and 1.4.x-style locks; we don't know what the - // iptables-restore version is if it doesn't support --wait, so we - // can't assume which lock method it'll use. - - // Roughly duplicate iptables 1.6.x xtables_lock() function. - l.lock16, err = os.OpenFile(lockfilePath16x, os.O_CREATE, 0600) - if err != nil { - return nil, fmt.Errorf("failed to open iptables lock %s: %v", lockfilePath16x, err) - } - - if err := wait.PollImmediate(200*time.Millisecond, 2*time.Second, func() (bool, error) { - if err := grabIptablesFileLock(l.lock16); err != nil { - return false, nil - } - return true, nil - }); err != nil { - return nil, fmt.Errorf("failed to acquire new iptables lock: %v", err) - } - - // Roughly duplicate iptables 1.4.x xtables_lock() function. - if err := wait.PollImmediate(200*time.Millisecond, 2*time.Second, func() (bool, error) { - l.lock14, err = net.ListenUnix("unix", &net.UnixAddr{Name: lockfilePath14x, Net: "unix"}) - if err != nil { - return false, nil - } - return true, nil - }); err != nil { - return nil, fmt.Errorf("failed to acquire old iptables lock: %v", err) - } - - success = true - return l, nil -} - -func grabIptablesFileLock(f *os.File) error { - return unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB) -} diff --git a/backends/ipvs-as-sink/util/save_restore.go b/backends/ipvs-as-sink/util/save_restore.go deleted file mode 100644 index 33abc1258..000000000 --- a/backends/ipvs-as-sink/util/save_restore.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "bytes" - "fmt" -) - -var ( - commitBytes = []byte("COMMIT") - spaceBytes = []byte(" ") -) - -// MakeChainLine return an iptables-save/restore formatted chain line given a Chain -func MakeChainLine(chain Chain) string { - return fmt.Sprintf(":%s - [0:0]", chain) -} - -// GetChainLines parses a table's iptables-save data to find chains in the table. -// It returns a map of iptables.Chain to []byte where the []byte is the chain line -// from save (with counters etc.). -// Note that to avoid allocations memory is SHARED with save. -func GetChainLines(table Table, save []byte) map[Chain][]byte { - chainsMap := make(map[Chain][]byte) - tablePrefix := []byte("*" + string(table)) - readIndex := 0 - // find beginning of table - for readIndex < len(save) { - line, n := readLine(readIndex, save) - readIndex = n - if bytes.HasPrefix(line, tablePrefix) { - break - } - } - // parse table lines - for readIndex < len(save) { - line, n := readLine(readIndex, save) - readIndex = n - if len(line) == 0 { - continue - } - if bytes.HasPrefix(line, commitBytes) || line[0] == '*' { - break - } else if line[0] == '#' { - continue - } else if line[0] == ':' && len(line) > 1 { - // We assume that the contains space - chain lines have 3 fields, - // space delimited. If there is no space, this line will panic. - spaceIndex := bytes.Index(line, spaceBytes) - if spaceIndex == -1 { - panic(fmt.Sprintf("Unexpected chain line in iptables-save output: %v", string(line))) - } - chain := Chain(line[1:spaceIndex]) - chainsMap[chain] = line - } - } - return chainsMap -} - -func readLine(readIndex int, byteArray []byte) ([]byte, int) { - currentReadIndex := readIndex - - // consume left spaces - for currentReadIndex < len(byteArray) { - if byteArray[currentReadIndex] == ' ' { - currentReadIndex++ - } else { - break - } - } - - // leftTrimIndex stores the left index of the line after the line is left-trimmed - leftTrimIndex := currentReadIndex - - // rightTrimIndex stores the right index of the line after the line is right-trimmed - // it is set to -1 since the correct value has not yet been determined. - rightTrimIndex := -1 - - for ; currentReadIndex < len(byteArray); currentReadIndex++ { - if byteArray[currentReadIndex] == ' ' { - // set rightTrimIndex - if rightTrimIndex == -1 { - rightTrimIndex = currentReadIndex - } - } else if (byteArray[currentReadIndex] == '\n') || (currentReadIndex == (len(byteArray) - 1)) { - // end of line or byte buffer is reached - if currentReadIndex <= leftTrimIndex { - return nil, currentReadIndex + 1 - } - // set the rightTrimIndex - if rightTrimIndex == -1 { - rightTrimIndex = currentReadIndex - if currentReadIndex == (len(byteArray)-1) && (byteArray[currentReadIndex] != '\n') { - // ensure that the last character is part of the returned string, - // unless the last character is '\n' - rightTrimIndex = currentReadIndex + 1 - } - } - // Avoid unnecessary allocation. - return byteArray[leftTrimIndex:rightTrimIndex], currentReadIndex + 1 - } else { - // unset rightTrimIndex - rightTrimIndex = -1 - } - } - return nil, currentReadIndex -} diff --git a/backends/ipvs-as-sink/util/types.go b/backends/ipvs-as-sink/util/types.go deleted file mode 100644 index b97d7ff70..000000000 --- a/backends/ipvs-as-sink/util/types.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -// Type represents the ipset type -type Type string - -const ( - // HashIPPort represents the `hash:ip,port` type ipset. The hash:ip,port is similar to hash:ip but - // you can store IP address and protocol-port pairs in it. TCP, SCTP, UDP, UDPLITE, ICMP and ICMPv6 are supported - // with port numbers/ICMP(v6) types and other protocol numbers without port information. - HashIPPort Type = "hash:ip,port" - // HashIPPortIP represents the `hash:ip,port,ip` type ipset. The hash:ip,port,ip set type uses a hash to store - // IP address, port number and a second IP address triples. The port number is interpreted together with a - // protocol (default TCP) and zero protocol number cannot be used. - HashIPPortIP Type = "hash:ip,port,ip" - // HashIPPortNet represents the `hash:ip,port,net` type ipset. The hash:ip,port,net set type uses a hash to store IP address, port number and IP network address triples. The port - // number is interpreted together with a protocol (default TCP) and zero protocol number cannot be used. Network address - // with zero prefix size cannot be stored either. - HashIPPortNet Type = "hash:ip,port,net" - // BitmapPort represents the `bitmap:port` type ipset. The bitmap:port set type uses a memory range, where each bit - // represents one TCP/UDP port. A bitmap:port type of set can store up to 65535 ports. - BitmapPort Type = "bitmap:port" -) - -// DefaultPortRange defines the default bitmap:port valid port range. -const DefaultPortRange string = "0-65535" - -const ( - // ProtocolFamilyIPV4 represents IPv4 protocol. - ProtocolFamilyIPV4 = "inet" - // ProtocolFamilyIPV6 represents IPv6 protocol. - ProtocolFamilyIPV6 = "inet6" - // ProtocolTCP represents TCP protocol. - ProtocolTCP = "tcp" - // ProtocolUDP represents UDP protocol. - ProtocolUDP = "udp" - // ProtocolSCTP represents SCTP protocol. - ProtocolSCTP = "sctp" -) - -// ValidIPSetTypes defines the supported ip set type. -var ValidIPSetTypes = []Type{ - HashIPPort, - HashIPPortIP, - BitmapPort, - HashIPPortNet, -} diff --git a/backends/ipvs-as-sink/util/utils.go b/backends/ipvs-as-sink/util/utils.go deleted file mode 100644 index e7099ef38..000000000 --- a/backends/ipvs-as-sink/util/utils.go +++ /dev/null @@ -1,194 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "path" - "regexp" - "strconv" - "strings" - - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/klog/v2" - "sigs.k8s.io/kpng/backends/ipvs-as-sink/exec" -) - -const ( - sysctlBase = "/proc/sys" -) - -// Interface is an injectable interface for running sysctl commands. -type sysInterface interface { - // GetSysctl returns the value for the specified sysctl setting - GetSysctl(sysctl string) (int, error) - // SetSysctl modifies the specified sysctl flag to the new value - SetSysctl(sysctl string, newVal int) error -} - -// New returns a new Interface for accessing sysctl -func NewSysInterface() sysInterface { - return &procSysctl{} -} - -// procSysctl implements Interface by reading and writing files under /proc/sys -type procSysctl struct { -} - -// GetSysctl returns the value for the specified sysctl setting -func (*procSysctl) GetSysctl(sysctl string) (int, error) { - data, err := ioutil.ReadFile(path.Join(sysctlBase, sysctl)) - if err != nil { - return -1, err - } - val, err := strconv.Atoi(strings.Trim(string(data), " \n")) - if err != nil { - return -1, err - } - return val, nil -} - -// SetSysctl modifies the specified sysctl flag to the new value -func (*procSysctl) SetSysctl(sysctl string, newVal int) error { - return ioutil.WriteFile(path.Join(sysctlBase, sysctl), []byte(strconv.Itoa(newVal)), 0640) -} - -// EnsureSysctl sets a kernel sysctl to a given numeric value. -func EnsureSysctl(sysctl sysInterface, name string, newVal int) error { - if oldVal, _ := sysctl.GetSysctl(name); oldVal != newVal { - if err := sysctl.SetSysctl(name, newVal); err != nil { - return fmt.Errorf("can't set sysctl %s to %d: %v", name, newVal, err) - } - klog.V(1).Info("Changed sysctl", "name", name, "before", oldVal, "after", newVal) - } - return nil -} - -// KernelHandler can handle the current installed kernel modules. -type KernelHandler interface { - GetModules() ([]string, error) - GetKernelVersion() (string, error) -} - -// LinuxKernelHandler implements KernelHandler interface. -type LinuxKernelHandler struct { - executor exec.Interface -} - -// NewLinuxKernelHandler initializes LinuxKernelHandler with exec. -func NewLinuxKernelHandler() *LinuxKernelHandler { - return &LinuxKernelHandler{ - executor: exec.New(), - } -} - -// GetModules returns all installed kernel modules. -func (handle *LinuxKernelHandler) GetModules() ([]string, error) { - // Check whether IPVS required kernel modules are built-in - kernelVersionStr, err := handle.GetKernelVersion() - if err != nil { - return nil, err - } - kernelVersion, err := version.ParseGeneric(kernelVersionStr) - if err != nil { - return nil, fmt.Errorf("error parsing kernel version %q: %v", kernelVersionStr, err) - } - ipvsModules := GetRequiredIPVSModules(kernelVersion) - - var bmods, lmods []string - - // Find out loaded kernel modules. If this is a full static kernel it will try to verify if the module is compiled using /boot/config-KERNELVERSION - modulesFile, err := os.Open("/proc/modules") - if err == os.ErrNotExist { - klog.Error(err, "Failed to read file /proc/modules, assuming this is a kernel without loadable modules support enabled") - kernelConfigFile := fmt.Sprintf("/boot/config-%s", kernelVersionStr) - kConfig, err := ioutil.ReadFile(kernelConfigFile) - if err != nil { - return nil, fmt.Errorf("failed to read Kernel Config file %s with error %w", kernelConfigFile, err) - } - for _, module := range ipvsModules { - if match, _ := regexp.Match("CONFIG_"+strings.ToUpper(module)+"=y", kConfig); match { - bmods = append(bmods, module) - } - } - return bmods, nil - } - if err != nil { - return nil, fmt.Errorf("failed to read file /proc/modules with error %w", err) - } - defer modulesFile.Close() - - mods, err := getFirstColumn(modulesFile) - if err != nil { - return nil, fmt.Errorf("failed to find loaded kernel modules: %v", err) - } - - builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersionStr) - b, err := ioutil.ReadFile(builtinModsFilePath) - if err != nil { - klog.Error(err, "Failed to read builtin modules file, you can ignore this message when kube-proxy is running inside container without mounting /lib/modules", "filePath", builtinModsFilePath) - } - - for _, module := range ipvsModules { - if match, _ := regexp.Match(module+".ko", b); match { - bmods = append(bmods, module) - } else { - // Try to load the required IPVS kernel modules if not built in - err := handle.executor.Command("modprobe", "--", module).Run() - if err != nil { - klog.Info("Failed to load kernel module with modprobe, "+ - "you can ignore this message when kube-proxy is running inside container without mounting /lib/modules", "moduleName", module) - } else { - lmods = append(lmods, module) - } - } - } - - mods = append(mods, bmods...) - mods = append(mods, lmods...) - return mods, nil -} - -// GetKernelVersion returns currently running kernel version. -func (handle *LinuxKernelHandler) GetKernelVersion() (string, error) { - kernelVersionFile := "/proc/sys/kernel/osrelease" - fileContent, err := ioutil.ReadFile(kernelVersionFile) - if err != nil { - return "", fmt.Errorf("error reading osrelease file %q: %v", kernelVersionFile, err) - } - - return strings.TrimSpace(string(fileContent)), nil -} - -// getFirstColumn reads all the content from r into memory and return a -// slice which consists of the first word from each line. -func getFirstColumn(r io.Reader) ([]string, error) { - b, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - - lines := strings.Split(string(b), "\n") - words := make([]string, 0, len(lines)) - for i := range lines { - fields := strings.Fields(lines[i]) - if len(fields) > 0 { - words = append(words, fields[0]) - } - } - return words, nil -} diff --git a/backends/ipvsfullstate/README.md b/backends/ipvs/README.md similarity index 97% rename from backends/ipvsfullstate/README.md rename to backends/ipvs/README.md index 32874314b..4b6be45b9 100644 --- a/backends/ipvsfullstate/README.md +++ b/backends/ipvs/README.md @@ -1,4 +1,4 @@ -# IPVS Fullstate Implementation +# IPVS Implementation This implementation simply requests the creation of all the network objects required to program Kubernetes services data path for every callback, leverages diffstore to maintain the states of all the network components, and operates only on the changes returned by the diffstore. @@ -12,6 +12,8 @@ On processing full state controller interacts with **Managers** which acts as a The logic for programing a NodePort service - **addServiceEndpointsForLoadBalancer** The logic for programing a LoadBalancer service + + ## 2. Managers Manager leverages diffstore for storing all the resource manipulation operations (create virtual server, add destination, add entry to ipset) required to render the full state and only acts on the changes in the store. - **ipvs** diff --git a/backends/ipvsfullstate/common.go b/backends/ipvs/common.go similarity index 98% rename from backends/ipvsfullstate/common.go rename to backends/ipvs/common.go index b90c99c06..83cf3c67c 100644 --- a/backends/ipvsfullstate/common.go +++ b/backends/ipvs/common.go @@ -1,10 +1,10 @@ -package ipvsfullsate +package ipvs import ( v1 "k8s.io/api/core/v1" netutils "k8s.io/utils/net" "sigs.k8s.io/kpng/api/localv1" - "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipvs" + "sigs.k8s.io/kpng/backends/ipvs/internal/ipvs" ) // getClusterIPs returns ClusterIPs for given IPFamily associated with the service. diff --git a/backends/ipvsfullstate/controller.go b/backends/ipvs/controller.go similarity index 93% rename from backends/ipvsfullstate/controller.go rename to backends/ipvs/controller.go index be786472b..8469b3b28 100644 --- a/backends/ipvsfullstate/controller.go +++ b/backends/ipvs/controller.go @@ -1,4 +1,4 @@ -package ipvsfullsate +package ipvs import ( "fmt" @@ -7,9 +7,9 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/server/mux" - "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipsets" - "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/iptables" - "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipvs" + "sigs.k8s.io/kpng/backends/ipvs/internal/ipsets" + "sigs.k8s.io/kpng/backends/ipvs/internal/iptables" + "sigs.k8s.io/kpng/backends/ipvs/internal/ipvs" "sigs.k8s.io/kpng/client" "k8s.io/klog/v2" diff --git a/backends/ipvsfullstate/flags.go b/backends/ipvs/flags.go similarity index 98% rename from backends/ipvsfullstate/flags.go rename to backends/ipvs/flags.go index 099381d80..b6da217eb 100644 --- a/backends/ipvsfullstate/flags.go +++ b/backends/ipvs/flags.go @@ -1,4 +1,4 @@ -package ipvsfullsate +package ipvs import ( "net" diff --git a/backends/ipvsfullstate/go.mod b/backends/ipvs/go.mod similarity index 75% rename from backends/ipvsfullstate/go.mod rename to backends/ipvs/go.mod index 1e0b91c1f..2fdbcc175 100644 --- a/backends/ipvsfullstate/go.mod +++ b/backends/ipvs/go.mod @@ -1,4 +1,4 @@ -module sigs.k8s.io/kpng/backends/ipvsfullstate +module sigs.k8s.io/kpng/backends/ipvs go 1.19 @@ -13,11 +13,12 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.5.9 // indirect + github.com/google/seesaw v0.0.0-20230304143336-8264e023fc51 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/spf13/cobra v1.4.0 // indirect - golang.org/x/net v0.0.0-20221004154528-8021a29435af // indirect - golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e // indirect google.golang.org/grpc v1.50.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/backends/ipvsfullstate/go.sum b/backends/ipvs/go.sum similarity index 89% rename from backends/ipvsfullstate/go.sum rename to backends/ipvs/go.sum index ad0e85512..a08767b2f 100644 --- a/backends/ipvsfullstate/go.sum +++ b/backends/ipvs/go.sum @@ -11,6 +11,8 @@ github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl76 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/seesaw v0.0.0-20230304143336-8264e023fc51 h1:Z3zKo8X3SG/dXSCyKVsY+afoYZfiNJLTXUJR4+eglU0= +github.com/google/seesaw v0.0.0-20230304143336-8264e023fc51/go.mod h1:URFhSAvwJ7nTm9NnWUaSkp8W/fF68Y9QgDMv449T64s= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -20,10 +22,13 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= golang.org/x/net v0.0.0-20221004154528-8021a29435af h1:wv66FM3rLZGPdxpYL+ApnDe2HzHcTFta3z5nsc13wI4= golang.org/x/net v0.0.0-20221004154528-8021a29435af/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e h1:halCgTFuLWDRD61piiNSxPsARANGD3Xl16hPrLgLiIg= google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= diff --git a/backends/ipvsfullstate/handle_clusterip_service.go b/backends/ipvs/handle_clusterip_service.go similarity index 97% rename from backends/ipvsfullstate/handle_clusterip_service.go rename to backends/ipvs/handle_clusterip_service.go index cf049455e..7dc7e661e 100644 --- a/backends/ipvsfullstate/handle_clusterip_service.go +++ b/backends/ipvs/handle_clusterip_service.go @@ -1,8 +1,8 @@ -package ipvsfullsate +package ipvs import ( v1 "k8s.io/api/core/v1" - "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipsets" + "sigs.k8s.io/kpng/backends/ipvs/internal/ipsets" "sigs.k8s.io/kpng/client" ) diff --git a/backends/ipvsfullstate/handle_loadbalancer_service.go b/backends/ipvs/handle_loadbalancer_service.go similarity index 95% rename from backends/ipvsfullstate/handle_loadbalancer_service.go rename to backends/ipvs/handle_loadbalancer_service.go index 8455b1a9c..b24a36415 100644 --- a/backends/ipvsfullstate/handle_loadbalancer_service.go +++ b/backends/ipvs/handle_loadbalancer_service.go @@ -1,8 +1,8 @@ -package ipvsfullsate +package ipvs import ( v1 "k8s.io/api/core/v1" - ipsets2 "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipsets" + "sigs.k8s.io/kpng/backends/ipvs/internal/ipsets" "sigs.k8s.io/kpng/client" ) @@ -14,8 +14,8 @@ func (c *Controller) addServiceEndpointsForLoadBalancer(serviceEndpoints *client // will further create ClusterIP components. c.addServiceEndpointsForNodePort(serviceEndpoints) - var entry *ipsets2.Entry - var set *ipsets2.Set + var entry *ipsets.Entry + var set *ipsets.Set service := serviceEndpoints.Service endpoints := serviceEndpoints.Endpoints diff --git a/backends/ipvsfullstate/handle_nodeport_service.go b/backends/ipvs/handle_nodeport_service.go similarity index 97% rename from backends/ipvsfullstate/handle_nodeport_service.go rename to backends/ipvs/handle_nodeport_service.go index 2c4997b80..1ef5dd1be 100644 --- a/backends/ipvsfullstate/handle_nodeport_service.go +++ b/backends/ipvs/handle_nodeport_service.go @@ -1,9 +1,9 @@ -package ipvsfullsate +package ipvs import ( v1 "k8s.io/api/core/v1" "sigs.k8s.io/kpng/api/localv1" - "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipsets" + "sigs.k8s.io/kpng/backends/ipvs/internal/ipsets" "sigs.k8s.io/kpng/client" ) diff --git a/backends/ipvsfullstate/internal/ipsets/entry.go b/backends/ipvs/internal/ipsets/entry.go similarity index 100% rename from backends/ipvsfullstate/internal/ipsets/entry.go rename to backends/ipvs/internal/ipsets/entry.go diff --git a/backends/ipvsfullstate/internal/ipsets/ipset.go b/backends/ipvs/internal/ipsets/ipset.go similarity index 100% rename from backends/ipvsfullstate/internal/ipsets/ipset.go rename to backends/ipvs/internal/ipsets/ipset.go diff --git a/backends/ipvsfullstate/internal/ipsets/ipset_util.go b/backends/ipvs/internal/ipsets/ipset_util.go similarity index 100% rename from backends/ipvsfullstate/internal/ipsets/ipset_util.go rename to backends/ipvs/internal/ipsets/ipset_util.go diff --git a/backends/ipvsfullstate/internal/ipsets/manager.go b/backends/ipvs/internal/ipsets/manager.go similarity index 100% rename from backends/ipvsfullstate/internal/ipsets/manager.go rename to backends/ipvs/internal/ipsets/manager.go diff --git a/backends/ipvsfullstate/internal/ipsets/runner.go b/backends/ipvs/internal/ipsets/runner.go similarity index 100% rename from backends/ipvsfullstate/internal/ipsets/runner.go rename to backends/ipvs/internal/ipsets/runner.go diff --git a/backends/ipvsfullstate/internal/ipsets/setup.go b/backends/ipvs/internal/ipsets/setup.go similarity index 100% rename from backends/ipvsfullstate/internal/ipsets/setup.go rename to backends/ipvs/internal/ipsets/setup.go diff --git a/backends/ipvsfullstate/internal/ipsets/types.go b/backends/ipvs/internal/ipsets/types.go similarity index 100% rename from backends/ipvsfullstate/internal/ipsets/types.go rename to backends/ipvs/internal/ipsets/types.go diff --git a/backends/ipvsfullstate/internal/iptables/manager.go b/backends/ipvs/internal/iptables/manager.go similarity index 100% rename from backends/ipvsfullstate/internal/iptables/manager.go rename to backends/ipvs/internal/iptables/manager.go diff --git a/backends/ipvsfullstate/internal/iptables/templates.go b/backends/ipvs/internal/iptables/templates.go similarity index 100% rename from backends/ipvsfullstate/internal/iptables/templates.go rename to backends/ipvs/internal/iptables/templates.go diff --git a/backends/ipvsfullstate/internal/iptables/types.go b/backends/ipvs/internal/iptables/types.go similarity index 100% rename from backends/ipvsfullstate/internal/iptables/types.go rename to backends/ipvs/internal/iptables/types.go diff --git a/backends/ipvsfullstate/internal/ipvs/destination.go b/backends/ipvs/internal/ipvs/destination.go similarity index 100% rename from backends/ipvsfullstate/internal/ipvs/destination.go rename to backends/ipvs/internal/ipvs/destination.go diff --git a/backends/ipvsfullstate/internal/ipvs/manager.go b/backends/ipvs/internal/ipvs/manager.go similarity index 100% rename from backends/ipvsfullstate/internal/ipvs/manager.go rename to backends/ipvs/internal/ipvs/manager.go diff --git a/backends/ipvsfullstate/internal/ipvs/setup.go b/backends/ipvs/internal/ipvs/setup.go similarity index 100% rename from backends/ipvsfullstate/internal/ipvs/setup.go rename to backends/ipvs/internal/ipvs/setup.go diff --git a/backends/ipvsfullstate/internal/ipvs/types.go b/backends/ipvs/internal/ipvs/types.go similarity index 100% rename from backends/ipvsfullstate/internal/ipvs/types.go rename to backends/ipvs/internal/ipvs/types.go diff --git a/backends/ipvsfullstate/internal/ipvs/util.go b/backends/ipvs/internal/ipvs/util.go similarity index 99% rename from backends/ipvsfullstate/internal/ipvs/util.go rename to backends/ipvs/internal/ipvs/util.go index 9b9322c3e..64ab03df9 100644 --- a/backends/ipvsfullstate/internal/ipvs/util.go +++ b/backends/ipvs/internal/ipvs/util.go @@ -25,7 +25,7 @@ import ( "k8s.io/apimachinery/pkg/util/version" "k8s.io/klog/v2" - "sigs.k8s.io/kpng/backends/ipvs-as-sink/exec" + "k8s.io/utils/exec" ) const ( diff --git a/backends/ipvsfullstate/internal/ipvs/virtual_server.go b/backends/ipvs/internal/ipvs/virtual_server.go similarity index 100% rename from backends/ipvsfullstate/internal/ipvs/virtual_server.go rename to backends/ipvs/internal/ipvs/virtual_server.go diff --git a/backends/ipvsfullstate/ipset_definitions.go b/backends/ipvs/ipset_definitions.go similarity index 98% rename from backends/ipvsfullstate/ipset_definitions.go rename to backends/ipvs/ipset_definitions.go index b982fac6b..862496d94 100644 --- a/backends/ipvsfullstate/ipset_definitions.go +++ b/backends/ipvs/ipset_definitions.go @@ -14,11 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ipvsfullsate +package ipvs import ( v1 "k8s.io/api/core/v1" - "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipsets" + "sigs.k8s.io/kpng/backends/ipvs/internal/ipsets" ) var ( diff --git a/backends/ipvsfullstate/ipset_entry_builder.go b/backends/ipvs/ipset_entry_builder.go similarity index 97% rename from backends/ipvsfullstate/ipset_entry_builder.go rename to backends/ipvs/ipset_entry_builder.go index 7e6c3fe4a..a88f6a6ae 100644 --- a/backends/ipvsfullstate/ipset_entry_builder.go +++ b/backends/ipvs/ipset_entry_builder.go @@ -1,8 +1,8 @@ -package ipvsfullsate +package ipvs import ( "sigs.k8s.io/kpng/api/localv1" - "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipsets" + "sigs.k8s.io/kpng/backends/ipvs/internal/ipsets" ) // newEntryForClusterIP creates entry for ClusterIP for kubeClusterIPSet. diff --git a/backends/ipvsfullstate/iptable_rules.go b/backends/ipvs/iptable_rules.go similarity index 99% rename from backends/ipvsfullstate/iptable_rules.go rename to backends/ipvs/iptable_rules.go index 77c90f66d..b18adb051 100644 --- a/backends/ipvsfullstate/iptable_rules.go +++ b/backends/ipvs/iptable_rules.go @@ -1,9 +1,9 @@ -package ipvsfullsate +package ipvs import ( "fmt" v1 "k8s.io/api/core/v1" - ipt "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/iptables" + ipt "sigs.k8s.io/kpng/backends/ipvs/internal/iptables" ) // masqueradeMark should match the mark of kubelet diff --git a/backends/ipvsfullstate/register.go b/backends/ipvs/register.go similarity index 95% rename from backends/ipvsfullstate/register.go rename to backends/ipvs/register.go index 6a867298b..35563d85e 100644 --- a/backends/ipvsfullstate/register.go +++ b/backends/ipvs/register.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ipvsfullsate +package ipvs import ( "github.com/spf13/pflag" @@ -41,7 +41,7 @@ func init() { } klog.V(3).Infof("Registering host %s", hostname) - backendcmd.Register("to-ipvsfullstate", func() backendcmd.Cmd { + backendcmd.Register("to-ipvs", func() backendcmd.Cmd { return &backend{ cfg: localsink.Config{NodeName: hostname}, } diff --git a/backends/ipvsfullstate/setup.go b/backends/ipvs/setup.go similarity index 96% rename from backends/ipvsfullstate/setup.go rename to backends/ipvs/setup.go index ead3a365f..311ad7bcb 100644 --- a/backends/ipvsfullstate/setup.go +++ b/backends/ipvs/setup.go @@ -14,13 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ipvsfullsate +package ipvs import ( v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" - "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/ipsets" - "sigs.k8s.io/kpng/backends/ipvsfullstate/internal/iptables" + "sigs.k8s.io/kpng/backends/ipvs/internal/ipsets" + "sigs.k8s.io/kpng/backends/ipvs/internal/iptables" ) // Setup is used for setting up the backend, initialize ipvs, ipsets and iptales. diff --git a/backends/ipvsfullstate/types.go b/backends/ipvs/types.go similarity index 98% rename from backends/ipvsfullstate/types.go rename to backends/ipvs/types.go index 8c6fd8746..2b93ba589 100644 --- a/backends/ipvsfullstate/types.go +++ b/backends/ipvs/types.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ipvsfullsate +package ipvs import ( "sigs.k8s.io/kpng/api/localv1" diff --git a/cmd/kpng/storecmds/storecmds_linux.go b/cmd/kpng/storecmds/storecmds_linux.go index 76bcaa3b2..2cadc2fdc 100644 --- a/cmd/kpng/storecmds/storecmds_linux.go +++ b/cmd/kpng/storecmds/storecmds_linux.go @@ -21,8 +21,7 @@ package storecmds import ( _ "sigs.k8s.io/kpng/backends/ebpf" _ "sigs.k8s.io/kpng/backends/iptables" - _ "sigs.k8s.io/kpng/backends/ipvs-as-sink" - _ "sigs.k8s.io/kpng/backends/ipvsfullstate" + _ "sigs.k8s.io/kpng/backends/ipvs" _ "sigs.k8s.io/kpng/backends/nft" _ "sigs.k8s.io/kpng/backends/userspacelin" ) diff --git a/go.work b/go.work index 29182634d..cb17fd7d4 100644 --- a/go.work +++ b/go.work @@ -4,8 +4,7 @@ use ( ./api ./backends/ebpf ./backends/iptables - ./backends/ipvs-as-sink - ./backends/ipvsfullstate + ./backends/ipvs ./backends/nft ./backends/userspacelin ./backends/windows/kernelspace diff --git a/go.work.sum b/go.work.sum index ad346f63a..7baeb86b1 100644 --- a/go.work.sum +++ b/go.work.sum @@ -12,80 +12,136 @@ cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWc cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0 h1:3DXvAyifywvq64LfkKaMOmkWPS1CikIQdMe2lY9vxU8= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go/aiplatform v1.24.0 h1:QqHZT1IMldf/daXoSnkJWBIqGBsw50X+xP6HSVzLRPo= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/analytics v0.12.0 h1:NKw6PpQi6V1O+KsjuTd+bhip9d0REYu4NevC45vtGp8= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/area120 v0.6.0 h1:TCMhwWEWhCn8d44/Zs7UCICTWje9j3HuV6nVGMjdpYw= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/artifactregistry v1.7.0 h1:9yKYCozdh29v7QMx3QBuksZGtPNICFb5SVnyNvkKRGg= cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/asset v1.7.0 h1:1zumX+e2xh1qTuBJE0ExxU45s7SmRaZQhp9EJEdQou8= cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/assuredworkloads v1.6.0 h1:2RVswX57rLj/3SneMwVk6inceggf+6wl4THRru/Joi8= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/automl v1.6.0 h1:U+kHmeKGXgBvTlrecPJhwkItWaIpIscG5DUpQxBQZZg= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= +cloud.google.com/go/bigquery v1.42.0 h1:JuTk8po4bCKRwObdT0zLb1K0BGkGHJdtgs2GK3j2Gws= cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/billing v1.5.0 h1:4RESn+mA7eGPBr5eQ4B/hbkHNivzYHbgRWpdlNeNjiE= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/binaryauthorization v1.2.0 h1:5F7dowxGuYQlX3LjfjH/sKf+IvI1TsItTw0sDZmoec4= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/cloudtasks v1.6.0 h1:IL5W4fh6dAq9x1mO+4evrWCISOmPJegdaO0hZRZmWNE= cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/containeranalysis v0.6.0 h1:2824iym832ljKdVpCBnpqm5K94YT/uHTVhNF+dRTXPI= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/datacatalog v1.6.0 h1:xzXGAE2fAuMh+ksODKr9nRv9ega1vHjFwRqMA8tRrVE= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/dataflow v0.7.0 h1:CW3541Fm7KPTyZjJdnX6NtaGXYFn5XbFC5UcjgALKvU= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataform v0.4.0 h1:fnwkyzCVcPI/TmBheGgpmK2h+hWUIDHcZBincHRhrQ0= cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/datalabeling v0.6.0 h1:dp8jOF21n/7jwgo/uuA0RN8hvLcKO4q6s/yvwevs2ZM= cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/dataqna v0.6.0 h1:gx9jr41ytcA3dXkbbd409euEaWtofCVXYBvJz3iYm18= cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= +cloud.google.com/go/datastream v1.3.0 h1:ula4YR2K66o5wifLdPQMtR2I6KP+zvqdSEb6ncd1e0g= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/dialogflow v1.16.1 h1:4ccwvAOnFdgliUJyQSeyG9Yy3HW4jF24xDIX6ftbqvY= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/documentai v1.8.0 h1:CipwaecNhtsWUSneV2J5y8OqudHqvqPlcMHgSyh8vak= cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/domains v0.7.0 h1:pu3JIgC1rswIqi5romW0JgNO6CTUydLYX8zyjiAvO1c= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/edgecontainer v0.1.0 h1:PS3AxjRqMu2XS01YLG/teWFQlZrP7+a+LJBj37zBMAQ= cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/firestore v1.1.0 h1:9x7Bx0A9R5/M9jibeJeZWqjeVEIxYW9fZYqB9a70/bY= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/functions v1.7.0 h1:s3Snbr2O4j4p7CuwImBas8rNNmkHS1YJANcCpKGqQSE= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/gaming v1.6.0 h1:PKggmegChZulPW8yvtziF8P9UOuVFwbvylbEucTNups= cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gkeconnect v0.6.0 h1:zAcvDa04tTnGdu6TEZewaLN2tdMtUOJJ7fEceULjguA= cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkehub v0.10.0 h1:JTcTaYQRGsVm+qkah7WzHb6e9sf1C0laYdRPn9aN+vg= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/language v1.6.0 h1:Fb2iua/5/UBvUuW9PgBinwsCRDi1qoQJEuekOinHFCs= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/lifesciences v0.6.0 h1:tIqhivE2LMVYkX0BLgG7xL64oNpDaFFI7teunglt1tI= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/mediatranslation v0.6.0 h1:qAJzpxmEX+SeND10Y/4868L5wfZpo4Y3BIEnIieP4dk= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/memcache v1.5.0 h1:qTBOiSnVw7rnW6GVeH5Br8qs80ILoflNgFZySvaT4ek= cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/metastore v1.6.0 h1:wzJ9HslsybiJ3HL2168dVonr9D/eBq0VqObiMSCrE6c= cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/networkconnectivity v1.5.0 h1:mtIQewrz1ewMU3J0vVkUIJtAkpOqgkz4+UmcreeAm08= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networksecurity v0.6.0 h1:qDEX/3sipg9dS5JYsAY+YvgTjPR63cozzAWop8oZS94= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/notebooks v1.3.0 h1:YfPI4pOYQDcqJ+thM2cGtR9oRoRv42vRfubSPZnk3DI= cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/osconfig v1.8.0 h1:fkFlXCxkUt3tE8LYtF6CipuPbC/HIrciwDTjFpsTf88= cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/oslogin v1.5.0 h1:/7sVaMdtqSm6AjxW8KzoM6UKawkg3REr0XJ1zKtidpc= cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/phishingprotection v0.6.0 h1:OrwHLSRSZyaiOt3tnY33dsKSedxbMzsXvqB21okItNQ= cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/privatecatalog v0.6.0 h1:Vz86uiHCtNGm1DeC32HeG2VXmOq5JRYA3VRPf8ZEcSg= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0 h1:CM59ZlNZxxpn24nwXfxhL4JtnGFTJ6gfSkoFbM8FKxE= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recommendationengine v0.6.0 h1:6w+WxPf2LmUEqX0YyvfCoYb8aBYOcbIV25Vg6R0FLGw= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommender v1.6.0 h1:C1tw+Qa/bgm6LoH1wuxYdoyinwKkW/jDJ0GpSJf58cE= cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/redis v1.8.0 h1:gtPd4pG/Go5mrdGQ4MJXxPHtjxtoWUBkrWLXNV1L2TA= cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/retail v1.9.0 h1:Q3W/JsQupZWaoFxUOugZd1Eq590R+Dk6dhacLK2h7+w= cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/scheduler v1.5.0 h1:Fe1Upic/q4cwqXbInCzgAW35QSerj8JlNwATIxDdfOI= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/secretmanager v1.6.0 h1:5v0zegRMlytVnN7J+bg5Ipqah3I2RZ67ysy00mvA+lA= cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/security v1.8.0 h1:linnRc3/gJYDfKbAtNixVQ52+66DpOx5MmCz0NNxal8= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/securitycenter v1.14.0 h1:hKIggnv2eCAPjsVnFcZbytMOsFOk6p4ut0iAUDoNsNA= cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/servicedirectory v1.5.0 h1:QmCWml/qvNOYyiPP4G52srYcsHSLCXuvydJDVLTFSe8= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/speech v1.7.0 h1:bRI2QczZGpcPfuhHr63VOdfyyfYp/43N0wRuBKrd0nQ= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/talent v1.2.0 h1:6c4pvu3k2idEhJRZnZ2HdVLWZUuT9fsns2gQtCzRtqA= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/videointelligence v1.7.0 h1:w56i2xl1jHX2tz6rHXBPHd6xujevhImzbc16Kl+V/zQ= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/vision/v2 v2.3.0 h1:eEyIDJ5/98UmQrYZ6eQExUT3iHyDjzzPX29UP6x7ZQo= cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/webrisk v1.5.0 h1:WdHJmLSAs5bIis/WWO7pIfiRBD1PiWe1OAlPrWeM9Tk= cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/workflows v1.7.0 h1:0MjX5ugKmTdbRG2Vai5aAgNAOe2wzvs/XQwFDSowy9c= cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037 h1:+PdD6GLKejR9DizMAKT5DpSAkKswvZrurk1/eEt9+pw= @@ -96,13 +152,16 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg6 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A= github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.20 h1:gJ3E98kMpFB1MFqQCvA1yFab8vthOeD4VlFRQULxahg= github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/validation v0.1.0 h1:ISSNzGUh+ZSzizJWOWzs8bwpXIePbGLW4z/AmUFGH5A= @@ -117,51 +176,173 @@ github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3 h1:4FA+QBaydEHlwxg0lMN3rhwoDaQy6LKhVWR4qvq4BuA= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= +github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae h1:AMzIhMUqU3jMrZiTuW0zkYeKlKDAFD+DG20IoO421/Y= +github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/aws/aws-sdk-go v1.15.11 h1:m45+Ru/wA+73cOZXiEGLDH2d9uLN3iHqMc0/z4noDXE= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= +github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.1.0+incompatible h1:7hqmJYuaEK3qwVjWubYiht3j93YI0WQBuysxHIfUriU= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bshuster-repo/logrus-logstash-hook v0.4.1 h1:pgAtgj+A31JBVtEHu2uHuEx0n+2ukqUJnS2vVe5pQNA= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o= +github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/checkpoint-restore/go-criu/v4 v4.1.0 h1:WW2B2uxx9KWF6bGlHqhm8Okiafwwx7Y2kcpn8lCpjgo= +github.com/checkpoint-restore/go-criu/v5 v5.0.0 h1:TW8f/UvntYoVDMN1K2HlT82qH1rb0sOjpGw3m6Ym+i4= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 h1:hzAQntlaYRkVSFEfj9OTWlVV1H155FMD8BTKktLv0QI= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1 h1:zH8ljVhhq7yC0MIeUL/IviMtY8hx2mK8cN9wEYb8ggw= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/containerd/aufs v1.0.0 h1:2oeJiwX5HstO7shSrPZjrohJZLzK36wvpdmzDRkL/LY= +github.com/containerd/btrfs v1.0.0 h1:osn1exbzdub9L5SouXO5swW4ea/xVdJZ3wokxN5GrnA= +github.com/containerd/console v1.0.2 h1:Pi6D+aZXM+oUw1czuKgH5IJ+y0jhYcwBJfx5/Ghn9dE= +github.com/containerd/containerd v1.5.7 h1:rQyoYtj4KddB3bxG6SAqd4+08gePNyJjRqvOIfV3rkM= +github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8= +github.com/containerd/fifo v1.0.0 h1:6PirWBr9/L7GDamKr+XM0IeUFXu5mf3M/BPpH9gaLBU= +github.com/containerd/go-cni v1.0.2 h1:YbJAhpTevL2v6u8JC1NhCYRwf+3Vzxcc5vGnYoJ7VeE= +github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0= +github.com/containerd/imgcrypt v1.1.1 h1:LBwiTfoUsdiEGAR1TpvxE+Gzt7469oVu87iR3mv3Byc= +github.com/containerd/nri v0.1.0 h1:6QioHRlThlKh2RkRTR4kIT3PKAcrLo3gIWnjkM4dQmQ= +github.com/containerd/stargz-snapshotter/estargz v0.4.1 h1:5e7heayhB7CcgdTkqfZqrNaNv15gABwr3Q2jBTbLlt4= +github.com/containerd/ttrpc v1.1.0 h1:GbtyLRxb0gOLR0TYQWt3O6B0NvT8tMdorEHqIQo/lWI= +github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= +github.com/containerd/zfs v1.0.0 h1:cXLJbx+4Jj7rNsTiqVfm6i+RNLx6FFA2fMmDlEf+Wm8= +github.com/containernetworking/cni v0.8.1 h1:7zpDnQ3T3s4ucOuJ/ZCLrYBxzkg0AELFfII3Epo9TmI= +github.com/containernetworking/plugins v0.9.1 h1:FD1tADPls2EEi3flPc2OegIY1M9pUa9r2Quag7HMLV8= +github.com/containers/ocicrypt v1.1.1 h1:prL8l9w3ntVqXvNH1CiNn5ENjcCnr38JqpSyvKKB4GI= +github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= +github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-iptables v0.5.0 h1:mw6SAibtHKZcNzAsOxjoHIG0gy5YFHhypWSSNc6EjbQ= +github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= +github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c h1:Xo2rK1pzOm0jO6abTPIQwbAmqBIOj132otexc1mmzFc= +github.com/d2g/dhcp4client v1.0.0 h1:suYBsYZIkSlUMEz4TAYCczKf62IA2UWC+O8+KtdOhCo= +github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5 h1:+CpLbZIeUn94m02LdEKPcgErLJ347NUwxPKs5u8ieiY= +github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4 h1:itqmmf1PFpC4n5JW+j4BU7X4MTfVurhYRTjODoPb2Y8= +github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba h1:p6poVbjHDkKa+wtC8frBMwQtT3BmqGYBjzMwJ63tuR4= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= +github.com/dlintw/goconf v0.0.0-20120228082610-dcc070983490 h1:I8/Qu5NTaiXi1TsEYmTeLDUlf7u9pEdbG+azjDvx8Vg= github.com/dlintw/goconf v0.0.0-20120228082610-dcc070983490/go.mod h1:jWlUIP63OLr0cV2FGN2IEzSFsMAe58if8rk/SAE0JRE= +github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017 h1:2HQmlpI3yI9deH18Q6xiSOIjXD4sLI55Y/gfpa8/558= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7 h1:Cvj7S8I4Xpx78KAl6TwTmMHuHlZ/0SM60NUneGJQ7IE= +github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 h1:xvqufLtNVwAhN8NMyWklVgxnWohi+wtMGQMhtxexlm0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flowstack/go-jsonschema v0.1.1 h1:dCrjGJRXIlbDsLAgTJZTjhwUJnnxVWl1OgNyYh5nyDc= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa h1:RDBNVkRviHZtvDvId8XSGPu3rmpmSe+wKRcEWNgsfWU= +github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7 h1:LofdAjjjqCSXMwLGgOgnE+rdPuvX9DxCqaHwKy7i/ko= +github.com/getkin/kin-openapi v0.76.0 h1:j77zg3Ec+k+r+GA3d8hBoXpAc6KX9TbBPrwQGBIy2sY= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= +github.com/go-ini/ini v1.25.4 h1:Mujh4R/dH6YL8bxuISne3xX2+qcQ9p0IxKAP6ExWoUo= +github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= +github.com/go-kit/log v0.1.0 h1:DGJh0Sm43HbOeYDNnVZFl8BvcYVvjD5bqYJvp0REbwQ= +github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8= +github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= +github.com/gogo/googleapis v1.4.0 h1:zgVt4UpGxcqVOw97aRGxT4svlcmdK35fynLNctY32zI= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.5.1 h1:/+mFTs4AlwsJ/mJe8NDtKb7BxLtbZFpcn8vDsneEkwQ= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -170,43 +351,122 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/seesaw v0.0.0-20220321203705-0e93b4c33bc6/go.mod h1:IkMRi+zMNoZu3BRbJ6UlNRdgJkfD5yiB41AR5KlyxJE= +github.com/googleapis/gax-go/v2 v2.1.0 h1:6DWmvNpomjL1+3liNSZbVns3zsYzzCjm6pRBO1tLeso= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33 h1:893HsJqtxp9z1SF76gg6hY70hRY1wVlTSnC/h1yUDCo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/hashicorp/consul/api v1.1.0 h1:BNQPM9ytxj6jbjjdRPioQ94T6YXriSopn0i8COv6SRA= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1 h1:LnuDWGNsoajlhGyHJvuWW6FVqRl8JOTPqS6CPTsYjhY= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0 h1:WhIgCr5a7AaVH6jPUwjtRuuE7/RDufnUvzIr48smyxs= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56 h1:742eGXur0715JMq73aD95/FU0XpVKXqNuTnEfXsLOYQ= +github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7 h1:SMvOWPJCES2GdFracYbBQh93GXac8fq7HeN6JnpduB8= +github.com/joefitzgerald/rainbow-reporter v0.1.0 h1:AuMG652zjdzI0YCCnXAqATtRBpGXMcAnrajcaTrSeuo= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= +github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3 h1:jUp75lepDg0phMUJBCmvaeFDldD2N3S1lBuPwUTszio= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/marstr/guid v1.1.0 h1:/M4H/1G4avsieL6BbUwCOBzulmoeKVP5ux/3mQNnbyI= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= +github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-shellwords v1.0.6 h1:9Jok5pILi5S1MnDirGVTufYGtksUs/V2BWUP3ZkeUUI= +github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2 h1:g+4J5sZg6osfvEfkRZxJ1em0VT95/UOZgi/l7zi1/oE= +github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.51 h1:0+Xg7vObnhrz/4ZCZcZh7zPXlmU0aveS2HDBd0m0qSo= +github.com/miekg/pkcs11 v1.0.3 h1:iMwmD7I5225wv84WxIG/bmxz9AXjWvTWIbM/TYHvWtw= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= +github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0 h1:lfGJxY7ToLJQjHHwi0EX6uYBdK78egf954SQl13PQJc= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f h1:2+myh5ml7lgEU/51gbeLHfKGNfgEQQIWrlbdaOsidbQ= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM= +github.com/moby/sys/symlink v0.1.0 h1:MTFZ74KtNI6qQQpuBxU+uKCim4WtOMokr03hCfJcazE= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/mrunalp/fileutils v0.5.0 h1:NKzVxiH7eSk+OQ4M+ZYW1K6h27RUV3MI6NUTsHhU6Z4= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/ncw/swift v1.0.47 h1:4DQRPj35Y41WogBxyhOXlrI37nzGlyEcsforeudyYPQ= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= @@ -215,48 +475,124 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/runc v1.0.2 h1:opHZMaswlyxz1OuGpBE53Dwe4/xF7EZTY0A2L/FpCOg= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39 h1:H7DMc6FAjgwZZi8BRqjrAAHWoqEr5e5L6pS4V0ezet4= +github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc= github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 h1:2c1EFnZHIPCW8qKWgHMH/fX2PkSabFc5mrVzfUNdg5U= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/sclevine/spec v1.2.0 h1:1Jwdf9jSfDl9NVmt8ndHqbTZ7XCCPbh1jI3hkDBHVYA= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 h1:lIOOHPEbXzO3vnmx2gok1Tfs31Q8GQqKLc8vVqyQq/I= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= +github.com/tchap/go-patricia v2.2.6+incompatible h1:JvoDL7JSoIP2HDE8AbDH3zC8QBPxmzYe32HHy5yQ+Ck= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= +github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= +github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA= +github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 h1:ESFSdwYZvkeru3RtdrYueztKhOBCSAAzS4Gf+k0tEow= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1:ERexzlUfuTvpE74urLSbIQW0Z/6hF9t8U4NsJLaioAY= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc= go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg= go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.4 h1:Dcx3/MYyfKcPNLpR4VVQUP5KgYrBeJtktBwEKkw08Ao= go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU= +go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4= go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= +go.etcd.io/etcd/pkg/v3 v3.5.4 h1:V5Dvl7S39ZDwjkKqJG2BfXgxZ3QREqqKifWQgIw5IM0= go.etcd.io/etcd/pkg/v3 v3.5.4/go.mod h1:OI+TtO+Aa3nhQSppMbwE4ld3uF1/fqqwbpfndbbrEe0= +go.etcd.io/etcd/raft/v3 v3.5.4 h1:YGrnAgRfgXloBNuqa+oBI/aRZMcK/1GS6trJePJ/Gqc= go.etcd.io/etcd/raft/v3 v3.5.4/go.mod h1:SCuunjYvZFC0fBX0vxMSPjuZmpcSk+XaAcMrD6Do03w= +go.etcd.io/etcd/server/v3 v3.5.4 h1:CMAZd0g8Bn5NRhynW6pKhc4FRg41/0QYy3d7aNm9874= go.etcd.io/etcd/server/v3 v3.5.4/go.mod h1:S5/YTU15KxymM5l3T6b09sNOHPXqGYIZStpuuGbb65c= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 h1:A/5uWzF44DlIgdm/PQFwfMkW0JX+cIcQi/SwLAmZP5M= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 h1:Q3C9yzW6I9jqEc8sawxzxZmY48fs9u220KXq6d5s3XU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0 h1:JsxtGXd06J8jrnya7fdI/U/MR6yXA5DtbZy+qoHQlr8= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0 h1:c5VRjxCXdQlx1HjzwGdQHzZaVI82b5EbBgOu2ljD92g= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0 h1:7ao1wpzHRVKf0OQ7GIxiQJA6X7DLX9o14gmVon7mMK8= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -264,13 +600,21 @@ golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd h1:XcWmESyNjXJMLahc3mqVQJcgSTDxFxhETVlfk9uGc38= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -287,6 +631,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -297,7 +643,10 @@ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -323,7 +672,12 @@ golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -341,7 +695,11 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= @@ -353,7 +711,9 @@ google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNe google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0 h1:4t9zuDlHLcIx0ZEhmXEeFVCRsiOgpgn2QOH9N0MNjPI= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8 h1:Cpp2P6TPjujNoC5M2KHY6g7wfyLYfIWRZaSdIKfDasA= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -393,22 +753,43 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= +gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= k8s.io/apiserver v0.25.2/go.mod h1:30r7xyQTREWCkG2uSjgjhQcKVvAAlqoD+YyrqR6Cn+I= +k8s.io/code-generator v0.25.2 h1:qEHux0+E1c+j1MhsWn9+4Z6av8zrZBixOTPW064rSiY= k8s.io/code-generator v0.25.2/go.mod h1:f61OcU2VqVQcjt/6TrU0sta1TA5hHkOO6ZZPwkL9Eys= k8s.io/component-base v0.25.2/go.mod h1:90W21YMr+Yjg7MX+DohmZLzjsBtaxQDDwaX4YxDkl60= k8s.io/component-helpers v0.25.2 h1:A4xQEFq7tbnhB3CTwZTLcQtyEhFFZN2TyQjNgziuSEI= k8s.io/component-helpers v0.25.2/go.mod h1:iuyfZG2jGWYvR5F/yGFUYNdL/IFz2smcwpNaOqP+YNM= +k8s.io/cri-api v0.25.2 h1:WH8NM2IExOjqU9ColGHKK7+RlUOCmbB/c58aAT/DVpw= k8s.io/cri-api v0.25.2/go.mod h1:LFvFRFKuO7cFzrcmLle/cwnuVMgoIZ6bMQ+jrzk24Yc= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 h1:TT1WdmqqXareKxZ/oNXEUSwKlLiHzPMyB0t8BaFeBYI= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= +rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= +rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.32 h1:2WjukG7txtEsbXsSKWtTibCdsyYAhcu6KFnttyDdZOQ= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.32/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= sigs.k8s.io/kpng/backends/iptables v0.0.0-20220824013548-88b8a1d9bc62/go.mod h1:IAnXygTQhnHewpyy3zeV0kLVRmPexdxB0Tdgch5Fz2o= diff --git a/hack/test_backend_build.sh b/hack/test_backend_build.sh index 23013b1e5..f67b3f492 100755 --- a/hack/test_backend_build.sh +++ b/hack/test_backend_build.sh @@ -41,8 +41,7 @@ function build_all_backends { case $package in "iptables") build_package backends/iptables ;; - "ipvs") build_package backends/ipvs-as-sink ;; - "ipvsfullstate") build_package backends/ipvsfullstate ;; + "ipvs") build_package backends/ipvs ;; "nft") build_package backends/nft ;; "ebpf") build_package backends/ebpf ;; "userspacelin") build_package backends/userspacelin;; diff --git a/hack/test_e2e.sh b/hack/test_e2e.sh index afbbd0d2a..5c32b77e2 100755 --- a/hack/test_e2e.sh +++ b/hack/test_e2e.sh @@ -1029,7 +1029,7 @@ if [[ "${cluster_count}" -lt "2" ]] && ${print_report}; then help fi -if ! [[ "${backend}" =~ ^(iptables|nft|ipvs|ipvsfullstate|ebpf|userspacelin|not-kpng)$ ]]; then +if ! [[ "${backend}" =~ ^(iptables|nft|ipvs|ebpf|userspacelin|not-kpng)$ ]]; then echo "user must specify the supported backend" help fi diff --git a/hack/test_skip_list.sh b/hack/test_skip_list.sh index 1c85d690e..10259efda 100644 --- a/hack/test_skip_list.sh +++ b/hack/test_skip_list.sh @@ -40,85 +40,12 @@ GINKGO_SKIP_dual_iptables_TEST="should be updated after adding or deleting ports |should serve multiport endpoints from pods\ |should be able to preserve UDP traffic when server pod cycles for a NodePort service\ |should check kube-proxy urls" - -GINKGO_SKIP_ipv4_ipvs_TEST="should check kube-proxy urls\ -|should work after the service has been recreated\ -|should be able to preserve UDP traffic when server pod cycles for a NodePort service\ -|should not be able to connect to terminating and unready endpoints if PublishNotReadyAddresses is false\ -|should implement service.kubernetes.io/service-proxy-name\ -|should be able to update service type to NodePort listening on same port number but different protocols\ -|hould have session affinity timeout work for service with type clusterIP\ -|should be able to preserve UDP traffic when initial unready endpoints get ready\ -|should be able to switch session affinity for NodePort service\ -|should drop INVALID conntrack entries\ -|should have session affinity timeout work for NodePort service\ -|should be able to preserve UDP traffic when server pod cycles for a ClusterIP service\ -|should be able to switch session affinity for service with type clusterIP\ -|should have session affinity work for NodePort service\ -|should be able to change the type from ExternalName to NodePort\ -|should implement service.kubernetes.io/headless\ -|should be able to create a functioning NodePort service\ -|should create endpoints for unready pods\ -|should be able to connect to terminating and unready endpoints if PublishNotReadyAddresses is true\ -|should function for multiple endpoint-Services with same selecto" -GINKGO_SKIP_ipv6_ipvs_TEST="should have session affinity work for NodePort service\ -|should have session affinity timeout work for NodePort service\ -|should be able to update service type to NodePort listening on same port number but different protocols\ -|should be able to switch session affinity for NodePort service\ -|should be able to create a functioning NodePort service\ -|should be able to connect to terminating and unready endpoints if PublishNotReadyAddresses is true\ -|should be able to change the type from ExternalName to NodePort\ -|should be able to preserve UDP traffic when server pod cycles for a NodePort service\ -|should function for endpoint-Service: udp\ -|should function for service endpoints using hostNetwork\ -|should support basic nodePort: udp functionality\ -|should function for node-Service: http\ -|should update nodePort: http\ -|should function for pod-Service: udp\ -|should function for node-Service: udp\ -|should function for multiple endpoint-Services with same selector\ -|should check kube-proxy urls\ -|should work after the service has been recreated\ -|should update nodePort: udp\ -|should be able to change the type from ClusterIP to ExternalName\ -|should create endpoints for unready pods\ -|should be able to change the type from NodePort to ExternalName\ -|should provide DNS for pods for Subdomain\ -|should provide DNS for services\ -|should provide DNS for the cluster\ -|should run through the lifecycle of Pods and PodStatus\ -|should function for endpoint-Service: http\ -|should function for pod-Service: http" - -GINKGO_SKIP_dual_ipvs_TEST="should work after the service has been recreated\ -|should be able to update service type to NodePort listening on same port number but different protocols\ -|should not be able to connect to terminating and unready endpoints if PublishNotReadyAddresses is false\ -|should be able to preserve UDP traffic when server pod cycles for a NodePort service\ -|should be able to preserve UDP traffic when server pod cycles for a ClusterIP service\ -|should implement service.kubernetes.io/service-proxy-name\ -|should have session affinity timeout work for service with type clusterIP\ -|should have session affinity work for NodePort service\ -|should be able to preserve UDP traffic when server pod cycles for a ClusterIP service\ -|should have session affinity timeout work for service with type clusterIP\ -|should be able to preserve UDP traffic when initial unready endpoints\ -|should be able to switch session affinity for service with type clusterIP\ -|should create endpoints for unready pods\ -|should have session affinity timeout work for NodePort service\ -|should be able to connect to terminating and unready endpoints if PublishNotReadyAddresses is true\ -|should drop INVALID conntrack entries\ -|should implement service.kubernetes.io/headless\ -|should check kube-proxy urls\ -|should be able to switch session affinity for NodePort service\ -|should function for multiple endpoint-Services with same selector\ -|should be able to create a functioning NodePort service\ -|should be able to change the type from ExternalName to NodePort" - -GINKGO_SKIP_ipv4_ipvsfullstate_TEST="should have session affinity work for NodePort service\ +GINKGO_SKIP_ipv4_ipvs_TEST="should have session affinity work for NodePort service\ |should have session affinity timeout work for NodePort service\ |should be able to switch session affinity for NodePort service" -GINKGO_SKIP_ipv6_ipvsfullstate_TEST="should have session affinity work for NodePort service\ +GINKGO_SKIP_ipv6_ipvs_TEST="should have session affinity work for NodePort service\ |should have session affinity timeout work for NodePort service\ |should be able to switch session affinity for NodePort service\ |should be able to change the type from NodePort to ExternalName\ @@ -128,7 +55,7 @@ GINKGO_SKIP_ipv6_ipvsfullstate_TEST="should have session affinity work for NodeP |should provide DNS for pods for Subdomain\ |should provide DNS for the cluster" -GINKGO_SKIP_dual_ipvsfullstate_TEST="should have session affinity work for NodePort service\ +GINKGO_SKIP_dual_ipvs_TEST="should have session affinity work for NodePort service\ |should have session affinity timeout work for NodePort service\ |should be able to switch session affinity for NodePort service" diff --git a/hack/tilt/deploy.sh b/hack/tilt/deploy.sh index 686edb362..2e764764b 100755 --- a/hack/tilt/deploy.sh +++ b/hack/tilt/deploy.sh @@ -33,7 +33,7 @@ function build_kpng_and_load_image { $BIN_DIR/kind load docker-image kpng --name kpng-proxy } -if ! [[ "${backend}" =~ ^(iptables|nft|ipvs|ipvsfullstate|ebpf|userspacelin|not-kpng)$ ]]; then +if ! [[ "${backend}" =~ ^(iptables|nft|ipvs|ebpf|userspacelin|not-kpng)$ ]]; then echo "skipping re-deployment due to unsupported backend" exit 0 fi diff --git a/hack/tilt/setup.sh b/hack/tilt/setup.sh index a7318874d..f53f8c3bb 100755 --- a/hack/tilt/setup.sh +++ b/hack/tilt/setup.sh @@ -207,7 +207,7 @@ do esac done -if ! [[ "${backend}" =~ ^(iptables|nft|ipvs|ipvsfullstate|ebpf|userspacelin|not-kpng)$ ]]; then +if ! [[ "${backend}" =~ ^(iptables|nft|ipvs|ebpf|userspacelin|not-kpng)$ ]]; then echo "user must specify the supported backend" help fi @@ -222,7 +222,7 @@ if [ ! "$deployment_model" ];then fi if [[ -n "${ip_family}" && -n "${backend}" ]]; then - if ! [[ "${backend}" =~ ^(iptables|nft|ipvs|ipvsfullstate|ebpf|userspacelin|not-kpng)$ ]]; then + if ! [[ "${backend}" =~ ^(iptables|nft|ipvs|ebpf|userspacelin|not-kpng)$ ]]; then echo "user must specify the supported backend" help exit 1