From 474b85384a5330df13d34c79237cda0535f2089c Mon Sep 17 00:00:00 2001 From: ywc689 Date: Wed, 6 Sep 2023 16:37:15 +0800 Subject: [PATCH 1/5] healthcheck: add healthcheck method 'udpping' and make it the default checker for UDP targets The 'udpping' checker first checks the l3 network connectivity with ping checker, and performs udp checker only ping checker responds ok. It's an enhancement for the simple udp checker when udp port probe times out. Signed-off-by: ywc689 --- .../pkg/helthcheck/ping_checker_test.go | 7 ++- tools/healthcheck/pkg/helthcheck/server.go | 4 +- .../healthcheck/pkg/helthcheck/udp_checker.go | 4 +- .../pkg/helthcheck/udp_ping_checker.go | 62 +++++++++++++++++++ .../pkg/helthcheck/udp_ping_checker_test.go | 48 ++++++++++++++ tools/healthcheck/pkg/lb/types.go | 1 + 6 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 tools/healthcheck/pkg/helthcheck/udp_ping_checker.go create mode 100644 tools/healthcheck/pkg/helthcheck/udp_ping_checker_test.go diff --git a/tools/healthcheck/pkg/helthcheck/ping_checker_test.go b/tools/healthcheck/pkg/helthcheck/ping_checker_test.go index 686ff15cf..7b86e665a 100644 --- a/tools/healthcheck/pkg/helthcheck/ping_checker_test.go +++ b/tools/healthcheck/pkg/helthcheck/ping_checker_test.go @@ -18,12 +18,13 @@ package hc import ( + "fmt" "net" "testing" "time" ) -var targets = []Target{ +var ping_targets = []Target{ {net.ParseIP("127.0.0.1"), 0, 0}, {net.ParseIP("192.168.88.30"), 0, 0}, {net.ParseIP("11.22.33.44"), 0, 0}, @@ -33,13 +34,13 @@ var targets = []Target{ } func TestPingChecker(t *testing.T) { - for _, target := range targets { + for _, target := range ping_targets { checker := NewPingChecker() id := Id(target.IP.String()) config := NewCheckerConfig(&id, checker, &target, StateUnknown, 0, 3*time.Second, 1*time.Second, 3) result := checker.Check(target, config.Timeout) - t.Logf("%v", result) + fmt.Printf("[ Ping ]%s ==>%v\n", target, result) } } diff --git a/tools/healthcheck/pkg/helthcheck/server.go b/tools/healthcheck/pkg/helthcheck/server.go index bd26265cb..2c3d6c55b 100644 --- a/tools/healthcheck/pkg/helthcheck/server.go +++ b/tools/healthcheck/pkg/helthcheck/server.go @@ -78,13 +78,15 @@ func (s *Server) NewChecker(typ lb.Checker, proto utils.IPProto) CheckMethod { checker = NewUDPChecker("", "") case lb.CheckerPING: checker = NewPingChecker() + case lb.CheckerUDPPing: + checker = NewUDPPingChecker("", "") case lb.CheckerNone: if s.config.LbAutoMethod { switch proto { case utils.IPProtoTCP: checker = NewTCPChecker("", "") case utils.IPProtoUDP: - checker = NewUDPChecker("", "") + checker = NewUDPPingChecker("", "") } } } diff --git a/tools/healthcheck/pkg/helthcheck/udp_checker.go b/tools/healthcheck/pkg/helthcheck/udp_checker.go index ceb6d4467..4cf385da6 100644 --- a/tools/healthcheck/pkg/helthcheck/udp_checker.go +++ b/tools/healthcheck/pkg/helthcheck/udp_checker.go @@ -47,7 +47,7 @@ func (hc *UDPChecker) String() string { return fmt.Sprintf("UDP checker for %v", hc.Config.Id) } -// Check executes a UDP healthcheck. +// Check executes an UDP healthcheck. func (hc *UDPChecker) Check(target Target, timeout time.Duration) *Result { msg := fmt.Sprintf("UDP check to %s", target.Addr()) start := time.Now() @@ -81,7 +81,7 @@ func (hc *UDPChecker) Check(target Target, timeout time.Duration) *Result { return NewResult(start, msg, false, err) } - buf := make([]byte, len(hc.Receive)+1) + buf := make([]byte, len(hc.Receive)) n, _, err := udpConn.ReadFrom(buf) if err != nil { if hc.Send == "" && hc.Receive == "" { diff --git a/tools/healthcheck/pkg/helthcheck/udp_ping_checker.go b/tools/healthcheck/pkg/helthcheck/udp_ping_checker.go new file mode 100644 index 000000000..6fbcee13d --- /dev/null +++ b/tools/healthcheck/pkg/helthcheck/udp_ping_checker.go @@ -0,0 +1,62 @@ +// Copyright 2023 IQiYi Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The healthcheck package refers to the framework of "github.com/google/ +// seesaw/healthcheck" heavily, with only some adaption changes for DPVS. + +package hc + +import ( + "fmt" + "time" +) + +var _ CheckMethod = (*UDPPingChecker)(nil) + +// UDPPingChecker contains configuration specific to a UDPPing healthcheck. +type UDPPingChecker struct { + Config *CheckerConfig + + *UDPChecker + *PingChecker +} + +// NewUDPPingChecker returns an initialised UDPPingChecker. +func NewUDPPingChecker(recv, send string) *UDPPingChecker { + return &UDPPingChecker{UDPChecker: NewUDPChecker(recv, send), + PingChecker: NewPingChecker()} +} + +func (hc *UDPPingChecker) BindConfig(conf *CheckerConfig) { + hc.Config = conf +} + +// String returns the string representation of a UDPPing healthcheck. +func (hc *UDPPingChecker) String() string { + return fmt.Sprintf("UDPPing checker for %v", hc.Config.Id) +} + +// Check executes an UDPPing healthcheck. +func (hc *UDPPingChecker) Check(target Target, timeout time.Duration) *Result { + start := time.Now() + + result := hc.PingChecker.Check(target, timeout) + if result.Success != true { + return result + } + + result = hc.UDPChecker.Check(target, time.Until(start.Add(timeout))) + result.Duration = time.Since(start) + return result +} diff --git a/tools/healthcheck/pkg/helthcheck/udp_ping_checker_test.go b/tools/healthcheck/pkg/helthcheck/udp_ping_checker_test.go new file mode 100644 index 000000000..f20e69ac7 --- /dev/null +++ b/tools/healthcheck/pkg/helthcheck/udp_ping_checker_test.go @@ -0,0 +1,48 @@ +// Copyright 2023 IQiYi Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The healthcheck package refers to the framework of "github.com/google/ +// seesaw/healthcheck" heavily, with only some adaption changes for DPVS. + +package hc + +import ( + "fmt" + "net" + "testing" + "time" + + "github.com/iqiyi/dpvs/tools/healthcheck/pkg/utils" +) + +var udpping_targets = []Target{ + {net.ParseIP("192.168.88.30"), 6601, utils.IPProtoUDP}, + {net.ParseIP("11.22.33.44"), 6601, utils.IPProtoUDP}, + {net.ParseIP("192.168.88.30"), 6602, utils.IPProtoUDP}, + {net.ParseIP("2001::30"), 6601, utils.IPProtoUDP}, + {net.ParseIP("1234:5678::9"), 6601, utils.IPProtoUDP}, + {net.ParseIP("2001::30"), 6602, utils.IPProtoUDP}, +} + +func TestUDPPingChecker(t *testing.T) { + for _, target := range udpping_targets { + checker := NewUDPPingChecker("", "") + id := Id(target.String()) + config := NewCheckerConfig(&id, checker, + &target, StateUnknown, 0, + 3*time.Second, 2*time.Second, 3) + result := checker.Check(target, config.Timeout) + fmt.Printf("[ UDPPing ] %s ==> %v\n", target, result) + } +} diff --git a/tools/healthcheck/pkg/lb/types.go b/tools/healthcheck/pkg/lb/types.go index bc3f8a02e..092f4fb22 100644 --- a/tools/healthcheck/pkg/lb/types.go +++ b/tools/healthcheck/pkg/lb/types.go @@ -27,6 +27,7 @@ const ( CheckerTCP CheckerUDP CheckerPING + CheckerUDPPing ) type RealServer struct { From ff4962b2727d55c3acf117031a80ee42c84dfc6c Mon Sep 17 00:00:00 2001 From: ywc689 Date: Mon, 11 Sep 2023 17:14:56 +0800 Subject: [PATCH 2/5] healthcheck: add HTTP(S) check method Signed-off-by: ywc689 --- .../pkg/helthcheck/http_checker.go | 182 ++++++++++++++++++ .../pkg/helthcheck/http_checker_test.go | 78 ++++++++ 2 files changed, 260 insertions(+) create mode 100644 tools/healthcheck/pkg/helthcheck/http_checker.go create mode 100644 tools/healthcheck/pkg/helthcheck/http_checker_test.go diff --git a/tools/healthcheck/pkg/helthcheck/http_checker.go b/tools/healthcheck/pkg/helthcheck/http_checker.go new file mode 100644 index 000000000..4432a9187 --- /dev/null +++ b/tools/healthcheck/pkg/helthcheck/http_checker.go @@ -0,0 +1,182 @@ +// Copyright 2023 IQiYi Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The healthcheck package refers to the framework of "github.com/google/ +// seesaw/healthcheck" heavily, with only some adaption changes for DPVS. + +package hc + +import ( + "crypto/tls" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +var _ CheckMethod = (*HttpChecker)(nil) + +type HttpCodeRange struct { + start int // inclusive + end int // inclusive +} + +// HttpChecker contains configuration specific to a HTTP(S) healthcheck. +type HttpChecker struct { + Config *CheckerConfig + + Method string + Host string + Uri string + ResponseCodes []HttpCodeRange + Response string + + Secure bool + TLSVerify bool + Proxy bool +} + +// NewHttpChecker returns an initialised HttpChecker. +func NewHttpChecker(method, host, uri string) *HttpChecker { + if len(method) == 0 { + method = "GET" + } + if len(uri) == 0 { + uri = "/" + } + return &HttpChecker{ + Method: method, + Uri: uri, + ResponseCodes: []HttpCodeRange{{200, 299}, {300, 399}, {400, 499}}, + Response: "", + Secure: false, + TLSVerify: true, + Proxy: false, + } +} + +func (hc *HttpChecker) BindConfig(conf *CheckerConfig) { + hc.Config = conf + if len(hc.Host) == 0 { + hc.Host = conf.Target.Addr() + } +} + +// String returns the string representation of a HTTP healthcheck. +func (hc *HttpChecker) String() string { + attr := []string{hc.Method, hc.Host, hc.Uri} + if hc.Secure { + attr = append(attr, "secure") + if hc.TLSVerify { + attr = append(attr, "tls-verify") + } + } + if hc.Proxy { + attr = append(attr, "proxy") + } + + return fmt.Sprintf("HTTP checker for %v [%s]", hc.Config.Id, strings.Join(attr, ", ")) +} + +// Check executes a HTTP healthcheck. +func (hc *HttpChecker) Check(target Target, timeout time.Duration) *Result { + var msg string + if hc.Secure { + msg = fmt.Sprintf("HTTPS %s to %s", hc.Method, hc.Host) + } else { + msg = fmt.Sprintf("HTTP %s to %s", hc.Method, hc.Host) + } + + start := time.Now() + if timeout == time.Duration(0) { + timeout = DefaultCheckConfig.Timeout + } + + u, err := url.Parse(hc.Uri) + if err != nil { + return NewResult(start, fmt.Sprintf("%s; url parse failed", msg), false, err) + } + if hc.Secure { + u.Scheme = "https" + } else { + u.Scheme = "http" + } + if len(u.Host) == 0 { + u.Host = hc.Host + } + + proxy := (func(*http.Request) (*url.URL, error))(nil) + if hc.Proxy { + proxy = http.ProxyURL(u) + } + + tlsConfig := &tls.Config{ + InsecureSkipVerify: !hc.TLSVerify, + } + client := &http.Client{ + Transport: &http.Transport{ + Proxy: proxy, + TLSClientConfig: tlsConfig, + }, + Timeout: timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return errors.New("redirect not permitted") + }, + } + + req, err := http.NewRequest(hc.Method, hc.Uri, nil) + req.URL = u + + // If we received a response we want to process it, even in the + // presence of an error - a redirect 3xx will result in both the + // response and an error being returned. + resp, err := client.Do(req) + if resp == nil { + return NewResult(start, fmt.Sprintf("%s; got no response", msg), false, err) + } + if resp.Body != nil { + defer resp.Body.Close() + } + + // Check response code. + codeOk := false + for _, cr := range hc.ResponseCodes { + if resp.StatusCode >= cr.start && resp.StatusCode <= cr.end { + codeOk = true + break + } + } + + // Check response body. + bodyOk := false + msg = fmt.Sprintf("%s; got %s", msg, resp.Status) + if len(hc.Response) == 0 { + bodyOk = true + } else if resp.Body != nil { + buf := make([]byte, len(hc.Response)) + n, err := io.ReadFull(resp.Body, buf) + if err != nil && err != io.ErrUnexpectedEOF { + msg = fmt.Sprintf("%s; failed to read HTTP response", msg) + } else if string(buf) != hc.Response { + msg = fmt.Sprintf("%s; unexpected response - %q", msg, string(buf[0:n])) + } else { + bodyOk = true + } + } + + return NewResult(start, msg, codeOk && bodyOk, nil) +} diff --git a/tools/healthcheck/pkg/helthcheck/http_checker_test.go b/tools/healthcheck/pkg/helthcheck/http_checker_test.go new file mode 100644 index 000000000..809b43bd3 --- /dev/null +++ b/tools/healthcheck/pkg/helthcheck/http_checker_test.go @@ -0,0 +1,78 @@ +// Copyright 2023 IQiYi Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// The healthcheck package refers to the framework of "github.com/google/ +// seesaw/healthcheck" heavily, with only some adaption changes for DPVS. + +package hc + +import ( + "fmt" + "net" + "strings" + "testing" + "time" + + "github.com/iqiyi/dpvs/tools/healthcheck/pkg/utils" +) + +var http_targets = []Target{ + {net.ParseIP("192.168.88.30"), 80, utils.IPProtoTCP}, + {net.ParseIP("192.168.88.30"), 443, utils.IPProtoTCP}, + {net.ParseIP("2001::30"), 80, utils.IPProtoTCP}, + {net.ParseIP("2001::30"), 443, utils.IPProtoTCP}, +} + +var http_url_targets = []string{ + "http://www.baidu.com", + "https://www.baidu.com", + "http://www.iqiyi.com", + "https://www.iqiyi.com", +} + +func TestHttpChecker(t *testing.T) { + for _, target := range http_targets { + checker := NewHttpChecker("", "", "") + checker.Host = target.Addr() + /* + if target.Port == 443 { + checker.Secure = true + } + */ + id := Id(target.String()) + config := NewCheckerConfig(&id, checker, &target, StateUnknown, + 0, 3*time.Second, 2*time.Second, 3) + result := checker.Check(target, config.Timeout) + fmt.Printf("[ HTTP ] %s ==> %v\n", target, result) + } + + for _, target := range http_url_targets { + host := target[strings.Index(target, "://")+3:] + checker := NewHttpChecker("GET", target, "") + checker.Host = host + checker.ResponseCodes = []HttpCodeRange{{200, 200}} + if strings.HasPrefix(target, "https") { + checker.Secure = true + } + id := Id(host) + config := NewCheckerConfig(&id, checker, &Target{}, StateUnknown, + 0, 3*time.Second, 2*time.Second, 3) + result := checker.Check(Target{}, config.Timeout) + if result.Success == false { + t.Errorf("[ HTTP ] %s ==> %v\n", target, result) + } else { + fmt.Printf("[ HTTP ] %s ==> %v\n", target, result) + } + } +} From e11055489d5e75a5a06556ff5ca8830a7b0b81bb Mon Sep 17 00:00:00 2001 From: ywc689 Date: Tue, 12 Sep 2023 15:33:42 +0800 Subject: [PATCH 3/5] healthcheck: set config's health state from dpvs properly Note that realserver's weight and inhibited flag can be modified by dpvs-agent independently. So we cannot assume that the weight is zero when inhibited flag is set. Signed-off-by: ywc689 --- tools/healthcheck/pkg/helthcheck/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/healthcheck/pkg/helthcheck/server.go b/tools/healthcheck/pkg/helthcheck/server.go index 2c3d6c55b..9e5712570 100644 --- a/tools/healthcheck/pkg/helthcheck/server.go +++ b/tools/healthcheck/pkg/helthcheck/server.go @@ -111,9 +111,9 @@ func (s *Server) getHealthchecks() (*Checkers, error) { } weight := rs.Weight state := StateUnknown - if weight > 0 { + if weight > 0 && rs.Inhibited == false { state = StateHealthy - } else if rs.Inhibited { + } else if weight == 0 && rs.Inhibited == true { state = StateUnhealthy } // TODO: allow users to specify check interval, timeout and retry From e3a7ee0d0746f663d31d6c5ad4a37eb9640714b7 Mon Sep 17 00:00:00 2001 From: ywc689 Date: Tue, 12 Sep 2023 15:39:28 +0800 Subject: [PATCH 4/5] dpvs-agent: fix confusions for realserver's inhibited and overloaded flags Signed-off-by: ywc689 --- tools/dpvs-agent/cmd/ipvs/post_vs_vip_port_rs.go | 2 +- tools/dpvs-agent/cmd/ipvs/put_vs_vip_port_rs.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/dpvs-agent/cmd/ipvs/post_vs_vip_port_rs.go b/tools/dpvs-agent/cmd/ipvs/post_vs_vip_port_rs.go index 76520bc12..a7c9378a1 100644 --- a/tools/dpvs-agent/cmd/ipvs/post_vs_vip_port_rs.go +++ b/tools/dpvs-agent/cmd/ipvs/post_vs_vip_port_rs.go @@ -56,7 +56,7 @@ func (h *postVsRs) Handle(params apiVs.PostVsVipPortRsParams) middleware.Respond rss[i].SetProto(front.GetProto()) rss[i].SetAddr(rs.IP) rss[i].SetInhibited(rs.Inhibited) - rss[i].SetOverloaded(rs.Inhibited) + rss[i].SetOverloaded(rs.Overloaded) rss[i].SetFwdMode(fwdmode) } diff --git a/tools/dpvs-agent/cmd/ipvs/put_vs_vip_port_rs.go b/tools/dpvs-agent/cmd/ipvs/put_vs_vip_port_rs.go index a472be1e7..acfeb572f 100644 --- a/tools/dpvs-agent/cmd/ipvs/put_vs_vip_port_rs.go +++ b/tools/dpvs-agent/cmd/ipvs/put_vs_vip_port_rs.go @@ -58,7 +58,7 @@ func (h *putVsRs) Handle(params apiVs.PutVsVipPortRsParams) middleware.Responder rss[i].SetWeight(uint32(rs.Weight)) rss[i].SetFwdMode(fwdmode) rss[i].SetInhibited(rs.Inhibited) - rss[i].SetOverloaded(rs.Inhibited) + rss[i].SetOverloaded(rs.Overloaded) } } From 1ca55d8c5db4ca701822b38e4d5ca58e0ae276ef Mon Sep 17 00:00:00 2001 From: ywc689 Date: Tue, 12 Sep 2023 15:40:50 +0800 Subject: [PATCH 5/5] ipvs: lower 'no dest found' log message level when scheduling a service without available backends Signed-off-by: ywc689 --- src/ipvs/ip_vs_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ipvs/ip_vs_core.c b/src/ipvs/ip_vs_core.c index 2e7851c40..85fa206eb 100644 --- a/src/ipvs/ip_vs_core.c +++ b/src/ipvs/ip_vs_core.c @@ -299,7 +299,7 @@ struct dp_vs_conn *dp_vs_schedule(struct dp_vs_service *svc, dest = svc->scheduler->schedule(svc, mbuf, iph); if (!dest) { - RTE_LOG(WARNING, IPVS, "%s: no dest found.\n", __func__); + RTE_LOG(INFO, IPVS, "%s: no dest found.\n", __func__); #ifdef CONFIG_DPVS_MBUF_DEBUG dp_vs_mbuf_dump("found dest failed.", iph->af, mbuf); #endif