From 5bc8b47a09f401b6078b9f90f34e9badc2d4f557 Mon Sep 17 00:00:00 2001 From: jacksontong Date: Tue, 16 Jul 2024 21:05:54 +0800 Subject: [PATCH] feat: add metrics and record time of app processing procedure --- go.mod | 10 +++---- go.sum | 26 ++++++++++++------- .../controller/app/action/install.go | 26 ++++++++++++++++++- .../controller/app/action/uninstall.go | 22 ++++++++++++++++ .../controller/app/action/upgrade.go | 25 ++++++++++++++++++ .../controller/app/app_controller.go | 1 + pkg/application/helm/action/upgrade.go | 2 +- pkg/controller/server.go | 4 +-- pkg/util/metrics/util.go | 24 ++++++++--------- 9 files changed, 109 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 1993df0446..3a3f186cc4 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/emicklei/go-restful v2.11.1+incompatible github.com/evanphx/json-patch v4.12.0+incompatible github.com/fatih/color v1.13.0 - github.com/fsnotify/fsnotify v1.4.9 + github.com/fsnotify/fsnotify v1.5.1 github.com/go-openapi/inflect v0.19.0 github.com/gogo/protobuf v1.3.2 github.com/google/gofuzz v1.2.0 @@ -67,8 +67,8 @@ require ( github.com/json-iterator/go v1.1.12 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de github.com/moul/http2curl v1.0.0 // indirect - github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.15.0 + github.com/onsi/ginkgo v1.16.5 + github.com/onsi/gomega v1.18.1 github.com/parnurzeal/gorequest v0.2.15 github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.10.1 @@ -89,7 +89,7 @@ require ( github.com/thoas/go-funk v0.4.0 go.etcd.io/etcd/client/pkg/v3 v3.5.1 go.etcd.io/etcd/client/v3 v3.5.1 - go.uber.org/zap v1.19.0 + go.uber.org/zap v1.19.1 golang.org/x/crypto v0.19.0 golang.org/x/net v0.21.0 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 @@ -119,7 +119,7 @@ require ( k8s.io/kubectl v0.24.0 k8s.io/metrics v0.24.0 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 - sigs.k8s.io/controller-runtime v0.6.0 + sigs.k8s.io/controller-runtime v0.12.0 sigs.k8s.io/yaml v1.3.0 yunion.io/x/pkg v0.0.0-20200603123312-ad58e621aec0 ) diff --git a/go.sum b/go.sum index 9ea2b0a3c4..09766da0c3 100644 --- a/go.sum +++ b/go.sum @@ -596,8 +596,9 @@ github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVB github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= @@ -651,7 +652,6 @@ github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= @@ -864,6 +864,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210715191844-86eeefc3e471/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -983,7 +984,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df/go.mod h1:QMZY7/J/KSQEhKWFeDesPjMj+wCHReeknARU3wqlyN4= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -1275,20 +1275,24 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -1726,6 +1730,7 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -1741,8 +1746,9 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -2175,7 +2181,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= @@ -2511,8 +2517,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyz sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30 h1:dUk62HQ3ZFhD48Qr8MIXCiKA8wInBQCtuE4QGfFW7yA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= -sigs.k8s.io/controller-runtime v0.6.0 h1:Fzna3DY7c4BIP6KwfSlrfnj20DJ+SeMBK8HSFvOk9NM= -sigs.k8s.io/controller-runtime v0.6.0/go.mod h1:CpYf5pdNY/B352A1TFLAS2JVSlnGQ5O2cftPHndTroo= +sigs.k8s.io/controller-runtime v0.12.0 h1:gA4zphrmHFc7ihmY/+GyyE0BxKD+OYdb5+DjD2azFAQ= +sigs.k8s.io/controller-runtime v0.12.0/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= sigs.k8s.io/controller-tools v0.2.4/go.mod h1:m/ztfQNocGYBgTTCmFdnK94uVvgxeZeE3LtJvd/jIzA= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= diff --git a/pkg/application/controller/app/action/install.go b/pkg/application/controller/app/action/install.go index ca25902f42..33647dd93d 100644 --- a/pkg/application/controller/app/action/install.go +++ b/pkg/application/controller/app/action/install.go @@ -22,6 +22,7 @@ import ( "context" "errors" "fmt" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" applicationv1 "tkestack.io/tke/api/application/v1" @@ -46,7 +47,28 @@ func Install(ctx context.Context, updateStatusFunc applicationprovider.UpdateStatusFunc) (*applicationv1.App, error) { hooks := getHooks(app) + var beginInstallTime, preInstallTime, pullChartTime, installTime, postInstallTime time.Time + defer func() { + var preInstallCost, pullChartCost, installCost, postInstallCost int + if !preInstallTime.IsZero() { + preInstallCost = int(preInstallTime.Sub(beginInstallTime).Milliseconds()) + if !pullChartTime.IsZero() { + pullChartCost = int(pullChartTime.Sub(preInstallTime).Milliseconds()) + if !installTime.IsZero() { + installCost = int(installTime.Sub(pullChartTime).Milliseconds()) + if !postInstallTime.IsZero() { + postInstallCost = int(postInstallTime.Sub(installTime).Milliseconds()) + } + } + } + } + log.Infof("handle for %s/%s cost: %d %d %d %d", + app.Namespace, app.Name, preInstallCost, pullChartCost, installCost, postInstallCost) + }() + + beginInstallTime = time.Now() err := hooks.PreInstall(ctx, applicationClient, platformClient, app, repo, updateStatusFunc) + preInstallTime = time.Now() if err != nil { if updateStatusFunc != nil { newStatus := app.Status.DeepCopy() @@ -65,6 +87,7 @@ func Install(ctx context.Context, } destfile, err := Pull(ctx, applicationClient, platformClient, app, repo, updateStatusFunc) + pullChartTime = time.Now() if err != nil { newStatus := app.Status.DeepCopy() if updateStatusFunc != nil { @@ -131,7 +154,7 @@ func Install(ctx context.Context, Wait: app.Spec.Chart.InstallPara.Wait, WaitForJobs: app.Spec.Chart.InstallPara.WaitForJobs, }) - + installTime = time.Now() if err != nil { if errors.Is(err, errors.New("chart manifest is empty")) { log.Errorf(fmt.Sprintf("ERROR: install cluster %s app %s manifest is empty, file %s", app.Spec.TargetCluster, app.Name, destfile)) @@ -158,6 +181,7 @@ func Install(ctx context.Context, } err = hooks.PostInstall(ctx, applicationClient, platformClient, app, repo, updateStatusFunc) + postInstallTime = time.Now() // 先走完hook，在更新app状态为succeed if err != nil { if updateStatusFunc != nil { diff --git a/pkg/application/controller/app/action/uninstall.go b/pkg/application/controller/app/action/uninstall.go index b63d808d27..776dc53165 100644 --- a/pkg/application/controller/app/action/uninstall.go +++ b/pkg/application/controller/app/action/uninstall.go @@ -21,6 +21,7 @@ package action import ( "context" "errors" + "time" "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/storage/driver" @@ -30,6 +31,7 @@ import ( appconfig "tkestack.io/tke/pkg/application/config" helmaction "tkestack.io/tke/pkg/application/helm/action" "tkestack.io/tke/pkg/application/util" + "tkestack.io/tke/pkg/util/log" ) // Uninstall provides the implementation of 'helm uninstall'. @@ -39,7 +41,25 @@ func Uninstall(ctx context.Context, app *applicationv1.App, repo appconfig.RepoConfiguration) (*release.UninstallReleaseResponse, error) { hooks := getHooks(app) + + var beginUninstallTime, preUninstallTime, uninstallTime, postUninstallTime time.Time + defer func() { + var preUninstallCost, uninstallCost, postUninstallCost int + if !preUninstallTime.IsZero() { + preUninstallCost = int(preUninstallTime.Sub(beginUninstallTime).Milliseconds()) + if !uninstallTime.IsZero() { + uninstallCost = int(uninstallTime.Sub(preUninstallTime).Milliseconds()) + if !postUninstallTime.IsZero() { + postUninstallCost = int(postUninstallTime.Sub(uninstallTime).Milliseconds()) + } + } + } + log.Infof("handle for %s/%s cost: %d %d %d", + app.Namespace, app.Name, preUninstallCost, uninstallCost, postUninstallCost) + }() + beginUninstallTime = time.Now() err := hooks.PreUninstall(ctx, applicationClient, platformClient, app, repo) + preUninstallTime = time.Now() if err != nil { return nil, err } @@ -52,11 +72,13 @@ func Uninstall(ctx context.Context, ReleaseName: app.Spec.Name, Timeout: defaultTimeout, }) + uninstallTime = time.Now() if err != nil && !errors.Is(err, driver.ErrReleaseNotFound) { return resp, err } err = hooks.PostUninstall(ctx, applicationClient, platformClient, app, repo) + postUninstallTime = time.Now() return resp, err } diff --git a/pkg/application/controller/app/action/upgrade.go b/pkg/application/controller/app/action/upgrade.go index e3772c813a..aff9e17f6d 100644 --- a/pkg/application/controller/app/action/upgrade.go +++ b/pkg/application/controller/app/action/upgrade.go @@ -22,6 +22,7 @@ import ( "context" "errors" "fmt" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" applicationv1 "tkestack.io/tke/api/application/v1" @@ -46,7 +47,28 @@ func Upgrade(ctx context.Context, updateStatusFunc applicationprovider.UpdateStatusFunc) (*applicationv1.App, error) { hooks := getHooks(app) + var beginUpgradeTime, preUpgradeTime, pullChartTime, upgradeTime, postupgradeTime time.Time + defer func() { + var preUpgradeCost, pullChartCost, upgradeCost, postUpgradeCost int + if !preUpgradeTime.IsZero() { + preUpgradeCost = int(preUpgradeTime.Sub(beginUpgradeTime).Milliseconds()) + if !pullChartTime.IsZero() { + pullChartCost = int(pullChartTime.Sub(preUpgradeTime).Milliseconds()) + if !upgradeTime.IsZero() { + upgradeCost = int(upgradeTime.Sub(pullChartTime).Milliseconds()) + if !postupgradeTime.IsZero() { + postUpgradeCost = int(postupgradeTime.Sub(upgradeTime).Milliseconds()) + } + } + } + } + log.Infof("handle for %s/%s cost: %d %d %d %d", + app.Namespace, app.Name, preUpgradeCost, pullChartCost, upgradeCost, postUpgradeCost) + }() + + beginUpgradeTime = time.Now() err := hooks.PreUpgrade(ctx, applicationClient, platformClient, app, repo, updateStatusFunc) + preUpgradeTime = time.Now() if err != nil { if updateStatusFunc != nil { newStatus := app.Status.DeepCopy() @@ -65,6 +87,7 @@ func Upgrade(ctx context.Context, } destfile, err := Pull(ctx, applicationClient, platformClient, app, repo, updateStatusFunc) + pullChartTime = time.Now() if err != nil { newStatus := app.Status.DeepCopy() if updateStatusFunc != nil { @@ -124,6 +147,7 @@ func Upgrade(ctx context.Context, Wait: app.Spec.Chart.UpgradePara.Wait, WaitForJobs: app.Spec.Chart.UpgradePara.WaitForJobs, }) + upgradeTime = time.Now() if err != nil { if errors.Is(err, errors.New("chart manifest is empty")) { log.Errorf(fmt.Sprintf("ERROR: upgrade cluster %s app %s manifest is empty, file %s", app.Spec.TargetCluster, app.Name, destfile)) @@ -191,6 +215,7 @@ func Upgrade(ctx context.Context, } err = hooks.PostUpgrade(ctx, applicationClient, platformClient, app, repo, updateStatusFunc) + postupgradeTime = time.Now() // 先走完hook，在更新app状态为succeed if err != nil { if updateStatusFunc != nil { diff --git a/pkg/application/controller/app/app_controller.go b/pkg/application/controller/app/app_controller.go index 8aa7078350..2b0bc49914 100644 --- a/pkg/application/controller/app/app_controller.go +++ b/pkg/application/controller/app/app_controller.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + _ "k8s.io/component-base/metrics/prometheus/workqueue" applicationv1 "tkestack.io/tke/api/application/v1" clientset "tkestack.io/tke/api/client/clientset/versioned" platformversionedclient "tkestack.io/tke/api/client/clientset/versioned/typed/platform/v1" diff --git a/pkg/application/helm/action/upgrade.go b/pkg/application/helm/action/upgrade.go index eb4dc73b36..b008ffeaf0 100644 --- a/pkg/application/helm/action/upgrade.go +++ b/pkg/application/helm/action/upgrade.go @@ -80,7 +80,7 @@ func (c *Client) Upgrade(ctx context.Context, options *UpgradeOptions) (*release histClient.Max = 1 rels, err := histClient.Run(options.ReleaseName) if errors.Is(err, driver.ErrReleaseNotFound) { - log.Infof("Release %d does not exist. Installing it now.", options.ReleaseName) + log.Infof("Release %s does not exist. Installing it now.", options.ReleaseName) return c.Install(ctx, &InstallOptions{ DryRun: options.DryRun, DependencyUpdate: options.DependencyUpdate, diff --git a/pkg/controller/server.go b/pkg/controller/server.go index 0cedb37c7d..cc2087af8c 100644 --- a/pkg/controller/server.go +++ b/pkg/controller/server.go @@ -22,7 +22,6 @@ import ( "net/http" goruntime "runtime" - "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/apimachinery/pkg/runtime" genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" apirequest "k8s.io/apiserver/pkg/endpoints/request" @@ -32,6 +31,7 @@ import ( "k8s.io/apiserver/pkg/server/mux" "k8s.io/apiserver/pkg/server/routes" componentconfig "k8s.io/component-base/config" + "k8s.io/component-base/metrics/legacyregistry" ) // BuildHandlerChain builds a handler chain with a base handler and CompletedConfig. @@ -62,7 +62,7 @@ func NewBaseHandler(c *componentconfig.DebuggingConfiguration, checks ...healthz goruntime.SetBlockProfileRate(1) } } - m.Handle("/metrics", promhttp.Handler()) + m.Handle("/metrics", legacyregistry.HandlerWithReset()) return m } diff --git a/pkg/util/metrics/util.go b/pkg/util/metrics/util.go index 41e89c2ce3..c04609118f 100644 --- a/pkg/util/metrics/util.go +++ b/pkg/util/metrics/util.go @@ -26,8 +26,8 @@ import ( "sync" "k8s.io/client-go/util/flowcontrol" - - "github.com/prometheus/client_golang/prometheus" + compbasemetrics "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" ) var ( @@ -36,7 +36,7 @@ var ( ) type rateLimiterMetric struct { - metric prometheus.Gauge + metric *compbasemetrics.Gauge stopCh chan struct{} } @@ -48,12 +48,12 @@ func registerRateLimiterMetric(ownerName string) error { // only register once in Prometheus. We happen to see an ownerName reused in parallel integration tests. return nil } - metric := prometheus.NewGauge(prometheus.GaugeOpts{ + metric := compbasemetrics.NewGauge(&compbasemetrics.GaugeOpts{ Name: "rate_limiter_use", Subsystem: ownerName, Help: fmt.Sprintf("A metric measuring the saturation of the rate limiter for %v", ownerName), }) - if err := prometheus.Register(metric); err != nil { + if err := legacyregistry.Register(metric); err != nil { return fmt.Errorf("error registering rate limiter usage metric: %v", err) } stopCh := make(chan struct{}) @@ -81,27 +81,27 @@ func RegisterMetricAndTrackRateLimiterUsage(ownerName string, rateLimiter flowco } var ( - GaugeApplicationInstallFailed = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + GaugeApplicationInstallFailed = compbasemetrics.NewGaugeVec(&compbasemetrics.GaugeOpts{ Name: "applicationInstallFailed", Help: "application install failed count", }, []string{"cluster", "application"}) - GaugeApplicationManifestFailed = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + GaugeApplicationManifestFailed = compbasemetrics.NewGaugeVec(&compbasemetrics.GaugeOpts{ Name: "applicationManifestFailed", Help: "application manifest failed", }, []string{"cluster", "application"}) - GaugeApplicationUpgradeFailed = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + GaugeApplicationUpgradeFailed = compbasemetrics.NewGaugeVec(&compbasemetrics.GaugeOpts{ Name: "applicationUpgradeFailed", Help: "application upgrade failed count", }, []string{"cluster", "application"}) - GaugeApplicationDaemonsetUpgradeFailed = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + GaugeApplicationDaemonsetUpgradeFailed = compbasemetrics.NewGaugeVec(&compbasemetrics.GaugeOpts{ Name: "applicationDaemonsetUpgradeFailed", Help: "application daemonset upgrade failed count", }, []string{"cluster", "application"}) - GaugeApplicationRollbackFailed = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + GaugeApplicationRollbackFailed = compbasemetrics.NewGaugeVec(&compbasemetrics.GaugeOpts{ Name: "applicationRollbackFailed", Help: "application rollback failed count", }, []string{"cluster", "application"}) - GaugeApplicationSyncFailed = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + GaugeApplicationSyncFailed = compbasemetrics.NewGaugeVec(&compbasemetrics.GaugeOpts{ Name: "applicationSyncFailed", Help: "application sync failed count", }, []string{"cluster", "application"}) @@ -109,5 +109,5 @@ var ( func init() { // Register the summary and the histogram with Prometheus's default registry. - prometheus.MustRegister(GaugeApplicationInstallFailed, GaugeApplicationUpgradeFailed, GaugeApplicationDaemonsetUpgradeFailed, GaugeApplicationRollbackFailed, GaugeApplicationSyncFailed, GaugeApplicationManifestFailed) + legacyregistry.MustRegister(GaugeApplicationInstallFailed, GaugeApplicationUpgradeFailed, GaugeApplicationDaemonsetUpgradeFailed, GaugeApplicationRollbackFailed, GaugeApplicationSyncFailed, GaugeApplicationManifestFailed) }