diff --git a/controllers/appmgr_controller.go b/controllers/appmgr_controller.go index 2de8772..9d9a4ec 100644 --- a/controllers/appmgr_controller.go +++ b/controllers/appmgr_controller.go @@ -5,8 +5,6 @@ import ( "errors" "fmt" "math" - "net/http" - "path/filepath" "reflect" "strconv" "strings" @@ -17,14 +15,11 @@ import ( "bytetrade.io/web3os/app-service/pkg/apiserver/api" "bytetrade.io/web3os/app-service/pkg/appinstaller" "bytetrade.io/web3os/app-service/pkg/constants" - "bytetrade.io/web3os/app-service/pkg/generated/clientset/versioned" "bytetrade.io/web3os/app-service/pkg/helm" - "bytetrade.io/web3os/app-service/pkg/tapr" "bytetrade.io/web3os/app-service/pkg/task" "bytetrade.io/web3os/app-service/pkg/users/userspace" "bytetrade.io/web3os/app-service/pkg/utils" - "github.com/go-resty/resty/v2" "helm.sh/helm/v3/pkg/action" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -33,7 +28,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/retry" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" @@ -56,8 +50,6 @@ type ApplicationManagerController struct { client.Client } -var middlewareTypes = []string{tapr.TypePostgreSQL.String(), tapr.TypeMongoDB.String(), tapr.TypeRedis.String(), tapr.TypeNats.String()} - // SetupWithManager sets up the ApplicationManagerController with the provided controller manager func (r *ApplicationManagerController) SetupWithManager(mgr ctrl.Manager) error { c, err := controller.New("appmgr-controller", mgr, controller.Options{ @@ -485,10 +477,6 @@ func (r *ApplicationManagerController) uninstall(ctx context.Context, appMgr *ap if err != nil { return err } - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return err - } token := appMgr.Status.Payload["token"] version = appMgr.Status.Payload["version"] @@ -499,126 +487,15 @@ func (r *ApplicationManagerController) uninstall(ctx context.Context, appMgr *ap } ops, err := appinstaller.NewHelmOps(ctx, config, appConfig, token, appinstaller.Opt{}) - appCacheDirs, _ := tryToGetAppdataDirFromDeployment(ctx, appConfig, appMgr.Spec.AppOwner) err = ops.Uninstall() if err != nil { return err } - // delete middleware requests crd - namespace := fmt.Sprintf("%s-%s", "user-system", appConfig.OwnerName) - for _, mt := range middlewareTypes { - name := fmt.Sprintf("%s-%s", appConfig.AppName, mt) - err = tapr.DeleteMiddlewareRequest(ctx, config, namespace, name) - if err != nil && !apierrors.IsNotFound(err) { - klog.Errorf("Failed to delete middleware request namespace=%s name=%s err=%v", namespace, name, err) - } - } - - if len(appCacheDirs) > 0 { - terminusNonce, e := utils.GenTerminusNonce() - if e != nil { - klog.Errorf("Failed to generate terminus nonce err=%v", e) - } else { - c := resty.New().SetTimeout(2*time.Second). - SetHeader(constants.AuthorizationTokenKey, token). - SetHeader("Terminus-Nonce", terminusNonce) - nodes, e := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if e == nil { - for _, dir := range appCacheDirs { - for _, n := range nodes.Items { - URL := fmt.Sprintf(constants.AppDataDirURL, appMgr.Spec.AppOwner, dir) - c.SetHeader("X-Terminus-Node", n.Name) - c.SetHeader("x-bfl-user", appMgr.Spec.AppOwner) - res, e := c.R().Delete(URL) - if e != nil { - klog.Errorf("Failed to delete dir err=%v", e) - } - if res.StatusCode() != http.StatusOK { - klog.Infof("delete app cache failed with: %v", res.String()) - } - } - } - } else { - klog.Error("Failed to get nodes err=%v", e) - } - } - } return nil } -func tryToGetAppdataDirFromDeployment(ctx context.Context, appconfig *appinstaller.ApplicationConfig, owner string) (dirs []string, err error) { - userspaceNs := utils.UserspaceName(owner) - config, err := ctrl.GetConfig() - if err != nil { - return dirs, err - } - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return dirs, err - } - sts, err := clientset.AppsV1().StatefulSets(userspaceNs).Get(ctx, "bfl", metav1.GetOptions{}) - if err != nil { - return dirs, err - } - appName := fmt.Sprintf("%s-%s", appconfig.Namespace, appconfig.AppName) - appCachePath := sts.GetAnnotations()["appcache_hostpath"] - if len(appCachePath) == 0 { - return dirs, errors.New("empty appcache_hostpath") - } - if !strings.HasSuffix(appCachePath, "/") { - appCachePath += "/" - } - dClient, err := versioned.NewForConfig(config) - if err != nil { - return dirs, err - } - appCRD, err := dClient.AppV1alpha1().Applications().Get(ctx, appName, metav1.GetOptions{}) - if err != nil { - return dirs, err - } - deploymentName := appCRD.Spec.DeploymentName - deployment, err := clientset.AppsV1().Deployments(appconfig.Namespace). - Get(context.Background(), deploymentName, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return tryToGetAppdataDirFromSts(ctx, appconfig.Namespace, deploymentName, appCachePath) - } - return dirs, err - } - - for _, v := range deployment.Spec.Template.Spec.Volumes { - if v.HostPath != nil && strings.HasPrefix(v.HostPath.Path, appCachePath) && len(v.HostPath.Path) > len(appCachePath) { - dirs = append(dirs, filepath.Base(v.HostPath.Path)) - } - } - return dirs, nil -} - -func tryToGetAppdataDirFromSts(ctx context.Context, namespace, stsName, baseDir string) (dirs []string, err error) { - config, err := ctrl.GetConfig() - if err != nil { - return dirs, err - } - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return dirs, err - } - - sts, err := clientset.AppsV1().StatefulSets(namespace). - Get(ctx, stsName, metav1.GetOptions{}) - if err != nil { - return dirs, err - } - for _, v := range sts.Spec.Template.Spec.Volumes { - if v.HostPath != nil && strings.HasPrefix(v.HostPath.Path, baseDir) && len(v.HostPath.Path) > len(baseDir) { - dirs = append(dirs, filepath.Base(v.HostPath.Path)) - } - } - return dirs, nil -} - func (r *ApplicationManagerController) upgrade(ctx context.Context, appMgr *appv1alpha1.ApplicationManager) (err error) { var version string var revision int diff --git a/pkg/apiserver/handler_middleware.go b/pkg/apiserver/handler_middleware.go index 9cf4c50..21635a6 100644 --- a/pkg/apiserver/handler_middleware.go +++ b/pkg/apiserver/handler_middleware.go @@ -211,6 +211,7 @@ func (h *Handler) installMiddleware(req *restful.Request, resp *restful.Response if e != nil { klog.Error(e) } + delete(middlewareManager, name) return } case <-ctx.Done(): @@ -314,6 +315,17 @@ func (h *Handler) cancelMiddleware(req *restful.Request, resp *restful.Response) if cancelType == "" { cancelType = "operate" } + name, err := utils.FmtAppMgrName(app, owner, "") + if err != nil { + api.HandleError(resp, req, err) + return + } + cancel, ok := middlewareManager[name] + if !ok { + api.HandleError(resp, req, errors.New("can not execute cancel")) + return + } + cancel() now := metav1.Now() status := v1alpha1.ApplicationManagerStatus{ OpType: v1alpha1.CancelOp, @@ -322,23 +334,14 @@ func (h *Handler) cancelMiddleware(req *restful.Request, resp *restful.Response) StatusTime: &now, UpdateTime: &now, } - name, err := utils.FmtAppMgrName(app, owner, "") - if err != nil { - api.HandleError(resp, req, err) - return - } + _, err = utils.UpdateAppMgrStatus(name, status) if err != nil { api.HandleError(resp, req, err) return } - cancel, ok := middlewareManager[name] - if !ok { - api.HandleError(resp, req, errors.New("can not execute cancel")) - return - } - cancel() + resp.WriteAsJson(api.InstallationResponse{ Response: api.Response{Code: 200}, Data: api.InstallationResponseData{UID: app}, diff --git a/pkg/apiserver/handler_webhook.go b/pkg/apiserver/handler_webhook.go index 44cf876..b248257 100644 --- a/pkg/apiserver/handler_webhook.go +++ b/pkg/apiserver/handler_webhook.go @@ -443,7 +443,7 @@ func (h *Handler) eviction2stop(ctx context.Context, req *admissionv1.AdmissionR } podName := pod.GetName() namespace := pod.GetNamespace() - klog.Infof("pod.Name=%s, pod.Namespace=%s,pod.Status.Reason=%s", podName, namespace, pod.Status.Reason) + klog.Infof("pod.Name=%s, pod.Namespace=%s,pod.Status.Reason=%s,message=%s", podName, namespace, pod.Status.Reason, pod.Status.Message) if pod.Status.Reason != "Evicted" { klog.Infof("skip pod admission request pod=%s, namespace=%s", podName, namespace) diff --git a/pkg/appinstaller/helm.go b/pkg/appinstaller/helm.go index 5be6876..1744d05 100644 --- a/pkg/appinstaller/helm.go +++ b/pkg/appinstaller/helm.go @@ -6,6 +6,7 @@ import ( "errors" "flag" "fmt" + "net/http" "net/http/httputil" "os" "path/filepath" @@ -46,6 +47,7 @@ import ( var ( systemServerHost = "" + middlewareTypes = []string{tapr.TypePostgreSQL.String(), tapr.TypeMongoDB.String(), tapr.TypeRedis.String(), tapr.TypeNats.String()} ) func init() { @@ -649,6 +651,9 @@ func (h *HelmOps) Uninstall() error { } } } + + appCacheDirs, _ := utils.TryToGetAppdataDirFromDeployment(context.TODO(), h.app.Namespace, h.app.AppName, h.app.OwnerName) + err = helm.UninstallCharts(h.actionConfig, h.app.AppName) if err != nil { return err @@ -658,6 +663,46 @@ func (h *HelmOps) Uninstall() error { klog.Errorf("Failed to unregister app err=%v", err) } + // delete middleware requests crd + namespace := fmt.Sprintf("%s-%s", "user-system", h.app.OwnerName) + for _, mt := range middlewareTypes { + name := fmt.Sprintf("%s-%s", h.app.AppName, mt) + err = tapr.DeleteMiddlewareRequest(context.TODO(), h.kubeConfig, namespace, name) + if err != nil && !apierrors.IsNotFound(err) { + klog.Errorf("Failed to delete middleware request namespace=%s name=%s err=%v", namespace, name, err) + } + } + + if len(appCacheDirs) > 0 { + terminusNonce, e := utils.GenTerminusNonce() + if e != nil { + klog.Errorf("Failed to generate terminus nonce err=%v", e) + } else { + c := resty.New().SetTimeout(2*time.Second). + SetHeader(constants.AuthorizationTokenKey, h.token). + SetHeader("Terminus-Nonce", terminusNonce) + nodes, e := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if e == nil { + for _, dir := range appCacheDirs { + for _, n := range nodes.Items { + URL := fmt.Sprintf(constants.AppDataDirURL, h.app.OwnerName, dir) + c.SetHeader("X-Terminus-Node", n.Name) + c.SetHeader("x-bfl-user", h.app.OwnerName) + res, e := c.R().Delete(URL) + if e != nil { + klog.Errorf("Failed to delete dir err=%v", e) + } + if res.StatusCode() != http.StatusOK { + klog.Infof("delete app cache failed with: %v", res.String()) + } + } + } + } else { + klog.Error("Failed to get nodes err=%v", e) + } + } + } + if !utils.IsProtectedNamespace(h.app.Namespace) { return client.CoreV1().Namespaces().Delete(context.TODO(), h.app.Namespace, metav1.DeleteOptions{}) } diff --git a/pkg/utils/app.go b/pkg/utils/app.go index 12362d4..9a31d24 100644 --- a/pkg/utils/app.go +++ b/pkg/utils/app.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "net/netip" + "path/filepath" "strings" "time" @@ -18,6 +19,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" @@ -502,3 +504,74 @@ func parseDestination(dest string) (string, string, error) { return alias, tokens[len(tokens)-1], nil } + +func TryToGetAppdataDirFromDeployment(ctx context.Context, namespace, name, owner string) (dirs []string, err error) { + userspaceNs := UserspaceName(owner) + config, err := ctrl.GetConfig() + if err != nil { + return dirs, err + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return dirs, err + } + sts, err := clientset.AppsV1().StatefulSets(userspaceNs).Get(ctx, "bfl", metav1.GetOptions{}) + if err != nil { + return dirs, err + } + appName := fmt.Sprintf("%s-%s", namespace, name) + appCachePath := sts.GetAnnotations()["appcache_hostpath"] + if len(appCachePath) == 0 { + return dirs, errors.New("empty appcache_hostpath") + } + if !strings.HasSuffix(appCachePath, "/") { + appCachePath += "/" + } + dClient, err := versioned.NewForConfig(config) + if err != nil { + return dirs, err + } + appCRD, err := dClient.AppV1alpha1().Applications().Get(ctx, appName, metav1.GetOptions{}) + if err != nil { + return dirs, err + } + deploymentName := appCRD.Spec.DeploymentName + deployment, err := clientset.AppsV1().Deployments(namespace). + Get(context.Background(), deploymentName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return tryToGetAppdataDirFromSts(ctx, namespace, deploymentName, appCachePath) + } + return dirs, err + } + + for _, v := range deployment.Spec.Template.Spec.Volumes { + if v.HostPath != nil && strings.HasPrefix(v.HostPath.Path, appCachePath) && len(v.HostPath.Path) > len(appCachePath) { + dirs = append(dirs, filepath.Base(v.HostPath.Path)) + } + } + return dirs, nil +} + +func tryToGetAppdataDirFromSts(ctx context.Context, namespace, stsName, baseDir string) (dirs []string, err error) { + config, err := ctrl.GetConfig() + if err != nil { + return dirs, err + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return dirs, err + } + + sts, err := clientset.AppsV1().StatefulSets(namespace). + Get(ctx, stsName, metav1.GetOptions{}) + if err != nil { + return dirs, err + } + for _, v := range sts.Spec.Template.Spec.Volumes { + if v.HostPath != nil && strings.HasPrefix(v.HostPath.Path, baseDir) && len(v.HostPath.Path) > len(baseDir) { + dirs = append(dirs, filepath.Base(v.HostPath.Path)) + } + } + return dirs, nil +}