Skip to content

Commit

Permalink
fix: delete app cache when cancel installation (#128)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hysyeah
hysyeah authored Jan 10, 2025
1 parent bcad95f commit 7b2ab3b
Show file tree
Hide file tree
Showing 5 changed files with 133 additions and 135 deletions.
123 changes: 0 additions & 123 deletions controllers/appmgr_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ import (
"errors"
"fmt"
"math"
"net/http"
"path/filepath"
"reflect"
"strconv"
"strings"
Expand All @@ -17,14 +15,11 @@ import (
"bytetrade.io/web3os/app-service/pkg/apiserver/api"
"bytetrade.io/web3os/app-service/pkg/appinstaller"
"bytetrade.io/web3os/app-service/pkg/constants"
"bytetrade.io/web3os/app-service/pkg/generated/clientset/versioned"
"bytetrade.io/web3os/app-service/pkg/helm"
"bytetrade.io/web3os/app-service/pkg/tapr"
"bytetrade.io/web3os/app-service/pkg/task"
"bytetrade.io/web3os/app-service/pkg/users/userspace"
"bytetrade.io/web3os/app-service/pkg/utils"

"github.com/go-resty/resty/v2"
"helm.sh/helm/v3/pkg/action"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand All @@ -33,7 +28,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/util/retry"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog/v2"
Expand All @@ -56,8 +50,6 @@ type ApplicationManagerController struct {
client.Client
}

var middlewareTypes = []string{tapr.TypePostgreSQL.String(), tapr.TypeMongoDB.String(), tapr.TypeRedis.String(), tapr.TypeNats.String()}

// SetupWithManager sets up the ApplicationManagerController with the provided controller manager
func (r *ApplicationManagerController) SetupWithManager(mgr ctrl.Manager) error {
c, err := controller.New("appmgr-controller", mgr, controller.Options{
Expand Down Expand Up @@ -485,10 +477,6 @@ func (r *ApplicationManagerController) uninstall(ctx context.Context, appMgr *ap
if err != nil {
return err
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return err
}

token := appMgr.Status.Payload["token"]
version = appMgr.Status.Payload["version"]
Expand All @@ -499,126 +487,15 @@ func (r *ApplicationManagerController) uninstall(ctx context.Context, appMgr *ap
}
ops, err := appinstaller.NewHelmOps(ctx, config, appConfig, token, appinstaller.Opt{})

appCacheDirs, _ := tryToGetAppdataDirFromDeployment(ctx, appConfig, appMgr.Spec.AppOwner)
err = ops.Uninstall()
if err != nil {
return err
}

// delete middleware requests crd
namespace := fmt.Sprintf("%s-%s", "user-system", appConfig.OwnerName)
for _, mt := range middlewareTypes {
name := fmt.Sprintf("%s-%s", appConfig.AppName, mt)
err = tapr.DeleteMiddlewareRequest(ctx, config, namespace, name)
if err != nil && !apierrors.IsNotFound(err) {
klog.Errorf("Failed to delete middleware request namespace=%s name=%s err=%v", namespace, name, err)
}
}

if len(appCacheDirs) > 0 {
terminusNonce, e := utils.GenTerminusNonce()
if e != nil {
klog.Errorf("Failed to generate terminus nonce err=%v", e)
} else {
c := resty.New().SetTimeout(2*time.Second).
SetHeader(constants.AuthorizationTokenKey, token).
SetHeader("Terminus-Nonce", terminusNonce)
nodes, e := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
if e == nil {
for _, dir := range appCacheDirs {
for _, n := range nodes.Items {
URL := fmt.Sprintf(constants.AppDataDirURL, appMgr.Spec.AppOwner, dir)
c.SetHeader("X-Terminus-Node", n.Name)
c.SetHeader("x-bfl-user", appMgr.Spec.AppOwner)
res, e := c.R().Delete(URL)
if e != nil {
klog.Errorf("Failed to delete dir err=%v", e)
}
if res.StatusCode() != http.StatusOK {
klog.Infof("delete app cache failed with: %v", res.String())
}
}
}
} else {
klog.Error("Failed to get nodes err=%v", e)
}
}
}
return nil

}

func tryToGetAppdataDirFromDeployment(ctx context.Context, appconfig *appinstaller.ApplicationConfig, owner string) (dirs []string, err error) {
userspaceNs := utils.UserspaceName(owner)
config, err := ctrl.GetConfig()
if err != nil {
return dirs, err
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return dirs, err
}
sts, err := clientset.AppsV1().StatefulSets(userspaceNs).Get(ctx, "bfl", metav1.GetOptions{})
if err != nil {
return dirs, err
}
appName := fmt.Sprintf("%s-%s", appconfig.Namespace, appconfig.AppName)
appCachePath := sts.GetAnnotations()["appcache_hostpath"]
if len(appCachePath) == 0 {
return dirs, errors.New("empty appcache_hostpath")
}
if !strings.HasSuffix(appCachePath, "/") {
appCachePath += "/"
}
dClient, err := versioned.NewForConfig(config)
if err != nil {
return dirs, err
}
appCRD, err := dClient.AppV1alpha1().Applications().Get(ctx, appName, metav1.GetOptions{})
if err != nil {
return dirs, err
}
deploymentName := appCRD.Spec.DeploymentName
deployment, err := clientset.AppsV1().Deployments(appconfig.Namespace).
Get(context.Background(), deploymentName, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
return tryToGetAppdataDirFromSts(ctx, appconfig.Namespace, deploymentName, appCachePath)
}
return dirs, err
}

for _, v := range deployment.Spec.Template.Spec.Volumes {
if v.HostPath != nil && strings.HasPrefix(v.HostPath.Path, appCachePath) && len(v.HostPath.Path) > len(appCachePath) {
dirs = append(dirs, filepath.Base(v.HostPath.Path))
}
}
return dirs, nil
}

func tryToGetAppdataDirFromSts(ctx context.Context, namespace, stsName, baseDir string) (dirs []string, err error) {
config, err := ctrl.GetConfig()
if err != nil {
return dirs, err
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return dirs, err
}

sts, err := clientset.AppsV1().StatefulSets(namespace).
Get(ctx, stsName, metav1.GetOptions{})
if err != nil {
return dirs, err
}
for _, v := range sts.Spec.Template.Spec.Volumes {
if v.HostPath != nil && strings.HasPrefix(v.HostPath.Path, baseDir) && len(v.HostPath.Path) > len(baseDir) {
dirs = append(dirs, filepath.Base(v.HostPath.Path))
}
}
return dirs, nil
}

func (r *ApplicationManagerController) upgrade(ctx context.Context, appMgr *appv1alpha1.ApplicationManager) (err error) {
var version string
var revision int
Expand Down
25 changes: 14 additions & 11 deletions pkg/apiserver/handler_middleware.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@ func (h *Handler) installMiddleware(req *restful.Request, resp *restful.Response
if e != nil {
klog.Error(e)
}
delete(middlewareManager, name)
return
}
case <-ctx.Done():
Expand Down Expand Up @@ -314,6 +315,17 @@ func (h *Handler) cancelMiddleware(req *restful.Request, resp *restful.Response)
if cancelType == "" {
cancelType = "operate"
}
name, err := utils.FmtAppMgrName(app, owner, "")
if err != nil {
api.HandleError(resp, req, err)
return
}
cancel, ok := middlewareManager[name]
if !ok {
api.HandleError(resp, req, errors.New("can not execute cancel"))
return
}
cancel()
now := metav1.Now()
status := v1alpha1.ApplicationManagerStatus{
OpType: v1alpha1.CancelOp,
Expand All @@ -322,23 +334,14 @@ func (h *Handler) cancelMiddleware(req *restful.Request, resp *restful.Response)
StatusTime: &now,
UpdateTime: &now,
}
name, err := utils.FmtAppMgrName(app, owner, "")
if err != nil {
api.HandleError(resp, req, err)
return
}

_, err = utils.UpdateAppMgrStatus(name, status)

if err != nil {
api.HandleError(resp, req, err)
return
}
cancel, ok := middlewareManager[name]
if !ok {
api.HandleError(resp, req, errors.New("can not execute cancel"))
return
}
cancel()

resp.WriteAsJson(api.InstallationResponse{
Response: api.Response{Code: 200},
Data: api.InstallationResponseData{UID: app},
Expand Down
2 changes: 1 addition & 1 deletion pkg/apiserver/handler_webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,7 +457,7 @@ func (h *Handler) eviction2stop(ctx context.Context, req *admissionv1.AdmissionR
}
podName := pod.GetName()
namespace := pod.GetNamespace()
klog.Infof("pod.Name=%s, pod.Namespace=%s,pod.Status.Reason=%s", podName, namespace, pod.Status.Reason)
klog.Infof("pod.Name=%s, pod.Namespace=%s,pod.Status.Reason=%s,message=%s", podName, namespace, pod.Status.Reason, pod.Status.Message)

if pod.Status.Reason != "Evicted" {
klog.Infof("skip pod admission request pod=%s, namespace=%s", podName, namespace)
Expand Down
45 changes: 45 additions & 0 deletions pkg/appinstaller/helm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"flag"
"fmt"
"net/http"
"net/http/httputil"
"os"
"path/filepath"
Expand Down Expand Up @@ -46,6 +47,7 @@ import (

var (
systemServerHost = ""
middlewareTypes = []string{tapr.TypePostgreSQL.String(), tapr.TypeMongoDB.String(), tapr.TypeRedis.String(), tapr.TypeNats.String()}
)

func init() {
Expand Down Expand Up @@ -649,6 +651,9 @@ func (h *HelmOps) Uninstall() error {
}
}
}

appCacheDirs, _ := utils.TryToGetAppdataDirFromDeployment(context.TODO(), h.app.Namespace, h.app.AppName, h.app.OwnerName)

err = helm.UninstallCharts(h.actionConfig, h.app.AppName)
if err != nil {
return err
Expand All @@ -658,6 +663,46 @@ func (h *HelmOps) Uninstall() error {
klog.Errorf("Failed to unregister app err=%v", err)
}

// delete middleware requests crd
namespace := fmt.Sprintf("%s-%s", "user-system", h.app.OwnerName)
for _, mt := range middlewareTypes {
name := fmt.Sprintf("%s-%s", h.app.AppName, mt)
err = tapr.DeleteMiddlewareRequest(context.TODO(), h.kubeConfig, namespace, name)
if err != nil && !apierrors.IsNotFound(err) {
klog.Errorf("Failed to delete middleware request namespace=%s name=%s err=%v", namespace, name, err)
}
}

if len(appCacheDirs) > 0 {
terminusNonce, e := utils.GenTerminusNonce()
if e != nil {
klog.Errorf("Failed to generate terminus nonce err=%v", e)
} else {
c := resty.New().SetTimeout(2*time.Second).
SetHeader(constants.AuthorizationTokenKey, h.token).
SetHeader("Terminus-Nonce", terminusNonce)
nodes, e := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
if e == nil {
for _, dir := range appCacheDirs {
for _, n := range nodes.Items {
URL := fmt.Sprintf(constants.AppDataDirURL, h.app.OwnerName, dir)
c.SetHeader("X-Terminus-Node", n.Name)
c.SetHeader("x-bfl-user", h.app.OwnerName)
res, e := c.R().Delete(URL)
if e != nil {
klog.Errorf("Failed to delete dir err=%v", e)
}
if res.StatusCode() != http.StatusOK {
klog.Infof("delete app cache failed with: %v", res.String())
}
}
}
} else {
klog.Error("Failed to get nodes err=%v", e)
}
}
}

if !utils.IsProtectedNamespace(h.app.Namespace) {
return client.CoreV1().Namespaces().Delete(context.TODO(), h.app.Namespace, metav1.DeleteOptions{})
}
Expand Down
Loading

0 comments on commit 7b2ab3b

Please sign in to comment.