feat: add tailscale acls support for OlaresManifest.yaml

api/app.bytetrade.io/v1alpha1/application_types.go

@@ -44,12 +44,20 @@ type ApplicationSpec struct {
 	//Entrances []Entrance `json:"entrances,omitempty"`
 	Entrances []Entrance `json:"entrances,omitempty"`
 
-	Ports []ServicePort `json:"ports,omitempty"`
+	Ports         []ServicePort `json:"ports,omitempty"`
+	TailScaleACLs []ACL         `json:"tailscaleAcls,omitempty"`
 
 	// the extend settings of the application
 	Settings map[string]string `json:"settings,omitempty"`
 }
 
+type ACL struct {
+	Action string   `json:"action,omitempty"`
+	Src    []string `json:"src,omitempty"`
+	Proto  string   `json:"proto"`
+	Dst    []string `json:"dst"`
+}
+
 type EntranceState string
 
 const (

cmd/app-service/main.go

@@ -129,6 +129,13 @@ func main() {
 		os.Exit(1)
 	}
 
+	if err = (&controllers.TailScaleACLController{
+		Client: mgr.GetClient(),
+	}).SetUpWithManager(mgr); err != nil {
+		setupLog.Error(err, "Unable to create controller", "controller", "tailScaleACLA manager")
+		os.Exit(1)
+	}
+
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {

config/crd/bases/app.bytetrade.io_applications.yaml

@@ -127,8 +127,8 @@ spec:
                       format: int32
                       type: integer
                     protocol:
-                      description: The protocol for this entrance. Supports "TCP"
-                        and "UDP". Default is TCP.
+                      description: The protocol for this entrance. Supports "tcp"
+                        and "udp". Default is tcp.
                       type: string
                   required:
                   - host
@@ -141,6 +141,26 @@ spec:
                   type: string
                 description: the extend settings of the application
                 type: object
+              tailscaleAcls:
+                items:
+                  properties:
+                    action:
+                      type: string
+                    dst:
+                      items:
+                        type: string
+                      type: array
+                    proto:
+                      type: string
+                    src:
+                      items:
+                        type: string
+                      type: array
+                  required:
+                  - dst
+                  - proto
+                  type: object
+                type: array
             required:
             - appid
             - isSysApp

controllers/application_controller.go

@@ -330,6 +330,10 @@ func (r *ApplicationReconciler) createApplication(ctx context.Context, req ctrl.
 	if err != nil {
 		klog.Errorf("failed to get app ports err=%v", err)
 	}
+	tailScaleACLs, err := r.getAppACLs(deployment)
+	if err != nil {
+		klog.Errorf("failed to get app tailscale acls err=%v", err)
+	}
 
 	var appid string
 	var isSysApp bool
@@ -355,6 +359,7 @@ func (r *ApplicationReconciler) createApplication(ctx context.Context, req ctrl.
 			DeploymentName: deployment.GetName(),
 			Entrances:      entrancesMap[name],
 			Ports:          servicePortsMap[name],
+			TailScaleACLs:  tailScaleACLs,
 			Icon:           icon[name],
 			Settings:       settings,
 		},
@@ -415,6 +420,11 @@ func (r *ApplicationReconciler) updateApplication(ctx context.Context, req ctrl.
 	deployment client.Object, app *appv1alpha1.Application, name string) error {
 	appCopy := app.DeepCopy()
 
+	tailScaleACLs, err := r.getAppACLs(deployment)
+	if err != nil {
+		klog.Errorf("failed to get tailscale err=%v", err)
+	}
+
 	owner := deployment.GetLabels()[constants.ApplicationOwnerLabel]
 	klog.Infof("in updateApplication ....")
 	icons := getAppIcon(deployment)
@@ -428,6 +438,8 @@ func (r *ApplicationReconciler) updateApplication(ctx context.Context, req ctrl.
 	appCopy.Spec.DeploymentName = deployment.GetName()
 	appCopy.Spec.Icon = icon
 
+	appCopy.Spec.TailScaleACLs = tailScaleACLs
+
 	actionConfig, _, err := helm.InitConfig(r.Kubeconfig, appCopy.Spec.Namespace)
 	if err != nil {
 		ctrl.Log.Error(err, "init helm config error")
@@ -708,6 +720,16 @@ func (r *ApplicationReconciler) getAppPorts(ctx context.Context, deployment clie
 	return portsMap, nil
 }
 
+func (r *ApplicationReconciler) getAppACLs(deployment client.Object) ([]appv1alpha1.ACL, error) {
+	acls := make([]appv1alpha1.ACL, 0)
+	aclsString := deployment.GetAnnotations()[constants.ApplicationTailScaleACLKey]
+	err := json.Unmarshal([]byte(aclsString), &acls)
+	if err != nil {
+		return nil, err
+	}
+	return acls, nil
+}
+
 func checkPortOfService(s *corev1.Service, port int32) bool {
 	for _, p := range s.Spec.Ports {
 		if p.Port == port {

controllers/appmgr_controller.go

@@ -449,9 +449,6 @@ func (r *ApplicationManagerController) install(ctx context.Context, appMgr *appv
 		klog.Errorf("Failed to update applicationmanagers status name=%s err=%v", appMgr.Name, err)
 	}
 
-	err = r.Get(ctx, types.NamespacedName{Name: appMgr.Name}, appMgr)
-	var curAppMgr appv1alpha1.ApplicationManager
-	err = r.Get(ctx, types.NamespacedName{Name: appMgr.Name}, &curAppMgr)
 	return err
 }
 

controllers/tailscale_acl_controller.go

@@ -0,0 +1,194 @@
+package controllers
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"bytetrade.io/web3os/app-service/api/app.bytetrade.io/v1alpha1"
+	"bytetrade.io/web3os/app-service/pkg/utils"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+const tailScaleACLPolicyMd5Key = "tailscale-acl-md5"
+
+var defaultHTTPSACL = v1alpha1.ACL{
+	Action: "accept",
+	Src:    []string{"*"},
+	Proto:  "",
+	Dst:    []string{"*:443"},
+}
+
+type ACLPolicy struct {
+	ACLs          []v1alpha1.ACL `json:"acls"`
+	AutoApprovers AutoApprovers  `json:"autoApprovers"`
+}
+
+type AutoApprovers struct {
+	Routes   map[string][]string `json:"routes"`
+	ExitNode []string            `json:"exitNode"`
+}
+
+type TailScaleACLController struct {
+	client.Client
+}
+
+func (r *TailScaleACLController) SetUpWithManager(mgr ctrl.Manager) error {
+	c, err := controller.New("app's tailscale acls manager controller", mgr, controller.Options{
+		Reconciler: r,
+	})
+	if err != nil {
+		return err
+	}
+	err = c.Watch(
+		&source.Kind{Type: &v1alpha1.Application{}},
+		handler.EnqueueRequestsFromMapFunc(
+			func(obj client.Object) []reconcile.Request {
+				app, ok := obj.(*v1alpha1.Application)
+				if !ok {
+					return nil
+				}
+				return []reconcile.Request{{NamespacedName: types.NamespacedName{
+					Name:      app.Name,
+					Namespace: app.Spec.Owner,
+				}}}
+			}),
+		predicate.Funcs{
+			CreateFunc: func(e event.CreateEvent) bool {
+				return true
+			},
+			UpdateFunc: func(e event.UpdateEvent) bool {
+				return true
+			},
+			DeleteFunc: func(e event.DeleteEvent) bool {
+				return true
+			},
+		},
+	)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r *TailScaleACLController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	_ = log.FromContext(ctx)
+	klog.Infof("reconcile tailscale acls request name=%v, owner=%v", req.Name, req.Namespace)
+
+	// for this request req.Namespace is owner
+	// list all apps by owner and generate acls by owner
+	var apps v1alpha1.ApplicationList
+	err := r.List(ctx, &apps)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	filteredApps := make([]v1alpha1.Application, 0)
+	for _, app := range apps.Items {
+		if app.Spec.Owner != req.Namespace {
+			continue
+		}
+		filteredApps = append(filteredApps, app)
+	}
+
+	tailScaleACLConfig := "tailscale-acl"
+	headScaleNamespace := fmt.Sprintf("user-space-%s", req.Namespace)
+
+	// calculate acls
+	acls := make([]v1alpha1.ACL, 0)
+	for _, app := range filteredApps {
+		acls = append(acls, app.Spec.TailScaleACLs...)
+	}
+	aclPolicyByte, err := makeACLPolicy(acls)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	klog.Infof("aclPolicyByte:string: %s", string(aclPolicyByte))
+	configMap := &corev1.ConfigMap{}
+	err = r.Get(ctx, types.NamespacedName{Name: tailScaleACLConfig, Namespace: headScaleNamespace}, configMap)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	oldTailScaleACLPolicyMd5Sum := ""
+	if configMap.Annotations != nil {
+		oldTailScaleACLPolicyMd5Sum = configMap.Annotations[tailScaleACLPolicyMd5Key]
+	}
+	curTailScaleACLPolicyMd5Sum := utils.Md5String(string(aclPolicyByte))
+
+	if curTailScaleACLPolicyMd5Sum != oldTailScaleACLPolicyMd5Sum {
+		if configMap.Annotations == nil {
+			configMap.Annotations = make(map[string]string)
+		}
+		if configMap.Data == nil {
+			configMap.Data = make(map[string]string)
+		}
+
+		configMap.Annotations[tailScaleACLPolicyMd5Key] = curTailScaleACLPolicyMd5Sum
+		configMap.Data["acl.json"] = string(aclPolicyByte)
+		err = r.Update(ctx, configMap)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
+	deploy := &appsv1.Deployment{}
+	err = r.Get(ctx, types.NamespacedName{Namespace: headScaleNamespace, Name: "headscale"}, deploy)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	headScaleACLMd5 := ""
+	if deploy.Spec.Template.Annotations != nil {
+		headScaleACLMd5 = deploy.Spec.Template.Annotations[tailScaleACLPolicyMd5Key]
+	}
+	if headScaleACLMd5 != curTailScaleACLPolicyMd5Sum {
+		if deploy.Spec.Template.Annotations == nil {
+			deploy.Spec.Template.Annotations = make(map[string]string)
+		}
+
+		// update headscale deploy template annotations for rolling update
+		deploy.Spec.Template.Annotations[tailScaleACLPolicyMd5Key] = curTailScaleACLPolicyMd5Sum
+		err = r.Update(ctx, deploy)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		klog.Infof("rolling update headscale...")
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func makeACLPolicy(acls []v1alpha1.ACL) ([]byte, error) {
+	acls = append(acls, defaultHTTPSACL)
+	for i := range acls {
+		acls[i].Action = "accept"
+		acls[i].Src = []string{"*"}
+	}
+	aclPolicy := ACLPolicy{
+		ACLs: acls,
+		AutoApprovers: AutoApprovers{
+			Routes: map[string][]string{
+				"10.0.0.0/8":     {"default"},
+				"172.16.0.0/12":  {"default"},
+				"192.168.0.0/16": {"default"},
+			},
+			ExitNode: []string{},
+		},
+	}
+	aclPolicyByte, err := json.Marshal(aclPolicy)
+	if err != nil {
+		return nil, err
+	}
+	return aclPolicyByte, nil
+}

pkg/apiserver/handler_appupgrade.go

@@ -80,6 +80,11 @@ func (h *Handler) appUpgrade(req *restful.Request, resp *restful.Response) {
 		api.HandleError(resp, req, err)
 		return
 	}
+	err = utils.CheckTailScaleACLs(appConfig.TailScaleACLs)
+	if err != nil {
+		api.HandleError(resp, req, err)
+		return
+	}
 
 	if !utils.MatchVersion(appConfig.CfgFileVersion, MinCfgFileVersion) {
 		api.HandleBadRequest(resp, req, fmt.Errorf("olaresManifest.version must %s", MinCfgFileVersion))