feat: add custom-columns capability in rest-api/v1/search (#618)

## What type of PR is this?

/kind feature

## What this PR does / why we need it:

The current search interface returns the full data stored in
Elasticsearch to the front-end. The front-end needs to perform an
operation to extract certain fields from the complete object and display
them to the user.

This PR moves the field extraction operation to the back-end, similar to
`kubectl get -o 'custom-columns=<spec>`. This will parameterize the data
returned by the back-end, simplify the integration workload for
third-party front-ends, and reduce the amount of data transmitted to the
front-end.

For example:

`curl -k
'https://127.0.0.1:7443/rest-api/v1/search?query=select+*+from+resources+where+kind+%3D+%27Pod%27+and+name+%3D+%27karpor-syncer-6dfddf556-djn9c%27&pattern=sql&page=1&pageSize=20&format=custom-columns%3DNAME%3Ametadata.name%2CAPI_VERSION%3AapiVersion'`

Output:

```json
{
  "success": true,
  "message": "OK",
  "data": {
    "items": [
      {
        "cluster": "kind",
        "object": {
          "fields": {
            "API_VERSION": "v1",
            "NAME": "karpor-syncer-6dfddf556-djn9c"
          },
          "titles": ["NAME", "API_VERSION"]
        }
      }
    ],
    "total": 1,
    "currentPage": 1,
    "pageSize": 20
  },
  "traceID": "karpor-server-db4c78b4b-6h628/mBPrQuHocQ-000034",
  "startTime": "2024-09-19T09:43:37.033057526Z",
  "endTime": "2024-09-19T09:43:37.618264576Z",
  "costTime": "585.20705ms"
}
```

## Which issue(s) this PR fixes:

<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related
issues/tests in a comment and do not use `Fixes`_*
-->

Fixes #

pkg/core/handler/search/formatter.go

@@ -0,0 +1,140 @@
+// Copyright The Karpor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package search
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	jp "github.com/KusionStack/karpor/pkg/util/jsonpath"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/util/jsonpath"
+)
+
+type Formatter interface {
+	Format(obj runtime.Object) (any, error)
+}
+
+// ParseObjectFormatter parses a format string and returns a Formatter.
+func ParseObjectFormatter(format string) (Formatter, error) {
+	parts := strings.SplitN(format, "=", 2)
+	if parts[0] == "" || parts[0] == "origin" {
+		return &NopFormatter{}, nil
+	}
+
+	spec := ""
+	if len(parts) > 1 {
+		spec = parts[1]
+	}
+
+	switch parts[0] {
+	case "custom-columns":
+		return NewCustomColumnsFormatter(spec)
+	default:
+		return nil, fmt.Errorf("unsupported format: %s", parts[0])
+	}
+}
+
+type NopFormatter struct{}
+
+// Format keeps the obj unchanged.
+func (f *NopFormatter) Format(obj runtime.Object) (any, error) {
+	return obj, nil
+}
+
+// NewCustomColumnsFormatter creates a custom columns formatter from a comma separated list of <header>:<jsonpath-field-spec> pairs.
+// e.g. NAME:metadata.name,API_VERSION:apiVersion creates a formatter that formats the input to:
+//
+// {"fields":{"API_VERSION":"v1","NAME":"foo"},"titles":["NAME","API_VERSION"]}
+func NewCustomColumnsFormatter(spec string) (*customColumnsFormatter, error) {
+	if len(spec) == 0 {
+		return nil, fmt.Errorf("custom-columns format specified but no custom columns given")
+	}
+
+	parts := strings.Split(spec, ",")
+	parsers := make([]*jsonpath.JSONPath, len(parts))
+	titles := make([]string, len(parts))
+
+	for ix := range parts {
+		colSpec := strings.SplitN(parts[ix], ":", 2)
+		if len(colSpec) != 2 {
+			return nil, fmt.Errorf("unexpected custom-columns spec: %s, expected <header>:<json-path-expr>", parts[ix])
+		}
+
+		spec, err := jp.RelaxedJSONPathExpression(colSpec[1])
+		if err != nil {
+			return nil, err
+		}
+
+		parsers[ix] = jsonpath.New(fmt.Sprintf("column%d", ix)).AllowMissingKeys(true)
+		if err := parsers[ix].Parse(spec); err != nil {
+			return nil, err
+		}
+
+		titles[ix] = colSpec[0]
+	}
+
+	return &customColumnsFormatter{titles: titles, parsers: parsers}, nil
+}
+
+type customColumnsFormatter struct {
+	titles  []string
+	parsers []*jsonpath.JSONPath
+}
+
+type CustomColumnsOutput struct {
+	Fields map[string]any `json:"fields,omitempty"`
+
+	// Titles is to imply the keys order as the keys order of the fields is random.
+	Titles []string `json:"titles,omitempty"`
+}
+
+// Format likes the `kubectl get -o 'custom-columns=<spec>'`, extracts and returns specified fields from input.
+func (f *customColumnsFormatter) Format(obj runtime.Object) (any, error) {
+	fields := map[string]any{}
+
+	for ix, parser := range f.parsers {
+		var values [][]reflect.Value
+		var err error
+		if unstructured, ok := obj.(runtime.Unstructured); ok {
+			values, err = parser.FindResults(unstructured.UnstructuredContent())
+		} else {
+			values, err = parser.FindResults(reflect.ValueOf(obj).Elem().Interface())
+		}
+
+		if err != nil {
+			return nil, err
+		}
+
+		if len(values) == 0 || len(values[0]) == 0 {
+			fields[f.titles[ix]] = nil
+			continue
+		}
+
+		typed := make([]any, len(values[0]))
+		for valIx, val := range values[0] {
+			typed[valIx] = val.Interface()
+		}
+
+		if len(typed) == 1 {
+			fields[f.titles[ix]] = typed[0]
+		} else {
+			fields[f.titles[ix]] = typed
+		}
+	}
+
+	return CustomColumnsOutput{Fields: fields, Titles: f.titles}, nil
+}

pkg/core/handler/search/formatter_test.go

@@ -0,0 +1,185 @@
+// Copyright The Karpor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package search
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+var (
+	podData = []byte(`
+{
+    "apiVersion": "v1",
+    "kind": "Pod",
+    "metadata": {
+        "labels": {
+            "name": "pause",
+            "app": "pause"
+        },
+        "name": "pause",
+        "namespace": "default"
+    },
+    "spec": {
+        "containers": [
+            {
+                "image": "registry.k8s.io/pause:3.8",
+                "imagePullPolicy": "IfNotPresent",
+                "name": "pause1"
+            },
+            {
+                "image": "registry.k8s.io/pause:3.8",
+                "imagePullPolicy": "IfNotPresent",
+                "name": "pause2"
+            }
+        ]
+    }
+}`)
+
+	eventData = []byte(`
+{
+    "apiVersion": "v1",
+	"count": 1,
+    "involvedObject": {
+        "apiVersion": "v1",
+        "kind": "Pod",
+        "name": "karpor-server-db4c78b4b-5jhhn",
+        "namespace": "karpor"
+    },
+    "kind": "Event",
+    "metadata": {
+        "creationTimestamp": "2024-09-18T08:52:22Z",
+        "name": "karpor-server-db4c78b4b-5jhhn.17f64a9c530cdb7c",
+        "namespace": "karpor"
+    },
+    "reason": "Scheduled"
+}`)
+
+	pod      corev1.Pod
+	event    corev1.Event
+	unObjPod unstructured.Unstructured
+)
+
+func init() {
+	json.Unmarshal(podData, &pod)
+	json.Unmarshal(eventData, &event)
+	json.Unmarshal(podData, &unObjPod)
+}
+
+func Test_customColumnsFormatter_Format(t *testing.T) {
+	type input struct {
+		spec string
+		objs []runtime.Object
+	}
+
+	tests := []struct {
+		name    string
+		input   input
+		want    any
+		wantErr bool
+	}{
+		{name: "containers name", input: input{spec: "CONTAINER_NAME:spec.containers[*].name", objs: []runtime.Object{&unObjPod}}, want: CustomColumnsOutput{Fields: map[string]any{"CONTAINER_NAME": []any{"pause1", "pause2"}}, Titles: []string{"CONTAINER_NAME"}}, wantErr: false},
+		{name: "dual containers name", input: input{spec: "CONTAINER_NAME:spec.containers[*].name", objs: []runtime.Object{&pod, &unObjPod}}, want: CustomColumnsOutput{Fields: map[string]any{"CONTAINER_NAME": []any{"pause1", "pause2"}}, Titles: []string{"CONTAINER_NAME"}}, wantErr: false},
+		{name: "first container name", input: input{spec: "CONTAINER_NAME:spec.containers[0].name", objs: []runtime.Object{&unObjPod}}, want: CustomColumnsOutput{Fields: map[string]any{"CONTAINER_NAME": "pause1"}, Titles: []string{"CONTAINER_NAME"}}, wantErr: false},
+		{name: "invalid path", input: input{spec: "NAME:{.kind} {.apiVersion}", objs: []runtime.Object{&unObjPod}}, want: nil, wantErr: true},
+		{name: "name and apiVersion", input: input{spec: "NAME:metadata.name,API_VERSION:apiVersion", objs: []runtime.Object{&pod, &pod}}, want: CustomColumnsOutput{Fields: map[string]any{"NAME": "pause", "API_VERSION": "v1"}, Titles: []string{"NAME", "API_VERSION"}}, wantErr: false},
+		{name: "count", input: input{spec: "COUNT:count", objs: []runtime.Object{&event}}, want: CustomColumnsOutput{Fields: map[string]any{"COUNT": int32(1)}, Titles: []string{"COUNT"}}, wantErr: false},
+		{name: "not exist key", input: input{spec: "NOT_EXIST:spec.containers[*].xx.yy", objs: []runtime.Object{&event}}, want: CustomColumnsOutput{Fields: map[string]any{"NOT_EXIST": nil}, Titles: []string{"NOT_EXIST"}}, wantErr: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			formatter, err := NewCustomColumnsFormatter(tt.input.spec)
+			if err != nil {
+				if !tt.wantErr {
+					t.Errorf("NewCustomColumnsFormatter error = %v, wantErr %v", err, tt.wantErr)
+				}
+				return
+			}
+
+			for _, obj := range tt.input.objs {
+				got, err := formatter.Format(obj)
+
+				if (err != nil) != tt.wantErr {
+					t.Errorf("customColumnsFormatter.Format() error = %v, wantErr %v", err, tt.wantErr)
+					return
+				}
+
+				if !reflect.DeepEqual(got, tt.want) {
+					t.Errorf("customColumnsFormatter.Format() = %v, want %v", got, tt.want)
+				}
+			}
+		})
+	}
+}
+
+func TestParseObjectFormatter(t *testing.T) {
+	tests := []struct {
+		name    string
+		format  string
+		want    Formatter
+		wantErr bool
+	}{
+		{name: "empty format", format: "", want: &NopFormatter{}, wantErr: false},
+		{name: "origin format", format: "origin", want: &NopFormatter{}, wantErr: false},
+		{name: "unsupported format", format: "yaml", want: nil, wantErr: true},
+		{name: "empty custom-columns spec", format: "custom-columns=", want: &customColumnsFormatter{}, wantErr: true},
+		{name: "custom-columns", format: "custom-columns=NAME:.metadata.name", want: &customColumnsFormatter{}, wantErr: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseObjectFormatter(tt.format)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseObjectFormatter() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			if reflect.TypeOf(got) != reflect.TypeOf(tt.want) {
+				t.Errorf("ParseObjectFormatter() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestNopFormatter_Format(t *testing.T) {
+	tests := []struct {
+		name    string
+		obj     runtime.Object
+		want    any
+		wantErr bool
+	}{
+		{name: "pod", obj: &pod, want: &pod, wantErr: false},
+		{name: "event", obj: &event, want: &event, wantErr: false},
+		{name: "nil", obj: nil, want: nil, wantErr: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := &NopFormatter{}
+			got, err := f.Format(tt.obj)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("NopFormatter.Format() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("NopFormatter.Format() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/core/handler/search/search.go

@@ -56,6 +56,14 @@ func SearchForResource(searchMgr *search.SearchManager, searchStorage storage.Se
 		searchPattern := r.URL.Query().Get("pattern")
 		searchPageSize, _ := strconv.Atoi(r.URL.Query().Get("pageSize"))
 		searchPage, _ := strconv.Atoi(r.URL.Query().Get("page"))
+
+		format := r.URL.Query().Get("format")
+		formatter, err := ParseObjectFormatter(format)
+		if err != nil {
+			handler.FailureWithCodeRender(ctx, w, r, err, http.StatusBadRequest)
+			return
+		}
+
 		if searchPageSize <= 1 {
 			searchPageSize = 10
 		}
@@ -72,12 +80,19 @@ func SearchForResource(searchMgr *search.SearchManager, searchStorage storage.Se
 		}
 
 		rt := &search.UniResourceList{}
-		for _, r := range res.Resources {
+		for _, res := range res.Resources {
 			unObj := &unstructured.Unstructured{}
-			unObj.SetUnstructuredContent(r.Object)
+			unObj.SetUnstructuredContent(res.Object)
+
+			obj, err := formatter.Format(unObj)
+			if err != nil {
+				handler.FailureRender(ctx, w, r, err)
+				return
+			}
+
 			rt.Items = append(rt.Items, search.UniResource{
-				Cluster: r.Cluster,
-				Object:  unObj,
+				Cluster: res.Cluster,
+				Object:  obj,
 			})
 		}
 		rt.Total = res.Total

pkg/core/manager/search/types.go

@@ -16,7 +16,6 @@ package search
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 )
 
 type SearchManager struct{}
@@ -27,8 +26,8 @@ func NewSearchManager() *SearchManager {
 }
 
 type UniResource struct {
-	Cluster string         `json:"cluster"`
-	Object  runtime.Object `json:"object"`
+	Cluster string `json:"cluster"`
+	Object  any    `json:"object"`
 }
 
 type UniResourceList struct {