Update to version v3.16.0

LICENSE

@@ -10,7 +10,7 @@
       "License" shall mean the terms and conditions for use, reproduction,
       and distribution as defined by Sections 1 through 9 of this document.
 
-      "Restream" shall mean the copyright owner or entity authorized by
+      "Licensor" shall mean the copyright owner or entity authorized by
       the copyright owner that is granting the License.
 
       "Legal Entity" shall mean the union of the acting entity and all
@@ -199,4 +199,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.

bindings.go

@@ -5,9 +5,11 @@ import (
 	"fmt"
 	"reflect"
 	"strconv"
-	"strings"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	otelattr "go.opentelemetry.io/otel/attribute"
+
 	"github.com/restream/reindexer/v3/bindings"
 	"github.com/restream/reindexer/v3/bindings/builtinserver/config"
 	"github.com/restream/reindexer/v3/cjson"
@@ -112,27 +114,13 @@ func packItem(ns *reindexerNamespace, item interface{}, json []byte, ser *cjson.
 func (db *reindexerImpl) getNS(namespace string) (*reindexerNamespace, error) {
 	db.lock.RLock()
 	defer db.lock.RUnlock()
-	ns, ok := db.ns[strings.ToLower(namespace)]
+
+	ns, ok := db.ns[namespace]
 	if !ok {
 		return nil, errNsNotFound
 	}
-	return ns, nil
-}
-
-func (db *reindexerImpl) putMeta(ctx context.Context, namespace, key string, data []byte) error {
-	return db.binding.PutMeta(ctx, namespace, key, string(data))
-}
-
-func (db *reindexerImpl) getMeta(ctx context.Context, namespace, key string) ([]byte, error) {
 
-	out, err := db.binding.GetMeta(ctx, namespace, key)
-	if err != nil {
-		return nil, err
-	}
-	defer out.Free()
-	ret := make([]byte, len(out.GetBuf()))
-	copy(ret, out.GetBuf())
-	return ret, nil
+	return ns, nil
 }
 
 func unpackItem(ns *nsArrayEntry, params *rawResultItemParams, allowUnsafe bool, nonCacheableData bool, item interface{}) (interface{}, error) {
@@ -317,15 +305,31 @@ func (db *reindexerImpl) prepareQuery(ctx context.Context, q *Query, asJson bool
 
 // Execute query
 func (db *reindexerImpl) execQuery(ctx context.Context, q *Query) *Iterator {
+	if db.otelTracer != nil {
+		defer db.startTracingSpan(ctx, "Reindexer.Query.Exec", otelattr.String("rx.ns", q.Namespace)).End()
+	}
+
+	if db.promMetrics != nil {
+		defer prometheus.NewTimer(db.promMetrics.clientCallsLatency.WithLabelValues("Query.Exec", q.Namespace)).ObserveDuration()
+	}
+
 	result, err := db.prepareQuery(ctx, q, false)
 	if err != nil {
 		return errIterator(err)
 	}
-	iter := newIterator(ctx, q, result, q.nsArray, q.joinToFields, q.joinHandlers, q.context)
+	iter := newIterator(ctx, q.db, q.Namespace, q, result, q.nsArray, q.joinToFields, q.joinHandlers, q.context)
 	return iter
 }
 
-func (db *reindexerImpl) execJSONQuery(ctx context.Context, q *Query, jsonRoot string) *JSONIterator {
+func (db *reindexerImpl) execToJsonQuery(ctx context.Context, q *Query, jsonRoot string) *JSONIterator {
+	if db.otelTracer != nil {
+		defer db.startTracingSpan(ctx, "Reindexer.Query.ExecToJson", otelattr.String("rx.ns", q.Namespace)).End()
+	}
+
+	if q.db.promMetrics != nil {
+		defer prometheus.NewTimer(q.db.promMetrics.clientCallsLatency.WithLabelValues("Query.ExecToJson", q.Namespace)).ObserveDuration()
+	}
+
 	result, err := db.prepareQuery(ctx, q, true)
 	if err != nil {
 		return errJSONIterator(err)
@@ -360,6 +364,13 @@ func (db *reindexerImpl) prepareSQL(ctx context.Context, namespace, query string
 
 // Execute query
 func (db *reindexerImpl) deleteQuery(ctx context.Context, q *Query) (int, error) {
+	if db.otelTracer != nil {
+		defer db.startTracingSpan(ctx, "Reindexer.Query.Delete", otelattr.String("rx.ns", q.Namespace)).End()
+	}
+
+	if db.promMetrics != nil {
+		defer prometheus.NewTimer(db.promMetrics.clientCallsLatency.WithLabelValues("Query.Delete", q.Namespace)).ObserveDuration()
+	}
 
 	ns, err := db.getNS(q.Namespace)
 	if err != nil {
@@ -393,6 +404,13 @@ func (db *reindexerImpl) deleteQuery(ctx context.Context, q *Query) (int, error)
 
 // Execute query
 func (db *reindexerImpl) updateQuery(ctx context.Context, q *Query) *Iterator {
+	if db.otelTracer != nil {
+		defer db.startTracingSpan(ctx, "Reindexer.Query.Update", otelattr.String("rx.ns", q.Namespace)).End()
+	}
+
+	if db.promMetrics != nil {
+		defer prometheus.NewTimer(db.promMetrics.clientCallsLatency.WithLabelValues("Query.Update", q.Namespace)).ObserveDuration()
+	}
 
 	ns, err := db.getNS(q.Namespace)
 	if err != nil {
@@ -424,17 +442,33 @@ func (db *reindexerImpl) updateQuery(ctx context.Context, q *Query) *Iterator {
 	}
 
 	q.nsArray = append(q.nsArray, nsArrayEntry{ns, ns.cjsonState.Copy()})
-	return newIterator(ctx, q, result, q.nsArray, nil, nil, nil)
+	return newIterator(ctx, q.db, q.Namespace, q, result, q.nsArray, nil, nil, nil)
 }
 
 // Execute query
 func (db *reindexerImpl) updateQueryTx(ctx context.Context, q *Query, tx *Tx) *Iterator {
+	if db.otelTracer != nil {
+		defer db.startTracingSpan(ctx, "Reindexer.Tx.Query.Update", otelattr.String("rx.ns", q.Namespace)).End()
+	}
+
+	if db.promMetrics != nil {
+		defer prometheus.NewTimer(db.promMetrics.clientCallsLatency.WithLabelValues("Tx.Query.Update", q.Namespace)).ObserveDuration()
+	}
+
 	err := db.binding.UpdateQueryTx(&tx.ctx, q.ser.Bytes())
 	return errIterator(err)
 }
 
 // Execute query
 func (db *reindexerImpl) deleteQueryTx(ctx context.Context, q *Query, tx *Tx) (int, error) {
+	if db.otelTracer != nil {
+		defer db.startTracingSpan(ctx, "Reindexer.Tx.Query.Delete", otelattr.String("rx.ns", q.Namespace)).End()
+	}
+
+	if db.promMetrics != nil {
+		defer prometheus.NewTimer(db.promMetrics.clientCallsLatency.WithLabelValues("Tx.Query.Delete", q.Namespace)).ObserveDuration()
+	}
+
 	err := db.binding.DeleteQueryTx(&tx.ctx, q.ser.Bytes())
 	return 0, err
 }
@@ -500,3 +534,11 @@ func WithDedicatedServerThreads() interface{} {
 func WithAppName(appName string) interface{} {
 	return bindings.OptionAppName{AppName: appName}
 }
+
+func WithPrometheusMetrics() interface{} {
+	return bindings.OptionPrometheusMetrics{EnablePrometheusMetrics: true}
+}
+
+func WithOpenTelemetry() interface{} {
+	return bindings.OptionOpenTelemetry{EnableTracing: true}
+}

bindings/builtin/builtin.go

@@ -177,6 +177,10 @@ func (binding *Builtin) Init(u []url.URL, options ...interface{}) error {
 	connectOptions.Opts |= bindings.ConnectOptWarnVersion
 	for _, option := range options {
 		switch v := option.(type) {
+		case bindings.OptionPrometheusMetrics:
+			// nothing
+		case bindings.OptionOpenTelemetry:
+			// nothing
 		case bindings.OptionBuiltinWithServer:
 			// nothing
 		case bindings.OptionCgoLimit:

bindings/builtinserver/builtinserver.go

@@ -80,6 +80,8 @@ func (server *BuiltinServer) Init(u []url.URL, options ...interface{}) error {
 
 	for _, option := range options {
 		switch v := option.(type) {
+		case bindings.OptionPrometheusMetrics:
+		case bindings.OptionOpenTelemetry:
 		case bindings.OptionCgoLimit:
 		case bindings.OptionBuiltintCtxWatch:
 		case bindings.ConnectOptions:

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v3.15.0"
+const ReindexerVersion = "v3.16.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (
@@ -53,6 +53,7 @@ const (
 	ValueUndefined = 9
 	ValueComposite = 10
 	ValueTuple     = 11
+	ValueUuid      = 12
 
 	QueryCondition              = 0
 	QueryDistinct               = 1

bindings/cproto/cproto.go

@@ -76,6 +76,10 @@ func (binding *NetCProto) Init(u []url.URL, options ...interface{}) (err error)
 
 	for _, option := range options {
 		switch v := option.(type) {
+		case bindings.OptionPrometheusMetrics:
+			// nothing
+		case bindings.OptionOpenTelemetry:
+			// nothing
 		case bindings.OptionConnPoolSize:
 			connPoolSize = v.ConnPoolSize
 

bindings/interface.go

@@ -302,6 +302,17 @@ type OptionAppName struct {
 	AppName string
 }
 
+// OptionPrometheusMetrics - enables collection of Reindexer's client side metrics (for example,
+// information about latency and rpc of all rx client calls like Upsert, Select, etc).
+type OptionPrometheusMetrics struct {
+	EnablePrometheusMetrics bool
+}
+
+// OptionOpenTelemetry - enables OpenTelemetry integration.
+type OptionOpenTelemetry struct {
+	EnableTracing bool
+}
+
 type Status struct {
 	Err     error
 	CProto  StatusCProto

changelog.md

@@ -1,3 +1,31 @@
+# Version 3.16.0 (09.06.2023)
+## Core
+- [fea] Add UUID field type for indexes
+- [fix] Fixed fulltext areas highlighting for the queries with prefixes/suffixes/postfixes
+- [fix] Fixed segfault on max unique json names overflowing
+- [fix] Dissalowed system namespaces deletion/renaming
+
+## Go connector
+- [fea] Added support for OpenTelemetry traces. [Details](readme.md#tracing)
+- [fea] Added prometheus client-side metrics. [Details](cpp_src/readme.md#prometheus-client-side-go)
+- [fix] Fixed bool value parsing/validation in DSL package
+
+## Face
+- [fea] Added the Namespace name length limitation
+- [fea] Added the UUID Index type
+- [fea] Added the notification about doubles in the Query Builder filter
+- [fix] Fixed the scroll in the Index table
+- [fix] Fixed the empty table in Statistics->Memory
+- [fix] Removed the select_functions parameter from the JSON preview of Query Builder
+- [fix] Fixed the URL clearing after clicking by opened Namespace tabs
+- [fix] Fixed the Memory table layout
+- [fix] Fixed the console errors on the replication config page
+- [fix] Fixed the column displaying for the Query -> Facet request
+- [fix] Fixed the synonyms displaying for the UI view
+- [fix] Fixed the console errors on the namespace pin icon clicking 
+- [fix] Added the bottom padding to the legend of the Statistics
+- [fix] Fixed the NavigationDuplicated console error on the pagination using on the Queries page
+
 # Version 3.15.0 (24.04.2023)
 ## Core
 - [fea] Improved typos handling agorithm for `text`-indexes. New options: `max_typo_distance`,`max_symbol_permutation_distance`,`max_missing_letters` and `max_extra_letters` were added

cjson/creflect.go

@@ -3,6 +3,7 @@ package cjson
 import (
 	"fmt"
 	"reflect"
+	"strings"
 	"unsafe"
 
 	"github.com/restream/reindexer/v3/bindings"
@@ -14,6 +15,7 @@ const (
 	valueInt64  = bindings.ValueInt64
 	valueDouble = bindings.ValueDouble
 	valueString = bindings.ValueString
+	valueUuid   = bindings.ValueUuid
 )
 
 // to avoid gcc toolchain requirement
@@ -102,6 +104,22 @@ func (pl *payloadIface) ptr(field, idx, typ int) unsafe.Pointer {
 	return unsafe.Pointer(pl.p + uintptr(arr.offset) + uintptr(idx)*f.Size)
 }
 
+const hexChars = "0123456789abcdef"
+
+func createUuid(v [2]uint64) string {
+	var b strings.Builder
+	b.Grow(36)
+	for i, j := 0, 0; i < 36; i++ {
+		switch i {
+		case 8, 13, 18, 23: b.WriteByte('-')
+		default:
+			b.WriteByte(hexChars[(v[j / 16] >> ((15 - j % 16) * 4)) & 0xF])
+			j++
+		}
+	}
+	return b.String()
+}
+
 func (pl *payloadIface) getInt(field, idx int) int {
 	p := pl.ptr(field, idx, valueInt)
 	return int(*(*Cint)(p))
@@ -112,6 +130,11 @@ func (pl *payloadIface) getInt64(field, idx int) int64 {
 	return *(*int64)(p)
 }
 
+func (pl *payloadIface) getUuid(field, idx int) string {
+	p := pl.ptr(field, idx, valueUuid)
+	return createUuid(*(*[2]uint64)(p))
+}
+
 func (pl *payloadIface) getFloat64(field, idx int) float64 {
 	p := pl.ptr(field, idx, valueDouble)
 	return float64(*(*Cdouble)(p))
@@ -177,6 +200,8 @@ func (pl *payloadIface) getValue(field int, idx int, v reflect.Value) {
 		v.SetFloat(pl.getFloat64(field, idx))
 	case valueString:
 		v.SetString(pl.getString(field, idx))
+	case valueUuid:
+		v.SetString(pl.getUuid(field, idx))
 	default:
 		panic(fmt.Errorf("Unknown key value type %d", pl.t.Fields[field].Type))
 	}
@@ -413,6 +438,27 @@ func (pl *payloadIface) getArray(field int, startIdx int, cnt int, v reflect.Val
 			}
 			v.Set(slice)
 		}
+	case valueUuid:
+		pi := (*[1 << 27]uint64)(ptr)[:l * 2:l * 2]
+		if a, ok := v.Addr().Interface().(*[]string); ok {
+			*a = make([]string, cnt, cnt)
+			for i := 0; i < cnt; i++ {
+				(*a)[i] = createUuid([2]uint64{pi[i * 2], pi[i * 2 + 1]})
+			}
+		} else {
+			slice := reflect.MakeSlice(v.Type(), cnt, cnt)
+			for i := 0; i < cnt; i++ {
+				sv := slice.Index(i)
+				if sv.Type().Kind() == reflect.Ptr {
+					el := reflect.New(reflect.New(sv.Type().Elem()).Elem().Type())
+					el.Elem().SetString(createUuid([2]uint64{pi[i * 2], pi[i * 2 + 1]}))
+					sv.Set(el)
+				} else {
+					sv.SetString(createUuid([2]uint64{pi[i * 2], pi[i * 2 + 1]}))
+				}
+			}
+			v.Set(slice)
+		}
 	default:
 		panic(fmt.Errorf("Got C array with elements of unknown C type %d in field '%s' for go type '%s'", pl.t.Fields[field].Type, pl.t.Fields[field].Name, v.Type().Elem().Kind().String()))
 	}
@@ -432,6 +478,8 @@ func (pl *payloadIface) getIface(field int) interface{} {
 			return pl.getFloat64(field, 0)
 		case valueString:
 			return pl.getString(field, 0)
+		case valueUuid:
+			return pl.getUuid(field, 0)
 		}
 	}
 
@@ -462,6 +510,12 @@ func (pl *payloadIface) getIface(field int) interface{} {
 			a[i] = pl.getString(field, i)
 		}
 		return a
+	case valueUuid:
+		a := make([]string, l, l)
+		for i := 0; i < l; i++ {
+			a[i] = pl.getUuid(field, i)
+		}
+		return a
 	}
 
 	return nil