Merge branch 'apache:main' into main

.github/workflows/ci.yml

@@ -68,6 +68,8 @@ jobs:
         run: make check-req
       - name: Generate codes
         run: make generate
+      - name: Build
+        run: make build
       - name: Lint
         run: make lint
       - name: Generate dependencies licenses

.github/workflows/publish-docker.yml

@@ -85,7 +85,8 @@ jobs:
         if: github.ref != 'refs/heads/main' # Only build docker image on PR(Push image when pushed to main branch)
         run: |
           make docker.build || make docker.build
-          make -C test/docker build | make -C test/docker build
+          BINARYTYPE=slim make -C banyand docker || BINARYTYPE=slim make -C banyand docker
+          make -C test/docker build || make -C test/docker build
           docker image ls
       - name: Log in to the Container registry
         uses: docker/[email protected]
@@ -97,5 +98,6 @@ jobs:
       - name: Push docker image
         if: github.ref == 'refs/heads/main'
         run: |
-          PLATFROMS=linux/amd64,linux/arm64,windows/amd64 make docker.push || make docker.push
-          make -C test/docker push | make -C test/docker push
+          PLATFORMS=linux/amd64,linux/arm64,windows/amd64 make docker.push || PLATFROMS=linux/amd64,linux/arm64,windows/amd64 make docker.push
+          PLATFORMS=linux/amd64,linux/arm64,windows/amd64 BINARYTYPE=slim make -C banyand docker.push || PLATFORMS=linux/amd64,linux/arm64,windows/amd64 BINARYTYPE=slim make -C banyand docker.push
+          make -C test/docker push || make -C test/docker push

.github/workflows/test.yml

@@ -25,7 +25,7 @@ on:
       timeout-minutes:
         required: false
         type: number
-        default: 30
+        default: 60
 
 jobs:
   test:
@@ -72,6 +72,8 @@ jobs:
         run: GOPROXY=https://proxy.golang.org go mod download
       - name: Generate mocks
         run: make generate
+      - name: Build
+        run: make build
       - name: Test integration and banyand
         run: TEST_CI_OPTS="--cover --covermode atomic --coverprofile=coverage.out ${{ inputs.options }}" make test-ci
       - name: Upload coverage to Codecov

.golangci.yml

@@ -29,7 +29,6 @@ linters:
     - exhaustive
     # TODO:// enable this lint
     # - exhaustruct
-    - exportloopref
     - gci
     - goconst
     - gocritic
@@ -88,7 +87,7 @@ linters-settings:
     sections:
       - standard
       - default
-      - prefix(github.com/apache/skywalking-banyandb/) 
+      - prefix(github.com/apache/skywalking-banyandb/)
   gocritic:
     enabled-checks:
       - appendCombine
@@ -116,6 +115,9 @@ linters-settings:
     #   toplevel     - for top level comments;
     #   all          - for all comments.
     scope: toplevel
+  gosec:
+    excludes:
+      - G115 # integer overflow conversion (TODO: verify these)
   staticcheck:
     checks: ["all", "-ST1000", "-ST1016", "-ST1020", "-ST1021", "-ST1022"]
   exhaustive:
@@ -129,7 +131,7 @@ linters-settings:
       - 'cobra\.Command$'
 
 run:
-  go: "1.22"
+  go: "1.23"
 issues:
   exclude-rules:
     - path: ".*\\.pb\\.go"

CHANGES.md

@@ -26,6 +26,9 @@ Release Notes.
 - Add liaison, remote queue, storage(rotation), time-series tables, metadata cache and scheduler metrics.
 - Add HTTP health check endpoint for the data node.
 - Add slow query log for the distributed query and local query.
+- Support applying the index rule to the tag belonging to the entity.
+- Add search analyzer "url" which breaks test into tokens at any non-letter and non-digit character.
+- Introduce "match_option" to the "match" query.
 
 ### Bugs
 
@@ -42,6 +45,7 @@ Release Notes.
 - Fix several "sync.Pool" leak issues by adding a tracker to the pool.
 - Fix panic when removing a expired segment.
 - Fix panic when reading a disorder block of measure. This block's versions are not sorted in descending order.
+- Fix the bug that the etcd client doesn't reconnect when facing the context timeout in the startup phase.
 
 ### Documentation
 
@@ -60,8 +64,10 @@ Release Notes.
 - Separate the monolithic Docker image into two images: banyand and bydbctl.
 - Update CI to publish linux/amd64 and linux/arm64 Docker images.
 - Make the build system compiles the binary based on the platform which is running on.
-- Push "skywalking-banyandb-test" image for e2e and stress test. This image contains bydbctl to do a health check.
-- Set etcd-client log level to "error" and etcd-server log level to "error".
+- Push "skywalking-banyandb:<tag>-testing" image for e2e and stress test. This image contains bydbctl to do a health check.
+- Set etcd-client log level to "error" and etcd-server log level to "warn".
+- Push "skywalking-banyandb:<tag>-slim" image for the production environment. This image doesn't contain bydbctl and Web UI.
+- Bump go to 1.23.
 
 ## 0.6.1
 
@@ -121,6 +127,7 @@ Release Notes.
 - Bump all dependencies of Go and Node.
 - Combine banyand and bydbctl Dockerfile.
 - Update readme for bydbctl
+- Introduce the go vulnerability check to "pre-push" task.
 
 ## 0.5.0
 

CONTRIBUTING.md

@@ -41,7 +41,7 @@ Once we've discussed your changes and you've got your code ready, make sure that
 
 Users who want to build a binary from sources have to set up:
 
-* Go 1.22
+* Go 1.23
 * Node 20.12
 * Git >= 2.30
 * Linux, macOS or Windows + WSL2

Makefile

@@ -40,6 +40,9 @@ clean: default  ## Clean artifacts in all projects
 	rm -f .env
 	rm -f *.out
 
+clean-build: TARGET=clean-build
+clean-build: default  ## Clean build artifacts in all projects
+
 generate: TARGET=generate
 generate: PROJECTS:=api $(PROJECTS) pkg
 generate: default  ## Generate API codes
@@ -100,6 +103,11 @@ check-req: ## Check the requirements
 	@$(MAKE) -C scripts/ci/check test
 	@$(MAKE) -C ui check-version
 
+include scripts/build/vuln.mk
+
+vuln-check: $(GOVULNCHECK)
+	$(GOVULNCHECK) -show color,verbose ./...	
+
 check: ## Check that the status is consistent with CI
 	$(MAKE) license-check
 	$(MAKE) format
@@ -120,6 +128,7 @@ pre-push: ## Check source files before pushing to the remote repo
 	$(MAKE) lint
 	$(MAKE) license-dep
 	$(MAKE) check
+	$(MAKE) vuln-check
 
 ##@ License targets
 
@@ -177,7 +186,7 @@ RELEASE_SCRIPTS := $(mk_dir)/scripts/release.sh
 release-binary: release-source ## Package binary archive
 	${RELEASE_SCRIPTS} -b
 
-release-source: clean ## Package source archive
+release-source: ## Package source archive
 	${RELEASE_SCRIPTS} -s
 
 release-sign: ## Sign artifacts
@@ -187,7 +196,11 @@ release-sign: ## Sign artifacts
 
 release-assembly: release-binary release-sign ## Generate release package
 
+PUSH_RELEASE_SCRIPTS := $(mk_dir)/scripts/push-release.sh
 
+release-push-candidate: ## Push release candidate
+	${PUSH_RELEASE_SCRIPTS}
+
 .PHONY: all $(PROJECTS) clean build  default nuke
 .PHONY: lint check tidy format pre-push
 .PHONY: test test-race test-coverage test-ci

api/proto/banyandb/database/v1/schema.proto

@@ -168,19 +168,16 @@ message IndexRule {
   Type type = 3 [(validate.rules).enum.defined_only = true];
   // updated_at indicates when the IndexRule is updated
   google.protobuf.Timestamp updated_at = 4;
-  enum Analyzer {
-    ANALYZER_UNSPECIFIED = 0;
-    // Keyword analyzer is a “noop” analyzer which returns the entire input string as a single token.
-    ANALYZER_KEYWORD = 1;
-    // Standard analyzer provides grammar based tokenization
-    ANALYZER_STANDARD = 2;
-    // Simple analyzer breaks text into tokens at any non-letter character,
-    // such as numbers, spaces, hyphens and apostrophes, discards non-letter characters,
-    // and changes uppercase to lowercase.
-    ANALYZER_SIMPLE = 3;
-  }
+
   // analyzer analyzes tag value to support the full-text searching for TYPE_INVERTED indices.
-  Analyzer analyzer = 5;
+  // available analyzers are:
+  // - "standard" provides grammar based tokenization
+  // - "simple" breaks text into tokens at any non-letter character,
+  //            such as numbers, spaces, hyphens and apostrophes, discards non-letter characters,
+  //            and changes uppercase to lowercase.
+  // - "keyword" is a “noop” analyzer which returns the entire input string as a single token.
+  // - "url" breaks test into tokens at any non-letter and non-digit character.
+  string analyzer = 5;
   // no_sort indicates whether the index is not for sorting.
   bool no_sort = 6;
 }

api/proto/banyandb/model/v1/query.proto

@@ -67,6 +67,16 @@ message Condition {
   string name = 1;
   BinaryOp op = 2;
   TagValue value = 3;
+  message MatchOption {
+    string analyzer = 1;
+    enum Operator {
+      OPERATOR_UNSPECIFIED = 0;
+      OPERATOR_AND = 1;
+      OPERATOR_OR = 2;
+    }
+    Operator operator = 2;
+  }
+  MatchOption match_option = 4;
 }
 
 // tag_families are indexed.

api/proto/banyandb/property/v1/rpc.proto

@@ -103,7 +103,7 @@ service PropertyService {
   rpc List(ListRequest) returns (ListResponse) {
     option (google.api.http) = {
       get: "/v1/property/lists/{container.group}/{container.name}/{ids}/{tags}"
-      additional_bindings {get: "/v1/property/lists/{container.group}"}
+      additional_bindings: {get: "/v1/property/lists/{container.group}"}
     };
   }
   rpc KeepAlive(KeepAliveRequest) returns (KeepAliveResponse) {

api/proto/buf.yaml

@@ -24,7 +24,7 @@ deps:
   - buf.build/envoyproxy/protoc-gen-validate
 lint:
   use:
-    - DEFAULT
+    - STANDARD
 breaking:
   use:
-    - FILE
+    - FILE

banyand/Dockerfile

@@ -21,8 +21,9 @@ RUN apk add --no-cache ca-certificates && update-ca-certificates
 FROM busybox:stable-glibc AS build-linux
 
 ARG TARGETARCH
+ARG BINARYTYPE
 
-COPY build/bin/linux/${TARGETARCH}/banyand-server-static /banyand
+COPY build/bin/linux/${TARGETARCH}/banyand-server-${BINARYTYPE} /banyand
 COPY --from=certs /etc/ssl/certs /etc/ssl/certs
 
 FROM mcr.microsoft.com/windows/servercore:ltsc2022 AS build-windows

banyand/Makefile

@@ -17,8 +17,7 @@
 #
 
 NAME := banyand
-SERVER := $(NAME)-server
-BINARIES := $(SERVER)
+BINARIES := $(NAME)-server
 
 IMG_NAME := skywalking-banyandb
 
@@ -34,7 +33,4 @@ include ../scripts/build/help.mk
 
 prepare-build: generate
 
-docker.dev:
-	@echo "Building $(IMG) with platform $(PLATFORMS)"
-	@pwd
-	time docker buildx build $(DOCKER_BUILD_ARGS) --platform $(PLATFORMS) --load --no-cache -t $(IMG) -f Dockerfile.dev --provenance=false .
+release: $(STATIC_BINARIES) $(SLIM_BINARIES)

banyand/liaison/http/rpath_empty.go

@@ -0,0 +1,25 @@
+//go:build slim
+// +build slim
+
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package http
+
+func (p *server) setRootPath() error {
+	return nil
+}

banyand/liaison/http/rpath_ui.go

@@ -0,0 +1,40 @@
+//go:build !slim
+// +build !slim
+
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package http
+
+import (
+	"io/fs"
+	"net/http"
+
+	"github.com/apache/skywalking-banyandb/ui"
+)
+
+func (p *server) setRootPath() error {
+	fSys, err := fs.Sub(ui.DistContent, "dist")
+	if err != nil {
+		return err
+	}
+	httpFS := http.FS(fSys)
+	fileServer := http.FileServer(http.FS(fSys))
+	serveIndex := serveFileContents("index.html", httpFS)
+	p.mux.Mount("/", intercept404(fileServer, serveIndex))
+	return nil
+}

banyand/liaison/http/server.go

@@ -21,7 +21,6 @@ package http
 import (
 	"context"
 	"fmt"
-	"io/fs"
 	"net"
 	"net/http"
 	"strconv"
@@ -43,7 +42,6 @@ import (
 	"github.com/apache/skywalking-banyandb/pkg/healthcheck"
 	"github.com/apache/skywalking-banyandb/pkg/logger"
 	"github.com/apache/skywalking-banyandb/pkg/run"
-	"github.com/apache/skywalking-banyandb/ui"
 )
 
 var (
@@ -137,14 +135,9 @@ func (p *server) PreRun(_ context.Context) error {
 	p.l = logger.GetLogger(p.Name())
 	p.mux = chi.NewRouter()
 
-	fSys, err := fs.Sub(ui.DistContent, "dist")
-	if err != nil {
+	if err := p.setRootPath(); err != nil {
 		return err
 	}
-	httpFS := http.FS(fSys)
-	fileServer := http.FileServer(http.FS(fSys))
-	serveIndex := serveFileContents("index.html", httpFS)
-	p.mux.Mount("/", intercept404(fileServer, serveIndex))
 	p.srv = &http.Server{
 		Addr:              p.listenAddr,
 		Handler:           p.mux,

banyand/measure/block.go

@@ -258,7 +258,7 @@ func (b *block) unmarshalTagFamilyFromSeqReaders(decoder *encoding.BytesBlockDec
 func (b *block) uncompressedSizeBytes() uint64 {
 	dataPointsCount := uint64(b.Len())
 
-	n := dataPointsCount * 8
+	n := dataPointsCount * (8 + 8) // 8 bytes for timestamp and 8 bytes for version
 
 	tff := b.tagFamilies
 	for i := range tff {