Add slim and testing images (#529)

* Add slim and testing images

---------

Signed-off-by: Gao Hongtao <[email protected]>

.github/workflows/ci.yml

@@ -68,6 +68,8 @@ jobs:
         run: make check-req
       - name: Generate codes
         run: make generate
+      - name: Build
+        run: make build
       - name: Lint
         run: make lint
       - name: Generate dependencies licenses

.github/workflows/publish-docker.yml

@@ -85,7 +85,8 @@ jobs:
         if: github.ref != 'refs/heads/main' # Only build docker image on PR(Push image when pushed to main branch)
         run: |
           make docker.build || make docker.build
-          make -C test/docker build | make -C test/docker build
+          BINARYTYPE=slim make -C banyand docker || BINARYTYPE=slim make -C banyand docker
+          make -C test/docker build || make -C test/docker build
           docker image ls
       - name: Log in to the Container registry
         uses: docker/[email protected]
@@ -97,5 +98,6 @@ jobs:
       - name: Push docker image
         if: github.ref == 'refs/heads/main'
         run: |
-          PLATFROMS=linux/amd64,linux/arm64,windows/amd64 make docker.push || make docker.push
-          make -C test/docker push | make -C test/docker push
+          PLATFORMS=linux/amd64,linux/arm64,windows/amd64 make docker.push || PLATFROMS=linux/amd64,linux/arm64,windows/amd64 make docker.push
+          PLATFORMS=linux/amd64,linux/arm64,windows/amd64 BINARYTYPE=slim make -C banyand docker.push || PLATFORMS=linux/amd64,linux/arm64,windows/amd64 BINARYTYPE=slim make -C banyand docker.push
+          make -C test/docker push || make -C test/docker push

.github/workflows/test.yml

@@ -72,6 +72,8 @@ jobs:
         run: GOPROXY=https://proxy.golang.org go mod download
       - name: Generate mocks
         run: make generate
+      - name: Build
+        run: make build
       - name: Test integration and banyand
         run: TEST_CI_OPTS="--cover --covermode atomic --coverprofile=coverage.out ${{ inputs.options }}" make test-ci
       - name: Upload coverage to Codecov

CHANGES.md

@@ -60,8 +60,9 @@ Release Notes.
 - Separate the monolithic Docker image into two images: banyand and bydbctl.
 - Update CI to publish linux/amd64 and linux/arm64 Docker images.
 - Make the build system compiles the binary based on the platform which is running on.
-- Push "skywalking-banyandb-test" image for e2e and stress test. This image contains bydbctl to do a health check.
-- Set etcd-client log level to "error" and etcd-server log level to "error".
+- Push "skywalking-banyandb:<tag>-testing" image for e2e and stress test. This image contains bydbctl to do a health check.
+- Set etcd-client log level to "error" and etcd-server log level to "warn".
+- Push "skywalking-banyandb:<tag>-slim" image for the production environment. This image doesn't contain bydbctl and Web UI.
 
 ## 0.6.1
 

Makefile

@@ -40,6 +40,9 @@ clean: default  ## Clean artifacts in all projects
 	rm -f .env
 	rm -f *.out
 
+clean-build: TARGET=clean-build
+clean-build: default  ## Clean build artifacts in all projects
+
 generate: TARGET=generate
 generate: PROJECTS:=api $(PROJECTS) pkg
 generate: default  ## Generate API codes

banyand/Dockerfile

@@ -21,8 +21,9 @@ RUN apk add --no-cache ca-certificates && update-ca-certificates
 FROM busybox:stable-glibc AS build-linux
 
 ARG TARGETARCH
+ARG BINARYTYPE
 
-COPY build/bin/linux/${TARGETARCH}/banyand-server-static /banyand
+COPY build/bin/linux/${TARGETARCH}/banyand-server-${BINARYTYPE} /banyand
 COPY --from=certs /etc/ssl/certs /etc/ssl/certs
 
 FROM mcr.microsoft.com/windows/servercore:ltsc2022 AS build-windows

banyand/Makefile

@@ -17,8 +17,7 @@
 #
 
 NAME := banyand
-SERVER := $(NAME)-server
-BINARIES := $(SERVER)
+BINARIES := $(NAME)-server
 
 IMG_NAME := skywalking-banyandb
 
@@ -34,7 +33,4 @@ include ../scripts/build/help.mk
 
 prepare-build: generate
 
-docker.dev:
-	@echo "Building $(IMG) with platform $(PLATFORMS)"
-	@pwd
-	time docker buildx build $(DOCKER_BUILD_ARGS) --platform $(PLATFORMS) --load --no-cache -t $(IMG) -f Dockerfile.dev --provenance=false .
+release: $(STATIC_BINARIES) $(SLIM_BINARIES)

banyand/liaison/http/rpath_empty.go

@@ -0,0 +1,25 @@
+//go:build slim
+// +build slim
+
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package http
+
+func (p *server) setRootPath() error {
+	return nil
+}

banyand/liaison/http/rpath_ui.go

@@ -0,0 +1,40 @@
+//go:build !slim
+// +build !slim
+
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package http
+
+import (
+	"io/fs"
+	"net/http"
+
+	"github.com/apache/skywalking-banyandb/ui"
+)
+
+func (p *server) setRootPath() error {
+	fSys, err := fs.Sub(ui.DistContent, "dist")
+	if err != nil {
+		return err
+	}
+	httpFS := http.FS(fSys)
+	fileServer := http.FileServer(http.FS(fSys))
+	serveIndex := serveFileContents("index.html", httpFS)
+	p.mux.Mount("/", intercept404(fileServer, serveIndex))
+	return nil
+}

banyand/liaison/http/server.go

@@ -21,7 +21,6 @@ package http
 import (
 	"context"
 	"fmt"
-	"io/fs"
 	"net"
 	"net/http"
 	"strconv"
@@ -43,7 +42,6 @@ import (
 	"github.com/apache/skywalking-banyandb/pkg/healthcheck"
 	"github.com/apache/skywalking-banyandb/pkg/logger"
 	"github.com/apache/skywalking-banyandb/pkg/run"
-	"github.com/apache/skywalking-banyandb/ui"
 )
 
 var (
@@ -137,14 +135,9 @@ func (p *server) PreRun(_ context.Context) error {
 	p.l = logger.GetLogger(p.Name())
 	p.mux = chi.NewRouter()
 
-	fSys, err := fs.Sub(ui.DistContent, "dist")
-	if err != nil {
+	if err := p.setRootPath(); err != nil {
 		return err
 	}
-	httpFS := http.FS(fSys)
-	fileServer := http.FileServer(http.FS(fSys))
-	serveIndex := serveFileContents("index.html", httpFS)
-	p.mux.Mount("/", intercept404(fileServer, serveIndex))
 	p.srv = &http.Server{
 		Addr:              p.listenAddr,
 		Handler:           p.mux,

docs/installation/binaries.md

@@ -26,8 +26,10 @@ The banyand and bydbctl directory structure is as follows.
 ├── NOTICE
 ├── README.md
 ├── bin
-│   ├── banyand-linux-arm64
-│   └── banyand-linux-amd64
+│   ├──banyand-server-slim-linux-amd64 
+│   ├──banyand-server-slim-linux-arm64 
+│   ├──banyand-server-static-linux-amd64 
+│   └──banyand-server-static-linux-arm64
 └── licenses
 ```
 
@@ -38,13 +40,13 @@ The banyand and bydbctl directory structure is as follows.
 ├── NOTICE
 ├── README.md
 ├── bin
-│   ├── bydbctl-linux-386
-│   ├── bydbctl-linux-amd64
-│   ├── bydbctl-linux-arm64
-│   ├── bydbctl-windows-386
-│   ├── bydbctl-windows-amd64
-│   ├── bydbctl-darwin-amd64
-│   └── bydbctl-darwin-arm64
+│   ├── bydbctl-cli-static-linux-386
+│   ├── bydbctl-cli-static-linux-amd64
+│   ├── bydbctl-cli-static-linux-arm64
+│   ├── bydbctl-cli-static-windows-386
+│   ├── bydbctl-cli-static-windows-amd64
+│   ├── bydbctl-cli-static-darwin-amd64
+│   └── bydbctl-cli-static-darwin-arm64
 └── licenses
 ```
 
@@ -72,7 +74,9 @@ To issue the below command to get basic binaries of banyand and bydbctl.
 make generate
 ...
 make build
+--- ui: all ---
 ...
+Done building ui
 --- banyand: all ---
 ...
 chmod +x build/bin/banyand-server;
@@ -84,26 +88,26 @@ chmod +x build/bin/dev/bydbctl-cli;
 Done building build/bin/dev/bydbctl-cli
 ```
 
-The build system provides a series of binary options as well.
-
-* `make -C banyand banyand-server` generates a basic `banyand-server`.
-* `make -C banyand release` or `make -C banyand banyand-server-static` builds out a static binary `banyand-server-static` for releasing.
-* `make -C bydbctl bydbctl-cli` generates a basic `bydbctl-cli`.
-* `make -C bydbctl release` or `make -C banyand bydbctl-cli-static` builds out a static binary `bydbctl-cli-static` for releasing.
-
 Then users get binaries as below
 
 ``` shell
 ls banyand/build/bin/dev
 banyand-server  
-banyand-server-static
 
 ls bydbctl/build/bin/dev
 bydbctl-cli
-bydbctl-cli-static
 ```
 
-> The build script now checks if the binary file exists before rebuilding. If you want to rebuild, please remove the binary file manually.
+The build system provides a series of binary options as well.
+
+* `make -C banyand banyand-server` generates a basic `banyand-server`.
+* `make -C banyand banyand-server-static` builds out a static binary `banyand-server-static` which is statically linked with all dependencies.
+* `make -C banyand banyand-server-slim` builds out a slim binary `banyand-server-slim` which doesn't include `UI`.
+* `make -C banyand release` builds out the static and slim binaries for releasing.
+* `make -C bydbctl bydbctl-cli` generates a basic `bydbctl-cli`.
+* `make -C bydbctl release` or `make -C banyand bydbctl-cli-static` builds out a static binary `bydbctl-cli-static` for releasing. This binary is statically linked with all dependencies.
+
+> The build script now checks if the binary file exists before rebuilding. If you want to rebuild, please remove the binary file manually by running `make clean-build`.
 
 ### Cross-compile Binaries
 
@@ -113,7 +117,7 @@ The build system supports cross-compiling binaries for different platforms. For
 TARGET_OS=windows PLATFORMS=windows/amd64 make release
 ```
 
-The `PLATFORMS` variable is a list of platforms separated by commas. The `TARGET_OS` variable is the target operating system. You could specify several platforms at once: 
+The `PLATFORMS` variable is a list of platforms separated by commas. The `TARGET_OS` variable is the target operating system. You could specify several platforms at once:
 
 ```shell
 TARGET_OS=linux PLATFORMS=linux/amd64,linux/arm64 make release

docs/installation/docker.md

@@ -1,5 +1,26 @@
 # Installation On Docker
 
+## Images on Docker Hub
+
+The BanyanDB images are hosted on Docker Hub. You can pull the images from the following links: [Apache SkyWalking BanyanDB](https://hub.docker.com/r/apache/skywalking-banyandb)
+
+There are two types of images:
+
+- `apache/skywalking-banyandb:<version>` - The specific version of the BanyanDB.
+- `apache/skywalking-banyandb:<version>-slim` - The slim version of the BanyanDB. It does not contain the Web UI.
+
+We pushed `linux/amd64` and `linux/arm64` for each type of image. You can pull the image for the specific architecture.
+
+## Images on GitHub Container Registry
+
+The BanyanDB images are hosted on GitHub Container Registry for development or testing. You can pull the images from the following links: [ghcr.io/apache/skywalking-banyandb](https://github.com/apache/skywalking-banyandb/pkgs/container/skywalking-banyandb)
+
+There are three types of images:
+
+- `ghcr.io/apache/skywalking-banyandb:<github-sha>` - The specific version of the BanyanDB. We pushed `linux/amd64`,  `linux/arm64` and `windows/amd64` for each type of image.
+- `ghcr.io/apache/skywalking-banyandb:<github-sha>-slim` - The slim version of the BanyanDB. It does not contain the Web UI. We pushed `linux/amd64`,  `linux/arm64` and `windows/amd64` for each type of image.
+- `ghcr.io/apache/skywalking-banyandb:<github-sha>-testing` - The testing version of the BanyanDB. It contains the Web UI and the `bydbctl`. We pushed `linux/amd64` and  `linux/arm64` for each type of image.
+
 ## Start a container in `standalone mode`
 The following commands pull the docker image and run the BanyanDB on Docker. Replace `latest` with the version of the BanyanDB you want to run.
 - pull the image

scripts/build/build.mk

@@ -25,6 +25,7 @@ $(error The BINARIES variable should be set to the name binaries to produce)
 endif
 
 STATIC_BINARIES ?= $(addsuffix -static,$(BINARIES))
+SLIM_BINARIES ?= $(addsuffix -slim,$(BINARIES))
 BUILD_DIR ?= build/bin
 TARGET_OS ?= linux
 OS := ${TARGET_OS}
@@ -61,8 +62,19 @@ $(STATIC_BINARIES_GOBUILD_TARGET): $(BUILD_DIR)/$(OS)/%-static: $(BUILD_LOCK)
 	$(call go_build_static_executable,,-s -w)
 	@echo "Done building static $*"
 
+SLIM_BINARIES_GOBUILD_TARGET_PATTERN := $(foreach goarch,$(GOBUILD_ARCHS),$(BUILD_DIR)/$(OS)/$(goarch)/$(NAME)-%-slim)
+SLIM_BINARIES_GOBUILD_TARGET := $(foreach goarch,$(GOBUILD_ARCHS),$(addprefix $(BUILD_DIR)/$(OS)/$(goarch)/,$(SLIM_BINARIES)))
+$(SLIM_BINARIES): $(NAME)-%-slim: $(SLIM_BINARIES_GOBUILD_TARGET_PATTERN)
+$(SLIM_BINARIES_GOBUILD_TARGET): $(BUILD_DIR)/$(OS)/%-slim: $(BUILD_LOCK)
+	$(call set_build_package,$*,$@)
+	@echo "Building slim $*"
+	$(MAKE) prepare-build
+	$(eval BUILD_TAGS := $(BUILD_TAGS) slim)
+	$(call go_build_static_executable,,-s -w)
+	@echo "Done building static $*"
+
 .PHONY: release
-release: $(STATIC_BINARIES)   ## Build the release binaries
+release: $(STATIC_BINARIES) ## Build the release binaries
 
 .PHONY: clean-build
 clean-build:  ## Clean all artifacts

scripts/build/docker.mk

@@ -27,6 +27,12 @@ endif
 # The tag of the docker image. The default value if latest.
 TAG ?= latest
 
+BINARYTYPE ?= static
+
+ifeq ($(BINARYTYPE),slim)
+	TAG := $(TAG)-slim
+endif
+
 IMG := $(HUB)/$(IMG_NAME):$(TAG)
 
 # Disable cache in CI environment
@@ -42,5 +48,5 @@ docker.push: DOCKER_TYPE = "Push"
 docker docker.push:
 	@echo "$(DOCKER_TYPE) $(IMG) with platform $(PLATFORMS)"
 	@pwd
-	time docker buildx build $(DOCKER_BUILD_ARGS) --platform $(PLATFORMS) $(LOAD_OR_PUSH) -t $(IMG) -f Dockerfile --provenance=false .
+	time docker buildx build $(DOCKER_BUILD_ARGS) --platform $(PLATFORMS) $(LOAD_OR_PUSH) -t $(IMG) -f Dockerfile --provenance=false --build-arg BINARYTYPE=$(BINARYTYPE) .
 

scripts/release.sh

@@ -67,7 +67,6 @@ copy_binaries() {
         # Extract os and arch from the path
         os_arch=$(echo ${binary} | awk -F'/' '{print $(NF-2)"/"$(NF-1)}')
         binary_name=$(basename ${binary})
-        binary_name=${binary_name%-*-*}
         cp -Rfv ${binary} ${bindir}/bin/${binary_name}-${os_arch//\//-}
     done
 }

test/docker/Makefile

@@ -23,7 +23,7 @@ HUB ?= apache
 # The tag of the docker image. The default value if latest.
 TAG ?= latest
 
-IMG := $(HUB)/skywalking-banyandb-test:$(TAG)
+IMG := $(HUB)/skywalking-banyandb:$(TAG)-testing
 
 build:
 	@echo "Building $(IMG)"