From 401613ad63bf67d56a3b5e5f7eff6509237b291b Mon Sep 17 00:00:00 2001 From: Gao Hongtao Date: Sun, 8 Sep 2024 16:19:05 +0800 Subject: [PATCH] Add slim and testing images (#529) * Add slim and testing images --------- Signed-off-by: Gao Hongtao --- .github/workflows/ci.yml | 2 + .github/workflows/publish-docker.yml | 8 ++-- .github/workflows/test.yml | 2 + CHANGES.md | 5 ++- Makefile | 3 ++ banyand/Dockerfile | 3 +- banyand/Makefile | 8 +--- banyand/liaison/http/rpath_empty.go | 25 +++++++++++ banyand/liaison/http/rpath_ui.go | 40 +++++++++++++++++ banyand/liaison/http/server.go | 9 +--- docs/installation/binaries.md | 44 ++++++++++--------- docs/installation/docker.md | 21 +++++++++ scripts/build/build.mk | 14 +++++- scripts/build/docker.mk | 8 +++- scripts/release.sh | 1 - test/docker/Makefile | 2 +- .../script/docker-compose/base-compose.yml | 6 +-- ui/Makefile | 9 ++-- 18 files changed, 160 insertions(+), 50 deletions(-) create mode 100644 banyand/liaison/http/rpath_empty.go create mode 100644 banyand/liaison/http/rpath_ui.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 828957d3e..110fd177f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,8 @@ jobs: run: make check-req - name: Generate codes run: make generate + - name: Build + run: make build - name: Lint run: make lint - name: Generate dependencies licenses diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 94db8db5e..c823da3fb 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -85,7 +85,8 @@ jobs: if: github.ref != 'refs/heads/main' # Only build docker image on PR(Push image when pushed to main branch) run: | make docker.build || make docker.build - make -C test/docker build | make -C test/docker build + BINARYTYPE=slim make -C banyand docker || BINARYTYPE=slim make -C banyand docker + make -C test/docker build || make -C test/docker build docker image ls - name: Log in to the Container registry uses: docker/login-action@v1.10.0 @@ -97,5 +98,6 @@ jobs: - name: Push docker image if: github.ref == 'refs/heads/main' run: | - PLATFROMS=linux/amd64,linux/arm64,windows/amd64 make docker.push || make docker.push - make -C test/docker push | make -C test/docker push \ No newline at end of file + PLATFORMS=linux/amd64,linux/arm64,windows/amd64 make docker.push || PLATFROMS=linux/amd64,linux/arm64,windows/amd64 make docker.push + PLATFORMS=linux/amd64,linux/arm64,windows/amd64 BINARYTYPE=slim make -C banyand docker.push || PLATFORMS=linux/amd64,linux/arm64,windows/amd64 BINARYTYPE=slim make -C banyand docker.push + make -C test/docker push || make -C test/docker push \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2de36378a..025d955ab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,6 +72,8 @@ jobs: run: GOPROXY=https://proxy.golang.org go mod download - name: Generate mocks run: make generate + - name: Build + run: make build - name: Test integration and banyand run: TEST_CI_OPTS="--cover --covermode atomic --coverprofile=coverage.out ${{ inputs.options }}" make test-ci - name: Upload coverage to Codecov diff --git a/CHANGES.md b/CHANGES.md index 4cafc211c..19e6f99ed 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -60,8 +60,9 @@ Release Notes. - Separate the monolithic Docker image into two images: banyand and bydbctl. - Update CI to publish linux/amd64 and linux/arm64 Docker images. - Make the build system compiles the binary based on the platform which is running on. -- Push "skywalking-banyandb-test" image for e2e and stress test. This image contains bydbctl to do a health check. -- Set etcd-client log level to "error" and etcd-server log level to "error". +- Push "skywalking-banyandb:-testing" image for e2e and stress test. This image contains bydbctl to do a health check. +- Set etcd-client log level to "error" and etcd-server log level to "warn". +- Push "skywalking-banyandb:-slim" image for the production environment. This image doesn't contain bydbctl and Web UI. ## 0.6.1 diff --git a/Makefile b/Makefile index c8389e229..644e822c5 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,9 @@ clean: default ## Clean artifacts in all projects rm -f .env rm -f *.out +clean-build: TARGET=clean-build +clean-build: default ## Clean build artifacts in all projects + generate: TARGET=generate generate: PROJECTS:=api $(PROJECTS) pkg generate: default ## Generate API codes diff --git a/banyand/Dockerfile b/banyand/Dockerfile index 5ba61d7c5..42db44e58 100644 --- a/banyand/Dockerfile +++ b/banyand/Dockerfile @@ -21,8 +21,9 @@ RUN apk add --no-cache ca-certificates && update-ca-certificates FROM busybox:stable-glibc AS build-linux ARG TARGETARCH +ARG BINARYTYPE -COPY build/bin/linux/${TARGETARCH}/banyand-server-static /banyand +COPY build/bin/linux/${TARGETARCH}/banyand-server-${BINARYTYPE} /banyand COPY --from=certs /etc/ssl/certs /etc/ssl/certs FROM mcr.microsoft.com/windows/servercore:ltsc2022 AS build-windows diff --git a/banyand/Makefile b/banyand/Makefile index 0ccedd6dd..f39cf0ee8 100644 --- a/banyand/Makefile +++ b/banyand/Makefile @@ -17,8 +17,7 @@ # NAME := banyand -SERVER := $(NAME)-server -BINARIES := $(SERVER) +BINARIES := $(NAME)-server IMG_NAME := skywalking-banyandb @@ -34,7 +33,4 @@ include ../scripts/build/help.mk prepare-build: generate -docker.dev: - @echo "Building $(IMG) with platform $(PLATFORMS)" - @pwd - time docker buildx build $(DOCKER_BUILD_ARGS) --platform $(PLATFORMS) --load --no-cache -t $(IMG) -f Dockerfile.dev --provenance=false . \ No newline at end of file +release: $(STATIC_BINARIES) $(SLIM_BINARIES) \ No newline at end of file diff --git a/banyand/liaison/http/rpath_empty.go b/banyand/liaison/http/rpath_empty.go new file mode 100644 index 000000000..d126093de --- /dev/null +++ b/banyand/liaison/http/rpath_empty.go @@ -0,0 +1,25 @@ +//go:build slim +// +build slim + +// Licensed to Apache Software Foundation (ASF) under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Apache Software Foundation (ASF) licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package http + +func (p *server) setRootPath() error { + return nil +} diff --git a/banyand/liaison/http/rpath_ui.go b/banyand/liaison/http/rpath_ui.go new file mode 100644 index 000000000..460239a29 --- /dev/null +++ b/banyand/liaison/http/rpath_ui.go @@ -0,0 +1,40 @@ +//go:build !slim +// +build !slim + +// Licensed to Apache Software Foundation (ASF) under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Apache Software Foundation (ASF) licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package http + +import ( + "io/fs" + "net/http" + + "github.com/apache/skywalking-banyandb/ui" +) + +func (p *server) setRootPath() error { + fSys, err := fs.Sub(ui.DistContent, "dist") + if err != nil { + return err + } + httpFS := http.FS(fSys) + fileServer := http.FileServer(http.FS(fSys)) + serveIndex := serveFileContents("index.html", httpFS) + p.mux.Mount("/", intercept404(fileServer, serveIndex)) + return nil +} diff --git a/banyand/liaison/http/server.go b/banyand/liaison/http/server.go index 5fa460b71..576b8289a 100644 --- a/banyand/liaison/http/server.go +++ b/banyand/liaison/http/server.go @@ -21,7 +21,6 @@ package http import ( "context" "fmt" - "io/fs" "net" "net/http" "strconv" @@ -43,7 +42,6 @@ import ( "github.com/apache/skywalking-banyandb/pkg/healthcheck" "github.com/apache/skywalking-banyandb/pkg/logger" "github.com/apache/skywalking-banyandb/pkg/run" - "github.com/apache/skywalking-banyandb/ui" ) var ( @@ -137,14 +135,9 @@ func (p *server) PreRun(_ context.Context) error { p.l = logger.GetLogger(p.Name()) p.mux = chi.NewRouter() - fSys, err := fs.Sub(ui.DistContent, "dist") - if err != nil { + if err := p.setRootPath(); err != nil { return err } - httpFS := http.FS(fSys) - fileServer := http.FileServer(http.FS(fSys)) - serveIndex := serveFileContents("index.html", httpFS) - p.mux.Mount("/", intercept404(fileServer, serveIndex)) p.srv = &http.Server{ Addr: p.listenAddr, Handler: p.mux, diff --git a/docs/installation/binaries.md b/docs/installation/binaries.md index 02cad7482..62610e199 100644 --- a/docs/installation/binaries.md +++ b/docs/installation/binaries.md @@ -26,8 +26,10 @@ The banyand and bydbctl directory structure is as follows. ├── NOTICE ├── README.md ├── bin -│   ├── banyand-linux-arm64 -│   └── banyand-linux-amd64 +│ ├──banyand-server-slim-linux-amd64 +│ ├──banyand-server-slim-linux-arm64 +│ ├──banyand-server-static-linux-amd64 +│   └──banyand-server-static-linux-arm64 └── licenses ``` @@ -38,13 +40,13 @@ The banyand and bydbctl directory structure is as follows. ├── NOTICE ├── README.md ├── bin -│   ├── bydbctl-linux-386 -│   ├── bydbctl-linux-amd64 -│   ├── bydbctl-linux-arm64 -│   ├── bydbctl-windows-386 -│   ├── bydbctl-windows-amd64 -│   ├── bydbctl-darwin-amd64 -│   └── bydbctl-darwin-arm64 +│   ├── bydbctl-cli-static-linux-386 +│   ├── bydbctl-cli-static-linux-amd64 +│   ├── bydbctl-cli-static-linux-arm64 +│   ├── bydbctl-cli-static-windows-386 +│   ├── bydbctl-cli-static-windows-amd64 +│   ├── bydbctl-cli-static-darwin-amd64 +│   └── bydbctl-cli-static-darwin-arm64 └── licenses ``` @@ -72,7 +74,9 @@ To issue the below command to get basic binaries of banyand and bydbctl. make generate ... make build +--- ui: all --- ... +Done building ui --- banyand: all --- ... chmod +x build/bin/banyand-server; @@ -84,26 +88,26 @@ chmod +x build/bin/dev/bydbctl-cli; Done building build/bin/dev/bydbctl-cli ``` -The build system provides a series of binary options as well. - -* `make -C banyand banyand-server` generates a basic `banyand-server`. -* `make -C banyand release` or `make -C banyand banyand-server-static` builds out a static binary `banyand-server-static` for releasing. -* `make -C bydbctl bydbctl-cli` generates a basic `bydbctl-cli`. -* `make -C bydbctl release` or `make -C banyand bydbctl-cli-static` builds out a static binary `bydbctl-cli-static` for releasing. - Then users get binaries as below ``` shell ls banyand/build/bin/dev banyand-server -banyand-server-static ls bydbctl/build/bin/dev bydbctl-cli -bydbctl-cli-static ``` -> The build script now checks if the binary file exists before rebuilding. If you want to rebuild, please remove the binary file manually. +The build system provides a series of binary options as well. + +* `make -C banyand banyand-server` generates a basic `banyand-server`. +* `make -C banyand banyand-server-static` builds out a static binary `banyand-server-static` which is statically linked with all dependencies. +* `make -C banyand banyand-server-slim` builds out a slim binary `banyand-server-slim` which doesn't include `UI`. +* `make -C banyand release` builds out the static and slim binaries for releasing. +* `make -C bydbctl bydbctl-cli` generates a basic `bydbctl-cli`. +* `make -C bydbctl release` or `make -C banyand bydbctl-cli-static` builds out a static binary `bydbctl-cli-static` for releasing. This binary is statically linked with all dependencies. + +> The build script now checks if the binary file exists before rebuilding. If you want to rebuild, please remove the binary file manually by running `make clean-build`. ### Cross-compile Binaries @@ -113,7 +117,7 @@ The build system supports cross-compiling binaries for different platforms. For TARGET_OS=windows PLATFORMS=windows/amd64 make release ``` -The `PLATFORMS` variable is a list of platforms separated by commas. The `TARGET_OS` variable is the target operating system. You could specify several platforms at once: +The `PLATFORMS` variable is a list of platforms separated by commas. The `TARGET_OS` variable is the target operating system. You could specify several platforms at once: ```shell TARGET_OS=linux PLATFORMS=linux/amd64,linux/arm64 make release diff --git a/docs/installation/docker.md b/docs/installation/docker.md index 8d9e17d1b..e4d7d9994 100644 --- a/docs/installation/docker.md +++ b/docs/installation/docker.md @@ -1,5 +1,26 @@ # Installation On Docker +## Images on Docker Hub + +The BanyanDB images are hosted on Docker Hub. You can pull the images from the following links: [Apache SkyWalking BanyanDB](https://hub.docker.com/r/apache/skywalking-banyandb) + +There are two types of images: + +- `apache/skywalking-banyandb:` - The specific version of the BanyanDB. +- `apache/skywalking-banyandb:-slim` - The slim version of the BanyanDB. It does not contain the Web UI. + +We pushed `linux/amd64` and `linux/arm64` for each type of image. You can pull the image for the specific architecture. + +## Images on GitHub Container Registry + +The BanyanDB images are hosted on GitHub Container Registry for development or testing. You can pull the images from the following links: [ghcr.io/apache/skywalking-banyandb](https://github.com/apache/skywalking-banyandb/pkgs/container/skywalking-banyandb) + +There are three types of images: + +- `ghcr.io/apache/skywalking-banyandb:` - The specific version of the BanyanDB. We pushed `linux/amd64`, `linux/arm64` and `windows/amd64` for each type of image. +- `ghcr.io/apache/skywalking-banyandb:-slim` - The slim version of the BanyanDB. It does not contain the Web UI. We pushed `linux/amd64`, `linux/arm64` and `windows/amd64` for each type of image. +- `ghcr.io/apache/skywalking-banyandb:-testing` - The testing version of the BanyanDB. It contains the Web UI and the `bydbctl`. We pushed `linux/amd64` and `linux/arm64` for each type of image. + ## Start a container in `standalone mode` The following commands pull the docker image and run the BanyanDB on Docker. Replace `latest` with the version of the BanyanDB you want to run. - pull the image diff --git a/scripts/build/build.mk b/scripts/build/build.mk index 29f8ec507..647f0a89d 100644 --- a/scripts/build/build.mk +++ b/scripts/build/build.mk @@ -25,6 +25,7 @@ $(error The BINARIES variable should be set to the name binaries to produce) endif STATIC_BINARIES ?= $(addsuffix -static,$(BINARIES)) +SLIM_BINARIES ?= $(addsuffix -slim,$(BINARIES)) BUILD_DIR ?= build/bin TARGET_OS ?= linux OS := ${TARGET_OS} @@ -61,8 +62,19 @@ $(STATIC_BINARIES_GOBUILD_TARGET): $(BUILD_DIR)/$(OS)/%-static: $(BUILD_LOCK) $(call go_build_static_executable,,-s -w) @echo "Done building static $*" +SLIM_BINARIES_GOBUILD_TARGET_PATTERN := $(foreach goarch,$(GOBUILD_ARCHS),$(BUILD_DIR)/$(OS)/$(goarch)/$(NAME)-%-slim) +SLIM_BINARIES_GOBUILD_TARGET := $(foreach goarch,$(GOBUILD_ARCHS),$(addprefix $(BUILD_DIR)/$(OS)/$(goarch)/,$(SLIM_BINARIES))) +$(SLIM_BINARIES): $(NAME)-%-slim: $(SLIM_BINARIES_GOBUILD_TARGET_PATTERN) +$(SLIM_BINARIES_GOBUILD_TARGET): $(BUILD_DIR)/$(OS)/%-slim: $(BUILD_LOCK) + $(call set_build_package,$*,$@) + @echo "Building slim $*" + $(MAKE) prepare-build + $(eval BUILD_TAGS := $(BUILD_TAGS) slim) + $(call go_build_static_executable,,-s -w) + @echo "Done building static $*" + .PHONY: release -release: $(STATIC_BINARIES) ## Build the release binaries +release: $(STATIC_BINARIES) ## Build the release binaries .PHONY: clean-build clean-build: ## Clean all artifacts diff --git a/scripts/build/docker.mk b/scripts/build/docker.mk index a3c6de5fc..7f5626a9b 100644 --- a/scripts/build/docker.mk +++ b/scripts/build/docker.mk @@ -27,6 +27,12 @@ endif # The tag of the docker image. The default value if latest. TAG ?= latest +BINARYTYPE ?= static + +ifeq ($(BINARYTYPE),slim) + TAG := $(TAG)-slim +endif + IMG := $(HUB)/$(IMG_NAME):$(TAG) # Disable cache in CI environment @@ -42,5 +48,5 @@ docker.push: DOCKER_TYPE = "Push" docker docker.push: @echo "$(DOCKER_TYPE) $(IMG) with platform $(PLATFORMS)" @pwd - time docker buildx build $(DOCKER_BUILD_ARGS) --platform $(PLATFORMS) $(LOAD_OR_PUSH) -t $(IMG) -f Dockerfile --provenance=false . + time docker buildx build $(DOCKER_BUILD_ARGS) --platform $(PLATFORMS) $(LOAD_OR_PUSH) -t $(IMG) -f Dockerfile --provenance=false --build-arg BINARYTYPE=$(BINARYTYPE) . diff --git a/scripts/release.sh b/scripts/release.sh index 0182a2f76..8e16d117c 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -67,7 +67,6 @@ copy_binaries() { # Extract os and arch from the path os_arch=$(echo ${binary} | awk -F'/' '{print $(NF-2)"/"$(NF-1)}') binary_name=$(basename ${binary}) - binary_name=${binary_name%-*-*} cp -Rfv ${binary} ${bindir}/bin/${binary_name}-${os_arch//\//-} done } diff --git a/test/docker/Makefile b/test/docker/Makefile index e1c79ffae..01c71efd3 100644 --- a/test/docker/Makefile +++ b/test/docker/Makefile @@ -23,7 +23,7 @@ HUB ?= apache # The tag of the docker image. The default value if latest. TAG ?= latest -IMG := $(HUB)/skywalking-banyandb-test:$(TAG) +IMG := $(HUB)/skywalking-banyandb:$(TAG)-testing build: @echo "Building $(IMG)" diff --git a/test/e2e-v2/script/docker-compose/base-compose.yml b/test/e2e-v2/script/docker-compose/base-compose.yml index 2f1f78a26..fe30d45c1 100644 --- a/test/e2e-v2/script/docker-compose/base-compose.yml +++ b/test/e2e-v2/script/docker-compose/base-compose.yml @@ -20,7 +20,7 @@ services: extends: file: ../../../docker/base-compose.yml service: banyandb - image: "apache/skywalking-banyandb-test:${TAG}" + image: "apache/skywalking-banyandb:${TAG}-testing" networks: - e2e @@ -28,7 +28,7 @@ services: extends: file: ../../../docker/base-compose.yml service: liaison - image: "apache/skywalking-banyandb-test:${TAG}" + image: "apache/skywalking-banyandb:${TAG}-testing" networks: - e2e @@ -36,7 +36,7 @@ services: extends: file: ../../../docker/base-compose.yml service: data - image: "apache/skywalking-banyandb-test:${TAG}" + image: "apache/skywalking-banyandb:${TAG}-testing" networks: - e2e diff --git a/ui/Makefile b/ui/Makefile index aed91b15e..60c1eee6d 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -37,7 +37,6 @@ all: build .PHONY: generate generate: install -generate: $(DIST_INDEX) $(DIST_INDEX): @echo "Building $(NAME)" @@ -45,8 +44,12 @@ $(DIST_INDEX): @echo "Done building $(NAME)" .PHONY: build -build: - @echo "Run generate to build the UI" +build: $(DIST_INDEX) + +.PHONY: clean-build +clean-build: + @echo "Cleaning build artifacts" + rm -rf dist .PHONY: test test: