Merge pull request #85 from cybozu-go/build-custom-rbd-cmd

Added the custom ceph command build workflow

.github/workflows/ceph-export-diff-release.yaml

@@ -0,0 +1,24 @@
+name: "ceph custom export-diff release"
+on:
+  push:
+    tags:
+      - "ceph-export-diff-v*.*.*.*"
+jobs:
+  release:
+    runs-on: mantle_large_runner_16core
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - run: sudo apt-get update
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: "go.mod"
+      - uses: ./.github/actions/set-up-kvm-for-e2e-tests
+      - run: make -C ceph setup
+      - run: make -C ceph build
+      - run: |
+          VERSION=${GITHUB_REF#refs/tags/}
+          zip -j packages.zip ceph/packages/*.deb
+          gh release create ${VERSION} -t "Release ${VERSION}"
+          gh release upload ${VERSION} packages.zip

.github/workflows/ceph-export-diff-test.yaml

@@ -0,0 +1,18 @@
+name: "ceph custom export-diff test"
+on:
+  pull_request:
+    paths:
+      - "ceph/**"
+jobs:
+  test:
+    runs-on: mantle_large_runner_16core
+    steps:
+      - run: sudo apt-get update
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: "go.mod"
+      - uses: ./.github/actions/set-up-kvm-for-e2e-tests
+      - run: make -C ceph setup
+      - run: make -C ceph build
+      - run: make -C ceph test

.gitignore

@@ -25,6 +25,9 @@ Dockerfile.cross
 *.swo
 *~
 
+ceph/go.mod
+ceph/go.sum
+ceph/packages/
 test/e2e/testdata/persistentvolumes.yaml
 test/e2e/testdata/values-mantle-primary.yaml
 test/e2e/testdata/values-mantle-secondary.yaml

ceph/Dockerfile

@@ -0,0 +1,87 @@
+FROM ghcr.io/cybozu/ubuntu:22.04 AS build-ceph
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ENV DEBIAN_FRONTEND=noninteractive
+ENV CEPH_TAG=v18.2.4
+ENV WORKSPACE=/workspace
+
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y \
+    build-essential curl git
+
+# Workaround for github actions runner.
+# Ceph depends on this library, but it is not automatically installed
+# because libraries that conflict with this library are installed.
+# Therefore, it should be installed explicitly.
+# See. https://github.com/actions/runner-images/issues/6399#issuecomment-1286050292
+RUN apt install --no-install-recommends -y libunwind-dev
+
+WORKDIR ${WORKSPACE}
+RUN git clone -b ${CEPH_TAG} --depth=1 --recurse-submodules --shallow-submodules https://github.com/ceph/ceph.git
+
+COPY *.patch ./
+WORKDIR ${WORKSPACE}/ceph
+
+# Ceph v18.2.4 has a bug that causes the ceph-volume command to fail on the OSD prepare pod.
+# This bug was caused by the wrong way of installing the Python package.
+# As a workaround, the following patch is applied ahead of upstream.
+# https://github.com/ceph/ceph/commit/729fd8e25ff2bfbcf99790d6cd08489d1c4e2ede
+# Apply the packing-1.patch(Commit 80edcd4) simply because the changes depend on it.
+RUN patch -p1 < ${WORKSPACE}/packing-1.patch && \
+    patch -p1 < ${WORKSPACE}/packing-2.patch
+
+RUN ./install-deps.sh && \
+    apt install --no-install-recommends -y python3-routes
+
+# This patch customizes the ceph rbd export-diff command. 
+RUN  patch -p1 < ${WORKSPACE}/export-diff.patch
+
+# Prebuild ceph source to generate files in `src/pybind/mgr/dashboard/frontend/dist` needed by CMake
+RUN ./make-dist
+
+# Build Ceph packages
+RUN sed -i -e 's/WITH_CEPHFS_JAVA=ON/WITH_CEPHFS_JAVA=OFF/' debian/rules && \
+    sed -i -e 's@usr/share/java/libcephfs-test.jar@@' debian/ceph-test.install
+# CMake in the self-build environment did not allow space-separated URLs.
+# As a workaround, the following patch is applied ahead of upstream.
+# https://github.com/ceph/ceph/commit/35435420781f84e9b71f72b10e6842a89c06de7f
+RUN patch -p1 < ${WORKSPACE}/boost-url.patch
+RUN rm debian/libcephfs-java.jlibs debian/libcephfs-jni.install debian/ceph-mgr-dashboard*
+RUN dpkg-buildpackage --build=binary -uc -us -j10
+RUN rm ${WORKSPACE}/*-dbg_*.deb ${WORKSPACE}/ceph-test_*.deb
+
+FROM ghcr.io/cybozu/ubuntu:22.04
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ENV DEBIAN_FRONTEND=noninteractive
+ENV WORKSPACE=/workspace
+ENV PACKAGES=/packages
+
+# Since the package file is extracted outside the docker image at the end, keep the files.
+COPY --from=build-ceph ${WORKSPACE}/*.deb ${PACKAGES}/
+
+WORKDIR ${WORKSPACE}
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/usr/local/bin/kubectl"
+RUN mkdir -p /var/run/ceph && \
+    apt-get update && apt-get install --no-install-recommends -y \
+        libstdc++-11-dev jq kmod lvm2 gdisk ca-certificates e2fsprogs attr udev libgflags2.2 ${PACKAGES}/*.deb && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    sed -i -e 's/udev_rules = 1/udev_rules = 0/' -e 's/udev_sync = 1/udev_sync = 0/' -e 's/obtain_device_list_from_udev = 1/obtain_device_list_from_udev = 0/' /etc/lvm/lvm.conf && \
+    # validate the sed command worked as expected
+    grep -sqo "udev_sync = 0" /etc/lvm/lvm.conf && \
+    grep -sqo "udev_rules = 0" /etc/lvm/lvm.conf && \
+    grep -sqo "obtain_device_list_from_udev = 0" /etc/lvm/lvm.conf && \
+    # Clean common files like /tmp, /var/lib, etc.
+    rm -rf \
+        /etc/{selinux,systemd,udev} \
+        /lib/{lsb,udev} \
+        /tmp/* \
+        /usr/lib{,64}/{locale,systemd,udev,dracut} \
+        /usr/share/{doc,info,locale,man} \
+        /usr/share/{bash-completion,pkgconfig/bash-completion.pc} \
+        /var/log/* \
+        /var/tmp/* && \
+    find / -xdev \( -name "*.pyc" -o -name "*.pyo" \) -exec rm -f {} \; && \
+    mkdir -p /usr/local/share/doc/ceph && \
+    chown ceph:ceph -R /run/ceph /var/lib/ceph

ceph/Makefile

@@ -0,0 +1,47 @@
+include ../versions.mk
+
+PROJECT_DIR := $(shell cd ..; pwd)
+E2E_DIR := $(PROJECT_DIR)/test/e2e
+MINIKUBE_HOME := $(E2E_DIR)/bin
+
+GO_FILES := $(shell find -name '*.go')
+PATCH_FILES := $(shell find -name '*.patch')
+
+KUBECTL_CMD := $(E2E_DIR)/bin/kubectl-$(KUBERNETES_VERSION)
+MINIKUBE_CMD := $(E2E_DIR)/bin/minikube
+
+export MINIKUBE_HOME
+
+.PHONY: build
+build: ceph-custom.tar packages/touch
+
+ceph-custom.tar: build-docker
+	docker save ceph-custom -o $@
+
+packages/touch: build-docker
+	rm -rf packages
+	docker create --name ceph-proc ceph-custom
+	docker cp ceph-proc:/packages .
+	docker rm ceph-proc
+	touch $@
+
+build-docker: Dockerfile $(PATCH_FILES)
+	docker build -f Dockerfile -t ceph-custom .
+
+.PHONY: test
+test: ceph-custom.tar
+	$(MAKE) -C ${E2E_DIR} launch-minikube MINIKUBE_PROFILE=minikube
+	$(MAKE) -C ${E2E_DIR} install-rook-ceph-operator
+	$(MAKE) -C ${E2E_DIR} install-rook-ceph-cluster1
+	$(MINIKUBE_CMD) image load ceph-custom.tar
+	$(KUBECTL_CMD) apply -k ./manifests/
+	go test -v -count=1 ./test/
+
+.PHONY: setup
+setup:
+	$(MAKE) -C $(E2E_DIR) setup
+
+.PHONY: clean
+clean:
+	rm -rf packages ceph-custom.tar
+	$(MAKE) -C $(E2E_DIR) clean

ceph/README.md

@@ -0,0 +1,50 @@
+# custom ceph rbd export-diff command
+
+This directory is for building & releasing custom Ceph RBD export-diff command. 
+
+## Build and test in your environment
+
+To build ceph, you need a lot of memory, disk, and time. You would prepare enough resources and use the tmux or screen command to keep the session alive.
+
+```sh
+tmux
+make setup
+make build
+make test
+```
+
+## Procedure to update
+### Update ceph version
+
+Imports all of the contents from upstream git, including tags.
+
+```sh
+git swtich main
+git remote
+# If there is no `upstream` in remotes, use the following command to add it.
+git remote add upstream https://github.com/ceph/ceph.git
+git fetch upstream
+git push origin --all
+git push origin --tags
+```
+
+### Update the patch
+TBD
+
+### Release
+
+1. The version is determined. The patch version is the same as the base ceph version. The fourth field starts at 0 for each ceph version and increases by 1 for each release.
+  ```sh
+  VERSION=16.2.4.0
+  ``` 
+2. Go to the [rule setting page](https://github.com/cybozu-go/mantle/settings/rules/3334068) and change the value of "Enforcement status" to Active.
+3. Add a new tag and push it.
+  ```sh
+  git switch main
+  git pull
+  git tag ceph-export-diff-v$VERSION
+  git push origin ceph-export-diff-v$VERSION
+  ```
+4. Go to the [rule setting page](https://github.com/cybozu-go/mantle/settings/rules/3334068) and change the value of "Enforcement status" to Disabled.
+5. Once a new tag is pushed, [GitHub Actions](https://github.com/cybozu-go/mantle/actions) automatically creates a draft release note for the tagged version, builds a tar archive for the new release, and attaches it to the release note.
+  Visit [https://github.com/cybozu-go/mantle/releases] to check the result.

ceph/boost-url.patch

@@ -0,0 +1,26 @@
+From 35435420781f84e9b71f72b10e6842a89c06de7f Mon Sep 17 00:00:00 2001
+From: Adam Emerson <[email protected]>
+Date: Wed, 22 May 2024 14:38:21 -0400
+Subject: [PATCH] build: Make boost_url a list
+
+Cmake 3.29 does not appreciate a space separated string being
+substituted in.
+
+Signed-off-by: Adam Emerson <[email protected]>
+---
+ cmake/modules/BuildBoost.cmake | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/cmake/modules/BuildBoost.cmake b/cmake/modules/BuildBoost.cmake
+index 3b3b2688426bb..52464ee299f39 100644
+--- a/cmake/modules/BuildBoost.cmake
++++ b/cmake/modules/BuildBoost.cmake
+@@ -164,7 +164,7 @@ function(do_build_boost root_dir version)
+     set(boost_version 1.82.0)
+     set(boost_sha256 a6e1ab9b0860e6a2881dd7b21fe9f737a095e5f33a3a874afc6a345228597ee6)
+     string(REPLACE "." "_" boost_version_underscore ${boost_version} )
+-    string(JOIN " " boost_url
++    list(APPEND boost_url
+       https://boostorg.jfrog.io/artifactory/main/release/${boost_version}/source/boost_${boost_version_underscore}.tar.bz2
+       https://download.ceph.com/qa/boost_${boost_version_underscore}.tar.bz2)
+     set(source_dir