From 8faf23f1a68d0fa907a52d96a3419b0fbe8aeaae Mon Sep 17 00:00:00 2001 From: Chen Jiali Date: Thu, 29 Aug 2024 13:43:40 +0800 Subject: [PATCH 1/9] fix: add patch to be compatible with Rockchip Linux Kernel --- ...ion-error-when-using-Rockchip-kernel.patch | 30 +++++++++++++++++++ debian/patches/series | 1 + 2 files changed, 31 insertions(+) create mode 100644 debian/patches/fix-DKMS-compilation-error-when-using-Rockchip-kernel.patch diff --git a/debian/patches/fix-DKMS-compilation-error-when-using-Rockchip-kernel.patch b/debian/patches/fix-DKMS-compilation-error-when-using-Rockchip-kernel.patch new file mode 100644 index 000000000..fa3b90794 --- /dev/null +++ b/debian/patches/fix-DKMS-compilation-error-when-using-Rockchip-kernel.patch @@ -0,0 +1,30 @@ +diff --git a/config/kernel.m4 b/config/kernel.m4 +index b51477b6..bfd57736 100644 +--- a/config/kernel.m4 ++++ b/config/kernel.m4 +@@ -679,6 +679,7 @@ $2 + MODULE_DESCRIPTION("conftest"); + MODULE_AUTHOR(ZFS_META_AUTHOR); + MODULE_VERSION(ZFS_META_VERSION "-" ZFS_META_RELEASE); ++MODULE_IMPORT_NS(VFS_internal_I_am_really_a_filesystem_and_am_NOT_a_driver); + MODULE_LICENSE($3); + ]) + +diff --git a/module/os/linux/spl/spl-generic.c b/module/os/linux/spl/spl-generic.c +index 986db151..13b38acb 100644 +--- a/module/os/linux/spl/spl-generic.c ++++ b/module/os/linux/spl/spl-generic.c +@@ -929,3 +929,4 @@ MODULE_DESCRIPTION("Solaris Porting Layer"); + MODULE_AUTHOR(ZFS_META_AUTHOR); + MODULE_LICENSE("GPL"); + MODULE_VERSION(ZFS_META_VERSION "-" ZFS_META_RELEASE); ++MODULE_IMPORT_NS(VFS_internal_I_am_really_a_filesystem_and_am_NOT_a_driver); +diff --git a/module/os/linux/zfs/zfs_ioctl_os.c b/module/os/linux/zfs/zfs_ioctl_os.c +index 663474ea..197d9364 100644 +--- a/module/os/linux/zfs/zfs_ioctl_os.c ++++ b/module/os/linux/zfs/zfs_ioctl_os.c +@@ -377,3 +377,4 @@ MODULE_LICENSE("Dual MIT/GPL"); /* lua */ + MODULE_LICENSE("Dual BSD/GPL"); /* zstd / misc */ + MODULE_LICENSE(ZFS_META_LICENSE); + MODULE_VERSION(ZFS_META_VERSION "-" ZFS_META_RELEASE); ++MODULE_IMPORT_NS(VFS_internal_I_am_really_a_filesystem_and_am_NOT_a_driver); diff --git a/debian/patches/series b/debian/patches/series index 785768526..ff13670ca 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -21,3 +21,4 @@ ubuntu/zfs-mount-container-start.patch #ubuntu/4751-suppress-types.patch fix-icp-sha2-on-armel.patch bump-linux-maximum.patch +fix-DKMS-compilation-error-when-using-Rockchip-kernel.patch From 2ff528f84cfa5bb0aee6420f241bb34f15acd0fc Mon Sep 17 00:00:00 2001 From: Chen Jiali Date: Thu, 29 Aug 2024 13:44:23 +0800 Subject: [PATCH 2/9] fix: add .gitignore --- .gitignore | 34 ++++++++++++++++++++++++++++++++++ debian/.gitignore | 11 +++++++++++ 2 files changed, 45 insertions(+) create mode 100644 .gitignore create mode 100644 debian/.gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..3b53714ef --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +build/ +install/ + +# Object files +*.o +*.lib +*.so +*.lo +*.la +*.pyc + +# Temp files +*.swp +*.swo +*~ + +# npmpub files +src/mraajsJAVASCRIPT_wrap.cxx +src/version.c +package.json +binding.gyp +READMEFIRST +npm-debug.log + +# Sonar +.scannerwork/ + +# Visual Studio Code +.vscode* + +# CMake object files +obj-aarch64-linux-gnu + +SOURCE \ No newline at end of file diff --git a/debian/.gitignore b/debian/.gitignore new file mode 100644 index 000000000..1ba4894e0 --- /dev/null +++ b/debian/.gitignore @@ -0,0 +1,11 @@ +/.debhelper +/debhelper-build-stamp +/files +/lib*/ +/*pyzfs*/ +/*zfs*/ +/tmp/ +/*.debhelper.log +/*.debhelper +/*.substvars +autoreconf.* From 372245c13dead3e24f3a3accf5c3d0f179dde8b1 Mon Sep 17 00:00:00 2001 From: Chen Jiali Date: Thu, 29 Aug 2024 13:44:58 +0800 Subject: [PATCH 3/9] feat: add CI/CD files * closes https://github.com/radxa/kernel/issues/54 --- .github/container/Dockerfile | 16 +++++ .github/container/build-deb | 98 +++++++++++++++++++++++++++++ .github/dependabot.yml | 7 +++ .github/workflows/new_version.yml | 37 +++++++++++ .github/workflows/release.yml | 100 ++++++++++++++++++++++++++++++ Makefile.mk | 63 +++++++++++++++++++ 6 files changed, 321 insertions(+) create mode 100644 .github/container/Dockerfile create mode 100755 .github/container/build-deb create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/new_version.yml create mode 100644 .github/workflows/release.yml create mode 100644 Makefile.mk diff --git a/.github/container/Dockerfile b/.github/container/Dockerfile new file mode 100644 index 000000000..eb6f5a02b --- /dev/null +++ b/.github/container/Dockerfile @@ -0,0 +1,16 @@ +# Some dependencies conflict with the compilation tools of amd64 architecture. Use arm64 directly. container image +FROM --platform=linux/arm64/v8 debian:bullseye + +RUN dpkg --add-architecture arm64 && \ + apt-get update && \ + apt-get full-upgrade --no-install-recommends -y \ + build-essential crossbuild-essential-arm64 git \ + debhelper devscripts lintian abigail-tools debhelper-compat \ + dh-python dh-sequence-dkms libaio-dev libblkid-dev \ + libcurl4-openssl-dev libelf-dev libpam0g-dev \ + libssl-dev libtirpc-dev libtool libudev-dev \ + lsb-release po-debconf python3-all-dev python3-cffi \ + python3-setuptools python3-sphinx uuid-dev zlib1g-dev \ + dkms linux-headers-arm64 && \ + adduser --gecos runner --disabled-password runner && \ + rm -rf /var/lib/apt/lists/* diff --git a/.github/container/build-deb b/.github/container/build-deb new file mode 100755 index 000000000..521e24310 --- /dev/null +++ b/.github/container/build-deb @@ -0,0 +1,98 @@ +#!/bin/bash + +set -euo pipefail +shopt -s nullglob + +build() { + local SCRIPT_DIR="$(dirname "$(realpath "$0")")" + + local NATIVE_BUILD="false" + local CONTAINER_BACKEND="podman" + local CONTAINER_REGISTRY="" + local NO_CONTAINER_UPDATE="false" + local CONTAINER_SHELL="false" + + if ! $NATIVE_BUILD + then + if [[ "$(basename "$CONTAINER_BACKEND")" == "docker" ]] && "$CONTAINER_BACKEND" -h | grep -q podman + then + echo "'$CONTAINER_BACKEND' backend is selected, but the functionality is actually provided by 'podman' backend. Updating accordingly..." + CONTAINER_BACKEND="$(command -v podman)" + fi + + local CONTAINER_IMAGE="$($CONTAINER_BACKEND image ls "-qf=reference=${CONTAINER_REGISTRY}zfs:master")" + local CONTAINER_EXIT_CODE=0 + + if ! $NO_CONTAINER_UPDATE + then + if [[ -z $CONTAINER_REGISTRY ]] + then + $CONTAINER_BACKEND build --force-rm -t "${CONTAINER_REGISTRY}zfs:master" "$SCRIPT_DIR" + else + $CONTAINER_BACKEND pull "${CONTAINER_REGISTRY}zfs:master" + fi + fi + + if [[ $CONTAINER_IMAGE != "$($CONTAINER_BACKEND image ls "-qf=reference=${CONTAINER_REGISTRY}zfs:master")" ]] + then + $CONTAINER_BACKEND container rm zfs &>/dev/null || true + $CONTAINER_BACKEND image rm "${CONTAINER_REGISTRY}zfs:builder" &>/dev/null || true + fi + + CONTAINER_BUILDER="$($CONTAINER_BACKEND image ls "-qf=reference=${CONTAINER_REGISTRY}zfs:builder")" + if [[ -z $CONTAINER_BUILDER ]] + then + $CONTAINER_BACKEND tag "${CONTAINER_REGISTRY}zfs:master" "${CONTAINER_REGISTRY}zfs:builder" + fi + + CONTAINER_OPTIONS=( "--name" "zfs" ) + CONTAINER_OPTIONS+=( "--workdir" "$PWD" ) + CONTAINER_OPTIONS+=( "--mount" "type=bind,source=$PWD,destination=$PWD" ) + CONTAINER_OPTIONS+=( "--mount" "type=bind,source=$(realpath $SCRIPT_DIR/../../..),destination=$(realpath $SCRIPT_DIR/../../..)" ) + if [[ -t 0 ]] + then + CONTAINER_OPTIONS+=( "-it" ) + fi + if [[ "$PWD" != "$SCRIPT_DIR" ]] + then + CONTAINER_OPTIONS+=( "--mount" "type=bind,source=$SCRIPT_DIR,destination=$SCRIPT_DIR" ) + fi + $CONTAINER_BACKEND container kill zfs &>/dev/null || true + $CONTAINER_BACKEND container rm zfs &>/dev/null || true + if [[ "$(basename "$CONTAINER_BACKEND")" == "podman" ]] + then + CONTAINER_OPTIONS+=( "--user" "root" ) + if $CONTAINER_SHELL + then + if ! $CONTAINER_BACKEND run "${CONTAINER_OPTIONS[@]}" "${CONTAINER_REGISTRY}zfs:builder" bash + then + CONTAINER_EXIT_CODE="$($CONTAINER_BACKEND inspect zfs --format='{{.State.ExitCode}}')" + fi + else + if ! $CONTAINER_BACKEND run "${CONTAINER_OPTIONS[@]}" "${CONTAINER_REGISTRY}zfs:builder" make -f Makefile.mk deb + then + CONTAINER_EXIT_CODE="$($CONTAINER_BACKEND inspect zfs --format='{{.State.ExitCode}}')" + fi + $CONTAINER_BACKEND container rm zfs + fi + else + local CONTAINER_SUDO="sed -i -E \"s/^(runner):(x?):([0-9]+):([0-9]+):(.*):(.*):(.*)$/\1:\2:$(id -u):$(id -g):\5:\6:\7/\" /etc/passwd && sudo -u runner" + if $CONTAINER_SHELL + then + if ! $CONTAINER_BACKEND run "${CONTAINER_OPTIONS[@]}" "${CONTAINER_REGISTRY}zfs:builder" bash -c "$CONTAINER_SUDO -i" + then + CONTAINER_EXIT_CODE="$($CONTAINER_BACKEND inspect zfs --format='{{.State.ExitCode}}')" + fi + else + if ! $CONTAINER_BACKEND run "${CONTAINER_OPTIONS[@]}" "${CONTAINER_REGISTRY}zfs:builder" bash -c "$CONTAINER_SUDO make -f Makefile.mk deb" + then + CONTAINER_EXIT_CODE="$($CONTAINER_BACKEND inspect zfs --format='{{.State.ExitCode}}')" + fi + $CONTAINER_BACKEND container rm zfs + fi + fi + return $CONTAINER_EXIT_CODE + fi +} + +build diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..2c7d17083 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/new_version.yml b/.github/workflows/new_version.yml new file mode 100644 index 000000000..f65a6f76e --- /dev/null +++ b/.github/workflows/new_version.yml @@ -0,0 +1,37 @@ +name: Create release +on: + workflow_dispatch: + +permissions: + contents: write + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + token: ${{secrets.GIT_PUSH_TOKEN}} + - name: Create release commit + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y git-buildpackage + export DEBEMAIL="dev@radxa.com" + export DEBFULLNAME='"Radxa Computer Co., Ltd"' + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + make -f Makefile.mk dch + - name: Test + run: | + make -f Makefile.mk test + sudo apt-get update + sudo apt-get build-dep --no-install-recommends -y . + make -f Makefile.mk all + sudo apt install -y binfmt-support qemu-user-static + .github/container/build-deb + - name: Push + run: | + git push diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..8044cf756 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,100 @@ +name: Build & Release +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + +permissions: + contents: write + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + - name: Test + run: | + make -f Makefile.mk test + - name: Build + run: | + sudo apt-get update + sudo apt-get build-dep --no-install-recommends -y . + make -f Makefile.mk all + sudo apt install -y binfmt-support qemu-user-static + .github/container/build-deb + - name: Workaround actions/upload-artifact#176 + run: | + echo "artifacts_path=$(realpath ..)" >> $GITHUB_ENV + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ github.event.repository.name }} + path: | + ${{ env.artifacts_path }}/*.deb + release: + runs-on: ubuntu-latest + needs: build + if: ${{ github.event_name != 'pull_request' && github.ref_name == 'master' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/download-artifact@v3 + with: + name: ${{ github.event.repository.name }} + - name: Check if the latest version is releasable + run: | + version="$(dpkg-parsechangelog -S Version)" + echo "version=$version" >> $GITHUB_ENV + echo "changes<> $GITHUB_ENV + echo '```' >> $GITHUB_ENV + echo "$(dpkg-parsechangelog -S Changes)" >> $GITHUB_ENV + echo '```' >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + if [[ -n "$(git tag -l "$version")" ]] + then + echo "distro=UNRELEASED" >> $GITHUB_ENV + else + echo "distro=$(dpkg-parsechangelog -S Distribution)" >> $GITHUB_ENV + fi + echo "$version" > VERSION + if [[ -f pkg.conf.template ]] + then + sed "s/VERSION/$(dpkg-parsechangelog -S Version)/g" pkg.conf.template > pkg.conf + fi + - name: Release + if: env.distro != 'UNRELEASED' + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ env.version }} + body_path: README.md + token: ${{ secrets.GITHUB_TOKEN }} + target_commitish: master + draft: false + fail_on_unmatched_files: true + files: | + *.deb + VERSION + - name: Append changelog + if: env.distro != 'UNRELEASED' + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ env.version }} + body: | + ## Changelog for ${{ env.version }} + ${{ env.changes }} + append_body: true + - name: Update Test repos + if: env.distro != 'UNRELEASED' + uses: radxa-repo/update-repo-action@main + with: + test-repo: true + token: ${{ secrets.RADXA_APT_TEST_REPO_TOKEN }} diff --git a/Makefile.mk b/Makefile.mk new file mode 100644 index 000000000..9d83e68e1 --- /dev/null +++ b/Makefile.mk @@ -0,0 +1,63 @@ +PROJECT ?= zfs + +.PHONY: all +all: build + +# +# Test +# +.PHONY: test +test: + +# +# Build +# +.PHONY: build +build: build-doc + +SRC-DOC := . +DOCS := $(SRC-DOC)/SOURCE +.PHONY: build-doc +build-doc: $(DOCS) + +$(SRC-DOC): + mkdir -p $(SRC-DOC) + +.PHONY: $(SRC-DOC)/SOURCE +$(SRC-DOC)/SOURCE: $(SRC-DOC) + echo -e "git clone $(shell git remote get-url origin)\ngit checkout $(shell git rev-parse HEAD)" > "$@" + +# +# Clean +# +.PHONY: distclean +distclean: clean + +.PHONY: clean +clean: clean-doc clean-deb + +.PHONY: clean-doc +clean-doc: + rm -rf $(DOCS) + +.PHONY: clean-deb +clean-deb: + # The packaging of the zfs team is not standard. There will be many *.install files generated during packaging, which need to be cleaned using GIT. + git clean -fx debian/*.install + dh_auto_clean + rm -rf debian/.debhelper debian/lib*/ debian/*pyzfs*/ debian/*zfs*/ debian/tmp/ debian/debhelper-build-stamp debian/files debian/*.debhelper.log debian/*.*.debhelper debian/*.substvars + +# +# Release +# +.PHONY: dch +dch: debian/changelog + EDITOR=true gbp dch --debian-branch=main --multimaint-merge --commit --release --dch-opt=--upstream + +.PHONY: deb +deb: debian + debuild --no-lintian --no-sign -b -aarm64 -Pcross + +.PHONY: release +release: + gh workflow run .github/workflows/new_version.yml From ed2847986e39b4b331ebe7dd7a4d7dbd401aa2a1 Mon Sep 17 00:00:00 2001 From: Chen Jiali Date: Thu, 29 Aug 2024 14:09:36 +0800 Subject: [PATCH 4/9] fix: remove apt repo arch amd64 for github docker debian image --- .github/container/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/container/Dockerfile b/.github/container/Dockerfile index eb6f5a02b..c24915608 100644 --- a/.github/container/Dockerfile +++ b/.github/container/Dockerfile @@ -1,7 +1,8 @@ # Some dependencies conflict with the compilation tools of amd64 architecture. Use arm64 directly. container image FROM --platform=linux/arm64/v8 debian:bullseye -RUN dpkg --add-architecture arm64 && \ +RUN dpkg --remove-architecture amd64 && \ + dpkg --add-architecture arm64 && \ apt-get update && \ apt-get full-upgrade --no-install-recommends -y \ build-essential crossbuild-essential-arm64 git \ From cc236026e3ccbfafb156ca0109bc2c7a648041e9 Mon Sep 17 00:00:00 2001 From: Chen Jiali Date: Thu, 29 Aug 2024 14:23:36 +0800 Subject: [PATCH 5/9] fix: try to pull arm64 docker image for CI --- .github/container/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/container/Dockerfile b/.github/container/Dockerfile index c24915608..a100b082a 100644 --- a/.github/container/Dockerfile +++ b/.github/container/Dockerfile @@ -1,9 +1,7 @@ # Some dependencies conflict with the compilation tools of amd64 architecture. Use arm64 directly. container image -FROM --platform=linux/arm64/v8 debian:bullseye +FROM --platform=linux/arm64 debian:bullseye -RUN dpkg --remove-architecture amd64 && \ - dpkg --add-architecture arm64 && \ - apt-get update && \ +RUN apt-get update && \ apt-get full-upgrade --no-install-recommends -y \ build-essential crossbuild-essential-arm64 git \ debhelper devscripts lintian abigail-tools debhelper-compat \ From 8f63c7692c164d5eabd8955993294f1fcd3302dd Mon Sep 17 00:00:00 2001 From: Chen Jiali Date: Thu, 29 Aug 2024 18:09:04 +0800 Subject: [PATCH 6/9] fix: use url to pull arm64 image --- .github/container/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/container/Dockerfile b/.github/container/Dockerfile index a100b082a..c9e1273f1 100644 --- a/.github/container/Dockerfile +++ b/.github/container/Dockerfile @@ -1,5 +1,5 @@ # Some dependencies conflict with the compilation tools of amd64 architecture. Use arm64 directly. container image -FROM --platform=linux/arm64 debian:bullseye +FROM docker.io/arm64v8/debian:bullseye RUN apt-get update && \ apt-get full-upgrade --no-install-recommends -y \ From a0267ac8e80f6c9f5eb91ee13bd3bf5091ff995f Mon Sep 17 00:00:00 2001 From: Chen Jiali Date: Thu, 29 Aug 2024 19:23:06 +0800 Subject: [PATCH 7/9] fix: add lintian-hook --- Makefile.mk | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile.mk b/Makefile.mk index 9d83e68e1..77a16ba80 100644 --- a/Makefile.mk +++ b/Makefile.mk @@ -44,7 +44,6 @@ clean-doc: clean-deb: # The packaging of the zfs team is not standard. There will be many *.install files generated during packaging, which need to be cleaned using GIT. git clean -fx debian/*.install - dh_auto_clean rm -rf debian/.debhelper debian/lib*/ debian/*pyzfs*/ debian/*zfs*/ debian/tmp/ debian/debhelper-build-stamp debian/files debian/*.debhelper.log debian/*.*.debhelper debian/*.substvars # @@ -56,7 +55,7 @@ dch: debian/changelog .PHONY: deb deb: debian - debuild --no-lintian --no-sign -b -aarm64 -Pcross + debuild --no-lintian --lintian-hook "lintian --fail-on error,warning --suppress-tags bad-distribution-in-changes-file -- %p_%v_*.changes" --no-sign -b -aarm64 -Pcross .PHONY: release release: From cb296c04e40278b2d3395fd2a8a69a9a756d7438 Mon Sep 17 00:00:00 2001 From: Chen Jiali Date: Thu, 29 Aug 2024 21:22:08 +0800 Subject: [PATCH 8/9] chore: change to bookworm for CI --- .github/container/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/container/Dockerfile b/.github/container/Dockerfile index c9e1273f1..67a47609d 100644 --- a/.github/container/Dockerfile +++ b/.github/container/Dockerfile @@ -1,5 +1,5 @@ # Some dependencies conflict with the compilation tools of amd64 architecture. Use arm64 directly. container image -FROM docker.io/arm64v8/debian:bullseye +FROM docker.io/arm64v8/debian:bookworm RUN apt-get update && \ apt-get full-upgrade --no-install-recommends -y \ From 8f9cc2f8362cb8bb47438f653f99f0c866038ab1 Mon Sep 17 00:00:00 2001 From: Chen Jiali Date: Fri, 30 Aug 2024 09:58:48 +0800 Subject: [PATCH 9/9] chore: change to unstable for CI --- .github/container/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/container/Dockerfile b/.github/container/Dockerfile index 67a47609d..0444ab1e7 100644 --- a/.github/container/Dockerfile +++ b/.github/container/Dockerfile @@ -1,5 +1,5 @@ # Some dependencies conflict with the compilation tools of amd64 architecture. Use arm64 directly. container image -FROM docker.io/arm64v8/debian:bookworm +FROM docker.io/arm64v8/debian:unstable RUN apt-get update && \ apt-get full-upgrade --no-install-recommends -y \