From 15de97e33f462434272f13e1ec3aa9cd5387441b Mon Sep 17 00:00:00 2001 From: Meng Zhuo Date: Thu, 30 May 2024 19:20:21 +0800 Subject: [PATCH 1/2] Add linux.riscv64 precompiled support --- .github/workflows/build.yml | 2 +- .multi_arch_docker | 1 + CHANGELOG.md | 1 + build/linux.riscv64/Dockerfile | 47 ++++++++++++++++++++++++++++++++++ build/linux.riscv64/build | 15 +++++++++++ build/linux.riscv64/tag | 1 + 6 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 build/linux.riscv64/Dockerfile create mode 100755 build/linux.riscv64/build create mode 100644 build/linux.riscv64/tag diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c49b25a3f..e30b2ac8f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,7 +47,7 @@ jobs: needs: package_source strategy: matrix: - build: [linux.x86_64, linux.aarch64, linux.armv6hf, darwin.x86_64, darwin.aarch64, windows.x86_64] + build: [linux.x86_64, linux.aarch64, linux.armv6hf, linux.riscv64, darwin.x86_64, darwin.aarch64, windows.x86_64] runs-on: ubuntu-latest steps: - name: Checkout repository diff --git a/.multi_arch_docker b/.multi_arch_docker index 1c5d32b9e..81048a229 100755 --- a/.multi_arch_docker +++ b/.multi_arch_docker @@ -80,6 +80,7 @@ function multi_arch_docker::main() { export DOCKER_PLATFORMS='linux/amd64' DOCKER_PLATFORMS+=' linux/arm64' DOCKER_PLATFORMS+=' linux/arm/v6' + DOCKER_PLATFORMS+=' linux/riscv64' multi_arch_docker::install_docker_buildx multi_arch_docker::login_to_docker_hub diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ae176d01..43db00cbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - SC2327/SC2328: Warn about capturing the output of redirected commands. - SC2329: Warn when (non-escaping) functions are never invoked. - SC2330: Warn about unsupported glob matches with [[ .. ]] in BusyBox. +- Precompiled binaries for Linux riscv64 (linux.riscv64) ### Changed - SC2015 about `A && B || C` no longer triggers when B is a test command. ### Fixed diff --git a/build/linux.riscv64/Dockerfile b/build/linux.riscv64/Dockerfile new file mode 100644 index 000000000..648ef2da5 --- /dev/null +++ b/build/linux.riscv64/Dockerfile @@ -0,0 +1,47 @@ +FROM ubuntu:22.04 + +ENV TARGETNAME linux.riscv64 +ENV TARGET riscv64-linux-gnu + +USER root +ENV DEBIAN_FRONTEND noninteractive + +# Init base +RUN apt update -y + +# Install qemu +RUN apt install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv +WORKDIR /qemu +RUN curl -Lv "https://download.qemu.org/qemu-9.0.0.tar.xz" | tar xJ --strip-components=1 +RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie +RUN cd build && ninja qemu-riscv64 +ENV QEMU_EXECVE 1 + + +# Set up a riscv64 userspace +RUN apt install -y --no-install-recommends debootstrap +RUN debootstrap --arch=riscv64 --foreign jammy /rvfs http://ports.ubuntu.com/ubuntu-ports +RUN cp /qemu/build/qemu-riscv64 /rvfs/usr/bin/qemu-riscv64-static + +RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"' +RUN chmod +x /bin/rv +RUN [ ! -e /rvfs/debootstrap ] || rv '/debootstrap/debootstrap' --second-stage + +# Install deps in the chroot +RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports jammy main universe' +RUN rv apt update -y +RUN rv apt install -y --no-install-recommends ghc cabal-install + +# Finally we can build the current dependencies. This takes hours. +# jobs must be 1, GHS riscv will use about 40G memory +RUN rv cabal update +RUN IFS=";" && rv cabal install --dependencies-only --jobs=1 ShellCheck +RUN IFS=';' && rv cabal install --lib --jobs=1 fgl + +# Clean up +RUN rm -rf /qemu + +# Copy the build script +WORKDIR /rvfs/scratch +COPY build /rvfs/usr/bin/build +ENTRYPOINT ["/bin/rv", "/usr/bin/build"] diff --git a/build/linux.riscv64/build b/build/linux.riscv64/build new file mode 100755 index 000000000..19cb143c3 --- /dev/null +++ b/build/linux.riscv64/build @@ -0,0 +1,15 @@ +#!/bin/sh +set -xe +{ + tar xzv --strip-components=1 + chmod +x striptests && ./striptests + mkdir "$TARGETNAME" + cabal update + ( IFS=';'; cabal build --enable-executable-static ) + find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; + ls -l "$TARGETNAME" + "$TARGET-strip" -s "$TARGETNAME/shellcheck" + ls -l "$TARGETNAME" + qemu-riscv64-static "$TARGETNAME/shellcheck" --version +} >&2 +tar czv "$TARGETNAME" diff --git a/build/linux.riscv64/tag b/build/linux.riscv64/tag new file mode 100644 index 000000000..901eaaa2b --- /dev/null +++ b/build/linux.riscv64/tag @@ -0,0 +1 @@ +koalaman/scbuilder-linux-riscv64 From 23e76de4f2a640ecd7c8c1da23495d985ef095e6 Mon Sep 17 00:00:00 2001 From: Vidar Holen Date: Tue, 11 Jun 2024 06:00:22 +0000 Subject: [PATCH 2/2] Allow riscv64 image to run without binfmt_misc --- build/linux.riscv64/Dockerfile | 44 ++++++----- build/linux.riscv64/build | 10 ++- build/linux.riscv64/cabal.project.freeze | 93 ++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 21 deletions(-) create mode 100644 build/linux.riscv64/cabal.project.freeze diff --git a/build/linux.riscv64/Dockerfile b/build/linux.riscv64/Dockerfile index 648ef2da5..0ac95e59d 100644 --- a/build/linux.riscv64/Dockerfile +++ b/build/linux.riscv64/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 ENV TARGETNAME linux.riscv64 ENV TARGET riscv64-linux-gnu @@ -7,39 +7,47 @@ USER root ENV DEBIAN_FRONTEND noninteractive # Init base -RUN apt update -y +RUN apt-get update -y # Install qemu -RUN apt install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv +RUN apt-get install -y --no-install-recommends build-essential ninja-build python3 pkg-config libglib2.0-dev libpixman-1-dev curl ca-certificates python3-virtualenv git python3-setuptools WORKDIR /qemu -RUN curl -Lv "https://download.qemu.org/qemu-9.0.0.tar.xz" | tar xJ --strip-components=1 -RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie +RUN git clone --depth 1 https://github.com/koalaman/qemu . +#RUN git clone https://github.com/balena-io/qemu . +# Release 7.0.0 +#RUN git checkout 639d1d8903f65d74eb04c49e0df7a4b2f014cd86 +RUN ./configure --target-list=riscv64-linux-user --static --disable-system --disable-pie --disable-werror RUN cd build && ninja qemu-riscv64 ENV QEMU_EXECVE 1 # Set up a riscv64 userspace -RUN apt install -y --no-install-recommends debootstrap -RUN debootstrap --arch=riscv64 --foreign jammy /rvfs http://ports.ubuntu.com/ubuntu-ports +RUN apt-get install -y --no-install-recommends debootstrap +RUN debootstrap --arch=riscv64 --foreign noble /rvfs http://ports.ubuntu.com/ubuntu-ports RUN cp /qemu/build/qemu-riscv64 /rvfs/usr/bin/qemu-riscv64-static -RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"' +# Command to run riscv binaries in the chroot. The Haskell runtime allocates 1TB +# vspace up front and QEmu has a RAM cost per vspace, so use ulimit to allocate +# less and reduce RAM usage. +RUN printf > /bin/rv '%s\n' '#!/bin/sh' 'ulimit -v $((10*1024*1024)); chroot /rvfs /usr/bin/qemu-riscv64-static /usr/bin/env "$@"' RUN chmod +x /bin/rv RUN [ ! -e /rvfs/debootstrap ] || rv '/debootstrap/debootstrap' --second-stage # Install deps in the chroot -RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports jammy main universe' -RUN rv apt update -y -RUN rv apt install -y --no-install-recommends ghc cabal-install +RUN printf > /rvfs/etc/apt/sources.list '%s\n' 'deb http://ports.ubuntu.com/ubuntu-ports noble main universe' +RUN rv apt-get update -y +RUN rv apt-get install -y --no-install-recommends ghc cabal-install -# Finally we can build the current dependencies. This takes hours. -# jobs must be 1, GHS riscv will use about 40G memory RUN rv cabal update -RUN IFS=";" && rv cabal install --dependencies-only --jobs=1 ShellCheck -RUN IFS=';' && rv cabal install --lib --jobs=1 fgl - -# Clean up -RUN rm -rf /qemu +# Generated with: cabal freeze -c 'hashable -arch-native'. We put it in /etc so cabal won't find it. +COPY cabal.project.freeze /rvfs/etc +# Awful hack to install everything from the freeze file +# This basically turns 'any.tagged ==0.8.8' into tagged-0.8.8 to install by version, +# and adds a -c before 'hashable -arch-native +integer-gmp' to make it a flag constraint. +RUN < /rvfs/etc/cabal.project.freeze sed 's/constraints:/&\n /' | grep -vw rts | sed -n -e 's/^ *\([^,]*\).*/\1/p' | sed -e 's/any\.\([^ ]*\) ==\(.*\)/\1-\2/; te; s/.*/-c\n&/; :e' > /tmp/preinstall-flags +# Finally we can build the current dependencies. This takes hours. +# There's apparently a random segfault during assembly, so retry a few times. +RUN set -x; IFS=${IFS# }; f() { rv cabal install --keep-going $(cat /tmp/preinstall-flags); }; for i in $(seq 5); do f; ret=$?; [ $ret = 0 ] && break; done; exit $ret # Copy the build script WORKDIR /rvfs/scratch diff --git a/build/linux.riscv64/build b/build/linux.riscv64/build index 19cb143c3..63c487faf 100755 --- a/build/linux.riscv64/build +++ b/build/linux.riscv64/build @@ -1,15 +1,19 @@ #!/bin/sh set -xe +IFS=';' { tar xzv --strip-components=1 chmod +x striptests && ./striptests + # Use a freeze file to ensure we use the same dependencies we cached during + # the docker image build. We don't want to spend time compiling anything new. + cp /etc/cabal.project.freeze . mkdir "$TARGETNAME" - cabal update - ( IFS=';'; cabal build --enable-executable-static ) + # Retry in case of random segfault + cabal build --enable-executable-static || cabal build --enable-executable-static find . -name shellcheck -type f -exec mv {} "$TARGETNAME/" \; ls -l "$TARGETNAME" "$TARGET-strip" -s "$TARGETNAME/shellcheck" ls -l "$TARGETNAME" - qemu-riscv64-static "$TARGETNAME/shellcheck" --version + "$TARGETNAME/shellcheck" --version } >&2 tar czv "$TARGETNAME" diff --git a/build/linux.riscv64/cabal.project.freeze b/build/linux.riscv64/cabal.project.freeze new file mode 100644 index 000000000..cbb42e10a --- /dev/null +++ b/build/linux.riscv64/cabal.project.freeze @@ -0,0 +1,93 @@ +active-repositories: hackage.haskell.org:merge +constraints: any.Diff ==0.5, + any.OneTuple ==0.4.2, + any.QuickCheck ==2.14.3, + QuickCheck -old-random +templatehaskell, + any.StateVar ==1.2.2, + any.aeson ==2.2.3.0, + aeson +ordered-keymap, + any.array ==0.5.4.0, + any.assoc ==1.1.1, + assoc -tagged, + any.base ==4.17.2.0, + any.base-orphans ==0.9.2, + any.bifunctors ==5.6.2, + bifunctors +tagged, + any.binary ==0.8.9.1, + any.bytestring ==0.11.5.2, + any.character-ps ==0.1, + any.comonad ==5.0.8, + comonad +containers +distributive +indexed-traversable, + any.containers ==0.6.7, + any.contravariant ==1.5.5, + contravariant +semigroups +statevar +tagged, + any.data-fix ==0.3.3, + any.deepseq ==1.4.8.0, + any.directory ==1.3.7.1, + any.distributive ==0.6.2.1, + distributive +semigroups +tagged, + any.dlist ==1.0, + dlist -werror, + any.exceptions ==0.10.5, + any.fgl ==5.8.2.0, + fgl +containers042, + any.filepath ==1.4.2.2, + any.foldable1-classes-compat ==0.1, + foldable1-classes-compat +tagged, + any.generically ==0.1.1, + any.ghc-bignum ==1.3, + any.ghc-boot-th ==9.4.7, + any.ghc-prim ==0.9.1, + any.hashable ==1.4.6.0, + hashable -arch-native +integer-gmp -random-initial-seed, + any.indexed-traversable ==0.1.4, + any.indexed-traversable-instances ==0.1.2, + any.integer-conversion ==0.1.1, + any.integer-logarithms ==1.0.3.1, + integer-logarithms -check-bounds +integer-gmp, + any.mtl ==2.2.2, + any.network-uri ==2.6.4.2, + any.os-string ==2.0.3, + any.parsec ==3.1.16.1, + any.pretty ==1.1.3.6, + any.primitive ==0.9.0.0, + any.process ==1.6.17.0, + any.random ==1.2.1.2, + any.regex-base ==0.94.0.2, + any.regex-tdfa ==1.3.2.2, + regex-tdfa +doctest -force-o2, + any.rts ==1.0.2, + any.scientific ==0.3.8.0, + scientific -integer-simple, + any.semialign ==1.3.1, + semialign +semigroupoids, + any.semigroupoids ==6.0.1, + semigroupoids +comonad +containers +contravariant +distributive +tagged +unordered-containers, + any.splitmix ==0.1.0.5, + splitmix -optimised-mixer, + any.stm ==2.5.1.0, + any.strict ==0.5, + any.tagged ==0.8.8, + tagged +deepseq +transformers, + any.template-haskell ==2.19.0.0, + any.text ==2.0.2, + any.text-iso8601 ==0.1.1, + any.text-short ==0.1.6, + text-short -asserts, + any.th-abstraction ==0.7.0.0, + any.th-compat ==0.1.5, + any.these ==1.2.1, + any.time ==1.12.2, + any.time-compat ==1.9.7, + any.transformers ==0.5.6.2, + any.transformers-compat ==0.7.2, + transformers-compat -five +five-three -four +generic-deriving +mtl -three -two, + any.unix ==2.7.3, + any.unordered-containers ==0.2.20, + unordered-containers -debug, + any.uuid-types ==1.0.6, + any.vector ==0.13.1.0, + vector +boundschecks -internalchecks -unsafechecks -wall, + any.vector-stream ==0.1.0.1, + any.witherable ==0.5 +index-state: hackage.haskell.org 2024-06-17T00:48:51Z