Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] riotbuild: Split out build logic #226

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

[WIP] riotbuild: Split out build logic #226

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ce7dd92
riotdocker-base: Split out build logic
maribu Feb 17, 2023
58bf7ae
static-test-tools: Split out build logic
maribu Feb 22, 2023
f6109b6
riotbuild: Split out and clean up build logic
maribu Feb 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
351 changes: 17 additions & 334 deletions riotbuild/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,341 +24,24 @@ FROM ${DOCKER_REGISTRY}/static-test-tools:latest

LABEL maintainer="Kaspar Schleiser <[email protected]>"

ENV DEBIAN_FRONTEND noninteractive

ENV LC_ALL C.UTF-8
ENV LANG C.UTF-8

# copy some included packages
RUN mkdir /pkgs
COPY files/libsocketcan-dev_0.0.11-1_i386.deb /pkgs/libsocketcan-dev_0.0.11-1_i386.deb
COPY files/libsocketcan2_0.0.11-1_i386.deb /pkgs/libsocketcan2_0.0.11-1_i386.deb

# The following package groups will be installed:
# - update the package index files to latest available version
# - native platform development and build system functionality (about 400 MB installed)
# - Cortex-M development (about 550 MB installed), through the gcc-arm-embedded PPA
# - MSP430 development (about 120 MB installed)
# - AVR development (about 110 MB installed)
# - LLVM/Clang build environment (about 125 MB installed)
# - QEMU
# All apt files will be deleted afterwards to reduce the size of the container image.
# The OS must not be updated by apt. Docker image should be build against the latest
# updated base OS image. This can be forced with `--pull` flag.
# This is all done in a single RUN command to reduce the number of layers and to
# allow the cleanup to actually save space.
# Total size without cleaning is approximately 1.525 GB (2016-03-08)
# After adding the cleanup commands the size is approximately 1.497 GB
ARG LLVM_VERSION=14
RUN \
dpkg --add-architecture i386 >&2 && \
echo 'Update the package index files to latest available versions' >&2 && \
apt-get update \
&& echo 'Installing native toolchain and build system functionality' >&2 && \
apt-get -y --no-install-recommends install \
afl++ \
automake \
bsdmainutils \
build-essential \
ca-certificates \
ccache \
cmake \
curl \
cython3 \
gcc-multilib \
gdb \
g++-multilib \
libffi-dev \
libpcre3 \
libtool \
libsdl2-dev:i386 \
m4 \
ninja-build \
parallel \
protobuf-compiler \
python2 \
python3-setuptools \
python3-wheel \
p7zip \
qemu-system-arm \
rsync \
socat \
ssh-client \
subversion \
unzip \
vim-common \
xsltproc \
&& echo 'Installing MSP430 toolchain' >&2 && \
apt-get -y --no-install-recommends install \
gcc-msp430 \
msp430-libc \
&& echo 'Installing AVR toolchain' >&2 && \
apt-get -y --no-install-recommends install \
gcc-avr \
binutils-avr \
avr-libc \
&& echo 'Installing LLVM/Clang toolchain' >&2 && \
apt-get -y --no-install-recommends install \
llvm-${LLVM_VERSION} \
clang-${LLVM_VERSION} \
clang-tools-${LLVM_VERSION} \
lld-${LLVM_VERSION} \
llvm \
clang \
clang-tools \
&& echo 'Installing C2Rust (build) dependencies' >&2 && \
apt-get -y --no-install-recommends install \
libclang-dev \
libssl-dev \
llvm-dev \
&& \
SYMS=$(find /usr/bin -type l) && \
for file in ${SYMS}; do \
SYMTARGET=$(readlink -f ${file}) && \
SYMNAME=${file%"-${LLVM_VERSION}"} && \
# Filter by symlinks starting with /usr/bin/llvm-${LLVM_VERSION}
case "${SYMTARGET}" in "/usr/lib/llvm-${LLVM_VERSION}"* ) ln -sf ${SYMTARGET} ${SYMNAME}; esac \
done \
&& echo 'Installing additional packages required for ESP32 toolchain' >&2 && \
apt-get -y --no-install-recommends install \
python3-serial \
libpython2.7 \
telnet \
&& echo 'Installing local packages' >&2 && \
apt-get install -y --no-install-recommends /pkgs/*.deb \
&& echo 'Cleaning up installation files' >&2 && \
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /pkgs

# Install ARM GNU embedded toolchain
# For updates, see https://developer.arm.com/open-source/gnu-toolchain/gnu-rm/downloads
ARG ARM_URLBASE=https://developer.arm.com/-/media/Files/downloads/gnu-rm
ARG ARM_URL=${ARM_URLBASE}/10.3-2021.10/gcc-arm-none-eabi-10.3-2021.10-x86_64-linux.tar.bz2
ARG ARM_MD5=2383e4eb4ea23f248d33adc70dc3227e
ARG ARM_FOLDER=gcc-arm-none-eabi-10.3-2021.10
RUN echo 'Installing arm-none-eabi toolchain from arm.com' >&2 && \
mkdir -p /opt && \
curl -L -o /opt/gcc-arm-none-eabi.tar.bz2 ${ARM_URL} && \
echo "${ARM_MD5} /opt/gcc-arm-none-eabi.tar.bz2" | md5sum -c && \
tar -C /opt -jxf /opt/gcc-arm-none-eabi.tar.bz2 && \
rm -f /opt/gcc-arm-none-eabi.tar.bz2 && \
echo 'Removing documentation' >&2 && \
rm -rf /opt/gcc-arm-none-eabi-*/share/doc
# No need to dedup, the ARM toolchain is already using hard links for the duplicated files

ENV PATH ${PATH}:/opt/${ARM_FOLDER}/bin

# Install MIPS binary toolchain
# For updates: https://www.mips.com/develop/tools/codescape-mips-sdk/ (select "Codescape GNU Toolchain")
ARG MIPS_VERSION=2020.06-01
RUN echo 'Installing mips-mti-elf toolchain from mips.com' >&2 && \
mkdir -p /opt && \
curl -L "https://codescape.mips.com/components/toolchain/${MIPS_VERSION}/Codescape.GNU.Tools.Package.${MIPS_VERSION}.for.MIPS.MTI.Bare.Metal.CentOS-6.x86_64.tar.gz" -o - \
| tar -C /opt -zx && \
echo 'Removing documentation and translations' >&2 && \
rm -rf /opt/mips-mti-elf/*/share/{doc,info,man,locale} && \
echo 'Deduplicating binaries' && \
cd /opt/mips-mti-elf/*/mips-mti-elf/bin && \
for f in *; do test -f "../../bin/mips-mti-elf-$f" && ln -f "../../bin/mips-mti-elf-$f" "$f"; done && cd -

ENV MIPS_ELF_ROOT /opt/mips-mti-elf/${MIPS_VERSION}

ENV PATH ${PATH}:${MIPS_ELF_ROOT}/bin

# Install RISC-V binary toolchain
ARG RISCV_VERSION=10.2.0-1.2
RUN mkdir -p /opt && \
wget -q https://github.com/xpack-dev-tools/riscv-none-embed-gcc-xpack/releases/download/v${RISCV_VERSION}/xpack-riscv-none-embed-gcc-${RISCV_VERSION}-linux-x64.tar.gz -O- \
| tar -C /opt -xz && \
echo 'Removing documentation' >&2 && \
rm -rf /opt/xpack-riscv-none-embed-gcc-${RISCV_VERSION}/share/doc && \
echo 'Deduplicating binaries' >&2 && \
cd /opt/xpack-riscv-none-embed-gcc-${RISCV_VERSION}/riscv-none-embed/bin && \
for f in *; do test -f "../../bin/riscv-none-embed-$f" && \
ln -f "../../bin/riscv-none-embed-$f" "$f"; \
done && \
cd -

ENV PATH $PATH:/opt/xpack-riscv-none-embed-gcc-${RISCV_VERSION}/bin

# Install complete ESP8266 toolchain in /opt/esp (139 MB after cleanup)
# remember https://github.com/RIOT-OS/RIOT/pull/10801 when updating
RUN echo 'Installing ESP8266 toolchain' >&2 && \
mkdir -p /opt/esp && \
cd /opt/esp && \
git clone https://github.com/gschorcht/xtensa-esp8266-elf && \
cd xtensa-esp8266-elf && \
git checkout -q 696257c2b43e2a107d3108b2c1ca6d5df3fb1a6f && \
rm -rf .git && \
cd /opt/esp && \
git clone https://github.com/gschorcht/RIOT-Xtensa-ESP8266-RTOS-SDK.git ESP8266_RTOS_SDK && \
cd ESP8266_RTOS_SDK/ && \
git checkout -q c0174eff7278eb5beea66ce1f65b7af57432d2a9 && \
rm -rf .git* docs examples Kconfig make README.md tools && \
cd components && \
rm -rf app_update aws_iot bootloader cjson coap espos esptool_py esp-tls \
freertos jsmn libsodium log mdns mqtt newlib partition_table \
pthread smartconfig_ack spiffs ssl tcpip_adapter vfs && \
find . -type f -name '*.[csS]' -exec rm {} \; && \
find . -type f -name '*.cpp' -exec rm {} \;

ENV PATH $PATH:/opt/esp/xtensa-esp8266-elf/bin
ENV ESP8266_RTOS_SDK_DIR /opt/esp/ESP8266_RTOS_SDK

# Install ESP32 toolchain in /opt/esp (289 MB)
ARG ESP32_GCC_RELEASE="esp-2021r2-patch3"
ARG ESP32_GCC_VERSION="gcc8_4_0"
ARG ESP32_GCC_REPO=https://github.com/espressif/crosstool-NG/releases/download
ARG ESP32_GCC_FILE=xtensa-esp32-elf-${ESP32_GCC_VERSION}-${ESP32_GCC_RELEASE}-linux-amd64.tar.gz
ARG ESP32_GCC_URL=${ESP32_GCC_REPO}/${ESP32_GCC_RELEASE}/${ESP32_GCC_FILE}

RUN echo 'Installing ESP32 toolchain' >&2 && \
curl -L ${ESP32_GCC_URL} | tar -C /opt/esp -xz && \
pip install --no-cache-dir pyserial
ENV PATH $PATH:/opt/esp/xtensa-esp32-elf/bin

# Install ESP32 QEMU in /opt/esp (89 MB)
ARG ESP32_QEMU_VERSION="esp-develop-20220203"
ARG ESP32_QEMU_REPO=https://github.com/espressif/qemu/releases/download
ARG ESP32_QEMU_FILE=qemu-${ESP32_QEMU_VERSION}.tar.bz2
ARG ESP32_QEMU_URL=${ESP32_QEMU_REPO}/${ESP32_QEMU_VERSION}/${ESP32_QEMU_FILE}

RUN echo 'Installing ESP32 QEMU' >&2 && \
curl -L ${ESP32_QEMU_URL} | tar -C /opt/esp -xj
ENV PATH $PATH:/opt/esp/qemu/bin

# Install ESP32-C3 toolchain in /opt/esp (344 MB)
ARG ESP32_GCC_FILE=riscv32-esp-elf-${ESP32_GCC_VERSION}-${ESP32_GCC_RELEASE}-linux-amd64.tar.gz
ARG ESP32_GCC_URL=${ESP32_GCC_REPO}/${ESP32_GCC_RELEASE}/${ESP32_GCC_FILE}

RUN echo 'Installing ESP32-C3 toolchain' >&2 && \
curl -L ${ESP32_GCC_URL} | tar -C /opt/esp -xz && \
pip install --no-cache-dir pyserial
ENV PATH $PATH:/opt/esp/riscv32-esp-elf/bin

# Install ESP32-S2 toolchain in /opt/esp (290 MB)
ARG ESP32_GCC_FILE=xtensa-esp32s2-elf-${ESP32_GCC_VERSION}-${ESP32_GCC_RELEASE}-linux-amd64.tar.gz
ARG ESP32_GCC_URL=${ESP32_GCC_REPO}/${ESP32_GCC_RELEASE}/${ESP32_GCC_FILE}

RUN echo 'Installing ESP32-S2 toolchain' >&2 && \
curl -L ${ESP32_GCC_URL} | tar -C /opt/esp -xz && \
pip install --no-cache-dir pyserial
ENV PATH $PATH:/opt/esp/xtensa-esp32s2-elf/bin

# Install ESP32-S3 toolchain in /opt/esp (292 MB)
ARG ESP32_GCC_FILE=xtensa-esp32s3-elf-${ESP32_GCC_VERSION}-${ESP32_GCC_RELEASE}-linux-amd64.tar.gz
ARG ESP32_GCC_URL=${ESP32_GCC_REPO}/${ESP32_GCC_RELEASE}/${ESP32_GCC_FILE}

RUN echo 'Installing ESP32-S3 toolchain' >&2 && \
curl -L ${ESP32_GCC_URL} | tar -C /opt/esp -xz && \
pip install --no-cache-dir pyserial
ENV PATH $PATH:/opt/esp/xtensa-esp32s3-elf/bin

ARG PICOLIBC_REPO=https://github.com/keith-packard/picolibc
ARG PICOLIBC_TAG=1.4.6
ARG PICOLIBC_URL=${PICOLIBC_REPO}/archive/${PICOLIBC_TAG}.tar.gz
ARG PICOLIBC_ARCHIVE=${PICOLIBC_TAG}.tar.gz

RUN echo 'Building and Installing PicoLIBC' >&2 && \
pip3 install --no-cache-dir meson && \
mkdir -p /usr/src/picolibc && \
cd /usr/src/picolibc/ &&\
curl -L -o ${PICOLIBC_ARCHIVE} ${PICOLIBC_URL} && \
tar -xf ${PICOLIBC_ARCHIVE} && \
cd picolibc-${PICOLIBC_TAG}

COPY cross-riscv-none-embed.txt /usr/src/picolibc/picolibc-${PICOLIBC_TAG}/

RUN cd /usr/src/picolibc/picolibc-${PICOLIBC_TAG} && \
which riscv-none-embed-gcc && \
ls -al /opt/xpack-riscv-none-embed-gcc-${RISCV_VERSION}/bin && \
mkdir build-arm build-riscv build-esp32 && \
cd build-riscv && \
meson .. -Dtests=true -Dmultilib=false -Dincludedir=picolibc/riscv-none-embed/include -Dlibdir=picolibc/riscv-none-embed/lib --cross-file ../cross-riscv-none-embed.txt && \
ninja && ninja install && \
cd ../build-esp32 && \
sh ../do-esp32-configure && \
ninja && ninja install && \
cd ../build-arm && \
sh ../do-arm-configure && \
ninja && ninja install

# No need to keep the sources around
RUN rm -rf /usr/src/picolibc

# RIOT toolchains
ARG RIOT_TOOLCHAIN_GCC_VERSION=10.1.0
ARG RIOT_TOOLCHAIN_PACKAGE_VERSION=18
ARG RIOT_TOOLCHAIN_TAG=20200722112854-64162e7
ARG RIOT_TOOLCHAIN_GCCPKGVER=${RIOT_TOOLCHAIN_GCC_VERSION}-${RIOT_TOOLCHAIN_PACKAGE_VERSION}
ARG RIOT_TOOLCHAIN_SUBDIR=${RIOT_TOOLCHAIN_GCCPKGVER}-${RIOT_TOOLCHAIN_TAG}

ARG MSP430_URL=https://github.com/RIOT-OS/toolchains/releases/download/${RIOT_TOOLCHAIN_SUBDIR}/riot-msp430-elf-${RIOT_TOOLCHAIN_GCCPKGVER}.tgz
RUN echo 'Installing RIOT MSP430 ELF toolchain' >&2 && \
wget -q ${MSP430_URL} -O- | tar -C /opt -xz
ENV PATH $PATH:/opt/riot-toolchain/msp430-elf/${RIOT_TOOLCHAIN_GCCPKGVER}/bin

# install required python packages from file
# numpy must be already installed before installing some other requirements (emlearn)
RUN pip3 install --no-cache-dir numpy==1.22.4
COPY requirements.txt /tmp/requirements.txt
RUN echo 'Installing python3 packages' >&2 \
&& pip3 install --no-cache-dir pybind11 \
&& pip3 install --no-cache-dir -r /tmp/requirements.txt \
&& rm /tmp/requirements.txt

# While sourcing ~/.cargo/env later would have the right short-term effect,
# we'd still need to set the right path even later when HOME is
# /data/riotbuild -- so setting it right away.
ENV PATH ${PATH}:/opt/rustup/.cargo/bin
# Install nightly Rust via rustup; this is needed for Rust-on-RIOT builds and
# contains all CARGO_TARGETs currently recognized for RIOT targets.
#
# *_HOMEs moved to /opt to make them world readable. RUSTUP_HOME is set
# persistently in case someone in their image wants to do a quick `sudo rustup
# toolchain add` or similar; CARGO_HOME is not because the user will need to
# write there, and all rustup does here is to place some binaries that later
# fan out to RUSTUP_HOME anyway.
#
# Components: rust-src is needed to run `-Z build-std=core`, which in turn is
# needed on AVR (which thus doesn't need the avr-unknown-gnu-atmega328 target;
# being able to build core might be useful for other targets as well).
ENV RUSTUP_HOME /opt/rustup/.rustup
RUN \
RUSTUP_HOME=/opt/rustup/.rustup \
CARGO_HOME=/opt/rustup/.cargo sh -c "\
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain nightly-2022-09-25 && \
rustup toolchain add stable && \
for T in nightly-2022-09-25 stable; do \
rustup component add rust-src --toolchain \$T && \
rustup target add i686-unknown-linux-gnu --toolchain \$T && \
rustup target add riscv32imac-unknown-none-elf --toolchain \$T && \
rustup target add thumbv7em-none-eabihf --toolchain \$T && \
rustup target add thumbv7em-none-eabi --toolchain \$T && \
rustup target add thumbv7m-none-eabi --toolchain \$T && \
rustup target add thumbv6m-none-eabi --toolchain \$T && \
rustup target add thumbv8m.main-none-eabihf --toolchain \$T && \
rustup target add thumbv8m.main-none-eabi --toolchain \$T && \
rustup target add thumbv8m.base-none-eabi --toolchain \$T && \
true; \
done"

RUN \
echo 'Installing C2Rust' >&2 && \
CARGO_HOME=/opt/rustup/.cargo cargo install --no-track --locked c2rust --git https://github.com/chrysn-pull-requests/c2rust --branch riscv-vector-types && \
echo 'Cleaning up root-owned crates.io cache' >&2 && \
rm -rf /opt/rustup/.cargo/{git,registry,.package-cache}

# get Dockerfile version from build args
ARG RIOTBUILD_VERSION=unknown
ENV RIOTBUILD_VERSION $RIOTBUILD_VERSION

ARG RIOTBUILD_COMMIT=unknown
ENV RIOTBUILD_COMMIT $RIOTBUILD_COMMIT

ARG RIOTBUILD_BRANCH=unknown
ENV RIOTBUILD_BRANCH $RIOTBUILD_BRANCH

# watch for single ">" vs double ">>"!
RUN echo "RIOTBUILD_VERSION=$RIOTBUILD_VERSION" > /etc/riotbuild
RUN echo "RIOTBUILD_COMMIT=$RIOTBUILD_COMMIT" >> /etc/riotbuild
RUN echo "RIOTBUILD_BRANCH=$RIOTBUILD_BRANCH" >> /etc/riotbuild
ENV DEBIAN_FRONTEND="noninteractive" \
LC_ALL="C.UTF-8" \
LANG="C.UTF-8" \
ESP8266_RTOS_SDK_DIR="/opt/esp/ESP8266_RTOS_SDK" \
RUSTUP_HOME="/opt/rustup/.rustup" \
PATH="${PATH}:/opt/arm-none-eabi/bin:/opt/riscv-none-elf/bin:/opt/esp/xtensa-esp8266-elf/bin:/opt/esp/xtensa-esp32-elf/bin:/opt/esp/qemu/bin:/opt/esp/riscv32-esp-elf/bin:/opt/esp/xtensa-esp32s2-elf/bin:/opt/esp/xtensa-esp32s3-elf/bin:/opt/msp430-elf/bin:/opt/rustup/.cargo/bin" \
RIOTBUILD_VERSION="$RIOTBUILD_VERSION" \
RIOTBUILD_COMMIT="$RIOTBUILD_COMMIT" \
RIOTBUILD_BRANCH="$RIOTBUILD_BRANCH"

RUN \
--mount=type=bind,source=build.sh,target=/root/build.sh \
--mount=type=bind,source=requirements.txt,target=/root/requirements.txt \
--mount=type=bind,source=files/libsocketcan-dev_0.0.11-1_i386.deb,target=/root/libsocketcan-dev_0.0.11-1_i386.deb \
--mount=type=bind,source=files/libsocketcan2_0.0.11-1_i386.deb,target=/root/libsocketcan2_0.0.11-1_i386.deb \
cd /root && ./build.sh

Loading