diff --git a/.cargo/config b/.cargo/config deleted file mode 100644 index 5d8aa3a8e2e06..0000000000000 --- a/.cargo/config +++ /dev/null @@ -1,5 +0,0 @@ -[target.x86_64-unknown-linux-musl] -rustflags = ["-C", "link-args=-rdynamic"] - -[target.x86_64-unknown-linux-gnu] -rustflags = ["-C", "link-args=-rdynamic"] diff --git a/.dockerignore b/.dockerignore index 7e128117c2be0..27351c9b68599 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,5 +7,6 @@ !shell.nix !default.nix !scripts/environment +scripts/environment/cross !.envrc !tests diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 0bf318b7e1097..8be49d7e8c23d 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -28,6 +28,6 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make bench diff --git a/.github/workflows/lints.yml b/.github/workflows/lints.yml index f70655acd7197..434a834626ed2 100644 --- a/.github/workflows/lints.yml +++ b/.github/workflows/lints.yml @@ -38,7 +38,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make check-markdown @@ -62,7 +62,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: echo "::add-path::/home/runner/.local/bin" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fd6a11710abbf..9049105f9e1ae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -306,7 +306,7 @@ jobs: with: name: vector-aarch64.rpm path: target/artifacts - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: | export VERSION=$(make version) @@ -377,7 +377,7 @@ jobs: with: name: vector-aarch64.rpm path: target/artifacts - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: | export VERSION=$(make version) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3b776fffc63ac..e611d5351443e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -63,7 +63,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test @@ -82,7 +82,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: bash scripts/environment/bootstrap-macos-10.sh + - run: bash scripts/environment/bootstrap/macos-10.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test @@ -100,6 +100,80 @@ jobs: RUSTFLAGS: "-D warnings" run: cargo test --no-default-features --features default-msvc + build-centos-7-x64_64-unknown-linux-gnu: + name: Release build - CentOS 7 - x64_64-unknown-linux-gnu + runs-on: ubuntu-20.04 + env: + RELEASE: false + # We need Centos 7 for Glibc 2.17 here! :) + ENVIRONMENT_BASEIMAGE: "centos:7" + ENVIRONMENT: true + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - run: make environment-prepare + - run: make build-x64_64-unknown-linux-gnu + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: vector-x64_64-unknown-linux-gnu + path: target/x64_64-unknown-linux-gnu/debug/vector + + build-ubuntu-20-04-x86_64-unknown-linux-musl: + name: Release build - Ubuntu 20.04 - x86_64-unknown-linux-musl + runs-on: ubuntu-20.04 + env: + RELEASE: false + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh + - run: bash scripts/environment/prepare.sh + - run: make slim-builds + - run: make build-cross-x86_64-unknown-linux-musl + - run: make clean-cross + - run: make build-x86_64-unknown-linux-musl + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: vector-x86_64-unknown-linux-musl + path: target/x86_64-unknown-linux-musl/debug/vector + + build-ubuntu-20-04-aarch64-unknown-linux-musl: + name: Release build - Ubuntu 20.04 - aarch64-unknown-linux-musl + runs-on: ubuntu-20.04 + env: + RELEASE: false + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh + - run: bash scripts/environment/prepare.sh + - run: make slim-builds + - run: make build-cross-aarch64-unknown-linux-musl + - run: make clean-cross + - run: make build-aarch64-unknown-linux-musl + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: vector-aarch64-unknown-linux-musl + path: target/aarch64-unknown-linux-musl/debug/vector + test-misc: name: Shutdown - Linux runs-on: ubuntu-20.04 @@ -111,7 +185,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test-shutdown @@ -129,7 +203,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test-integration-aws @@ -145,7 +219,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test-integration-clickhouse @@ -161,7 +235,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test-integration-docker @@ -177,7 +251,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test-integration-elasticsearch @@ -193,7 +267,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test-integration-gcp @@ -209,7 +283,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test-integration-influxdb @@ -225,7 +299,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test-integration-kafka @@ -241,7 +315,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test-integration-loki @@ -257,7 +331,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test-integration-pulsar @@ -273,7 +347,7 @@ jobs: ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - run: sudo bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: sudo bash scripts/environment/bootstrap/ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: make slim-builds - run: make test-integration-splunk diff --git a/.gitignore b/.gitignore index c479258247dba..9d50c04bf1a9e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ scripts/package-lock.json target node_modules tests/data/wasm/*/target +scripts/environment/cross diff --git a/Cargo.lock b/Cargo.lock index cd496e8491f73..56fff4ea6ea51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3502,9 +3502,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "2.0.0+1.4.2" +version = "2.1.0+1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c31c649704b732bb27e47748ff8a288b4f72eccc08b777b1a442b52152b11c" +checksum = "6d3f17044cba41c7309facedc72ca9bf25f177bf1e06756318e010f043713017" dependencies = [ "cmake", "libc", diff --git a/Cargo.toml b/Cargo.toml index 6829f36b06f18..bb2431c597e1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -201,20 +201,20 @@ k8s-test-framework = { version = "0.1", path = "lib/k8s-test-framework" } [features] # Default features for *-unknown-linux-gnu and *-apple-darwin default = ["sources", "transforms", "sinks", "vendor-all", "unix", "leveldb-plain", "rdkafka-plain"] -default-musl = ["sources", "transforms", "sinks", "vendor-all", "unix", "leveldb-cmake", "rdkafka-cmake"] +default-musl = ["sources", "transforms", "sinks", "vendor-all", "unix", "leveldb-plain", "rdkafka-plain"] # Default features for *-unknown-linux-* which make use of `cmake` for dependencies default-cmake = ["sources", "transforms", "sinks", "vendor-all", "unix", "leveldb-cmake", "rdkafka-cmake"] # Default features for *-pc-windows-msvc # TODO: Enable SASL https://github.com/timberio/vector/pull/3081#issuecomment-659298042 -default-msvc = ["sources", "transforms", "sinks", "vendor-openssl", "vendor-libz", "leveldb-cmake", "rdkafka-cmake"] +default-msvc = ["sources", "transforms", "sinks", "vendor-openssl", "vendor-libz", "rdkafka-cmake"] # Enables features that work only on systems providing `cfg(unix)` unix = ["jemallocator"] # These are **very** useful on Cross compilations! vendor-all = ["vendor-sasl", "vendor-openssl", "vendor-libz"] vendor-sasl = ["rdkafka/gssapi-vendored"] -vendor-openssl = ["openssl/vendored"] -vendor-libz = ["libz-sys/static"] +vendor-openssl = ["openssl/vendored", "rdkafka/ssl-vendored"] +vendor-libz = ["libz-sys/static", "rdkafka/libz-static"] sasl = ["rdkafka/gssapi"] # This feature is less portable, but doesn't require `cmake` as build dependency rdkafka-plain = ["rdkafka"] diff --git a/Makefile b/Makefile index 10ca09a4a73bb..b091304bc6aed 100644 --- a/Makefile +++ b/Makefile @@ -5,14 +5,19 @@ RUN := $(shell realpath $(shell dirname $(firstword $(MAKEFILE_LIST)))/scripts/r # Begin OS detection ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10... export OPERATING_SYSTEM := Windows - export RUST_TARGET ?= "x86_64-unknown-windows-msvc" + export HOST ?= "x86_64-unknown-windows-msvc" export DEFAULT_FEATURES = default-msvc else export OPERATING_SYSTEM := $(shell uname) # same as "uname -s" - export RUST_TARGET ?= "x86_64-unknown-linux-gnu" + export HOST ?= "x86_64-unknown-linux-gnu" export DEFAULT_FEATURES = default + export NUM_CPUS=$(awk '/^processor/ { N++} END { print N }' /proc/cpuinfo) endif +# Override this to use debug builds in release builders. +export RELEASE ?= true +override BUILD_MODE = $(if $(findstring true,$(RELEASE)),release,debug) +override MAYBE_RELEASE_FLAG = $(if $(findstring true,$(RELEASE)),--release,) # Override this with any scopes for testing/benching. export SCOPE ?= "" # Override to false to disable autospawning services on integration tests. @@ -28,8 +33,13 @@ export RUST_TOOLCHAIN ?= $(shell cat rust-toolchain) export CONTAINER_TOOL ?= docker # Override this to automatically enter a container containing the correct, full, official build environment for Vector, ready for development export ENVIRONMENT ?= false +# Override the baseimage of the environment +export ENVIRONMENT_BASEIMAGE ?= ubuntu:20.04 +export ENVIRONMENT_BOOTSTRAP ?= ./scripts/environment/bootstrap/$(subst :,-,${ENVIRONMENT_BASEIMAGE}).sh + # The upstream container we publish artifacts to on a successful master build. export ENVIRONMENT_UPSTREAM ?= docker.pkg.github.com/timberio/vector/environment +export ENVIRONMENT_TAG ?= $(subst :,-,${ENVIRONMENT_BASEIMAGE}) # Override to disable building the container, having it pull from the Github packages repo instead # TODO: Disable this by default. Blocked by `docker pull` from Github Packages requiring authenticated login export ENVIRONMENT_AUTOBUILD ?= true @@ -40,9 +50,179 @@ export WASM_MODULES = $(patsubst tests/data/wasm/%/,%,$(wildcard tests/data/wasm # The same WASM modules, by output path. export WASM_MODULE_OUTPUTS = $(patsubst %,/target/wasm32-wasi/%,$(WASM_MODULES)) + +# export CC ?= gcc +# export CXX ?= g++ +# export LD ?= ldd +# export HOST_CXX ?= g++ +# export HOST_CC ?= gcc +# export HOST_LD ?= ldd +# Some cross compile builds don't have access to TCL, since it's only used in some C SASL unit tests, we just make it null. +export TCLPATH ?= /dev/null +# # We like cross compiles! +export PKG_CONFIG_ALLOW_CROSS ?= true +# # We also like static packages! +export PKG_CONFIG_ALL_STATIC ?= true +# # In case it didn't get the memo +export LIBZ_SYS_STATIC ?= 1 +# In case it didn't get the memo +export OPENSSL_STATIC ?= 1 +export SASL2_STATIC ?= 1 +export CROSS_COMPILE ?= true +export krb5_cv_attr_constructor_destructor ?= yes +export ac_cv_func_regcomp ?= yes +export ac_cv_printf_positional ?= yes +export CRATE_CC_NO_DEFAULTS ?= true + +# In the environment, we have an established structure. Otherwise, configure it where you please! Absolute paths only. +ifeq ($(ENVIRONMENT), true) +export MUSL_CROSS_MAKE ?= /git/richfelker/musl-cross-make +else +export MUSL_CROSS_MAKE ?= ${abspath scripts/environment/cross} +endif + +export MUSL_CROSS_MAKE_PATH ?= ${MUSL_CROSS_MAKE}/output +export MUSL_CROSS_MAKE_x86_64-unknown-linux-musl ?= ${MUSL_CROSS_MAKE_PATH}/x86_64-unknown-linux-musl +export MUSL_CROSS_MAKE_aarch64-unknown-linux-musl ?= ${MUSL_CROSS_MAKE_PATH}/aarch64-unknown-linux-musl + +export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER ?= ${MUSL_CROSS_MAKE_PATH}/bin/x86_64-unknown-linux-musl-cc +export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS ?= \ + -C link-arg=-lgcc \ + -C link-arg=-lstdc++ \ + -C link-arg=-lresolv \ + -C link-arg=-lc +export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTC_LINK_LIB ?= \ + static=gcc \ + static=stdc++ \ + static=resolv \ + static=c \ + static=c++ \ + static=c++abi \ + static=unwind +export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTC_LINK_SEARCH ?= \ + ${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/include/ \ + ${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/include/c++/9.2.0/ \ + ${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/include/c++/9.2.0/x86_64-unknown-linux-musl/ +export CC_x86_64-unknown-linux-musl ?= ${MUSL_CROSS_MAKE_PATH}/bin/x86_64-unknown-linux-musl-gcc +export CXX_x86_64-unknown-linux-musl ?= ${MUSL_CROSS_MAKE_PATH}/bin/x86_64-unknown-linux-musl-g++ +export AR_x86_64-unknown-linux-musl ?= ${MUSL_CROSS_MAKE_PATH}/bin/x86_64-unknown-linux-musl-ar +export LD_x86_64-unknown-linux-musl ?= ${MUSL_CROSS_MAKE_PATH}/bin/x86_64-unknown-linux-musl-ld +export CFLAGS_x86_64-unknown-linux-musl ?= \ + -fPIC \ + -static \ + -nodefaultlibs \ + -static-libgcc \ + -nostdinc \ + -nostartfiles \ + -L${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/lib/ \ + -L${MUSL_CROSS_MAKE_PATH}/lib/gcc/x86_64-unknown-linux-musl/9.2.0/ \ + -I${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/include/ \ + -I${MUSL_CROSS_MAKE_PATH}/lib/gcc/x86_64-unknown-linux-musl/9.2.0/include/ \ + -lresolv \ + -lc \ + -lgcc \ + -lpthread \ + -latomic +export CXXFLAGS_x86_64-unknown-linux-musl ?= \ + ${CFLAGS_x86_64-unknown-linux-musl} \ + -nostdinc++ \ + -I${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/include/c++/9.2.0/ \ + -I${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/include/c++/9.2.0/x86_64-unknown-linux-musl/ \ + -I${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/include/c++/9.2.0/x86_64-unknown-linux-musl/bits \ + -lstdc++ +export LDFLAGS_x86_64-unknown-linux-musl ?= \ + -static \ + -nodefaultlibs \ + -static-libgcc \ + -nostdinc \ + -nostdinc++ \ + -nostartfiles \ + -L${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/lib/ \ + -L${MUSL_CROSS_MAKE_PATH}/lib/gcc/x86_64-unknown-linux-musl/9.2.0/ \ + -I${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/include/ \ + -I${MUSL_CROSS_MAKE_PATH}/lib/gcc/x86_64-unknown-linux-musl/9.2.0/include/ \ + -I${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/include/c++/9.2.0/ \ + -I${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/include/c++/9.2.0/x86_64-unknown-linux-musl/ \ + -I${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}/include/c++/9.2.0/x86_64-unknown-linux-musl/bits \ + -lresolv \ + -lc \ + -lstdc++ \ + -lgcc \ + -lpthread \ + -latomic + +export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER ?= ${MUSL_CROSS_MAKE_PATH}/bin/aarch64-unknown-linux-musl-cc +export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS ?= \ + -C link-arg=-lgcc \ + -C link-arg=-lstdc++ \ + -C link-arg=-lresolv \ + -C link-arg=-lc +export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTC_LINK_LIB ?= \ + static=gcc \ + static=stdc++ \ + static=resolv \ + static=c \ + static=c++ \ + static=c++abi \ + static=unwind +export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTC_LINK_SEARCH ?= \ + ${MUSL_CROSS_MAKE_AARCH64-unknown-linux-musl}/include/ \ + ${MUSL_CROSS_MAKE_AARCH64-unknown-linux-musl}/include/c++/9.2.0/ \ + ${MUSL_CROSS_MAKE_AARCH64-unknown-linux-musl}/include/c++/9.2.0/aarch64-unknown-linux-musl/ +export CC_aarch64-unknown-linux-musl ?= ${MUSL_CROSS_MAKE_PATH}/bin/aarch64-unknown-linux-musl-gcc +export CXX_aarch64-unknown-linux-musl ?= ${MUSL_CROSS_MAKE_PATH}/bin/aarch64-unknown-linux-musl-g++ +export AR_aarch64-unknown-linux-musl ?= ${MUSL_CROSS_MAKE_PATH}/bin/aarch64-unknown-linux-musl-ar +export LD_aarch64-unknown-linux-musl ?= ${MUSL_CROSS_MAKE_PATH}/bin/aarch64-unknown-linux-musl-ld +export CFLAGS_aarch64-unknown-linux-musl ?= \ + -fPIC \ + -static \ + -nostdlib \ + -nostdinc \ + -nostartfiles \ + -I${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl}/include/ \ + -I${MUSL_CROSS_MAKE_PATH}/lib/gcc/aarch64-unknown-linux-musl/9.2.0/include/ \ + -L${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl}/lib/ \ + -L${MUSL_CROSS_MAKE_PATH}/lib/gcc/aarch64-unknown-linux-musl/9.2.0/ \ + -lresolv \ + -lc \ + -lgcc \ + -lpthread \ + -latomic +export CXXFLAGS_aarch64-unknown-linux-musl ?= \ + ${CFLAGS_aarch64-unknown-linux-musl} \ + -nostdinc++ \ + -I${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl}/include/c++/9.2.0/ \ + -I${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl}/include/c++/9.2.0/aarch64-unknown-linux-musl/ \ + -I${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl}/include/c++/9.2.0/aarch64-unknown-linux-musl/bits \ + -lstdc++ +export LDFLAGS_aarch64-unknown-linux-musl ?= \ + -static \ + -nostdlib \ + -nostdinc \ + -nostdinc++ \ + -nostartfiles \ + -I${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl}/include/ \ + -I${MUSL_CROSS_MAKE_PATH}/lib/gcc/aarch64-unknown-linux-musl/9.2.0/include/ \ + -I${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl}/include/c++/9.2.0/ \ + -I${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl}/include/c++/9.2.0/aarch64-unknown-linux-musl/ \ + -I${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl}/include/c++/9.2.0/aarch64-unknown-linux-musl/bits \ + -L${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl}/lib/ \ + -L${MUSL_CROSS_MAKE_PATH}/lib/gcc/aarch64-unknown-linux-musl/9.2.0/ \ + -lresolv \ + -lc \ + -lstdc++ \ + -lgcc \ + -lpthread \ + -latomic + # Deprecated. export USE_CONTAINER ?= $(CONTAINER_TOOL) +# Important vars we don't really want folks changing without knowledge of what they're doing. +export VECTOR_BINARY_x86_64_unknown_linux_gnu ?= /target/x86_64-unknown-linux-gnu/${BUILD_MODE}/vector +export VECTOR_BINARY_x86_64-unknown-linux-musl ?= /target/x86_64-unknown-linux-musl/${BUILD_MODE}/vector +export VECTOR_BINARY_aarch64-unknown-linux-musl ?= /target/aarch64-unknown-linux-musl/${BUILD_MODE}/vector + FORMATTING_BEGIN_YELLOW = \033[0;33m FORMATTING_BEGIN_BLUE = \033[36m FORMATTING_END = \033[0m @@ -85,10 +265,11 @@ endef endif # We use a volume here as non-Linux hosts are extremely slow to share disks, and Linux hosts tend to get permissions clobbered. +# TODO: Clean up this env mess... Use an env file. define ENVIRONMENT_EXEC ${ENVIRONMENT_PREPARE} @echo "Entering environment..." - @mkdir -p target + @mkdir -p ./target ./scripts/environment/cross $(CONTAINER_TOOL) run \ --name vector-environment \ --rm \ @@ -96,27 +277,78 @@ define ENVIRONMENT_EXEC --init \ --interactive \ --env INSIDE_ENVIRONMENT=true \ + --env MUSL_CROSS_MAKE \ + --env MUSL_CROSS_MAKE_PATH \ + --env ENVIRONMENT_BASEIMAGE \ + --env ENVIRONMENT_BOOTSTRAP \ + --env ENVIRONMENT_TAG \ + --env RUST_BACKTRACE \ + --env MUSL_CROSS_MAKE \ + --env TARGET \ + --env HOST \ + --env MUSL_CROSS_MAKE_PATH \ + --env MUSL_CROSS_MAKE_x86_64-unknown-linux-musl \ + --env MUSL_CROSS_MAKE_aarch64-unknown-linux-musl \ + --env CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER \ + --env CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS \ + --env CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL__RUSTC_LINK_LIB \ + --env CC_x86_64-unknown-linux-musl \ + --env CXX_x86_64-unknown-linux-musl \ + --env AR_x86_64-unknown-linux-musl \ + --env LD_x86_64-unknown-linux-musl \ + --env CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER \ + --env CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS \ + --env CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTC_LINK_LIB \ + --env CC_aarch64-unknown-linux-musl \ + --env CXX_aarch64-unknown-linux-musl \ + --env AR_aarch64-unknown-linux-musl \ + --env LD_aarch64-unknown-linux-musl \ + --env CC \ + --env CXX \ + --env HOST_CXX \ + --env HOST_CC \ + --env TCLPATH \ + --env PKG_CONFIG_ALLOW_CROSS \ + --env PKG_CONFIG_ALL_STATIC \ + --env LIBZ_SYS_STATIC \ + --env OPENSSL_STATIC \ + --env CROSS_COMPILE \ + --env SASL2_STATIC \ + --env CRATE_CC_NO_DEFAULTS \ + --env CROSS_COMPILE \ + --env krb5_cv_attr_constructor_destructor \ + --env ac_cv_func_regcomp \ + --env ac_cv_printf_positional \ + --env LDFLAGS \ --network host \ --mount type=bind,source=${PWD},target=/git/timberio/vector \ --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ - --mount type=volume,source=vector-target,target=/git/timberio/vector/target \ - --mount type=volume,source=vector-cargo-cache,target=/root/.cargo \ - $(ENVIRONMENT_UPSTREAM) + --mount type=volume,source=vector-${ENVIRONMENT_TAG}-target,target=/git/timberio/vector/target \ + --mount type=volume,source=vector-${ENVIRONMENT_TAG}-rustup,target=/root/.rustup \ + --mount type=volume,source=vector-${ENVIRONMENT_TAG}-cross,target=${MUSL_CROSS_MAKE} \ + --mount type=volume,source=vector-${ENVIRONMENT_TAG}-cargo-cache,target=/root/.cargo \ + ${ENVIRONMENT_UPSTREAM}:${ENVIRONMENT_TAG} endef define ENVIRONMENT_COPY_ARTIFACTS @echo "Copying artifacts off volumes... (Docker errors below are totally okay)" - @mkdir -p ./target/release - @mkdir -p ./target/debug - @mkdir -p ./target/criterion + mkdir -p \ + ./target/criterion \ + ./target/${BUILD_MODE} \ + ./target/x86_64-unknown-linux-gnu/${BUILD_MODE} \ + ./target/x86_64-unknown-linux-musl/${BUILD_MODE} \ + ./target/aarch64-unknown-linux-musl/${BUILD_MODE} \ + ./aarch64-unknown-linux-musl @$(CONTAINER_TOOL) rm -f vector-build-outputs || true @$(CONTAINER_TOOL) run \ -d \ - -v vector-target:/target \ + -v vector-${ENVIRONMENT_TAG}-target:/target \ --name vector-build-outputs \ busybox true - @$(CONTAINER_TOOL) cp vector-build-outputs:/target/release/vector ./target/release/ || true - @$(CONTAINER_TOOL) cp vector-build-outputs:/target/debug/vector ./target/debug/ || true + $(CONTAINER_TOOL) cp vector-build-outputs:${VECTOR_BINARY_x86_64_unknown_linux_gnu} ./target/x86_64-unknown-linux-gnu/${BUILD_MODE} || true + @$(CONTAINER_TOOL) cp vector-build-outputs:${VECTOR_BINARY_x86_64-unknown-linux-musl} ./target/x86_64-unknown-linux-musl/${BUILD_MODE} || true + @$(CONTAINER_TOOL) cp vector-build-outputs:${VECTOR_BINARY_aarch64-unknown-linux-musl} ./target/aarch64-unknown-linux-musl/${BUILD_MODE} || true + @$(CONTAINER_TOOL) cp vector-build-outputs:/target/${BUILD_MODE}/vector ./target/${BUILD_MODE}/ || true @$(CONTAINER_TOOL) cp vector-build-outputs:/target/criterion ./target/criterion || true @$(CONTAINER_TOOL) rm -f vector-build-outputs endef @@ -127,7 +359,9 @@ define ENVIRONMENT_PREPARE @echo "Building the environment. (ENVIRONMENT_AUTOBUILD=true) This may take a few minutes..." $(CONTAINER_TOOL) build \ $(if $(findstring true,$(VERBOSE)),,--quiet) \ - --tag $(ENVIRONMENT_UPSTREAM) \ + --build-arg BASEIMAGE=${ENVIRONMENT_BASEIMAGE} \ + --build-arg BOOTSTRAP=${ENVIRONMENT_BOOTSTRAP} \ + --tag $(ENVIRONMENT_UPSTREAM):${ENVIRONMENT_TAG} \ --file scripts/environment/Dockerfile \ . endef @@ -138,6 +372,7 @@ endef endif environment: export ENVIRONMENT_TTY = true ## Enter a full Vector dev shell in $CONTAINER_TOOL, binding this folder to the container. +environment: override ENVIRONMENT = true environment: ${ENVIRONMENT_EXEC} @@ -145,33 +380,79 @@ environment-prepare: ## Prepare the Vector dev shell using $CONTAINER_TOOL. ${ENVIRONMENT_PREPARE} environment-clean: ## Clean the Vector dev shell using $CONTAINER_TOOL. - @$(CONTAINER_TOOL) volume rm -f vector-target vector-cargo-cache + @$(CONTAINER_TOOL) volume rm -f vector-target vector-cargo-cache vector-cross @$(CONTAINER_TOOL) rmi $(ENVIRONMENT_UPSTREAM) || true environment-push: environment-prepare ## Publish a new version of the container image. $(CONTAINER_TOOL) push $(ENVIRONMENT_UPSTREAM) -##@ Building -build: ## Build the project in release mode (Supports `ENVIRONMENT=true`) +##@ Building (Supports `ENVIRONMENT=true`) +build: ## Build the project in release mode ${MAYBE_ENVIRONMENT_EXEC} cargo build --release --no-default-features --features ${DEFAULT_FEATURES} ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} -build-dev: ## Build the project in development mode (Supports `ENVIRONMENT=true`) +build-dev: ## Build the project in development mode ${MAYBE_ENVIRONMENT_EXEC} cargo build --no-default-features --features ${DEFAULT_FEATURES} -build-all: build-x86_64-unknown-linux-musl build-armv7-unknown-linux-musleabihf build-aarch64-unknown-linux-musl ## Build the project in release mode for all supported platforms +build-all: build-x86_64-unknown-linux-gnu build-x86_64-unknown-linux-musl build-armv7-unknown-linux-musleabihf build-aarch64-unknown-linux-musl ## Build the project in release mode for all supported platforms -build-x86_64-unknown-linux-gnu: ## Build dynamically linked binary in release mode for the x86_64 architecture - $(RUN) build-x86_64-unknown-linux-gnu +build-x86_64-unknown-linux-gnu: ${VECTOR_BINARY_x86_64_unknown_linux_gnu} ## Build dynamically linked binary in release mode for the x86_64 architecture -build-x86_64-unknown-linux-musl: ## Build static binary in release mode for the x86_64 architecture - $(RUN) build-x86_64-unknown-linux-musl +${VECTOR_BINARY_x86_64_unknown_linux_gnu}: + ${MAYBE_ENVIRONMENT_EXEC} cargo build ${MAYBE_RELEASE_FLAG} --no-default-features --features default --target x86_64-unknown-linux-gnu + ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} -build-armv7-unknown-linux-musleabihf: load-qemu-binfmt ## Build static binary in release mode for the armv7 architecture - $(RUN) build-armv7-unknown-linux-musleabihf +.PHONY: ${MUSL_CROSS_MAKE} +${MUSL_CROSS_MAKE}: + ${MAYBE_ENVIRONMENT_EXEC} git clone https://github.com/richfelker/musl-cross-make.git ${MUSL_CROSS_MAKE} || true + +.PHONY: build-cross-x86_64-unknown-linux-musl +build-cross-x86_64-unknown-linux-musl: ${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl} + +.PHONY: ${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl} +${MUSL_CROSS_MAKE_x86_64-unknown-linux-musl}: ${MUSL_CROSS_MAKE} + ${MAYBE_ENVIRONMENT_EXEC} make --quiet --directory ${MUSL_CROSS_MAKE} install -j${NUM_CPUS} TARGET=x86_64-unknown-linux-musl + +.PHONY: build-x86_64-unknown-linux-musl +build-x86_64-unknown-linux-musl: ${VECTOR_BINARY_x86_64-unknown-linux-musl} ## Build static binary in release mode for the x86_64 architecture + +${VECTOR_BINARY_x86_64-unknown-linux-musl}: export CROSS_COMPILE = x86_64-unknown-linux-musl +${VECTOR_BINARY_x86_64-unknown-linux-musl}: export TARGET = x86_64-unknown-linux-musl +${VECTOR_BINARY_x86_64-unknown-linux-musl}: export CC = ${CC_x86_64-unknown-linux-musl} +${VECTOR_BINARY_x86_64-unknown-linux-musl}: export CXX = ${CXX_x86_64-unknown-linux-musl} +${VECTOR_BINARY_x86_64-unknown-linux-musl}: export LD = ${CARGO_TARGET_x86_64-UNKOWNN_LINUX_MUSL_LINKER} +${VECTOR_BINARY_x86_64-unknown-linux-musl}: export HOST_CXX = musl-g++ +${VECTOR_BINARY_x86_64-unknown-linux-musl}: export HOST_CC = musl-gcc +${VECTOR_BINARY_x86_64-unknown-linux-musl}: export PATH +=:${MUSL_CROSS_MAKE_PATH}/bin +${VECTOR_BINARY_x86_64-unknown-linux-musl}: + ${MAYBE_ENVIRONMENT_EXEC} rustup toolchain install nightly + ${MAYBE_ENVIRONMENT_EXEC} rustup target add x86_64-unknown-linux-musl --toolchain nightly + ${MAYBE_ENVIRONMENT_EXEC} cargo +nightly build ${MAYBE_RELEASE_FLAG} --no-default-features --features default-musl --target x86_64-unknown-linux-musl + ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} -build-aarch64-unknown-linux-musl: load-qemu-binfmt ## Build static binary in release mode for the aarch64 architecture - $(RUN) build-aarch64-unknown-linux-musl +.PHONY: build-cross-aarch64-unknown-linux-musl +build-cross-aarch64-unknown-linux-musl: ${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl} + +.PHONY: ${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl} +${MUSL_CROSS_MAKE_aarch64-unknown-linux-musl}: ${MUSL_CROSS_MAKE} + ${MAYBE_ENVIRONMENT_EXEC} ${MAKE} --quiet --directory ${MUSL_CROSS_MAKE} install -j${NUM_CPUS} TARGET=aarch64-unknown-linux-musl + +.PHONY: build-aarch64-unknown-linux-musl +build-aarch64-unknown-linux-musl: ${VECTOR_BINARY_aarch64-unknown-linux-musl} ## Build static binary in release mode for the aarch64 architecture + +${VECTOR_BINARY_aarch64-unknown-linux-musl}: export CROSS_COMPILE = aarch64-unknown-linux-musl +${VECTOR_BINARY_aarch64-unknown-linux-musl}: export TARGET = aarch64-unknown-linux-musl +${VECTOR_BINARY_aarch64-unknown-linux-musl}: export CC = ${CC_aarch64-unknown-linux-musl} +${VECTOR_BINARY_aarch64-unknown-linux-musl}: export CXX = ${CXX_aarch64-unknown-linux-musl} +${VECTOR_BINARY_aarch64-unknown-linux-musl}: export LD = ${CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER} +${VECTOR_BINARY_aarch64-unknown-linux-musl}: export HOST_CXX = musl-g++ +${VECTOR_BINARY_aarch64-unknown-linux-musl}: export HOST_CC = musl-gcc +${VECTOR_BINARY_aarch64-unknown-linux-musl}: export PATH +=:${MUSL_CROSS_MAKE_PATH}/bin +${VECTOR_BINARY_aarch64-unknown-linux-musl}: + ${MAYBE_ENVIRONMENT_EXEC} rustup toolchain install nightly + ${MAYBE_ENVIRONMENT_EXEC} rustup target add aarch64-unknown-linux-musl + ${MAYBE_ENVIRONMENT_EXEC} cargo +nightly build ${MAYBE_RELEASE_FLAG} --no-default-features --features default-musl --target aarch64-unknown-linux-musl + ${MAYBE_ENVIRONMENT_COPY_ARTIFACTS} ##@ Testing (Supports `ENVIRONMENT=true`) @@ -357,7 +638,11 @@ check-style: ## Check that all files are styled properly ${MAYBE_ENVIRONMENT_EXEC} ./scripts/check-style.sh check-markdown: ## Check that markdown is styled properly - ${MAYBE_ENVIRONMENT_EXEC} ./scripts/check-markdown.sh + ${MAYBE_ENVIRONMENT_EXEC} markdownlint \ + --config scripts/.markdownlintrc \ + --ignore scripts/node_modules \ + --ignore target \ + . check-meta: ## Check that all /.meta file are valid ${MAYBE_ENVIRONMENT_EXEC} ./scripts/check-meta.sh @@ -366,7 +651,7 @@ check-version: ## Check that Vector's version is correct accounting for recent c ${MAYBE_ENVIRONMENT_EXEC} ./scripts/check-version.rb check-examples: ## Check that the config/examples files are valid - ${MAYBE_ENVIRONMENT_EXEC} cargo run -- validate --topology --deny-warnings ./config/examples/*.toml + $(foreach EXAMPLE, $(wildcard ./config/examples/*.toml), ${MAYBE_ENVIRONMENT_EXEC} cargo run -- validate --deny-warnings ${EXAMPLE};) check-scripts: ## Check that scipts do not have common mistakes ${MAYBE_ENVIRONMENT_EXEC} ./scripts/check-scripts.sh @@ -517,6 +802,9 @@ build-ci-docker-images: ## Rebuilds all Docker images used in CI clean: environment-clean ## Clean everything cargo clean +clean-cross: + ${MAYBE_ENVIRONMENT_EXEC} ${MAKE} --quiet --directory ${MUSL_CROSS_MAKE} clean + fmt: ## Format code ${MAYBE_ENVIRONMENT_EXEC} cargo fmt ${MAYBE_ENVIRONMENT_EXEC} ./scripts/check-style.sh --fix @@ -551,3 +839,7 @@ target/wasm32-wasi/.obtained: @mkdir -p target/wasm32-wasi @touch target/wasm32-wasi/.obtained +.PHONY: rust-target-% +rust-target-%: override ARCH = $(@:rust-target-%=%) +rust-target-%: + @rustup target add ${ARCH} diff --git a/docker-compose.yml b/docker-compose.yml index 05f8cf3fba1bc..f04eb3b1625ab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,24 +4,6 @@ services: # Jobs # - build-x86_64-unknown-linux-musl: - build: - context: . - dockerfile: scripts/ci-docker-images/builder-x86_64-unknown-linux-musl/Dockerfile - environment: - NATIVE_BUILD: "false" - TARGET: x86_64-unknown-linux-musl - FEATURES: default-musl - CARGO_TERM_COLOR: always - volumes: - - $PWD:$PWD - - ./target/x86_64-unknown-linux-musl/cargo/registry:/opt/rust/cargo/registry - - ./target/x86_64-unknown-linux-musl/cargo/git:/opt/rust/cargo/git - - ./target/x86_64-unknown-linux-musl/rustup/tmp:/opt/rust/rustup/tmp - working_dir: $PWD - user: $USER - command: scripts/build.sh - build-x86_64-unknown-linux-gnu: build: context: . @@ -42,42 +24,6 @@ services: user: $USER command: scripts/build.sh - build-armv7-unknown-linux-musleabihf: - build: - context: . - dockerfile: scripts/ci-docker-images/builder-armv7-unknown-linux-musleabihf/Dockerfile - environment: - NATIVE_BUILD: "false" - TARGET: armv7-unknown-linux-musleabihf - FEATURES: default-musl - CARGO_TERM_COLOR: always - volumes: - - $PWD:$PWD - - ./target/armv7-unknown-linux-musleabihf/cargo/registry:/opt/rust/cargo/registry - - ./target/armv7-unknown-linux-musleabihf/cargo/git:/opt/rust/cargo/git - - ./target/armv7-unknown-linux-musleabihf/rustup/tmp:/opt/rust/rustup/tmp - working_dir: $PWD - user: $USER - command: scripts/build.sh - - build-aarch64-unknown-linux-musl: - build: - context: . - dockerfile: scripts/ci-docker-images/builder-aarch64-unknown-linux-musl/Dockerfile - environment: - NATIVE_BUILD: "false" - TARGET: aarch64-unknown-linux-musl - FEATURES: default-musl - CARGO_TERM_COLOR: always - volumes: - - $PWD:$PWD - - ./target/aarch64-unknown-linux-musl/cargo/registry:/opt/rust/cargo/registry - - ./target/aarch64-unknown-linux-musl/cargo/git:/opt/rust/cargo/git - - ./target/aarch64-unknown-linux-musl/rustup/tmp:/opt/rust/rustup/tmp - working_dir: $PWD - user: $USER - command: scripts/build.sh - package-archive-x86_64-unknown-linux-musl: build: context: . diff --git a/rust-toolchain b/rust-toolchain index 50aceaa7b715f..0fb53d6a6a24e 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.45.0 +1.45.2 diff --git a/scripts/check-markdown.sh b/scripts/check-markdown.sh deleted file mode 100755 index afe6f43d15359..0000000000000 --- a/scripts/check-markdown.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# check-markdown.sh -# -# SUMMARY -# -# Checks the markdown format within the Vector repo. -# This ensures that markdown is consistent and easy to read across the -# entire Vector repo. - -scripts/node_modules/.bin/markdownlint \ - --config scripts/.markdownlintrc \ - --ignore scripts/node_modules \ - . diff --git a/scripts/ci-docker-images/builder-armv7-unknown-linux-musleabihf/Dockerfile b/scripts/ci-docker-images/builder-armv7-unknown-linux-musleabihf/Dockerfile deleted file mode 100755 index f1bff0a844c6d..0000000000000 --- a/scripts/ci-docker-images/builder-armv7-unknown-linux-musleabihf/Dockerfile +++ /dev/null @@ -1,311 +0,0 @@ -# We use compiler-rt, libunwind, and libc++ from LLVM. -ARG LLVM_VERSION=9.0.0 -# The version of musl should be the same as the one Rust is compiled with. -# This version can be found in this file: -# https://github.com/rust-lang/rust/blob/master/src/ci/docker/scripts/musl-toolchain.sh. -# Exampel: "1.1.22". -ARG MUSL_VERSION=1.1.22 - -# LLVM target triple for Linux and MUSL. Full list of supported triples -# can be found here: -# https://forge.rust-lang.org/release/platform-support.html. -# Example: "armv7-unknown-linux-musleabihf". -ARG TARGET="armv7-unknown-linux-musleabihf" -# Target architecture for LLVM -ARG LLVM_TARGET="ARM" -# Target architecture for Linux kernel headres -ARG LINUX_TARGET="arm" -# We also need GCC for the target architecture to compile compier-rt. Because -# GCC target triples are different from LLVM, we need to specify it explicitly. -# Example: "arm-linux-gnueabihf". -ARG GNU_TARGET="arm-linux-gnueabihf" - -ARG LTO="" - -# Used by libc++. Programs compiled with this version should work with other -# kernel versions too. See https://wiki.gentoo.org/wiki/Linux-headers#FAQ. -ARG LINUX_HEADERS_VERSION=5.3.10 - -# Locations for produced tools and libraries. -ARG RUST_PREFIX=/opt/rust -ARG CLANG_PREFIX=/usr -ARG LIBS_PREFIX=/opt/libs - -# Libs -FROM ubuntu:19.10 AS libs-builder - -ARG CLANG_PREFIX -RUN apt-get update && apt-get install -y clang llvm lld - -# Get all sources at first, so that changes in arguments -# flags would not redownload them. -RUN apt-get update && apt-get -y install curl xz-utils - -ARG LLVM_VERSION -RUN curl --proto '=https' --tlsv1.2 -sSfL \ - https://github.com/llvm/llvm-project/archive/llvmorg-$LLVM_VERSION.tar.gz | \ - tar xzf - -ENV LLVM_DIR $SRC_DIR/llvm-project-llvmorg-$LLVM_VERSION - -ARG MUSL_VERSION -RUN curl --proto '=https' --tlsv1.2 -sSf \ - https://www.musl-libc.org/releases/musl-$MUSL_VERSION.tar.gz | \ - tar xzf - -ENV MUSL_DIR $SRC_DIR/musl-$MUSL_VERSION - -ARG LINUX_HEADERS_VERSION -RUN curl --proto '=https' --tlsv1.2 -sSf \ - https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-$LINUX_HEADERS_VERSION.tar.xz | \ - tar xJf - -ENV LINUX_DIR $SRC_DIR/linux-$LINUX_HEADERS_VERSION - -# Install build dependencies. -RUN apt-get update && apt-get -y install rsync ninja-build cmake python3-distutils clang llvm lld - -ARG TARGET -ARG LIBS_PREFIX - -# Compile Clang runtime (https://compiler-rt.llvm.org). -# We need crtbegin.o, crtend.o, and libclang_rt.builtins.a from there. -# Because Clang toolchain is not fully bootstrapped yet (no standard library), -# it is necessary to use GCC to compile it. -ARG GNU_TARGET -ARG CLANG_PREFIX -RUN apt-get install -y gcc${GNU_TARGET:+-${GNU_TARGET}} g++${GNU_TARGET:+-${GNU_TARGET}} -WORKDIR $LLVM_DIR/compiler-rt -ENV CC=${GNU_TARGET:+${GNU_TARGET}-}gcc -ENV CXX=${GNU_TARGET:+${GNU_TARGET}-}g++ -ENV CFLAGS="" -ENV CXXFLAGS="" -ENV LDFLAGS="" -RUN mkdir build && \ - cd build && \ - cmake \ - -DCMAKE_BUILD_TYPE=release \ - -DLLVM_CONFIG_PATH=$CLANG_PREFIX/bin/llvm-config \ - -DCOMPILER_RT_DEFAULT_TARGET_TRIPLE=$TARGET \ - -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ - -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \ - -DCOMPILER_RT_BUILD_XRAY=OFF \ - -DCOMPILER_RT_BUILD_PROFILE=OFF \ - -DCMAKE_INSTALL_PREFIX=$LIBS_PREFIX \ - -G Ninja \ - .. && \ - cmake --build . --target install - -# Compile musl. -WORKDIR $MUSL_DIR -ENV CC=clang -ENV CFLAGS="-target $TARGET -nostdinc -isystem $LIBS_PREFIX/include -D_Noreturn=" -ENV LDFLAGS="-fuse-ld=lld -static -nostdlib -nostartfiles" -ENV CROSS_COMPILE=$CLANG_PREFIX/bin/ -RUN cd $MUSL_DIR && \ - mkdir build && \ - cd build && \ - ../configure \ - --target=$TARGET \ - --prefix=$LIBS_PREFIX \ - --disable-shared \ - --disable-gcc-wrapper && \ - make -j$(nproc) install - -# Install Linux headers. -WORKDIR $LINUX_DIR -ARG LINUX_TARGET -ENV CC=clang -ENV CFLAGS="-target $TARGET -nostdinc -isystem $LIBS_PREFIX/include" -ENV LDFLAGS="-fuse-ld=lld -static -nostdlib -nostartfiles" -RUN make -j$(nproc) headers_install \ - KBUILD_VERBOSE=1 \ - HOSTCC="/usr/bin/gcc" \ - ARCH="$LINUX_TARGET" \ - CC="$CC" \ - INSTALL_HDR_PATH=$LIBS_PREFIX - -# Compile libc++ with musl using Clang as the compiler. See -# https://blogs.gentoo.org/gsoc2016-native-clang/2016/05/05/build-a-freestanding-libcxx/ -# for explanations. - -# libunwind -WORKDIR $LLVM_DIR/libunwind -ENV CC=clang -ENV CXX=clang++ -ENV CFLAGS="-target $TARGET -nostdinc -isystem $LIBS_PREFIX/include" -ENV CXXFLAGS="$CFLAGS -nostdinc++ -I$LLVM_DIR/libcxx/include" -ENV LDFLAGS="-fuse-ld=lld -static -nostdlib -nostartfiles -L$LIBS_PREFIX/lib -lc" -RUN mkdir build && \ - cd build && \ - cmake \ - -DCMAKE_BUILD_TYPE=release \ - -DLIBUNWIND_ENABLE_SHARED=OFF \ - -DLIBUNWIND_INSTALL_PREFIX=$LIBS_PREFIX/ \ - -DLLVM_PATH=$LLVM_DIR \ - -G Ninja \ - .. && \ - cmake --build . --target install - -# libc++abi -WORKDIR $LLVM_DIR/libcxxabi -ENV CC=clang -ENV CXX=clang++ -ENV CFLAGS="-target $TARGET -nostdinc -isystem $LIBS_PREFIX/include" -ENV CXXFLAGS="$CFLAGS -nostdinc++ -I$LLVM_DIR/libcxx/include" -ENV LDFLAGS="-fuse-ld=lld -static -nostdlib -nostartfiles -L$LIBS_PREFIX/lib -lunwind -lc" -RUN mkdir build && \ - cd build && \ - cmake \ - -DCMAKE_BUILD_TYPE=release \ - -DLIBCXXABI_ENABLE_SHARED=OFF \ - -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ - -DLIBCXXABI_LIBUNWIND_PATH=$LLVM_DIR/libunwind \ - -DLIBCXXABI_LIBCXX_INCLUDES=$LLVM_DIR/libcxx/include \ - -DLIBCXXABI_INSTALL_PREFIX=$LIBS_PREFIX/ \ - -DLLVM_PATH=$LLVM_DIR \ - -G Ninja \ - .. && \ - cmake --build . --target install - -# libc++ -WORKDIR $LLVM_DIR/libcxx -ENV CC=clang -ENV CXX=clang++ -ENV CFLAGS="-target $TARGET -nostdinc -isystem $LIBS_PREFIX/include" -ENV CXXFLAGS="$CFLAGS -nostdinc++ -I$LIBS_PREFIX/include/c++/v1" -ENV LDFLAGS="-fuse-ld=lld -static -nostdlib -nostartfiles -L$LIBS_PREFIX/lib -lc++abi -lunwind -lc" -RUN mkdir build && \ - cd build && \ - cmake \ - -DCMAKE_BUILD_TYPE=release \ - -DLIBCXX_ENABLE_SHARED=OFF \ - -DLIBCXX_HAS_MUSL_LIBC=ON \ - -DLIBCXX_HAS_GCC_S_LIB=OFF \ - -DLIBCXX_CXX_ABI=libcxxabi \ - -DLIBCXX_CXX_ABI_INCLUDE_PATHS=$LLVM_DIR/libcxxabi/include \ - -DLIBCXX_CXX_ABI_LIBRARY_PATH=$LIBS_PREFIX \ - -DLIBCXX_INSTALL_PREFIX=$LIBS_PREFIX/ \ - -DLIBCXX_INSTALL_HEADER_PREFIX=$LIBS_PREFIX/ \ - -DLLVM_PATH=$LLVM_DIR \ - -G Ninja \ - .. && \ - cmake --build . --target install - -# Create wrappers for Clang that automatically use correct libraries and start files. -# See -# https://blogs.gentoo.org/gsoc2016-native-clang/2016/05/31/build-gnu-free-executables-with-clang/ -# and -# https://renenyffenegger.ch/notes/development/languages/C-C-plus-plus/GCC/options/no/compare-nostartfiles-nodefaultlibs-nolibc-nostdlib. -ENV MUSL_CFLAGS="-target $TARGET -nostdinc -isystem $LIBS_PREFIX/include" -ENV MUSL_CXXFLAGS="$MUSL_CFLAGS -nostdinc++ -I$LIBS_PREFIX/include/c++/v1" - -ENV MUSL_LDFLAGS="-fuse-ld=lld -static -nostdlib -nostartfiles -L$LIBS_PREFIX/lib -L$LIBS_PREFIX/lib/linux -lc++ -lc++abi -lunwind -l$(basename /opt/libs/lib/linux/libclang_rt.builtins* | tail -c+4 | head -c-3) -lc" -ENV MUSL_STARTFILES="$LIBS_PREFIX/lib/crt1.o $(find $LIBS_PREFIX/lib/linux -iname 'clang_rt.crtbegin*.o') $(find $LIBS_PREFIX/lib/linux -iname 'clang_rt.crtend*.o')" - -RUN mkdir -p $LIBS_PREFIX/bin -RUN echo \ - "#!/bin/sh\n"\ - "case \"\$@\" in *-shared*);; *-nostdlib*);; *) STARTFILES=\"$MUSL_STARTFILES\";; esac\n"\ - "$CLANG_PREFIX/bin/clang -Qunused-arguments $MUSL_CFLAGS \$@ \$STARTFILES $MUSL_LDFLAGS\n"\ - "exit \$?" > $LIBS_PREFIX/bin/musl-cc -RUN echo \ - "#!/bin/sh\n"\ - "case \"\$@\" in *-shared*);; *-nostdlib*);; *) STARTFILES=\"$MUSL_STARTFILES\";; esac\n"\ - "$CLANG_PREFIX/bin/clang++ -Qunused-arguments $MUSL_CXXFLAGS \$@ \$STARTFILES $MUSL_LDFLAGS\n"\ - "exit \$?" > $LIBS_PREFIX/bin/musl-c++ -RUN chmod +x $LIBS_PREFIX/bin/* - -# At this point a fully functional C++ compiler that is able to produce -# static binaries linked with musl and libc++ is bootstrapped. -# It can be used by calling musl-c++ (or musl-cc for C) executable. -# However, we need to also create generic aliases to make it possible to -# use it as a drop-in replacement for the system-wide GCC. -RUN ln -s $LIBS_PREFIX/bin/musl-cc $LIBS_PREFIX/bin/cc -RUN ln -s $LIBS_PREFIX/bin/musl-cc $LIBS_PREFIX/bin/gcc -RUN if [ -n "$GNU_TARGET" ]; then ln -s $LIBS_PREFIX/bin/musl-cc $LIBS_PREFIX/bin/$(echo $GNU_TARGET | sed 's/gnu/musl/g')-gcc; fi -RUN ln -s $LIBS_PREFIX/bin/musl-cc $LIBS_PREFIX/bin/musl-gcc -RUN ln -s $LIBS_PREFIX/bin/musl-c++ $LIBS_PREFIX/bin/c++ -RUN ln -s $LIBS_PREFIX/bin/musl-c++ $LIBS_PREFIX/bin/g++ -RUN if [ -n "$GNU_TARGET" ]; then ln -s $LIBS_PREFIX/bin/musl-c++ $LIBS_PREFIX/bin/$(echo $GNU_TARGET | sed 's/gnu/musl/g')-g++; fi -RUN ln -s $LIBS_PREFIX/bin/musl-c++ $LIBS_PREFIX/bin/musl-g++ - -RUN ln -s $CLANG_PREFIX/bin/ld.lld $LIBS_PREFIX/bin/ld -RUN ln -s $CLANG_PREFIX/bin/llvm-ar $LIBS_PREFIX/bin/ar -RUN ln -s $CLANG_PREFIX/bin/llvm-strip $LIBS_PREFIX/bin/strip -RUN ln -s $CLANG_PREFIX/bin/llvm-ar $LIBS_PREFIX/bin/arm-linux-musleabihf-ar - -# Use stdatomic.h header provided by Clang because it is not present in musl. -RUN ln -s $CLANG_PREFIX/lib/clang/9.0.0/include/stdatomic.h $LIBS_PREFIX/include/stdatomic.h - -# Some build scripts hardcode -lstdc++ linker flag on all Linux systems. -# Because our linker is already configured to link with libc++ instead, -# we can just provide dummy libstdc++ to be compatible with GNU systems. -# For example, macOS provides similar experience, where -# `clang++ -o program program.cpp -lstdc++` would still link to libc++. -RUN echo > dummy.c && \ - $LIBS_PREFIX/bin/cc -c dummy.c && \ - $CLANG_PREFIX/bin/llvm-ar cr $LIBS_PREFIX/lib/libstdc++.a dummy.o && \ - rm dummy.c dummy.o - -# The actual builder. -FROM ubuntu:19.10 - -# Install Rust using rustup. -RUN apt-get update && apt-get install -y curl -ARG TARGET -ARG RUST_PREFIX -ENV RUSTUP_HOME=$RUST_PREFIX/rustup -ENV CARGO_HOME=$RUST_PREFIX/cargo -COPY rust-toolchain /tmp/ -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ - sh -s -- -y --profile minimal --default-toolchain $(cat /tmp/rust-toolchain) --target $TARGET -ENV PATH="$RUSTUP_HOME/bin:$CARGO_HOME/bin:$PATH" - -ARG CLANG_PREFIX -RUN apt-get update && apt-get install -y clang llvm lld - -ARG LIBS_PREFIX -COPY --from=libs-builder $LIBS_PREFIX $LIBS_PREFIX -ENV PATH="$LIBS_PREFIX/bin:$PATH" - -# Essential tools needed for compiling native dependencies -RUN apt-get update && apt-get install -y --install-recommends build-essential cmake git - -# Because the system GCC and binutils are shadowed by aliases, we need to -# instruct Cargo and cc crate to use GCC on the host system. They are used to -# compile dependencies of build scripts. -RUN echo \ - "#!/bin/sh\n"\ - "/usr/bin/gcc -B/usr/bin \$@\n"\ - "exit \$?" > $LIBS_PREFIX/bin/gnu-cc && chmod +x $LIBS_PREFIX/bin/gnu-cc -RUN echo \ - "#!/bin/sh\n"\ - "/usr/bin/g++ -B/usr/bin \$@\n"\ - "exit \$?" > $LIBS_PREFIX/bin/gnu-c++ && chmod +x $LIBS_PREFIX/bin/gnu-c++ - -ENV CC_x86_64_unknown_linux_gnu=gnu-cc -ENV CXX_x86_64_unknown_linux_gnu=gnu-c++ -ENV LD_x86_64_unknown_linux_gnu=/usr/bin/ld -ENV AR_x86_64_unknown_linux_gnu=/usr/bin/ar - -ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gnu-cc - -# Set up environment variables for Rust bindgen -# See https://github.com/rust-lang/rust-bindgen/issues/1229#issuecomment-473493753 -ENV BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/opt/libs -target $TARGET" - -# Set default Cargo target -RUN mkdir -p ~/.cargo -RUN printf "[build]\ntarget = \"$TARGET\"\n" > ~/.cargo/config - -# In case of non-native target, install QEMU-user to enable running `cargo test` -RUN if [ -n "$GNU_TARGET"]; then apt-get update && apt-get install -y qemu-user; fi - -# Add dependencies for running tests -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata ca-certificates - -ENV TARGET="$TARGET" - -# Install `cargo-deb from Git until -# https://github.com/mmstick/cargo-deb/pull/112 is published on crates.io -RUN cargo install --git https://github.com/mmstick/cargo-deb --rev cc879930c06fa22e99f2839bad620ac0d0da879e cargo-deb - -RUN apt-get install -y rename cmark-gfm diff --git a/scripts/ci-docker-images/builder-x86_64-unknown-linux-gnu/Dockerfile b/scripts/ci-docker-images/builder-x86_64-unknown-linux-gnu/Dockerfile deleted file mode 100644 index 04b1e222d3ea3..0000000000000 --- a/scripts/ci-docker-images/builder-x86_64-unknown-linux-gnu/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -FROM centos:7 as builder -# This is formatted "$UID:$GID" by the docker-compose/scripts. -ARG USER - -RUN yum install -y epel-release && \ - yum clean all && \ - yum makecache -RUN yum install -y \ - make openssl-devel cmake3 git \ - gcc gcc-c++ libstdc++-static sudo \ - perl-App-cpanminus && \ - yum clean all -RUN ln -s /usr/bin/cmake3 /usr/bin/cmake -RUN cpanm File::Rename \ - && rename --version - -RUN cd /tmp && \ - git clone https://github.com/github/cmark-gfm && \ - cd cmark-gfm && \ - git checkout 0.29.0.gfm.0 && \ - make install INSTALL_PREFIX=/usr && \ - ldconfig && \ - cd .. && \ - rm -rf cmark-gfm && \ - cmark-gfm --version - -RUN echo "%sudo ALL = (ALL) NOPASSWD: ALL" >> /etc/sudoers.d/sudo -RUN useradd -m -G wheel \ - -u $(echo "${USER}" | cut -d: -f1) \ - # -g $(echo "${USER}" | cut -d: -f2) \ - runner - -USER ${USER} -RUN curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path --default-toolchain none -y -ENV PATH=/home/runner/.cargo/bin:$PATH -RUN echo "export PATH=/home/runner/.cargo/bin:$PATH" >> ~/bashrc -ENV LIBRARY_PATH /usr/local/lib:$LIBRARY_PATH -ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH - -RUN rustup update stable -RUN rustup run stable cargo install cargo-deb --version '^1.24.0' - -ENTRYPOINT ["/bin/bash"] diff --git a/scripts/ci-docker-images/builder-x86_64-unknown-linux-musl/Dockerfile b/scripts/ci-docker-images/builder-x86_64-unknown-linux-musl/Dockerfile deleted file mode 100644 index b39a5d40fb895..0000000000000 --- a/scripts/ci-docker-images/builder-x86_64-unknown-linux-musl/Dockerfile +++ /dev/null @@ -1,317 +0,0 @@ -# We use compiler-rt, libunwind, and libc++ from LLVM. -ARG LLVM_VERSION=9.0.0 -ARG LLVM_GIT_COMMIT=71fe7ec06b85f612fc0e4eb4134c7a7d0f23fac5 -# The version of musl should be the same as the one Rust is compiled with. -# This version can be found in this file: -# https://github.com/rust-lang/rust/blob/master/src/ci/docker/scripts/musl-toolchain.sh. -# Exampel: "1.1.22". -ARG MUSL_VERSION=1.1.22 - -# LLVM target triple for Linux and MUSL. Full list of supported triples -# can be found here: -# https://forge.rust-lang.org/release/platform-support.html. -# Example: "armv7-unknown-linux-musleabihf". -ARG TARGET="x86_64-unknown-linux-musl" -# Target architecture for LLVM -ARG LLVM_TARGET="X86" -# Target architecture for Linux kernel headres -ARG LINUX_TARGET="x86" -# We also need GCC for the target architecture to compile compier-rt. Because -# GCC target triples are different from LLVM, we need to specify it explicitly. -# Example: "arm-linux-gnueabihf". -ARG GNU_TARGET="" - -ARG LTO="" - -# Used by libc++. Programs compiled with this version should work with other -# kernel versions too. See https://wiki.gentoo.org/wiki/Linux-headers#FAQ. -ARG LINUX_HEADERS_VERSION=5.3.10 - -# Locations for produced tools and libraries. -ARG RUST_PREFIX=/opt/rust -ARG CLANG_PREFIX=/usr -ARG LIBS_PREFIX=/opt/libs - -# Libs -FROM ubuntu:19.10 AS libs-builder - -ARG CLANG_PREFIX -RUN apt-get update && apt-get install -y llvm clang lld - -# Get all sources at first, so that changes in arguments -# flags would not redownload them. -RUN apt-get update && apt-get -y install curl xz-utils - -ARG LLVM_VERSION -RUN curl --proto '=https' --tlsv1.2 -sSfL \ - https://github.com/llvm/llvm-project/archive/llvmorg-$LLVM_VERSION.tar.gz | \ - tar xzf - -ENV LLVM_DIR $SRC_DIR/llvm-project-llvmorg-$LLVM_VERSION - -ARG MUSL_VERSION -RUN curl --proto '=https' --tlsv1.2 -sSf \ - https://www.musl-libc.org/releases/musl-$MUSL_VERSION.tar.gz | \ - tar xzf - -ENV MUSL_DIR $SRC_DIR/musl-$MUSL_VERSION - -ARG LINUX_HEADERS_VERSION -RUN curl --proto '=https' --tlsv1.2 -sSf \ - https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-$LINUX_HEADERS_VERSION.tar.xz | \ - tar xJf - -ENV LINUX_DIR $SRC_DIR/linux-$LINUX_HEADERS_VERSION - -# Install build dependencies. -RUN apt-get update && apt-get -y install rsync ninja-build cmake python3-distutils clang llvm lld - -ARG TARGET -ARG CLANG_PREFIX -ARG LIBS_PREFIX - -# Compile Clang runtime (https://compiler-rt.llvm.org). -# We need crtbegin.o, crtend.o, and libclang_rt.builtins.a from there. -# Because Clang toolchain is not fully bootstrapped yet (no standard library), -# it is necessary to use GCC to compile it. -ARG GNU_TARGET -RUN apt-get install -y gcc${GNU_TARGET:+-${GNU_TARGET}} g++${GNU_TARGET:+-${GNU_TARGET}} -WORKDIR $LLVM_DIR/compiler-rt -ENV CC=${GNU_TARGET:+${GNU_TARGET}-}gcc -ENV CXX=${GNU_TARGET:+${GNU_TARGET}-}g++ -ENV CFLAGS="" -ENV CXXFLAGS="" -ENV LDFLAGS="" -RUN mkdir build && \ - cd build && \ - cmake \ - -DCMAKE_BUILD_TYPE=release \ - -DLLVM_CONFIG_PATH=$CLANG_PREFIX/bin/llvm-config \ - -DCOMPILER_RT_DEFAULT_TARGET_TRIPLE=$TARGET \ - -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ - -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \ - -DCOMPILER_RT_BUILD_XRAY=OFF \ - -DCOMPILER_RT_BUILD_PROFILE=OFF \ - -DCMAKE_INSTALL_PREFIX=$LIBS_PREFIX \ - -G Ninja \ - .. && \ - cmake --build . --target install - -# Compile musl. -WORKDIR $MUSL_DIR -ENV CC=clang -ENV CFLAGS="-target $TARGET -nostdinc -isystem $LIBS_PREFIX/include -D_Noreturn=" -ENV LDFLAGS="-fuse-ld=lld -static -nostdlib -nostartfiles" -ENV CROSS_COMPILE=$CLANG_PREFIX/bin/ -RUN cd $MUSL_DIR && \ - mkdir build && \ - cd build && \ - ../configure \ - --target=$TARGET \ - --prefix=$LIBS_PREFIX \ - --disable-shared \ - --disable-gcc-wrapper && \ - make -j$(nproc) install - -# Install Linux headers. -WORKDIR $LINUX_DIR -ARG LINUX_TARGET -ENV CC=clang -ENV CFLAGS="-target $TARGET -nostdinc -isystem $LIBS_PREFIX/include" -ENV LDFLAGS="-fuse-ld=lld -static -nostdlib -nostartfiles" -RUN make -j$(nproc) headers_install \ - KBUILD_VERBOSE=1 \ - HOSTCC="/usr/bin/gcc" \ - ARCH="$LINUX_TARGET" \ - CC="$CC" \ - INSTALL_HDR_PATH=$LIBS_PREFIX - -# Compile libc++ with musl using Clang as the compiler. See -# https://blogs.gentoo.org/gsoc2016-native-clang/2016/05/05/build-a-freestanding-libcxx/ -# for explanations. - -# libunwind -WORKDIR $LLVM_DIR/libunwind -ENV CC=clang -ENV CXX=clang++ -ENV CFLAGS="-target $TARGET -nostdinc -isystem $LIBS_PREFIX/include" -ENV CXXFLAGS="$CFLAGS -nostdinc++ -I$LLVM_DIR/libcxx/include" -ENV LDFLAGS="-fuse-ld=lld -static -nostdlib -nostartfiles -L$LIBS_PREFIX/lib -lc" -RUN mkdir build && \ - cd build && \ - cmake \ - -DCMAKE_BUILD_TYPE=release \ - -DLIBUNWIND_ENABLE_SHARED=OFF \ - -DLIBUNWIND_INSTALL_PREFIX=$LIBS_PREFIX/ \ - -DLLVM_PATH=$LLVM_DIR \ - -G Ninja \ - .. && \ - cmake --build . --target install - -# libc++abi -WORKDIR $LLVM_DIR/libcxxabi -ENV CC=clang -ENV CXX=clang++ -ENV CFLAGS="-target $TARGET -nostdinc -isystem $LIBS_PREFIX/include" -ENV CXXFLAGS="$CFLAGS -nostdinc++ -I$LLVM_DIR/libcxx/include" -ENV LDFLAGS="-fuse-ld=lld -static -nostdlib -nostartfiles -L$LIBS_PREFIX/lib -lunwind -lc" -RUN mkdir build && \ - cd build && \ - cmake \ - -DCMAKE_BUILD_TYPE=release \ - -DLIBCXXABI_ENABLE_SHARED=OFF \ - -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ - -DLIBCXXABI_LIBUNWIND_PATH=$LLVM_DIR/libunwind \ - -DLIBCXXABI_LIBCXX_INCLUDES=$LLVM_DIR/libcxx/include \ - -DLIBCXXABI_INSTALL_PREFIX=$LIBS_PREFIX/ \ - -DLLVM_PATH=$LLVM_DIR \ - -G Ninja \ - .. && \ - cmake --build . --target install - -# libc++ -WORKDIR $LLVM_DIR/libcxx -ENV CC=clang -ENV CXX=clang++ -ENV CFLAGS="-target $TARGET -nostdinc -isystem $LIBS_PREFIX/include" -ENV CXXFLAGS="$CFLAGS -nostdinc++ -I$LIBS_PREFIX/include/c++/v1" -ENV LDFLAGS="-fuse-ld=lld -static -nostdlib -nostartfiles -L$LIBS_PREFIX/lib -lc++abi -lunwind -lc" -RUN mkdir build && \ - cd build && \ - cmake \ - -DCMAKE_BUILD_TYPE=release \ - -DLIBCXX_ENABLE_SHARED=OFF \ - -DLIBCXX_HAS_MUSL_LIBC=ON \ - -DLIBCXX_HAS_GCC_S_LIB=OFF \ - -DLIBCXX_CXX_ABI=libcxxabi \ - -DLIBCXX_CXX_ABI_INCLUDE_PATHS=$LLVM_DIR/libcxxabi/include \ - -DLIBCXX_CXX_ABI_LIBRARY_PATH=$LIBS_PREFIX \ - -DLIBCXX_INSTALL_PREFIX=$LIBS_PREFIX/ \ - -DLIBCXX_INSTALL_HEADER_PREFIX=$LIBS_PREFIX/ \ - -DLLVM_PATH=$LLVM_DIR \ - -G Ninja \ - .. && \ - cmake --build . --target install - -# Create wrappers for Clang that automatically use correct libraries and start files. -# See -# https://blogs.gentoo.org/gsoc2016-native-clang/2016/05/31/build-gnu-free-executables-with-clang/ -# and -# https://renenyffenegger.ch/notes/development/languages/C-C-plus-plus/GCC/options/no/compare-nostartfiles-nodefaultlibs-nolibc-nostdlib. -ENV MUSL_CFLAGS="-target $TARGET -nostdinc -isystem $LIBS_PREFIX/include" -ENV MUSL_CXXFLAGS="$MUSL_CFLAGS -nostdinc++ -I$LIBS_PREFIX/include/c++/v1" - -ENV MUSL_LDFLAGS="-fuse-ld=lld -static -nostdlib -nostartfiles -L$LIBS_PREFIX/lib -L$LIBS_PREFIX/lib/linux -lc++ -lc++abi -lunwind -l$(basename /opt/libs/lib/linux/libclang_rt.builtins* | tail -c+4 | head -c-3) -lc" -ENV MUSL_STARTFILES="$LIBS_PREFIX/lib/crt1.o $(find $LIBS_PREFIX/lib/linux -iname 'clang_rt.crtbegin*.o') $(find $LIBS_PREFIX/lib/linux -iname 'clang_rt.crtend*.o')" - -RUN mkdir -p $LIBS_PREFIX/bin -RUN echo \ - "#!/bin/sh\n"\ - "case \"\$@\" in *-shared*);; *-nostdlib*);; *) STARTFILES=\"$MUSL_STARTFILES\";; esac\n"\ - "$CLANG_PREFIX/bin/clang -Qunused-arguments $MUSL_CFLAGS \$@ \$STARTFILES $MUSL_LDFLAGS\n"\ - "exit \$?" > $LIBS_PREFIX/bin/musl-cc -RUN echo \ - "#!/bin/sh\n"\ - "case \"\$@\" in *-shared*);; *-nostdlib*);; *) STARTFILES=\"$MUSL_STARTFILES\";; esac\n"\ - "$CLANG_PREFIX/bin/clang++ -Qunused-arguments $MUSL_CXXFLAGS \$@ \$STARTFILES $MUSL_LDFLAGS\n"\ - "exit \$?" > $LIBS_PREFIX/bin/musl-c++ -RUN chmod +x $LIBS_PREFIX/bin/* - -# At this point a fully functional C++ compiler that is able to produce -# static binaries linked with musl and libc++ is bootstrapped. -# It can be used by calling musl-c++ (or musl-cc for C) executable. -# However, we need to also create generic aliases to make it possible to -# use it as a drop-in replacement for the system-wide GCC. -RUN ln -s $LIBS_PREFIX/bin/musl-cc $LIBS_PREFIX/bin/cc -RUN ln -s $LIBS_PREFIX/bin/musl-cc $LIBS_PREFIX/bin/gcc -RUN if [ -n "$GNU_TARGET" ]; then ln -s $LIBS_PREFIX/bin/musl-cc $LIBS_PREFIX/bin/$(echo $GNU_TARGET | sed 's/gnu/musl/g')-gcc; fi -RUN ln -s $LIBS_PREFIX/bin/musl-cc $LIBS_PREFIX/bin/musl-gcc -RUN ln -s $LIBS_PREFIX/bin/musl-c++ $LIBS_PREFIX/bin/c++ -RUN ln -s $LIBS_PREFIX/bin/musl-c++ $LIBS_PREFIX/bin/g++ -RUN if [ -n "$GNU_TARGET" ]; then ln -s $LIBS_PREFIX/bin/musl-c++ $LIBS_PREFIX/bin/$(echo $GNU_TARGET | sed 's/gnu/musl/g')-g++; fi -RUN ln -s $LIBS_PREFIX/bin/musl-c++ $LIBS_PREFIX/bin/musl-g++ - -RUN ln -s $CLANG_PREFIX/bin/llvm-ar $LIBS_PREFIX/bin/ar -RUN ln -s $CLANG_PREFIX/bin/llvm-strip $LIBS_PREFIX/bin/strip -RUN ln -s $CLANG_PREFIX/bin/ld.lld $LIBS_PREFIX/bin/ld - -# Use some headers provided by Clang because they are not present in musl. -RUN ln -s $CLANG_PREFIX/lib/clang/9.0.0/include/stdatomic.h $LIBS_PREFIX/include/stdatomic.h -RUN ln -s $CLANG_PREFIX/lib/clang/9.0.0/include/emmintrin.h $LIBS_PREFIX/include/emmintrin.h -RUN ln -s $CLANG_PREFIX/lib/clang/9.0.0/include/xmmintrin.h $LIBS_PREFIX/include/xmmintrin.h -RUN ln -s $CLANG_PREFIX/lib/clang/9.0.0/include/mmintrin.h $LIBS_PREFIX/include/mmintrin.h -RUN ln -s $CLANG_PREFIX/lib/clang/9.0.0/include/mm_malloc.h $LIBS_PREFIX/include/mm_malloc.h - -# Some build scripts hardcode -lstdc++ linker flag on all Linux systems. -# Because our linker is already configured to link with libc++ instead, -# we can just provide dummy libstdc++ to be compatible with GNU systems. -# For example, macOS provides similar experience, where -# `clang++ -o program program.cpp -lstdc++` would still link to libc++. -RUN echo > dummy.c && \ - $LIBS_PREFIX/bin/cc -c dummy.c && \ - $CLANG_PREFIX/bin/llvm-ar cr $LIBS_PREFIX/lib/libstdc++.a dummy.o && \ - rm dummy.c dummy.o - -# The actual builder. -FROM ubuntu:19.10 - -# Install Rust using rustup. -RUN apt-get update && apt-get install -y curl -ARG TARGET -ARG RUST_PREFIX -ENV RUSTUP_HOME=$RUST_PREFIX/rustup -ENV CARGO_HOME=$RUST_PREFIX/cargo -COPY rust-toolchain /tmp/ -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ - sh -s -- -y --profile minimal --default-toolchain $(cat /tmp/rust-toolchain) --target $TARGET -ENV PATH="$RUSTUP_HOME/bin:$CARGO_HOME/bin:$PATH" - -RUN apt-get update && apt-get install -y llvm clang lld - -ARG LIBS_PREFIX -COPY --from=libs-builder $LIBS_PREFIX $LIBS_PREFIX -ENV PATH="$LIBS_PREFIX/bin:$PATH" - -# Essential tools needed for compiling native dependencies -RUN apt-get update && apt-get install -y --install-recommends build-essential cmake git - -# Because the system GCC and binutils are shadowed by aliases, we need to -# instruct Cargo and cc crate to use GCC on the host system. They are used to -# compile dependencies of build scripts. -RUN echo \ - "#!/bin/sh\n"\ - "/usr/bin/gcc -B/usr/bin \$@\n"\ - "exit \$?" > $LIBS_PREFIX/bin/gnu-cc && chmod +x $LIBS_PREFIX/bin/gnu-cc -RUN echo \ - "#!/bin/sh\n"\ - "/usr/bin/g++ -B/usr/bin \$@\n"\ - "exit \$?" > $LIBS_PREFIX/bin/gnu-c++ && chmod +x $LIBS_PREFIX/bin/gnu-c++ - -ENV CC_x86_64_unknown_linux_gnu=gnu-cc -ENV CXX_x86_64_unknown_linux_gnu=gnu-c++ -ENV LD_x86_64_unknown_linux_gnu=/usr/bin/ld -ENV AR_x86_64_unknown_linux_gnu=/usr/bin/ar - -ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gnu-cc - -# Set up environment variables for Rust bindgen -# See https://github.com/rust-lang/rust-bindgen/issues/1229#issuecomment-473493753 -ENV BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/opt/libs -target $TARGET" - -# Set default Cargo target -RUN mkdir -p ~/.cargo -RUN printf "[build]\ntarget = \"$TARGET\"\n" > ~/.cargo/config - -# In case of non-native target, install QEMU-user to enable running `cargo test` -RUN if [ -n "$GNU_TARGET"]; then apt-get update && apt-get install -y qemu-user; fi - -# Add dependencies for running tests -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata ca-certificates - -ENV TARGET="$TARGET" - -# Install `cargo-deb from Git until -# https://github.com/mmstick/cargo-deb/pull/112 is published on crates.io -RUN cargo install --git https://github.com/mmstick/cargo-deb --rev cc879930c06fa22e99f2839bad620ac0d0da879e cargo-deb - -RUN apt-get install -y rename cmark-gfm - -# Install journalctl -RUN apt-get install -y systemd diff --git a/scripts/environment/Dockerfile b/scripts/environment/Dockerfile index 840051c36d53f..2f192b253399e 100644 --- a/scripts/environment/Dockerfile +++ b/scripts/environment/Dockerfile @@ -1,22 +1,32 @@ -FROM docker.io/ubuntu:20.04 AS ENVIRONMENT +ARG BASEIMAGE=docker.io/ubuntu:20.04 + +FROM ${BASEIMAGE} AS ENVIRONMENT +ARG BOOTSTRAP=scripts/environment/bootstraps/ubuntu-20.04.sh + ENV DEBIAN_FRONTEND=noninteractive \ + DOCKER_CLI_EXPERIMENTAL=enabled \ TZ='America/New York' \ PATH=/root/.cargo/bin:/root/.local/bin/:$PATH \ LANG=en_US.UTF-8 \ LANGUAGE=en_US.UTF-8 \ - LC_ALL=en_US.UTF-8 + LC_ALL=en_US.UTF-8 \ + PKG_CONFIG_ALLOW_CROSS=true \ + PKG_CONFIG_ALL_STATIC=true \ + LIBZ_SYS_STATIC=1 \ + OPENSSL_STATIC=1 # Container junk RUN echo $TZ > /etc/timezone # Setup the env RUN mkdir -p /git/timberio/vector/{scripts,scripts/environment} -ADD scripts/environment/bootstrap-ubuntu-20.04.sh scripts/environment/prepare.sh \ - /git/timberio/vector/scripts/environment/ # Setup the toolchain WORKDIR /git/timberio/vector -RUN ./scripts/environment/bootstrap-ubuntu-20.04.sh +ADD ${BOOTSTRAP} /git/timberio/vector/${BOOTSTRAP} +ADD scripts/environment/prepare.sh \ + /git/timberio/vector/scripts/environment/ +RUN ${BOOTSTRAP} ADD scripts/Gemfile scripts/Gemfile.lock \ /git/timberio/vector/scripts/ @@ -25,8 +35,8 @@ ADD rust-toolchain \ RUN ./scripts/environment/prepare.sh # Declare volumes -VOLUME /vector -VOLUME /vector/target +VOLUME /git/timberio/vector +VOLUME /git/timberio/vector/target VOLUME /root/.cargo # Prepare for use diff --git a/scripts/environment/bootstrap-macos-10.sh b/scripts/environment/bootstrap-macos-10.sh deleted file mode 100644 index 054020fbd79e6..0000000000000 --- a/scripts/environment/bootstrap-macos-10.sh +++ /dev/null @@ -1,10 +0,0 @@ -#! /usr/bin/env bash -set -e -o verbose - -brew install ruby - -echo "export PATH=\"/usr/local/opt/ruby/bin:\$PATH\"" >> "$HOME/.bash_profile" - -if [ -n "${CI-}" ] ; then - echo "::add-path::/usr/local/opt/ruby/bin" -fi diff --git a/scripts/environment/bootstrap/centos-7.sh b/scripts/environment/bootstrap/centos-7.sh new file mode 100755 index 0000000000000..af924e8dd8b82 --- /dev/null +++ b/scripts/environment/bootstrap/centos-7.sh @@ -0,0 +1,28 @@ +#! /usr/bin/env bash +set -e -o verbose + +# Deps +yum install -y epel-release +yum clean all +yum makecache +yum install -y \ + make \ + openssl-devel \ + cmake3 \ + autotools \ + wget \ + git \ + gcc \ + gcc-c++ \ + libstdc++-static \ + sudo \ + which \ + bzip2 \ + krb5-devel \ + perl-App-cpanminus +yum clean all + +ln -s /usr/bin/cmake3 /usr/bin/cmake + +cpanm File::Rename \ + && rename --version diff --git a/scripts/environment/bootstrap/macos-10.sh b/scripts/environment/bootstrap/macos-10.sh new file mode 100644 index 0000000000000..738b1cdbffcdb --- /dev/null +++ b/scripts/environment/bootstrap/macos-10.sh @@ -0,0 +1,2 @@ +#! /usr/bin/env bash +set -e -o verbose diff --git a/scripts/environment/bootstrap-ubuntu-20.04.sh b/scripts/environment/bootstrap/ubuntu-20.04.sh similarity index 77% rename from scripts/environment/bootstrap-ubuntu-20.04.sh rename to scripts/environment/bootstrap/ubuntu-20.04.sh index dbb8eaf705656..d03e0c1ffd0fa 100755 --- a/scripts/environment/bootstrap-ubuntu-20.04.sh +++ b/scripts/environment/bootstrap/ubuntu-20.04.sh @@ -8,6 +8,7 @@ apt upgrade --yes # Deps apt install --yes \ + wget \ build-essential \ pkg-config \ libssl-dev \ @@ -15,20 +16,24 @@ apt install --yes \ jq \ shellcheck \ software-properties-common \ + musl-tools \ locales \ apt-transport-https \ ca-certificates \ curl \ gnupg-agent \ - nodejs \ - npm \ - ruby-bundler \ libsasl2-dev \ gnupg2 \ - wget + wget \ + tcl-dev \ + cmake \ + crossbuild-essential-arm64 \ + gnupg2 -# Grease -# Grease is used for the `make release-github` task. +# Stupid, right? Sadly it works. +ln -s "/usr/bin/g++" "/usr/bin/musl-g++" || true + +# We use Grease for parts of release. TEMP=$(mktemp -d) curl \ -L https://github.com/timberio/grease/releases/download/v1.0.1/grease-1.0.1-linux-amd64.tar.gz \ @@ -38,20 +43,21 @@ tar \ -C "${TEMP}" cp "${TEMP}/grease/bin/grease" /usr/bin/grease + # Locales locale-gen en_US.UTF-8 dpkg-reconfigure locales -# Rust -curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal - # Docker curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - add-apt-repository \ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ - xenial \ + focal \ stable" # Install those new things apt update --yes apt install --yes yarn docker-ce docker-ce-cli containerd.io + +# Remarshal is particular +pip3 install remarshal==0.11.2 diff --git a/scripts/environment/entrypoint.sh b/scripts/environment/entrypoint.sh index 02deefb0b6ccc..d0a8e5fa16605 100755 --- a/scripts/environment/entrypoint.sh +++ b/scripts/environment/entrypoint.sh @@ -1,3 +1,4 @@ #! /usr/bin/env bash +export PATH+=:${MUSL_CROSS_MAKE_PATH}/bin; exec "$@" diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index e17e6af5a445e..0a8ff161beacf 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -1,18 +1,44 @@ #! /usr/bin/env bash set -e -o verbose -rustup default "$(cat rust-toolchain)" +WORKDIR=$(pwd) + +curl https://sh.rustup.rs -sSf | sh -s -- -y +source $HOME/.cargo/env + +rustup target add wasm32-wasi +rustup toolchain install nightly --target x86_64-unknown-linux-musl +rustup toolchain install nightly --target armv7-unknown-linux-musleabihf +rustup toolchain install nightly --target aarch64-unknown-linux-musl rustup component add rustfmt rustup component add clippy -rustup target add wasm32-wasi +rustup default "$(cat rust-toolchain)" + +# Ruby toolchain +export PATH="${HOME}/.rbenv/bin:${HOME}/.rbenv/shims:${PATH}" +git clone https://github.com/rbenv/rbenv.git ~/.rbenv +cd ~/.rbenv && src/configure && make -C src +eval "$(rbenv init -)" +mkdir -p "$(rbenv root)"/plugins +git clone https://github.com/rbenv/ruby-build.git "$(rbenv root)"/plugins/ruby-build +rbenv install 2.7.1 +rbenv global 2.7.1 -cd scripts +cd ${WORKDIR}/scripts bundle update --bundler bundle install cd .. -cd scripts -npm install markdownlint-cli +cd ${WORKDIR}/scripts +# Node toolchain +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash +export NVM_DIR="$HOME/.nvm" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm +[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion +nvm install 10.19.0 +nvm use 10.19.0 +nvm alias default 10.19.0 +npm install markdownlint-cli --global cd .. pip3 install jsonschema==3.2.0