chore: add support for x86_64-unknown-linux-musl releases (#654)

Signed-off-by: Ben Johnson <[email protected]>

Author: binarylogic

.circleci/config.yml

@@ -141,6 +141,23 @@ jobs:
             - "*-x86_64-unknown-linux-gnu.tar.gz"
             - "*-amd64.deb"
 
+  build-x86_64-unknown-linux-musl-archive:
+    docker:
+      - image: timberiodev/vector-builder-x86_64-unknown-linux-musl:latest
+    resource_class: xlarge
+    steps:
+      - checkout
+      - run:
+          name: Build archive
+          command: |
+            export VERSION=$(make version)
+            make build-archive FEATURES="jemallocator"
+      - persist_to_workspace:
+          root: target/artifacts
+          paths:
+            - "*-x86_64-unknown-linux-musl.tar.gz"
+            - "*-amd64.deb"
+
   #
   # Packaging
   #
@@ -414,6 +431,7 @@ build-requires: &build-requires
 package-requires: &package-requires
   requires:
     - build-x86_64-unknown-linux-gnu-archive
+    - build-x86_64-unknown-linux-musl-archive
     - build-x86_64-apple-darwin-archive
 
 verify-requires: &verify-requires
@@ -452,6 +470,9 @@ workflows:
       - build-x86_64-unknown-linux-gnu-archive:
           <<: *release-workflow-filters
           <<: *build-requires
+      - build-x86_64-unknown-linux-musl-archive:
+          <<: *release-workflow-filters
+          <<: *build-requires
       - build-x86_64-apple-darwin-archive:
           <<: *release-workflow-filters
           <<: *build-requires

.metadata.toml

@@ -1308,10 +1308,6 @@ toml_table = "https://github.com/toml-lang/toml#table"
 uuidv4 = "https://en.wikipedia.org/wiki/Universally_unique_identifier#Version_4_(random)"
 vector_repo = "https://github.com/timberio/vector"
 vector_initd_service = "https://github.com/timberio/vector/blob/master/distribution/init.d/vector"
-vector_edge_x86_64-apple-darwin = "https://packages.timber.io/vector/edge/vector-edge-x86_64-apple-darwin.tar.gz"
-vector_latest_x86_64-apple-darwin = "https://packages.timber.io/vector/latest/vector-latest-x86_64-apple-darwin.tar.gz"
-vector_edge_x86_64-unknown-linux-gnu = "https://packages.timber.io/vector/edge/vector-edge-x86_64-unknown-linux-gnu.tar.gz"
-vector_latest_x86_64-unknown-linux-gnu = "https://packages.timber.io/vector/latest/vector-latest-x86_64-unknown-linux-gnu.tar.gz"
 vector_systemd_file = "https://github.com/timberio/vector/blob/master/distribution/systemd/vector.service"
 vote_feature = "https://github.com/timberio/vector/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc+label%3A%22Type%3A+New+Feature%22"
 vector_chat = "https://chat.vector.dev"

Makefile

@@ -52,8 +52,9 @@ test: ## Spins up Docker resources and runs _every_ test
 
 ##@ Releasing
 
+build-archive: FEATURES = "default"
 build-archive: ## Build a Vector archive for a given $TARGET and $VERSION
-	@scripts/build-archive.sh
+	@FEATURES=$(FEATURES) scripts/build-archive.sh
 
 build-ci-docker-images: ## Build the various Docker images used for CI
 	@scripts/build-ci-docker-images.sh

docs/setup/installation/manual/from-archives.md

@@ -24,6 +24,7 @@ architectures. If you don't see an architecture, then we recommend
 | :------------| :-----: | :---- |
 | [`x86_64-apple-darwin`][url.vector_latest_x86_64-apple-darwin] | `latest` | 64-bit OSX (10.7+, Lion+) |
 | [`uknown-linux-gnu`][url.vector_latest_x86_64-unknown-linux-gnu] | `latest` | 64-bit Linux (2.6.18+) |
+| [`uknown-linux-musl`][url.vector_latest_x86_64-unknown-linux-musl] | `latest` | 64-bit Linux with MUSL |
 
 {% endtab %}
 {% tab title="Edge" %}
@@ -37,7 +38,8 @@ using them over the "latest" alternatives.
 | Architecture | Channel | Notes |
 | :------------| :-----: | :---- |
 | [`x86_64-apple-darwin`][url.vector_edge_x86_64-apple-darwin] | `edge` | 64-bit OSX (10.7+, Lion+) |
-| [`uknown-linux-gnu`][url.vector_edge_x86_64-unknown-linux-gnu] | `latest` | 64-bit Linux (2.6.18+) |
+| [`uknown-linux-gnu`][url.vector_edge_x86_64-unknown-linux-gnu] | `edge` | 64-bit Linux (2.6.18+) |
+| [`uknown-linux-musl`][url.vector_edge_x86_64-unknown-linux-musl] | `edge` | 64-bit Linux with MUSL |
 {% endtab %}
 {% endtabs %}
 

scripts/build-archive.sh

@@ -13,14 +13,18 @@
 
 set -eu
 
-echo "Building version $VERSION for target $TARGET"
+echo "Building -- version: $VERSION, target: $TARGET, features: $FEATURES"
 
 artifacts_dir="target/artifacts"
 target_dir="target/$TARGET"
 archive_dir_name="vector-$VERSION"
 archive_dir="$target_dir/$archive_dir_name"
 
-cargo build --target $TARGET --release
+if [ "$FEATURES" == "default" ]; then
+  cargo build --target $TARGET --release
+else
+  cargo build --no-default-features --features $FEATURES --target $TARGET --release
+fi
 
 # Build the archive directory
 rm -rf $archive_dir

scripts/build-ci-docker-images.sh

@@ -45,6 +45,7 @@ function extend_cross_base_image() {
 # The following images are basic Docker images that do not extend a
 # cross base image.
 build_image "builder-x86_64-unknown-linux-gnu"
+build_image "builder-x86_64-unknown-linux-musl"
 build_image "checker"
 build_image "packager-deb"
 build_image "packager-rpm"

scripts/ci-docker-images/builder-x86_64-unknown-linux-musl/Dockerfile

@@ -1,46 +1,20 @@
 FROM japaric/x86_64-unknown-linux-musl:latest
 
-# Install the git repository to ensure we get the latest version.
-# This is important for determining the current Vector version number.
 RUN apt-get update
 RUN apt-get install -y software-properties-common
 RUN add-apt-repository ppa:git-core/ppa
 
-RUN echo "deb http://dk.archive.ubuntu.com/ubuntu/ trusty main universe
-deb http://dk.archive.ubuntu.com/ubuntu/ trusty-updates main universe" | > /etc/apt/sources.list.d/fury.list
-
-RUN apt-get update
-
-# Note: We do not call `apt-get upgrade` because we do not want Vector to
-#       depend on newer versions of libraries. This is just an assumption,
-#       I could be completely wrong with how this works.
-
-RUN apt-get install -y --no-install-recommends \
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends \
   build-essential \
   curl \
-  g++-4.4 \
-  gcc-4.4 \
   git \
   openssh-server \
   vim \
-  libleveldb-dev \
-  libsnappy-dev
-
-RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.4 50
-RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.4 50
-
-RUN update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30
-RUN update-alternatives --set cc /usr/bin/gcc
-
-RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30
-RUN update-alternatives --set c++ /usr/bin/g++
-
-# Note: We do not compile or install leveldb or rdkafka libraries because
-#       those Rust creates automatically build and link the libraries for
-#       this target. Unfortunately, this is not the case for other targets,
-#       hence the reason we only support this target currently.
+  zlib1g-dev
 
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable
 ENV PATH="$PATH:/root/.cargo/bin"
+RUN cargo install cargo-deb
 ENV TARGET=x86_64-unknown-linux-musl
 RUN rustup target add x86_64-unknown-linux-musl

scripts/generate/metadata/links.rb

@@ -296,6 +296,11 @@ def parse(full_name)
         label = "#{type.titleize}: #{name}"
         VECTOR_ISSUES_ROOT + "/new?" + {"labels" => [label]}.to_query
 
+      when /^url\.vector_(edge|latest)_(.*)/
+        channel = $1
+        target = $2
+        "https://packages.timber.io/vector/#{channel}/vector-#{channel}-#{target}.tar.gz"
+
       when /^url\.(.*)_test$/
         name = $1
         "#{TEST_HARNESS_ROOT}/tree/master/cases/#{name}"

scripts/package-rpm.sh

@@ -21,7 +21,7 @@ export RELEASE=1
 export CLEANED_VERSION=$VERSION
 CLEANED_VERSION=$(echo $CLEANED_VERSION | sed 's/-/\./g')
 
-# The arch is the first part of the part
+# The arch is the first part of the target
 ARCH=$(echo $TARGET | cut -d'-' -f1)
 
 # Create source dir

scripts/release-s3.sh

@@ -22,6 +22,7 @@ if [ -z "$CHANNEL" ]; then
   exit 1
 fi
 
+escaped_version=$(echo $VERSION | sed "s/\./\\\./g")
 
 #
 # S3
@@ -34,19 +35,24 @@ aws s3 cp "target/artifacts/" "s3://packages.timber.io/vector/$VERSION/" --recur
 echo "Uploading all artifacts to s3://packages.timber.io/vector/edge/"
 td=$(mktemp -d)
 cp -a "target/artifacts/." "$td"
-escaped_version=$(echo $VERSION | sed "s/\./\\\./g")
 rename -v "s/$escaped_version/edge/" "$td/*"
+echo "Renamed all builds: via \"s/$escaped_version/edge/\""
+ls $td
 aws s3 rm --recursive "s3://packages.timber.io/vector/edge/"
 aws s3 cp "$td" "s3://packages.timber.io/vector/edge/" --recursive
 rm -rf $td
+echo "Uploaded edge archives"
 
 if [[ "$CHANNEL" == "latest" ]]; then
   # Update the "latest" files
   echo "Uploading all artifacts to s3://packages.timber.io/vector/latest/"
   td=$(mktemp -d)
   cp -a "target/artifacts/." "$td"
-  rename -v "s/edge/latest/" "$td/*"
+  rename -v "s/$escaped_version/latest/" "$td/*"
+  echo "Renamed all builds: via \"s/$escaped_version/latest/\""
+  ls $td
   aws s3 rm --recursive "s3://packages.timber.io/vector/latest/"
   aws s3 cp "$td" "s3://packages.timber.io/vector/latest/" --recursive
   rm -rf $td
+  echo "Uploaded latest archives"
 fi