Build docker images from travis-ci (wurstmeister#335)

sscaling · wurstmeister · commit 0efae8f04232 · 2018-06-28T12:16:01.000+01:00
* Fix up downloading binaries < 0.11 * Update kafkacat tests to account for older broker versions Fixes wurstmeister#163 Fixes wurstmeister#324 Fixes wurstmeister#210 Fixes wurstmeister#231 Fixes wurstmeister#210
diff --git a/.travis.yml b/.travis.yml
@@ -1,14 +1,52 @@
 sudo: required
 
-language: java
+language: scala
 
 services:
   - docker
 
+# An alternative approach is to build a full matrix.
+#scala:
+#  - "2.10"
+#  - 2.11
+#
+#env:
+#  - KAFKA_VERSION=0.8.2.2
+#  - KAFKA_VERSION=0.9.0.1
+#  - KAFKA_VERSION=0.10.2.1
+#  - KAFKA_VERSION=0.11.0.2
+#  - KAFKA_VERSION=1.0.1
+#  - KAFKA_VERSION=1.1.0
+
+# This version will be also tagged as 'latest'
+env:
+  global:
+    - LATEST="2.11-1.1.0"
+
+# Build recommended versions based on: http://kafka.apache.org/downloads
+matrix:
+  include:
+  - scala: "2.10"
+    env: KAFKA_VERSION=0.8.2.2
+  - scala: 2.11
+    env: KAFKA_VERSION=0.9.0.1
+  - scala: 2.11
+    env: KAFKA_VERSION=0.10.2.1
+  - scala: 2.11
+    env: KAFKA_VERSION=0.11.0.2
+  - scala: 2.11
+    env: KAFKA_VERSION=1.0.1
+  - scala: 2.11
+    env: KAFKA_VERSION=1.1.0
+
 install:
   - docker --version
   - docker-compose --version
-  - docker build -t wurstmeister/kafka .
+  - echo "KAFKA VERSION  $KAFKA_VERSION"
+  - echo "SCALA VERSION  $TRAVIS_SCALA_VERSION"
+  - echo "LATEST VERSION $LATEST"
+  - export CURRENT=${TRAVIS_SCALA_VERSION}-${KAFKA_VERSION}
+  - docker build --build-arg kafka_version=$KAFKA_VERSION --build-arg scala_version=$TRAVIS_SCALA_VERSION -t wurstmeister/kafka .
   - docker pull confluentinc/cp-kafkacat
 
 before_script:
@@ -17,15 +55,30 @@ before_script:
 
 script:
   # Shellcheck main source files
-  - shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh
+  - shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh versions.sh
   - cd test
   # Shellcheck the tests
-  - shellcheck -x -e SC1090 -s bash *sh
+  - shellcheck -x -e SC1090 -s bash *.sh **/*.sh
   - sleep 5 # Wait for containers to start
+  - docker ps -a
   - ./runAllTests.sh
   # End-to-End scenario tests
   - cd scenarios
   - ./runJmxScenario.sh
 
 after_script:
   - docker-compose stop
+
+# This will deploy from master. Might want to have a single release branch for a little more control
+deploy:
+  - provider: script
+    script: bash docker_push latest
+    on:
+      repo: wurstmeister/kafka-docker
+      branch: master
+      condition: $CURRENT = $LATEST
+  - provider: script
+    script: bash docker_push "${TRAVIS_SCALA_VERSION}-${KAFKA_VERSION}"
+    on:
+      repo: wurstmeister/kafka-docker
+      # branch: release
diff --git a/Dockerfile b/Dockerfile
@@ -13,12 +13,12 @@ ENV KAFKA_VERSION=$kafka_version \
 
 ENV PATH=${PATH}:${KAFKA_HOME}/bin
 
-COPY download-kafka.sh start-kafka.sh broker-list.sh create-topics.sh /tmp/
+COPY download-kafka.sh start-kafka.sh broker-list.sh create-topics.sh versions.sh /tmp/
 
 RUN apk add --no-cache bash curl jq docker \
  && mkdir /opt \
  && chmod a+x /tmp/*.sh \
- && mv /tmp/start-kafka.sh /tmp/broker-list.sh /tmp/create-topics.sh /usr/bin \
+ && mv /tmp/start-kafka.sh /tmp/broker-list.sh /tmp/create-topics.sh /tmp/versions.sh /usr/bin \
  && sync && /tmp/download-kafka.sh \
  && tar xfz /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt \
  && rm /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz \
diff --git a/README.md b/README.md
@@ -11,10 +11,25 @@ Dockerfile for [Apache Kafka](http://kafka.apache.org/)
 
 The image is available directly from [Docker Hub](https://hub.docker.com/r/wurstmeister/kafka/)
 
+Tags and releases
+-----------------
+
+All versions of the image are built from the same set of scripts with only minor variations (i.e. certain features are not supported on older versions). The version format mirrors the Kafka format, `<scala version>-<kafka version>`. Initially, all images are built with the recommended version of scala documented on [http://kafka.apache.org/downloads](http://kafka.apache.org/downloads). Available tags are:
+
+- `2.11-1.1.0`
+- `2.11-1.0.1`
+- `2.11-0.11.0.2`
+- `2.11-0.10.2.1`
+- `2.11-0.9.0.1`
+- `2.10-0.8.2.2`
+
+Everytime the image is updated, all tags will be pushed with the latest updates. This should allow for greater consistency across tags, as well as any security updates that have been made to the base image.
+
 ---
 
 ## Announcements
 
+* **28-May-2018** - New docker image tag format - see Readme.
 * **03-Apr-2018** - *BREAKING* - `KAFKA_ADVERTISED_PROTOCOL_NAME` and `KAFKA_PROTOCOL_NAME` removed. Please update to canonical kafka settings.
 * **03-Apr-2018** - *BREAKING* - `KAFKA_ZOOKEEPER_CONNECT` is now a mandatory environment var.
 
diff --git a/create-topics.sh b/create-topics.sh
@@ -26,6 +26,13 @@ if $start_timeout_exceeded; then
     exit 1
 fi
 
+# introduced in 0.10. In earlier versions, this will fail because the topic already exists.
+# shellcheck disable=SC1091
+source "/usr/bin/versions.sh"
+if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -gt "9" ]] || [[ "$MAJOR_VERSION" -gt "0" ]]; then
+    KAFKA_0_10_OPTS="--if-not-exists"
+fi
+
 # Expected format:
 #   name:partitions:replicas:cleanup.policy
 IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOPICS; do
@@ -35,14 +42,15 @@ IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOP
     if [ -n "${topicConfig[3]}" ]; then
         config="--config=cleanup.policy=${topicConfig[3]}"
     fi
+
     COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\
 		--create \\
 		--zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\
 		--topic ${topicConfig[0]} \\
 		--partitions ${topicConfig[1]} \\
 		--replication-factor ${topicConfig[2]} \\
 		${config} \\
-		--if-not-exists &"
+		${KAFKA_0_10_OPTS} &"
     eval "${COMMAND}"
 done
 
diff --git a/docker_push b/docker_push
@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+BASE_IMAGE="wurstmeister/kafka"
+IMAGE_VERSION="$1"
+
+if [ -z "$IMAGE_VERSION" ]; then
+  echo "No IMAGE_VERSION var specified"
+  exit 1
+fi
+
+echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+TARGET="$BASE_IMAGE:$IMAGE_VERSION"
+docker tag "$BASE_IMAGE" "$TARGET"
+docker push "$TARGET"
diff --git a/download-kafka.sh b/download-kafka.sh
@@ -1,11 +1,23 @@
 #!/bin/sh -e
 
-path="kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz"
-url=$(curl --stderr /dev/null "https://www.apache.org/dyn/closer.cgi?path=/${path}&as_json=1" | jq -r '"\(.preferred)\(.path_info)"')
+# shellcheck disable=SC1091
+source "/usr/bin/versions.sh"
+
+FILENAME="kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz"
+
+## Versions prior to 0.10.2.1 are not actively mirrored
+echo "Downloading kafka $MAJOR_VERSION.$MINOR_VERSION"
+if [[ "$MAJOR_VERSION" == "0" && "$MINOR_VERSION" -lt "11" ]]; then
+	echo "Version prior to 0.10.2.1 - downloading direct"
+	url="https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/${FILENAME}"
+else
+	url=$(curl --stderr /dev/null "https://www.apache.org/dyn/closer.cgi?path=/kafka/${KAFKA_VERSION}/${FILENAME}&as_json=1" | jq -r '"\(.preferred)\(.path_info)"')
+fi
 
 if [[ -z "$url" ]]; then
 	echo "Unable to determine mirror for downloading Kafka, the service may be down"
 	exit 1
 fi
 
-wget -q "${url}" -O "/tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz"
+echo "Downloading Kafka from $url"
+wget "${url}" -O "/tmp/${FILENAME}"
diff --git a/start-kafka.sh b/start-kafka.sh
@@ -22,7 +22,6 @@ fi
 create-topics.sh &
 unset KAFKA_CREATE_TOPICS
 
-# DEPRECATED: but maintained for compatibility with older brokers pre 0.9.0 (https://issues.apache.org/jira/browse/KAFKA-1809)
 if [[ -z "$KAFKA_ADVERTISED_PORT" && \
   -z "$KAFKA_LISTENERS" && \
   -z "$KAFKA_ADVERTISED_LISTENERS" && \
diff --git a/test/0.0/test.read-write.kafkacat.sh b/test/0.0/test.read-write.kafkacat.sh
@@ -1,8 +1,10 @@
 #!/bin/bash -e
 
+source version.functions
+
 testReadWrite() {
-	echo 'foo,bar' | kafkacat -b "$BROKER_LIST" -P -D, -t readwrite
-	kafkacat -b "$BROKER_LIST" -C -e -t readwrite
+	echo 'foo,bar' | eval "kafkacat -b $BROKER_LIST $KAFKACAT_OPTS -P -D, -t readwrite"
+	eval "kafkacat -b $BROKER_LIST $KAFKACAT_OPTS -C -e -t readwrite"
 }
 
 testReadWrite
diff --git a/test/0.9/test.snappy.kafkacat.sh b/test/0.9/test.snappy.kafkacat.sh
@@ -1,8 +1,10 @@
 #!/bin/bash -e
 
+source version.functions
+
 testSnappy() {
-	echo 'foo,bar' | kafkacat -X compression.codec=snappy -b "$BROKER_LIST" -P -D, -t snappy
-	kafkacat -X compression.codec=snappy -b "$BROKER_LIST" -C -e -t snappy
+	echo 'foo,bar' | eval "kafkacat -X compression.codec=snappy -b $BROKER_LIST $KAFKACAT_OPTS -P -D, -t snappy"
+	eval "kafkacat -X compression.codec=snappy -b $BROKER_LIST $KAFKACAT_OPTS -C -e -t snappy"
 }
 
 testSnappy
diff --git a/test/docker-compose.yml b/test/docker-compose.yml
@@ -9,7 +9,10 @@ services:
     ports:
       - "9092"
     environment:
-      KAFKA_LISTENERS: PLAINTEXT://:9092
+      HOSTNAME_COMMAND: "echo $$(hostname)"
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_PORT: 9092
+      BROKER_ID_COMMAND: "docker inspect --format '{{ .Name }}' $$(hostname) | awk -F_ '{ printf $$NF }'"
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
@@ -31,13 +34,12 @@ services:
   kafkacattest:
     image: confluentinc/cp-kafkacat
     environment:
-      BROKER_LIST: ${BROKER_LIST}
+       - BROKER_LIST
+       - KAFKA_VERSION=${KAFKA_VERSION-1.1.0}
     volumes:
       - .:/tests
     working_dir: /tests
     entrypoint:
       - ./runTestPattern.sh
     command:
-      - -v
-      - ${KAFKA_VERSION-1.1.0}
       - "*/*.kafkacat.sh"
diff --git a/test/runTestPattern.sh b/test/runTestPattern.sh
@@ -1,5 +1,7 @@
 #!/bin/bash -e
 
+source version.functions
+
 PATTERN=$1
 VERSION=$KAFKA_VERSION
 
@@ -21,32 +23,6 @@ echo ""
 echo ""
 echo "Running tests for Kafka $VERSION with pattern $PATTERN"
 
-# Modified from https://stackoverflow.com/a/4025065
-compareVersion() {
-    # Only care about major / minor
-    LEFT=$(echo "$1" | cut -d. -f1-2)
-    RIGHT=$(echo "$2" | cut -d. -f1-2)
-    if [[ "$LEFT" != "$RIGHT" ]]
-    then
-        local IFS=.
-        local i ver1=($LEFT) ver2=($RIGHT)
-        for ((i=0; i<${#ver1[@]}; i++))
-        do
-            if (( "${ver1[i]}" > "${ver2[i]}" ))
-            then
-                echo ">"
-                return
-            fi
-            if (( "${ver1[i]}" < "${ver2[i]}" ))
-            then
-                echo "<"
-                return
-            fi
-        done
-    fi
-    echo "="
-}
-
 runPattern() {
   for t in $PATTERN; do
     echo
@@ -58,7 +34,6 @@ runPattern() {
     echo "Kafka $VERSION is '$RESULT' target $TARGET of test $t"
     if [[ "$RESULT" != "<" ]]; then
       echo "  testing '$t'"
-      # shellcheck disable=SC1090
       ( source "$t" )
       status=$?
       if [[ -z "$status" || ! "$status" -eq 0 ]]; then
diff --git a/test/scenarios/jmx/docker-compose.yml b/test/scenarios/jmx/docker-compose.yml
@@ -10,7 +10,10 @@ services:
       - "9092"
       - "1099"
     environment:
-      KAFKA_LISTENERS: PLAINTEXT://kafka:9092
+      KAFKA_ADVERTISED_HOST_NAME: kafka
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_PORT: 9092
+      BROKER_ID_COMMAND: "docker inspect --format '{{ .Name }}' $$(hostname) | awk -F_ '{ printf $$NF }'"
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka -Dcom.sun.management.jmxremote.rmi.port=1099"
       JMX_PORT: 1099
diff --git a/test/version.functions b/test/version.functions
@@ -0,0 +1,37 @@
+#!/bin/bash -e
+
+# Modified from https://stackoverflow.com/a/4025065
+compareVersion() {
+    # Only care about major / minor
+    LEFT=$(echo "$1" | cut -d. -f1-2)
+    RIGHT=$(echo "$2" | cut -d. -f1-2)
+    if [[ "$LEFT" != "$RIGHT" ]]
+    then
+        local IFS=.
+        local i ver1=($LEFT) ver2=($RIGHT)
+        for ((i=0; i<${#ver1[@]}; i++))
+        do
+            if (( "${ver1[i]}" > "${ver2[i]}" ))
+            then
+                echo ">"
+                return
+            fi
+            if (( "${ver1[i]}" < "${ver2[i]}" ))
+            then
+                echo "<"
+                return
+            fi
+        done
+    fi
+    echo "="
+}
+
+# https://github.com/edenhill/librdkafka/wiki/Broker-version-compatibility
+# To support different broker versions, we need to configure kafkacat differently
+VERSION_8=$(compareVersion "$KAFKA_VERSION" "0.8")
+VERSION_9=$(compareVersion "$KAFKA_VERSION" "0.9")
+
+if [[ "$VERSION_8" == "=" || "$VERSION_9" == "=" ]]; then
+	export KAFKACAT_OPTS="-Xapi.version.request=false -Xbroker.version.fallback=$KAFKA_VERSION"
+	echo "[INFO] Using kafkacat opts on older version '$KAFKACAT_OPTS'"
+fi
diff --git a/versions.sh b/versions.sh
@@ -0,0 +1,7 @@
+#!/bin/bash -e
+
+MAJOR_VERSION=$(echo "$KAFKA_VERSION" | cut -d. -f1)
+export MAJOR_VERSION
+
+MINOR_VERSION=$(echo "$KAFKA_VERSION" | cut -d. -f2)
+export MINOR_VERSION
