diff --git a/.circleci/config.yml b/.circleci/config.yml index 646ccfc..f7ac425 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,7 +22,8 @@ jobs: name: Setup dependencies command: | latests=$(wget -qO - https://raw.githubusercontent.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/master/.latests) - version=${CIRCLE_TAG#v} + version=2.3.3 + # version=${CIRCLE_TAG#v} [ -n "$version" ] || version=$(echo "${latests}" | cut -f1 -d' ') printf 'export LATESTS=%s\n' ${latests} >> $BASH_ENV printf 'export VERSION=%s\n' ${version} >> $BASH_ENV @@ -41,15 +42,14 @@ jobs: command: | SHORT_GIT_HASH=$(echo $CIRCLE_SHA1 | cut -c -7) DATE_BUILD=$(date +'%Y-%m-%d') - + case "${CIRCLE_BRANCH}" in master|develop|devops) - docker image build -f Dockerfile \ + docker build -t actionml/spark:latest --no-cache\ --build-arg release=${CIRCLE_TAG:-build} \ --build-arg version=${VERSION%%-*} \ --build-arg GIT_HASH=$SHORT_GIT_HASH --build-arg DATE_BUILD=$DATE_BUILD \ - --build-arg BRANCH=${CIRCLE_BRANCH} \ - -t actionml/spark:latest ./ + --build-arg BRANCH=${CIRCLE_BRANCH} . ;; *) ;; diff --git a/Dockerfile b/Dockerfile index eca935b..afaae21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,6 @@ ARG GIT_HASH ARG DATE_BUILD ARG BRANCH - LABEL com.actionml.spark.vendor=ActionML \ com.actionml.spark.version=$version \ com.actionml.spark.release=$release @@ -19,9 +18,11 @@ ENV DATE_BUILD=${DATE_BUILD} ENV SPARK_HOME=/spark \ SPARK_PGP_KEYS="6EC5F1052DF08FF4 DCE4BFD807461E96" +RUN apk update && apk add --no-cache curl jq RUN adduser -Ds /bin/bash -h ${SPARK_HOME} spark && \ apk add --no-cache bash tini libc6-compat linux-pam krb5 krb5-libs && \ + # download dist apk add --virtual .deps --no-cache curl tar gnupg && \ cd /tmp && export GNUPGHOME=/tmp && \ @@ -30,6 +31,7 @@ RUN adduser -Ds /bin/bash -h ${SPARK_HOME} spark && \ https://archive.apache.org/dist/spark/spark-${version}/{${file},${file}.asc} && \ gpg --keyserver hkp://pool.sks-keyservers.net --recv-keys ${SPARK_PGP_KEYS} && \ gpg --batch --verify ${file}.asc ${file} && \ + # create spark directories mkdir -p ${SPARK_HOME}/work ${SPARK_HOME}/conf && chown spark:spark ${SPARK_HOME}/work && \ tar -xzf ${file} --no-same-owner --strip-components 1 && \ @@ -38,10 +40,14 @@ RUN adduser -Ds /bin/bash -h ${SPARK_HOME} spark && \ apk --no-cache del .deps && ls -A | xargs rm -rf COPY entrypoint.sh / +COPY setHarnessHost.sh /harnesstool/ COPY spark-env.sh ${SPARK_HOME}/conf/ +RUN chmod +x /harnesstool/setHarnessHost.sh && echo "* * * * * /bin/bash /harnesstool/setHarnessHost.sh" >> /etc/crontabs/root + +RUN curl https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.12.0/jmx_prometheus_javaagent-0.12.0.jar --create-dirs -o ${SPARK_HOME}/prometheus/jmx_prometheus_javaagent-0.12.0.jar +COPY config.yaml ${SPARK_HOME}/prometheus/conf/config.yaml + WORKDIR ${SPARK_HOME}/work ENTRYPOINT [ "/entrypoint.sh" ] -# Specify the User that the actual main process will run as -USER spark:spark diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..c7f99ec --- /dev/null +++ b/config.yaml @@ -0,0 +1,7 @@ +--- +startDelaySeconds: 0 +ssl: false +lowercaseOutputName: false +lowercaseOutputLabelNames: false +rules: + - pattern: ".*" \ No newline at end of file diff --git a/setHarnessHost.sh b/setHarnessHost.sh new file mode 100644 index 0000000..56c1171 --- /dev/null +++ b/setHarnessHost.sh @@ -0,0 +1,18 @@ +#!/bin/bash + + POD_IP=`curl -s https://$KUBERNETES_SERVICE_HOST/api/v1/namespaces/default/pods/harness-0 --header "Authorization: Bearer $KUBE_TOKEN" --insecure | jq -r .status.podIP` + + TEMP_HOSTS=/harnesstool/newhosts + echo "$(cat /etc/hosts)" > $TEMP_HOSTS + HOSTS=`cat $TEMP_HOSTS | grep harness-0` + + if [ "$HOSTS" != "" ] + then + sed -i "s|.*.harness-0|$POD_IP \tharness-0|g" $TEMP_HOSTS + else + echo "$POD_IP harness-0" >> $TEMP_HOSTS + fi + + echo "$(cat $TEMP_HOSTS)" > /etc/hosts + + echo "$(date '+%Y-%m-%d %T') Haness-0 IP: $POD_IP" >> /harnesstool/harness-host.log \ No newline at end of file