improve building process (Blazemeter#1397)

* Split the main Dockerfile into two - one main Dockerfile for taurus image creation, and the second tests/ci/Dockerfile.build for an artifact building process and for the site deployment.
* Removed commands with sed and changed versioning logic.
* Changed an agent for Jenkins job to be build from tests/ci/Dockerfile.build.
* Simplified the Jenkinsfile stages to be performed on one agent and moved the artifacts creation stage to the top.
Us674965

.dockerignore

@@ -2,6 +2,5 @@
 .dockerignore
 Dockerfile
 tests
-dist
 site
 **/*.pyc

Dockerfile

@@ -6,7 +6,7 @@ WORKDIR /tmp
 ADD https://dl-ssl.google.com/linux/linux_signing_key.pub /tmp
 ADD https://deb.nodesource.com/setup_12.x /tmp
 RUN apt-get -y update \
-  && apt-get -y install dirmngr git \
+  && apt-get -y install dirmngr \
   && $APT_INSTALL software-properties-common apt-utils \
   && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF \
   && cat /tmp/linux_signing_key.pub | apt-key add - \
@@ -40,11 +40,10 @@ RUN mv /opt/google/chrome/google-chrome /opt/google/chrome/_google-chrome \
   && mv /tmp/chrome_launcher.sh /opt/google/chrome/google-chrome \
   && chmod +x /opt/google/chrome/google-chrome
 
-COPY . /tmp/bzt-src
+COPY dist /tmp/bzt-src
 WORKDIR /tmp/bzt-src
 RUN google-chrome-stable --version && firefox --version && mono --version && nuget | head -1 \
-  && ./build-sdist.sh \
-  && python3 -m pip install dist/bzt-*.tar.gz \
+  && python3 -m pip install bzt-*.tar.gz \
   && echo '{"install-id": "Docker"}' > /etc/bzt.d/99-zinstallID.json \
   && echo '{"settings": {"artifacts-dir": "/tmp/artifacts"}}' > /etc/bzt.d/90-artifacts-dir.json \
   && bzt -install-tools -v && ls -la /tmp && cat /tmp/jpgc-*.log && ls -la ~/.bzt/jmeter-taurus/*/lib/ext && ls -la ~/.bzt/jmeter-taurus/*/lib/ext/jmeter-plugins-tst-*.jar

Jenkinsfile

@@ -1,36 +1,52 @@
 @Library("jenkins_library") _
 
 pipeline {
-    agent any
+    agent {
+        dockerfile {
+            filename 'tests/ci/Dockerfile.build'
+            args '-u root -v /var/run/docker.sock:/var/run/docker.sock'
+        }
+    }
     options {
         timestamps()
-        skipDefaultCheckout()
     }
     stages {
         stage('Checkout') {
             steps {
-                cleanWs()
                 script {
-                    scmVars = checkout scm
-                    commitHash = scmVars.GIT_COMMIT
                     tagName = sh(returnStdout: true, script: "git tag --points-at HEAD").trim()
                     isRelease = !tagName.isEmpty()
                     IMAGE_TAG = env.JOB_NAME + "." + env.BUILD_NUMBER
                     IMAGE_TAG = IMAGE_TAG.toLowerCase()
                     imageName = "blazemeter/taurus"
                     extraImageTag = isRelease ? "${imageName}:${tagName} -t ${imageName}:latest" : "${imageName}:unstable"
-
+                    VERSION = sh(returnStdout: true, script: "git describe --tags \$(git rev-list --tags --max-count=1)").trim()
+                    GIT_INFO = sh(returnStdout: true, script: "echo \$(git rev-parse --abbrev-ref HEAD) \$(git show --oneline -s)").trim()
+                    if (!isRelease) {
+                        VERSION = "${VERSION}.${BUILD_NUMBER}"
+                    }
                 }
                 sh """
-                   echo "BUILD_NUM = \"${BUILD_NUMBER}\"" > bzt/resources/version/build.py
+                   echo 'BUILD_NUM=\"${BUILD_NUMBER}\"' > bzt/resources/version/build.py
+                   echo 'VERSION=\"${VERSION}\"' > bzt/resources/version/version.py
+                   echo 'GIT_INFO=\"${GIT_INFO}\"' > bzt/resources/version/gitinfo.py
                    """
             }
         }
+        stage("Create artifacts") {
+            steps {
+                script {
+                    sh "./build-artifacts.sh"
+                }
+                archiveArtifacts artifacts: 'dist/*.whl', fingerprint: true
+                archiveArtifacts artifacts: 'build/nsis/*_x64.exe', fingerprint: true
+            }
+        }
         stage("Docker Image Build") {
             steps {
-                sh """
-                   docker build -t ${JOB_NAME} -t ${extraImageTag} .
-                   """
+                script {
+                    sh "docker build --no-cache -t ${JOB_NAME} -t ${extraImageTag} ."
+                }
             }
         }
         stage("Integration Tests") {
@@ -47,55 +63,26 @@ pipeline {
                 }
             }
         }
-        stage("Create Artifacts") {
-            steps {
-                script {
-                    sh """
-                       sed -ri "s/OS: /Rev: ${commitHash}; OS: /" bzt/cli.py
-                       """
-
-                    if (!isRelease) {
-                        sh """
-                           sed -ri "s/VERSION = .([^\\"]+)./VERSION = '\\1.${BUILD_NUMBER}'/" bzt/__init__.py
-                           """
-                    }
-
-                    sh """
-                       docker run --entrypoint /bzt-configs/build-artifacts.sh -v `pwd`:/bzt-configs ${JOB_NAME} ${BUILD_NUMBER}
-                       """
-                }
-                archiveArtifacts artifacts: 'dist/*.whl', fingerprint: true
-            }
-        }
         stage("Deploy site") {
             steps {
-                sh """
-                   docker build -t deploy-image -f site/Dockerfile.deploy .
-                   """
                 script {
-                    PROJECT_ID="blazemeter-taurus-website-prod"
-                    withCredentials([file(credentialsId: "${PROJECT_ID}", variable: 'CRED_JSON')]) {
-                        def WORKSPACE_JSON = 'Google_credentials.json'
-                        def input = readJSON file: CRED_JSON
-                        writeJSON file: WORKSPACE_JSON, json: input
+                    PROJECT_ID = "blazemeter-taurus-website-prod"
+                    withCredentials([file(credentialsId: PROJECT_ID, variable: 'CRED_JSON')]) {
                         sh """
-                           docker run --entrypoint /bzt/site/deploy-site.sh \
-                           -e KEY_FILE=${WORKSPACE_JSON} \
-                           -e PROJECT_ID=${PROJECT_ID} \
-                           -e BUILD_NUMBER=${BUILD_NUMBER} \
-                           -u root \
-                           -v /var/run/docker.sock:/var/run/docker.sock \
-                           -v `pwd`:/bzt -t deploy-image \
-                           ${isRelease}
-                          """
+                           gcloud auth activate-service-account --key-file ${CRED_JSON}
+                           gcloud config set project ${PROJECT_ID}
+                           gcloud config set compute/zone us-central1-a
+                           """
                     }
+                    sh "./site/deploy-site.sh ${isRelease}"
                 }
             }
         }
     }
     post {
         always {
             smartSlackNotification(channel: "taurus-dev", buildStatus:currentBuild.result ?: 'SUCCESS')
+            cleanWs()
         }
     }
 }

build-artifacts.sh

@@ -1,12 +1,24 @@
 #!/bin/bash -xe
-BUILD_NUMBER=$1
 
-apt-get update -y
-apt-get install -y --force-yes gcc-mingw-w64-x86-64 nsis
-apt-get install -y libssl-dev libncurses5-dev libsqlite3-dev libreadline-dev libtk8.5 libgdm-dev libdb4o-cil-dev libpcap-dev
+PREV_DIR=`pwd`
+cd "$(dirname $0)"
 
-# build source distribution
-./build-sdist.sh
+echo "Cleaning environment"
+python3 setup.py clean
+
+echo "Building NUnit plugin"
+pushd ./dotnet/NUnitRunner
+./rebuild.sh
+popd
+
+echo "Building chrome-loader.exe"
+rm -f bzt/resources/chrome-loader.exe
+x86_64-w64-mingw32-gcc -std=c99 -o bzt/resources/chrome-loader.exe bzt/resources/chrome-loader.c
+
+echo "Creating distribution packages"
+python3 ./setup.py sdist bdist_wheel
+
+cd "${PREV_DIR}"
 
 # build a windows installer
 pip3 install virtualenv

bzt/__init__.py

@@ -19,7 +19,7 @@
 from abc import abstractmethod
 
 
-VERSION = '1.15.0'
+from bzt.resources.version import VERSION
 
 
 class RCProvider(object):

scripts/installer/gen_installer.py

@@ -17,7 +17,7 @@ def run_cmd(label, cmdline, **kwargs):
 
 
 def extract_bzt_version(bzt_dist):
-    matches = re.findall(r'(\d+\.[\d.]+)', bzt_dist)
+    matches = re.findall(r'(\d+\.\d+\.\d+(?:\.\d+)?)', bzt_dist)
     if not matches:
         raise ValueError("Can't extract version from string %r" % bzt_dist)
     version = matches[0]

setup.py

@@ -16,7 +16,7 @@
 import pkg_resources
 from setuptools import setup
 
-from bzt import VERSION
+from bzt.resources.version import VERSION
 
 with open('requirements.txt') as _f:
     content = _f.read()

site/Taurus/kwindexer.py

@@ -33,7 +33,7 @@ def get_keys(struct, ignore_first_level=False):
 
 
 def index_file(fname):
-    with open(fname) as fhd:
+    with open(fname, encoding='utf8') as fhd:
         content = fhd.read()
     blocks_re = re.compile(r'```yaml([^`]+)```')
     blocks = blocks_re.findall(content)

site/deploy-site.sh

@@ -1,12 +1,8 @@
 #!/bin/bash -xe
 
-gcloud auth activate-service-account --key-file ${KEY_FILE}
-gcloud config set project ${PROJECT_ID}
-gcloud config set compute/zone us-central1-a
-
 GOOGLE_STORAGE="https:\/\/storage.cloud.google.com\/taurus-site\/"
 UNSTABLE_SNAPSHOT=""
-TAURUS_VERSION=$(python -c 'import bzt; print(bzt.VERSION)')
+TAURUS_VERSION=$(python3 -c 'from bzt.resources.version import VERSION; print(VERSION)')
 
 mkdir site/builds
 PREFIX="\/builds\/"
@@ -38,7 +34,7 @@ RELEASE_SNAPSHOT="${PREFIX}${STABLE_EXE}"
 sed -ri "s/RELEASE_SNAPSHOT/${RELEASE_SNAPSHOT}/" site/dat/docs/Installation.md
 sed -ri "s/UNSTABLE_SNAPSHOT/${UNSTABLE_SNAPSHOT}/" site/dat/docs/Installation.md
 
-python site/Taurus/kwindexer.py site/dat/docs site/dat/docs/KeywordIndex.md
+python3 site/Taurus/kwindexer.py site/dat/docs site/dat/docs/KeywordIndex.md
 
 gsutil cp gs://taurus-site/learn.zip learn.zip
 unzip -o learn.zip -d site
@@ -48,7 +44,7 @@ docker build -t taurus-site.${BUILD_NUMBER} site
 
 gcloud auth --quiet configure-docker
 
-docker tag  taurus-site.${BUILD_NUMBER} gcr.io/${PROJECT_ID}/taurus-site.${BUILD_NUMBER}
+docker tag taurus-site.${BUILD_NUMBER} gcr.io/${PROJECT_ID}/taurus-site.${BUILD_NUMBER}
 docker push gcr.io/${PROJECT_ID}/taurus-site.${BUILD_NUMBER}
 gcloud container clusters get-credentials taurus-site
 

tests/ci/Dockerfile.build

@@ -0,0 +1,48 @@
+FROM ubuntu:18.04
+
+ENV DBUS_SESSION_BUS_ADDRESS=/dev/null DEBIAN_FRONTEND=noninteractive APT_INSTALL="apt-get -y install --no-install-recommends"
+
+ADD https://dl-ssl.google.com/linux/linux_signing_key.pub /tmp
+RUN apt-get -y update \
+  && apt-get -y install dirmngr git \
+  && $APT_INSTALL software-properties-common apt-utils \
+  && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF \
+  && cat /tmp/linux_signing_key.pub | apt-key add - \
+  && apt-add-repository multiverse -y \
+  && echo "deb http://download.mono-project.com/repo/ubuntu bionic main" | tee /etc/apt/sources.list.d/mono-official.list \
+  && apt-add-repository ppa:yandex-load/main -y \
+  && apt-add-repository ppa:nilarimogard/webupd8 -y \
+  && $APT_INSTALL tzdata \
+  && dpkg-reconfigure --frontend noninteractive tzdata \
+  && $APT_INSTALL \
+    language-pack-en mc kmod unzip build-essential \
+    libxslt1-dev libffi-dev libxi6 libgconf-2-4 libexif12 libyaml-dev \
+    udev openjdk-8-jdk xvfb siege tsung apache2-utils phantom phantom-ssl \
+    pepperflashplugin-nonfree flashplugin-installer \
+    mono-complete nuget net-tools gcc-mingw-w64-x86-64 \
+  && $APT_INSTALL python3-dev python3-pip \
+  && python3 -m pip install --upgrade pip \
+  && python3 -m pip install --user --upgrade setuptools wheel \
+  && nuget update -self \
+  && apt-get clean
+
+RUN apt-get update && \
+    apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common && \
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
+    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \
+    apt-get update && \
+    apt-get install -y docker-ce docker-ce-cli containerd.io
+
+RUN apt-get update -y \
+    && apt-get install -y \
+    gcc-mingw-w64-x86-64 nsis libssl-dev libncurses5-dev libsqlite3-dev \
+    libreadline-dev libtk8.5 libgdm-dev libdb4o-cil-dev libpcap-dev curl \
+    && apt-get clean
+
+RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
+    echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
+    echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list && \
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
+    apt-get update && \
+    apt-get install -y google-cloud-sdk kubectl && \
+    python3 -m pip install pyyaml