From cbc881a09d511a931693ebc46de51e7d5f512d8a Mon Sep 17 00:00:00 2001 From: aleksandrOranskiy Date: Wed, 7 Oct 2020 13:22:19 +0300 Subject: [PATCH] improve building process (#1397) * Split the main Dockerfile into two - one main Dockerfile for taurus image creation, and the second tests/ci/Dockerfile.build for an artifact building process and for the site deployment. * Removed commands with sed and changed versioning logic. * Changed an agent for Jenkins job to be build from tests/ci/Dockerfile.build. * Simplified the Jenkinsfile stages to be performed on one agent and moved the artifacts creation stage to the top. Us674965 --- .dockerignore | 1 - Dockerfile | 7 ++- Jenkinsfile | 81 +++++++++++++----------------- build-artifacts.sh | 24 ++++++--- build-sdist.sh | 27 ---------- bzt/__init__.py | 2 +- scripts/installer/gen_installer.py | 2 +- setup.py | 2 +- site/Dockerfile.deploy | 15 ------ site/Taurus/kwindexer.py | 2 +- site/deploy-site.sh | 10 ++-- tests/ci/Dockerfile.build | 48 ++++++++++++++++++ 12 files changed, 110 insertions(+), 111 deletions(-) delete mode 100755 build-sdist.sh delete mode 100644 site/Dockerfile.deploy create mode 100644 tests/ci/Dockerfile.build diff --git a/.dockerignore b/.dockerignore index 84a41ebf67..cd59a26c45 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,5 @@ .dockerignore Dockerfile tests -dist site **/*.pyc \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 004b586c63..7080f1ee7a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ WORKDIR /tmp ADD https://dl-ssl.google.com/linux/linux_signing_key.pub /tmp ADD https://deb.nodesource.com/setup_12.x /tmp RUN apt-get -y update \ - && apt-get -y install dirmngr git \ + && apt-get -y install dirmngr \ && $APT_INSTALL software-properties-common apt-utils \ && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF \ && cat /tmp/linux_signing_key.pub | apt-key add - \ @@ -40,11 +40,10 @@ RUN mv /opt/google/chrome/google-chrome /opt/google/chrome/_google-chrome \ && mv /tmp/chrome_launcher.sh /opt/google/chrome/google-chrome \ && chmod +x /opt/google/chrome/google-chrome -COPY . /tmp/bzt-src +COPY dist /tmp/bzt-src WORKDIR /tmp/bzt-src RUN google-chrome-stable --version && firefox --version && mono --version && nuget | head -1 \ - && ./build-sdist.sh \ - && python3 -m pip install dist/bzt-*.tar.gz \ + && python3 -m pip install bzt-*.tar.gz \ && echo '{"install-id": "Docker"}' > /etc/bzt.d/99-zinstallID.json \ && echo '{"settings": {"artifacts-dir": "/tmp/artifacts"}}' > /etc/bzt.d/90-artifacts-dir.json \ && bzt -install-tools -v && ls -la /tmp && cat /tmp/jpgc-*.log && ls -la ~/.bzt/jmeter-taurus/*/lib/ext && ls -la ~/.bzt/jmeter-taurus/*/lib/ext/jmeter-plugins-tst-*.jar diff --git a/Jenkinsfile b/Jenkinsfile index 8542f320f8..8f8f6547bc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,36 +1,52 @@ @Library("jenkins_library") _ pipeline { - agent any + agent { + dockerfile { + filename 'tests/ci/Dockerfile.build' + args '-u root -v /var/run/docker.sock:/var/run/docker.sock' + } + } options { timestamps() - skipDefaultCheckout() } stages { stage('Checkout') { steps { - cleanWs() script { - scmVars = checkout scm - commitHash = scmVars.GIT_COMMIT tagName = sh(returnStdout: true, script: "git tag --points-at HEAD").trim() isRelease = !tagName.isEmpty() IMAGE_TAG = env.JOB_NAME + "." + env.BUILD_NUMBER IMAGE_TAG = IMAGE_TAG.toLowerCase() imageName = "blazemeter/taurus" extraImageTag = isRelease ? "${imageName}:${tagName} -t ${imageName}:latest" : "${imageName}:unstable" - + VERSION = sh(returnStdout: true, script: "git describe --tags \$(git rev-list --tags --max-count=1)").trim() + GIT_INFO = sh(returnStdout: true, script: "echo \$(git rev-parse --abbrev-ref HEAD) \$(git show --oneline -s)").trim() + if (!isRelease) { + VERSION = "${VERSION}.${BUILD_NUMBER}" + } } sh """ - echo "BUILD_NUM = \"${BUILD_NUMBER}\"" > bzt/resources/version/build.py + echo 'BUILD_NUM=\"${BUILD_NUMBER}\"' > bzt/resources/version/build.py + echo 'VERSION=\"${VERSION}\"' > bzt/resources/version/version.py + echo 'GIT_INFO=\"${GIT_INFO}\"' > bzt/resources/version/gitinfo.py """ } } + stage("Create artifacts") { + steps { + script { + sh "./build-artifacts.sh" + } + archiveArtifacts artifacts: 'dist/*.whl', fingerprint: true + archiveArtifacts artifacts: 'build/nsis/*_x64.exe', fingerprint: true + } + } stage("Docker Image Build") { steps { - sh """ - docker build -t ${JOB_NAME} -t ${extraImageTag} . - """ + script { + sh "docker build --no-cache -t ${JOB_NAME} -t ${extraImageTag} ." + } } } stage("Integration Tests") { @@ -47,48 +63,18 @@ pipeline { } } } - stage("Create Artifacts") { - steps { - script { - sh """ - sed -ri "s/OS: /Rev: ${commitHash}; OS: /" bzt/cli.py - """ - - if (!isRelease) { - sh """ - sed -ri "s/VERSION = .([^\\"]+)./VERSION = '\\1.${BUILD_NUMBER}'/" bzt/__init__.py - """ - } - - sh """ - docker run --entrypoint /bzt-configs/build-artifacts.sh -v `pwd`:/bzt-configs ${JOB_NAME} ${BUILD_NUMBER} - """ - } - archiveArtifacts artifacts: 'dist/*.whl', fingerprint: true - } - } stage("Deploy site") { steps { - sh """ - docker build -t deploy-image -f site/Dockerfile.deploy . - """ script { - PROJECT_ID="blazemeter-taurus-website-prod" - withCredentials([file(credentialsId: "${PROJECT_ID}", variable: 'CRED_JSON')]) { - def WORKSPACE_JSON = 'Google_credentials.json' - def input = readJSON file: CRED_JSON - writeJSON file: WORKSPACE_JSON, json: input + PROJECT_ID = "blazemeter-taurus-website-prod" + withCredentials([file(credentialsId: PROJECT_ID, variable: 'CRED_JSON')]) { sh """ - docker run --entrypoint /bzt/site/deploy-site.sh \ - -e KEY_FILE=${WORKSPACE_JSON} \ - -e PROJECT_ID=${PROJECT_ID} \ - -e BUILD_NUMBER=${BUILD_NUMBER} \ - -u root \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v `pwd`:/bzt -t deploy-image \ - ${isRelease} - """ + gcloud auth activate-service-account --key-file ${CRED_JSON} + gcloud config set project ${PROJECT_ID} + gcloud config set compute/zone us-central1-a + """ } + sh "./site/deploy-site.sh ${isRelease}" } } } @@ -96,6 +82,7 @@ pipeline { post { always { smartSlackNotification(channel: "taurus-dev", buildStatus:currentBuild.result ?: 'SUCCESS') + cleanWs() } } } diff --git a/build-artifacts.sh b/build-artifacts.sh index 10d0c8b7ab..7beaf0c9ff 100755 --- a/build-artifacts.sh +++ b/build-artifacts.sh @@ -1,12 +1,24 @@ #!/bin/bash -xe -BUILD_NUMBER=$1 -apt-get update -y -apt-get install -y --force-yes gcc-mingw-w64-x86-64 nsis -apt-get install -y libssl-dev libncurses5-dev libsqlite3-dev libreadline-dev libtk8.5 libgdm-dev libdb4o-cil-dev libpcap-dev +PREV_DIR=`pwd` +cd "$(dirname $0)" -# build source distribution -./build-sdist.sh +echo "Cleaning environment" +python3 setup.py clean + +echo "Building NUnit plugin" +pushd ./dotnet/NUnitRunner +./rebuild.sh +popd + +echo "Building chrome-loader.exe" +rm -f bzt/resources/chrome-loader.exe +x86_64-w64-mingw32-gcc -std=c99 -o bzt/resources/chrome-loader.exe bzt/resources/chrome-loader.c + +echo "Creating distribution packages" +python3 ./setup.py sdist bdist_wheel + +cd "${PREV_DIR}" # build a windows installer pip3 install virtualenv diff --git a/build-sdist.sh b/build-sdist.sh deleted file mode 100755 index af8977b2ad..0000000000 --- a/build-sdist.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -xe - -PREV_DIR=`pwd` -cd "$(dirname $0)" - -echo "Getting build info" -VERSION=$(git describe --tags $(git rev-list --tags --max-count=1)) -GIT_INFO="$(git rev-parse --abbrev-ref HEAD) $(git show --oneline -s)" -echo "VERSION=\"$VERSION\"" > bzt/resources/version/version.py -echo "GIT_INFO=\"$GIT_INFO\"" > bzt/resources/version/gitinfo.py - -echo "Cleaning environment" -python3 setup.py clean - -echo "Building NUnit plugin" -pushd ./dotnet/NUnitRunner -./rebuild.sh -popd - -echo "Building chrome-loader.exe" -rm -f bzt/resources/chrome-loader.exe -x86_64-w64-mingw32-gcc -std=c99 -o bzt/resources/chrome-loader.exe bzt/resources/chrome-loader.c - -echo "Creating distribution packages" -python3 ./setup.py sdist bdist_wheel - -cd "${PREV_DIR}" \ No newline at end of file diff --git a/bzt/__init__.py b/bzt/__init__.py index 8c06dc4d3c..7cf700000d 100644 --- a/bzt/__init__.py +++ b/bzt/__init__.py @@ -19,7 +19,7 @@ from abc import abstractmethod -VERSION = '1.15.0' +from bzt.resources.version import VERSION class RCProvider(object): diff --git a/scripts/installer/gen_installer.py b/scripts/installer/gen_installer.py index d2d9e2daca..7e9791d806 100644 --- a/scripts/installer/gen_installer.py +++ b/scripts/installer/gen_installer.py @@ -17,7 +17,7 @@ def run_cmd(label, cmdline, **kwargs): def extract_bzt_version(bzt_dist): - matches = re.findall(r'(\d+\.[\d.]+)', bzt_dist) + matches = re.findall(r'(\d+\.\d+\.\d+(?:\.\d+)?)', bzt_dist) if not matches: raise ValueError("Can't extract version from string %r" % bzt_dist) version = matches[0] diff --git a/setup.py b/setup.py index 869a797c6e..05c0b884da 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ import pkg_resources from setuptools import setup -from bzt import VERSION +from bzt.resources.version import VERSION with open('requirements.txt') as _f: content = _f.read() diff --git a/site/Dockerfile.deploy b/site/Dockerfile.deploy deleted file mode 100644 index 3df9051a26..0000000000 --- a/site/Dockerfile.deploy +++ /dev/null @@ -1,15 +0,0 @@ -FROM docker:18.09 - -RUN apk add --update python py-pip curl which bash zip -RUN pip install pyyaml -RUN curl -sSL https://sdk.cloud.google.com | bash - -ENV PATH $PATH:/root/google-cloud-sdk/bin - -RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl -RUN chmod +x ./kubectl && mv ./kubectl /usr/local/bin/kubectl - -RUN mkdir /bzt -WORKDIR /bzt - -ENTRYPOINT ["sh", "-c"] \ No newline at end of file diff --git a/site/Taurus/kwindexer.py b/site/Taurus/kwindexer.py index 05a63d6205..20a439bac1 100644 --- a/site/Taurus/kwindexer.py +++ b/site/Taurus/kwindexer.py @@ -33,7 +33,7 @@ def get_keys(struct, ignore_first_level=False): def index_file(fname): - with open(fname) as fhd: + with open(fname, encoding='utf8') as fhd: content = fhd.read() blocks_re = re.compile(r'```yaml([^`]+)```') blocks = blocks_re.findall(content) diff --git a/site/deploy-site.sh b/site/deploy-site.sh index 33023f0ecd..b62b7fa935 100755 --- a/site/deploy-site.sh +++ b/site/deploy-site.sh @@ -1,12 +1,8 @@ #!/bin/bash -xe -gcloud auth activate-service-account --key-file ${KEY_FILE} -gcloud config set project ${PROJECT_ID} -gcloud config set compute/zone us-central1-a - GOOGLE_STORAGE="https:\/\/storage.cloud.google.com\/taurus-site\/" UNSTABLE_SNAPSHOT="" -TAURUS_VERSION=$(python -c 'import bzt; print(bzt.VERSION)') +TAURUS_VERSION=$(python3 -c 'from bzt.resources.version import VERSION; print(VERSION)') mkdir site/builds PREFIX="\/builds\/" @@ -38,7 +34,7 @@ RELEASE_SNAPSHOT="${PREFIX}${STABLE_EXE}" sed -ri "s/RELEASE_SNAPSHOT/${RELEASE_SNAPSHOT}/" site/dat/docs/Installation.md sed -ri "s/UNSTABLE_SNAPSHOT/${UNSTABLE_SNAPSHOT}/" site/dat/docs/Installation.md -python site/Taurus/kwindexer.py site/dat/docs site/dat/docs/KeywordIndex.md +python3 site/Taurus/kwindexer.py site/dat/docs site/dat/docs/KeywordIndex.md gsutil cp gs://taurus-site/learn.zip learn.zip unzip -o learn.zip -d site @@ -48,7 +44,7 @@ docker build -t taurus-site.${BUILD_NUMBER} site gcloud auth --quiet configure-docker -docker tag taurus-site.${BUILD_NUMBER} gcr.io/${PROJECT_ID}/taurus-site.${BUILD_NUMBER} +docker tag taurus-site.${BUILD_NUMBER} gcr.io/${PROJECT_ID}/taurus-site.${BUILD_NUMBER} docker push gcr.io/${PROJECT_ID}/taurus-site.${BUILD_NUMBER} gcloud container clusters get-credentials taurus-site diff --git a/tests/ci/Dockerfile.build b/tests/ci/Dockerfile.build new file mode 100644 index 0000000000..93746749da --- /dev/null +++ b/tests/ci/Dockerfile.build @@ -0,0 +1,48 @@ +FROM ubuntu:18.04 + +ENV DBUS_SESSION_BUS_ADDRESS=/dev/null DEBIAN_FRONTEND=noninteractive APT_INSTALL="apt-get -y install --no-install-recommends" + +ADD https://dl-ssl.google.com/linux/linux_signing_key.pub /tmp +RUN apt-get -y update \ + && apt-get -y install dirmngr git \ + && $APT_INSTALL software-properties-common apt-utils \ + && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF \ + && cat /tmp/linux_signing_key.pub | apt-key add - \ + && apt-add-repository multiverse -y \ + && echo "deb http://download.mono-project.com/repo/ubuntu bionic main" | tee /etc/apt/sources.list.d/mono-official.list \ + && apt-add-repository ppa:yandex-load/main -y \ + && apt-add-repository ppa:nilarimogard/webupd8 -y \ + && $APT_INSTALL tzdata \ + && dpkg-reconfigure --frontend noninteractive tzdata \ + && $APT_INSTALL \ + language-pack-en mc kmod unzip build-essential \ + libxslt1-dev libffi-dev libxi6 libgconf-2-4 libexif12 libyaml-dev \ + udev openjdk-8-jdk xvfb siege tsung apache2-utils phantom phantom-ssl \ + pepperflashplugin-nonfree flashplugin-installer \ + mono-complete nuget net-tools gcc-mingw-w64-x86-64 \ + && $APT_INSTALL python3-dev python3-pip \ + && python3 -m pip install --upgrade pip \ + && python3 -m pip install --user --upgrade setuptools wheel \ + && nuget update -self \ + && apt-get clean + +RUN apt-get update && \ + apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + apt-get update && \ + apt-get install -y docker-ce docker-ce-cli containerd.io + +RUN apt-get update -y \ + && apt-get install -y \ + gcc-mingw-w64-x86-64 nsis libssl-dev libncurses5-dev libsqlite3-dev \ + libreadline-dev libtk8.5 libgdm-dev libdb4o-cil-dev libpcap-dev curl \ + && apt-get clean + +RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ + echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get update && \ + apt-get install -y google-cloud-sdk kubectl && \ + python3 -m pip install pyyaml