Add template job and script to generate new build jobs

The commit adds a new template job and detemplate.sh script that will
use the template and will help new users generate a new build job for an
upstream project.

Signed-off-by: Boris Ranto <[email protected]>

detemplate.sh

@@ -0,0 +1,30 @@
+#! /usr/bin/env bash
+
+set -e
+
+echo "Enter project name"
+read PROJECT
+echo "Enter github name as in github.com/ceph/<github_name>"
+read GITHUB
+
+echo "You entered:"
+echo "PROJECT=$PROJECT"
+echo "GITHUB=$GITHUB"
+
+if test -e ${PROJECT}; then
+	echo "The current directory already contains ${PROJECT} directory, please remove it first."
+	exit 1
+fi
+
+## Create the initial directory structure
+cp -a template ${PROJECT}
+
+## Fix up the filenames
+mv ${PROJECT}/config/definitions/template.yml ${PROJECT}/config/definitions/${PROJECT}.yml
+
+find ${PROJECT} -type f -exec sed -i -e "s/PROJECT/$PROJECT/g" {} \;
+find ${PROJECT} -type f -exec sed -i -e "s/GITHUB/$GITHUB/g" {} \;
+
+echo "The following job was created:" ${PROJECT}
+echo "Please follow all the TODO markers to complete the creation of the build job."
+exit 0

template/build/build_deb

@@ -0,0 +1,86 @@
+#! /usr/bin/bash
+set -ex
+
+# Only do actual work when we are a DEB distro
+if test "$DISTRO" != "debian" -a "$DISTRO" != "ubuntu"; then
+    exit 0
+fi
+
+
+## Install any setup-time deps
+# TODO -- upload install-deps.sh script that installs the setup time deps
+# to upstream repo or update this section to install these packages
+
+# We need this for mk-build-deps
+sudo apt-get install -y equivs
+
+# Run the install-deps.sh upstream script if it exists
+if [ -x install-deps.sh ]; then
+    echo "Ensuring dependencies are installed"
+    sudo ./install-deps.sh
+fi
+
+
+## Setup the pbuilder
+# TODO remove if you do not want to use pbuilders
+setup_pbuilder
+
+
+## Get some basic information about the system and the repository
+# TODO -- update this to get the proper VERSION/REVISION
+DEB_ARCH=$(dpkg-architecture -qDEB_BUILD_ARCH)
+VERSION="$(./get-version.sh)"
+REVISION="$(./get-revision.sh)"
+
+
+## Build the source tarball
+# TODO -- upload make-dist script that stores the tarball in dist/ to upstream repo or update this section
+echo "Building source distribution"
+if [ -x make-dist ]; then
+    echo "Ensuring dependencies are installed"
+    ./make-dist
+fi
+
+
+## Prepare the debian files
+# TODO -- Make sure the debian folder is tracked upstream
+# Bump the changelog
+dch -v "$VERSION" "New release ($VERSION)"
+
+# Install debian build-time dependencies
+yes | sudo mk-build-deps --install debian/control
+
+# Create .dsc and source tarball
+sudo dpkg-buildpackage -S -us -uc
+
+
+## Build with pbuilder
+echo "Building debs"
+
+PBUILDDIR="/srv/debian-base"
+
+sudo pbuilder --clean
+
+mkdir -p dist/deb
+
+echo "Building debs for $DIST"
+sudo pbuilder build \
+    --distribution $DIST \
+    --basetgz $PBUILDDIR/$DIST.tgz \
+    --buildresult dist/deb/ \
+    --debbuildopts "-j`grep -c processor /proc/cpuinfo`" \
+    dist/PROJECT_$VERSION.dsc
+
+
+## Upload the created RPMs to chacra
+chacra_endpoint="PROJECT/${BRANCH}/${GIT_COMMIT}/${DISTRO}/${DIST}"
+
+[ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags=""
+
+# push binaries to chacra
+find ../*.deb | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/${DEB_ARCH}/
+
+# start repo creation
+$VENV/chacractl repo update ${chacra_endpoint}
+
+echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint}

template/build/build_rpm

@@ -0,0 +1,83 @@
+#! /usr/bin/bash
+set -ex
+
+# Only do actual work when we are an RPM distro
+if test "$DISTRO" != "fedora" -a "$DISTRO" != "centos" -a "$DISTRO" != "rhel"; then
+    exit 0
+fi
+
+
+## Install any setup-time deps (to make dist package)
+# TODO -- upload install-deps.sh script that installs the setup time deps
+# to upstream repo or update this section to install these packages
+
+# We need this to get the major version from lsb_release
+yum install -y redhat-lsb-core mock
+
+# Run the install-deps.sh upstream script if it exists
+if [ -x install-deps.sh ]; then
+    echo "Ensuring dependencies are installed"
+    sudo ./install-deps.sh
+fi
+
+
+## Get some basic information about the system and the repository
+# TODO -- update this to get the proper VERSION/REVISION
+RELEASE="$(lsb_release --short -r | cut -d '.' -f 1)" # sytem release
+VERSION="$(./get-version.sh)"
+REVISION="$(./get-revision.sh)"
+RPM_RELEASE=$(echo $REVISION | tr '-' '_') # the '-' has a special meaning
+
+
+## Build the source tarball
+# TODO -- upload make-dist script that stores the tarball in dist/ to upstream repo or update this section
+echo "Building source distribution"
+if [ -x make-dist ]; then
+    echo "Ensuring dependencies are installed"
+    ./make-dist
+fi
+
+
+## Prepare the spec file for build
+# TODO -- Make sure the spec(.in) file is tracked upstream
+sed -e "s/@VERSION@/${VERSION}/g" -e "s/@RELEASE@/${RPM_RELEASE}/g" < PROJECT.spec.in > dist/PROJECT.spec
+
+
+## Install the build-time dependencies (for rpmbuild -bs)
+sudo yum-builddep -y dist/PROJECT.spec
+
+
+## Create the source rpm
+# TODO -- update the paths to match the location of source package (dist/ by
+# default) and the spec file (PWD by default)
+echo "Building SRPM"
+rpmbuild \
+    --define "_sourcedir ./dist" \
+    --define "_specdir ." \
+    --define "_builddir ." \
+    --define "_srcrpmdir ." \
+    --define "_rpmdir ." \
+    --define "dist .any" \
+    --define "fedora 21" \
+    --define "rhel 7" \
+    --nodeps -bs dist/PROJECT.spec
+SRPM=$(readlink -f *.src.rpm)
+
+
+## Build the binaries with mock
+echo "Building RPMs"
+sudo mock -r ${MOCK_TARGET}-${RELEASE}-${ARCH} --resultdir=./dist/rpm/ ${SRPM}
+
+
+## Upload the created RPMs to chacra
+chacra_endpoint="PROJECT/${BRANCH}/${GIT_COMMIT}/${DISTRO}/${RELEASE}"
+
+[ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags=""
+
+# push binaries to chacra
+find ./dist/rpm/ | egrep '\.rpm$' | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/$ARCH/
+
+# start repo creation
+$VENV/chacractl repo update ${chacra_endpoint}
+
+echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint}

template/build/setup

@@ -0,0 +1,44 @@
+#! /usr/bin/bash
+#
+# Ceph distributed storage system
+#
+# Copyright (C) 2016 Red Hat <[email protected]>
+#
+# Author: Boris Ranto <[email protected]>
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2.1 of the License, or (at your option) any later version.
+#
+set -ex
+
+# Make sure we execute at the top level directory before we do anything
+cd $WORKSPACE
+
+# This will set the DISTRO and MOCK_TARGET variables
+get_distro_and_target
+
+# Perform a clean-up
+git clean -fxd
+
+# Make sure the dist directory is clean
+rm -rf dist
+mkdir -p dist
+
+# Print some basic system info
+HOST=$(hostname --short)
+echo "Building on $(hostname) with the following env"
+echo "*****"
+env
+echo "*****"
+
+export LC_ALL=C # the following is vulnerable to i18n
+
+pkgs=( "chacractl>=0.0.4" )
+install_python_packages "pkgs[@]"
+
+# ask shaman which chacra instance to use
+chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/`
+# create the .chacractl config file using global variables
+make_chacractl_config $chacra_url

template/config/definitions/template.yml

@@ -0,0 +1,107 @@
+- job:
+    name: PROJECT
+    project-type: matrix
+    defaults: global
+    display-name: 'PROJECT'
+    block-downstream: false
+    block-upstream: false
+    concurrent: true
+    parameters:
+      - string:
+          name: BRANCH
+          description: "The git branch (or tag) to build"
+
+      - string:
+          name: DISTROS
+          description: "A list of distros to build for. Available options are: xenial, centos7, centos6, trusty, precise, wheezy, and jessie"
+          default: "centos7 precise xenial"
+
+      - string:
+          name: ARCHS
+          description: "A list of architectures to build for. Available options are: x86_64, and arm64"
+          default: "x86_64"
+
+      - bool:
+          name: FORCE
+          description: "
+If this is unchecked, then nothing is built or pushed if they already exist in chacra. This is the default.
+
+If this is checked, then the binaries will be built and pushed to chacra even if they already exist in chacra."
+
+      - string:
+          name: BUILD_VIRTUALENV
+          description: "Base parent path for virtualenv locations, set to avoid issues with extremely long paths that are incompatible with tools like pip. Defaults to '/tmp/' (note the trailing slash, which is required)."
+          default: "/tmp/"
+
+    execution-strategy:
+       combination-filter: DIST==AVAILABLE_DIST && ARCH==AVAILABLE_ARCH && (ARCH=="x86_64" || (ARCH == "arm64" && (DIST == "xenial" || DIST == "centos7")))
+    axes:
+      - axis:
+          type: label-expression
+          name: MACHINE_SIZE
+          values:
+            - huge
+      - axis:
+          type: label-expression
+          name: AVAILABLE_ARCH
+          values:
+            - x86_64
+            - arm64
+      - axis:
+          type: label-expression
+          name: AVAILABLE_DIST
+          values:
+            - centos6
+            - centos7
+            - trusty
+            - xenial
+            - jessie
+            - precise
+            - wheezy
+      - axis:
+          type: dynamic
+          name: DIST
+      - axis:
+          type: dynamic
+          name: DIST
+          values:
+            - DISTROS
+      - axis:
+          type: dynamic
+          name: ARCH
+          values:
+            - ARCHS
+
+    scm:
+      - git:
+          url: [email protected]:ceph/GITHUB.git
+          # Use the SSH key attached to the ceph-jenkins GitHub account.
+          credentials-id: '39fa150b-b2a1-416e-b334-29a9a2c0b32d'
+          branches:
+            - $BRANCH
+          skip-tag: true
+          wipe-workspace: true
+
+    builders:
+      - shell: |
+          echo "Cleaning up top-level workarea (shared among workspaces)"
+          rm -rf dist
+          rm -rf venv
+          rm -rf release
+      # debian build scripts
+      - shell:
+          !include-raw:
+            - ../../../scripts/build_utils.sh
+            - ../../build/setup
+            - ../../build/build_deb
+      # rpm build scripts
+      - shell:
+          !include-raw:
+            - ../../../scripts/build_utils.sh
+            - ../../build/setup
+            - ../../build/build_rpm
+
+    wrappers:
+      - inject-passwords:
+          global: true
+          mask-password-params: true