added script and templates to generate devel jobs

scripts/create_devel_jobs.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import argparse
+import shutil
+import sys
+import tempfile
+import urllib2
+import yaml
+
+from buildfarm import devel_jobs, jenkins_support
+from buildfarm.stack_of_remote_repository import get_stack_of_remote_repository
+
+#import pprint # for debugging only, remove
+
+URL_PROTOTYPE = 'https://raw.github.com/ros/rosdistro/master/releases/%s.yaml'
+
+
+def parse_options():
+    parser = argparse.ArgumentParser(
+             description='Create a set of jenkins jobs '
+             'for continuous integration for a catkin package.')
+    parser.add_argument('--fqdn', dest='fqdn',
+           help='The source repo to push to, fully qualified something...',
+           default='50.28.27.175')
+    parser.add_argument(dest='rosdistro',
+           help='The ros distro. electric, fuerte, groovy')
+    parser.add_argument('--distros', nargs='+',
+           help='A list of debian distros. Default: %(default)s',
+           default=[])
+    parser.add_argument('--commit', dest='commit',
+           help='Really?', action='store_true')
+    parser.add_argument('--delete', dest='delete',
+           help='Delete extra jobs', action='store_true')
+    parser.add_argument('--repo-workspace', dest='repos', action='store',
+           help='A directory into which all the repositories will be checked out into.')
+    return parser.parse_args()
+
+
+def doit(repo_map, stacks, distros, fqdn, rosdistro, commit=False, delete_extra_jobs=False):
+    jenkins_instance = None
+    if args.commit or delete_extra_jobs:
+        jenkins_instance = jenkins_support.JenkinsConfig_to_handle(jenkins_support.load_server_config_file(jenkins_support.get_default_catkin_debs_config()))
+
+    # What ROS distro are we configuring?
+    rosdistro = repo_map['release-name']
+
+    # Figure out default distros.  Command-line arg takes precedence; if
+    # it's not specified, then read targets.yaml.
+    if distros:
+        default_distros = distros
+    else:
+        print('Fetching "%s"' % (URL_PROTOTYPE % 'targets'))
+        targets_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % 'targets'))
+        my_targets = [x for x in targets_map if rosdistro in x]
+        if len(my_targets) != 1:
+            print('Must have exactly one entry for rosdistro "%s" in targets.yaml' % rosdistro)
+            sys.exit(1)
+        default_distros = my_targets[0][rosdistro]
+
+    # We take the intersection of repo-specific targets with default
+    # targets.
+    results = {}
+    for short_package_name, r in repo_map['repositories'].items():
+        if 'type' not in r or 'url' not in r:
+            print('"type" or "url" key missing for repository "%s"; skipping' % r)
+            continue
+        vcs_type = r['type']
+        url = r['url']
+        version = None
+        if vcs_type != 'svn':
+            if 'version' not in r:
+                print('"version" key missing for SVN repository "%s"; skipping' % r)
+                continue
+            else:
+                version = r['version']
+        if 'target' not in r or r['target'] == 'all':
+            target_distros = default_distros
+        else:
+            target_distros = list(set(r['target']) & set(default_distros))
+
+        print ('Configuring "%s" for "%s"' % (r['url'], target_distros))
+
+        results[short_package_name] = devel_jobs.doit(vcs_type, url, version,
+             short_package_name,
+             stacks[short_package_name],
+             target_distros,
+             fqdn,
+             rosdistro=rosdistro,
+             short_package_name=short_package_name,
+             commit=commit,
+             jenkins_instance=jenkins_instance)
+        print ('individual results', results[short_package_name])
+
+    if delete_extra_jobs:
+        # clean up extra jobs
+        configured_jobs = set()
+
+        for _, v in results.iteritems():
+            devel_jobs.summarize_results(*v)
+            for e in v:
+                configured_jobs.update(set(e))
+
+        existing_jobs = set([j['name'] for j in jenkins_instance.get_jobs()])
+        relevant_jobs = existing_jobs - configured_jobs
+        relevant_jobs = [j for j in relevant_jobs if j.startswith('ros-%s-' % rosdistro) and '_devel_' in j]
+
+        for j in relevant_jobs:
+            print('Job "%s" detected as extra' % j)
+            if commit:
+                jenkins_instance.delete_job(j)
+                print('Deleted job "%s"' % j)
+
+    return results
+
+
+if __name__ == '__main__':
+    args = parse_options()
+    repo = 'http://%s/repos/building' % args.fqdn
+
+    print('Fetching "%s"' % (URL_PROTOTYPE % (args.rosdistro + '-devel')))
+    repo_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % (args.rosdistro + '-devel')))
+    if 'release-name' not in repo_map:
+        print('No "release-name" key in yaml file')
+        sys.exit(1)
+    if repo_map['release-name'] != args.rosdistro:
+        print('release-name mismatch (%s != %s)' % (repo_map['release-name'], args.rosdistro))
+        sys.exit(1)
+    if 'repositories' not in repo_map:
+        print('No "repositories" key in yaml file')
+    if 'type' not in repo_map or repo_map['type'] != 'devel':
+        print('Wrong type value in yaml file')
+        sys.exit(1)
+
+    workspace = args.repos
+    try:
+        if not args.repos:
+            workspace = tempfile.mkdtemp()
+        stacks = {}
+        for name, repo in repo_map['repositories'].items():
+            stacks[name] = get_stack_of_remote_repository(name, repo['type'], repo['url'], repo['version'] if 'version' in repo else None)
+
+    finally:
+        if not args.repos:
+            shutil.rmtree(workspace)
+
+    results_map = doit(repo_map,
+        stacks,
+        args.distros,
+        args.fqdn,
+        rosdistro=args.rosdistro,
+        commit=args.commit,
+        delete_extra_jobs=args.delete)
+
+    if not args.commit:
+        print('This was not pushed to the server.  If you want to do so use "--commit" to do it for real.')

src/buildfarm/devel_jobs.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import em
+import pkg_resources
+import os
+from xml.sax.saxutils import escape
+
+import jenkins
+
+from buildfarm.rosdep_support import resolve_rosdeps
+
+
+class Templates(object):
+    template_dir = os.path.dirname(__file__)
+    config_devel = pkg_resources.resource_string('buildfarm', 'resources/templates/devel_job/config.devel.xml.em')  # A config.xml template for devel.
+    config_scm_devel = {}
+    config_scm_devel['git'] = pkg_resources.resource_string('buildfarm', 'resources/templates/devel_job/fragment.scm.git.devel.xml.em')
+    config_scm_devel['hg'] = pkg_resources.resource_string('buildfarm', 'resources/templates/devel_job/fragment.scm.hg.devel.xml.em')
+    config_scm_devel['svn'] = pkg_resources.resource_string('buildfarm', 'resources/templates/devel_job/fragment.scm.svn.devel.xml.em')
+    command_devel = pkg_resources.resource_string('buildfarm', 'resources/templates/devel_job/devel_build.sh.em')  # The bash script that the devel config.xml runs.
+
+
+def expand(config_template, d):
+    s = em.expand(config_template, **d)
+    return s
+
+
+def create_jenkins_job(jobname, config, jenkins_instance):
+    try:
+        jobs = jenkins_instance.get_jobs()
+        print("working on job", jobname)
+        if jobname in [job['name'] for job in jobs]:
+            jenkins_instance.reconfig_job(jobname, config)
+        else:
+            jenkins_instance.create_job(jobname, config)
+        return True
+    except jenkins.JenkinsException as ex:
+        print('Failed to configure "%s" with error: %s' % (jobname, ex))
+        return False
+
+
+def create_devel_config(d):
+    d['COMMAND'] = escape(expand(Templates.command_devel, d))
+    return expand(Templates.config_devel, d)
+
+
+def devel_job_name(rosdistro, packagename, distro, arch):
+    return "ros-%(rosdistro)s-%(packagename)s_devel_%(distro)s_%(arch)s" % locals()
+
+
+def devel_jobs(vcs_type, uri, version, package, stack, distros, rosdistro, fqdn, ros_package_repo="http://50.28.27.175/repos/building"):
+    d = dict(
+        URL=uri,
+        VERSION=version,
+        NAME=package,
+    )
+    scm_fragment = expand(Templates.config_scm_devel[vcs_type], d)
+
+    notification_email = ' '.join([m.email for m in stack.maintainers])
+    d = {
+        'NAME': package,
+        'SCM_FRAGMENT': scm_fragment,
+        'XUNIT_XML_FRAGMENT': '',
+        'NOTIFICATION_EMAIL': notification_email,
+    }
+
+    build_depends = [depends.name for depends in stack.build_depends]
+    jobs = []
+    for distro in [distros[1]]:  # for now only take one distro
+        for arch in ['amd64']:  # removed 'i386' to build devel only on one arch
+            job_name = devel_job_name(rosdistro, package, distro, arch)
+            d['ARCH'] = arch
+            d['BUILD_DEPENDS'] = ' '.join(build_depends)
+            d['DISTRO'] = distro
+            d['JOBNAME'] = job_name
+            d['PLATFORM'] = distro
+            d['ROSDISTRO'] = rosdistro
+            config = create_devel_config(d)
+            #print(config)
+            jobs.append((job_name, config))
+    return jobs
+
+
+def doit(vcs_type, uri, version, package, stack, distros, fqdn, rosdistro, short_package_name, commit, jenkins_instance):
+
+    #package = os.path.splitext(os.path.basename(uri))[0]
+
+    jobs = devel_jobs(vcs_type, uri, version, package, stack, distros, rosdistro, fqdn)
+    successful_jobs = []
+    failed_jobs = []
+    for job_name, config in jobs:
+        if commit:
+            if create_jenkins_job(job_name, config, jenkins_instance):
+                successful_jobs.append(job_name)
+            else:
+                failed_jobs.append(job_name)
+    unattempted_jobs = [job for (job, config) in jobs if job not in successful_jobs and job not in failed_jobs]
+
+    return (unattempted_jobs, successful_jobs, failed_jobs)
+
+
+def summarize_results(unattempted_jobs, successful_jobs, failed_jobs):
+    print('=' * 80)
+    jobs = set(unattempted_jobs).union(set(successful_jobs)).union(set(failed_jobs))
+    print ('Summary: %d jobs configured.  Listed below.' % len(jobs))
+    print ('Unexecuted: %d' % len(unattempted_jobs))
+    for job_name in unattempted_jobs:
+        print ("  %s" % job_name)
+    print ('Successful: %d' % len(successful_jobs))
+    for job_name in successful_jobs:
+        print ("  %s" % job_name)
+    print ('Failed: %d' % len(failed_jobs))
+    for job_name in failed_jobs:
+        print ("  %s" % job_name)
+    print('=' * 80)

src/buildfarm/resources/templates/devel_job/config.devel.xml.em

@@ -0,0 +1,44 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<project>
+  <actions/>
+  <description>Generated job to perform continuous integration for "@(NAME)". DO NOT EDIT BY HAND. Generated by catkin-debs/scripts/create_devel_job(s).py.</description>
+  <keepDependencies>false</keepDependencies>
+  <properties/>
+
+@(SCM_FRAGMENT)
+
+  <assignedNode>devel</assignedNode>
+
+  <canRoam>false</canRoam>
+  <disabled>false</disabled>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <authToken>buildit</authToken> <!-- This allows for users to post to HUDSON/job/NAME/build?token=buildit -->
+  <triggers class="vector">
+    <hudson.triggers.SCMTrigger>
+      <spec>*/10 * * * *</spec>
+    </hudson.triggers.SCMTrigger>
+
+  </triggers>
+  <concurrentBuild>false</concurrentBuild>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>@(COMMAND)</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers>
+    @(XUNIT_XML_FRAGMENT)
+
+    <hudson.tasks.junit.JUnitResultArchiver>
+      <testResults>build/test_results/_hudson/*.xml</testResults>
+      <keepLongStdio>true</keepLongStdio>
+      <testDataPublishers/>
+    </hudson.tasks.junit.JUnitResultArchiver>
+    <hudson.tasks.Mailer>
+      <recipients>@(NOTIFICATION_EMAIL)</recipients>
+      <dontNotifyEveryUnstableBuild>false</dontNotifyEveryUnstableBuild>
+      <sendToIndividuals>false</sendToIndividuals>
+    </hudson.tasks.Mailer>
+  </publishers>
+  <buildWrappers/>
+
+</project>

src/buildfarm/resources/templates/devel_job/devel_build.sh.em

@@ -0,0 +1,16 @@
+# THIS BUILD RECIPE WAS AUTOGENERATED
+
+export ARCH="@(ARCH)"
+export OS_PLATFORM="@(PLATFORM)"
+export ROSDISTRO_NAME="@(ROSDISTRO)"
+export STACK_BUILD_DEPENDS="@(BUILD_DEPENDS)"
+export STACK_NAME="@(NAME)"
+export UBUNTU_DISTRO="@(DISTRO)"
+
+export IMAGETYPE="all"
+export JOB_TYPE="catkin"
+export OS_NAME="ubuntu"
+export SCRIPT="rosci-catkin-cmake-builder.sh"
+
+wget https://raw.github.com/willowgarage/buildfarm/master/dispatch.sh -O $WORKSPACE/build.sh
+bash $WORKSPACE/build.sh

src/buildfarm/resources/templates/devel_job/fragment.scm.git.devel.xml.em

@@ -0,0 +1,37 @@
+  <scm class="hudson.plugins.git.GitSCM">
+    <configVersion>1</configVersion>
+    <remoteRepositories>
+      <org.spearce.jgit.transport.RemoteConfig>
+        <string>origin</string>
+        <int>5</int>
+        <string>fetch</string>
+        <string>+refs/heads/*:refs/remotes/origin/*</string>
+        <string>receivepack</string>
+        <string>git-upload-pack</string>
+        <string>uploadpack</string>
+        <string>git-upload-pack</string>
+        <string>url</string>
+        <string>@(URL)</string>
+        <string>tagopt</string>
+        <string/>
+      </org.spearce.jgit.transport.RemoteConfig>
+    </remoteRepositories>
+    <branches>
+      <hudson.plugins.git.BranchSpec>
+        <name>@(VERSION)</name>
+      </hudson.plugins.git.BranchSpec>
+    </branches>
+    <localBranch/>
+    <mergeOptions/>
+    <recursiveSubmodules>true</recursiveSubmodules>
+    <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+    <authorOrCommitter>false</authorOrCommitter>
+    <clean>false</clean>
+    <wipeOutWorkspace>false</wipeOutWorkspace>
+    <buildChooser class="hudson.plugins.git.util.DefaultBuildChooser"/>
+    <gitTool>Default</gitTool>
+    <submoduleCfg class="list"/>
+    <relativeTargetDir>@(NAME)</relativeTargetDir>
+    <excludedRegions/>
+    <excludedUsers/>
+  </scm>

src/buildfarm/resources/templates/devel_job/fragment.scm.hg.devel.xml.em

@@ -0,0 +1,8 @@
+  <scm class="hudson.plugins.mercurial.MercurialSCM">
+    <source>@(URL)</source>
+    <branch>@(VERSION)</branch>
+    <subdir>@(NAME)</subdir>
+    <modules/>
+    <clean>false</clean>
+    <forest>false</forest>
+  </scm>

src/buildfarm/resources/templates/devel_job/fragment.scm.svn.devel.xml.em

@@ -0,0 +1,15 @@
+  <scm class="hudson.scm.SubversionSCM">
+    <locations>
+      <hudson.scm.SubversionSCM_-ModuleLocation>
+        <remote>@(URL)</remote>
+        <local>@(NAME)</local>
+      </hudson.scm.SubversionSCM_-ModuleLocation>
+    </locations>
+    <useUpdate>true</useUpdate>
+    <doRevert>false</doRevert>
+    <excludedRegions/>
+    <includedRegions/>
+    <excludedUsers/>
+    <excludedRevprop/>
+    <excludedCommitMessages/>
+  </scm>