osc-checker.py

#
# (C) 2011 coolo@suse.de, Novell Inc, openSUSE.org
# Distribute under GPLv2 or GPLv3
#
# Copy this script to ~/.osc-plugins/ or /var/lib/osc-plugins .
# Then try to run 'osc checker --help' to see the usage.


import socket
import os
import traceback
import subprocess

try:
    fqdn = socket.gethostbyaddr(socket.gethostname())[0]
except:
    fqdn = os.uname()[1]

class HASH(object):
    def __init__(self,h=None):
        if h:
            for k in h.keys():
                setattr(self, k, h[k])
    def __repr__(self):
        return str(self.__dict__)
    def __str__(self):
        return str(self.__dict__)
    def delete(self, key):
        del self.__dict__[key]
    def insert(self, key, val):
        self.__dict__[key] = val
    def keys(self):
        return self.__dict__.keys()
    def has_key(self,name):
        return self.__dict__.has_key(name)

globals()['HASH'] = HASH                        # evil hack, needed as we are already inside osc.commandline

def _checker_fetch_rev_entry(self, apiurl, project, package, revision=None, brief=False, verbose=False):
    """ a misnomer. This fetches the revision number and the rpm license strings.
        option brief=True suppresses fetching of specfile and thus does not return License strings.
    """

    if revision:
        url = makeurl(apiurl, ['source', project, package], { 'view':'info', 'parse':1, 'rev':revision})
    else:
        url = makeurl(apiurl, ['source', project, package], { 'view':'info', 'parse':1  } )

    try:
        f = http_GET(url)
    except urllib2.HTTPError, err:
        return HASH({ 'version': None, 'name':None })
    xml = ET.parse(f)

    name = xml.find('name')
    if name is None or not name.text:
       return HASH({ 'version': None, 'name': None, 'error':'no error and no name'})

    vers = xml.find('version')
    if vers is None or not vers.text:
        return HASH({ 'version': None, 'name': name.text, 'error':'no error and no version'})

    r = { 'version': vers.text, 'name': name.text }

    return HASH(r)

def _checker_change_review_state(self, opts, id, newstate, by_group='', by_user='', message='', supersed=None):
    """ taken from osc/osc/core.py, improved:
        - verbose option added,
        - empty by_user=& removed.
        - numeric id can be int().
    """
    query = {'cmd': 'changereviewstate', 'newstate': newstate }
    if by_group:  query['by_group'] = by_group
    if by_user:   query['by_user'] = by_user
    if supersed: query['superseded_by'] = supersed
#    if message: query['comment'] = message
    u = makeurl(opts.apiurl, ['request', str(id)], query=query)
    f = http_POST(u, data=message)
    root = ET.parse(f).getroot()
    return root.attrib['code']

def _checker_checkout_add(self, prj, pkg, rev, opts):
    dir = opts.directory + '/' + re.sub('.*//', '', opts.apiurl) + '/' + prj + '/' + pkg
    if rev: dir = dir + '%r' + rev
    if opts.no_op:
        print "package NOT checked out to " + dir
    else:
        oldcwd = os.getcwd()
        do_co = True

        try:
            os.rmdir(dir)     # remove if empty.
        except:
            pass

        if os.path.exists(dir):
            print "Oops, %s already checked out.\n Please remove to pull a fresh copy." % dir
            return

        o_umask = os.umask(002)             # allow group writable
        try:
            os.makedirs(dir, mode=0777)       # ask for (at least) group writable
        except Exception,e:
            do_co = False
            print "os.makedirs(%s) failed: %s" % (dir, str(e))
        os.umask(o_umask)

        if do_co:
            os.chdir(dir)
            nc = conf.config['checker_checkout_no_colon']
            conf.config['checker_checkout_no_colon'] = False
            conf.config['checker_checkout_rooted'] = False
            conf.config['package_tracking'] = False

            checkout_package(opts.apiurl, prj, pkg, revision=rev, pathname=dir, server_service_files=True, expand_link=True)
            if opts.origin:
                f = open('.origin', 'wb')
                f.write(opts.origin)
                f.close()

            conf.config['checker_checkout_no_colon'] = nc
            os.chdir(oldcwd)

def _checker_find_submit_request(self, opts, project, package):
	xpath = "(action/target/@project='%s' and action/target/@package='%s')" % (project, package)
        url = makeurl(opts.apiurl, ['search','request'], 'match=%s' % quote_plus(xpath))
        f = http_GET(url)
        collection = ET.parse(f).getroot()
        for root in collection.findall('request'):
           r = Request()
           r.read(root)
	   return r.reqid
	return None
        
def _check_repo(self, repo):
    allfine = True
    founddisabled = False
    foundbuilding = None
    foundfailed = None
    foundoutdated = None
    missings = {}
    for arch in repo.findall('arch'):
        if arch.attrib.has_key('missing'):
            for pkg in arch.attrib['missing'].split(','):
                missings[pkg] = 1
        if not (arch.attrib['result'] in ['succeeded', 'excluded']):
            allfine = False
        if arch.attrib['result'] == 'disabled':
            founddisabled = True
        if arch.attrib['result'] == 'failed':
            foundfailed = repo.attrib['name']
        if arch.attrib['result'] == 'building':
            foundbuilding = repo.attrib['name']
        if arch.attrib['result'] == 'outdated':
            foundoutdated = repo.attrib['name']

    return [allfine, founddisabled, foundbuilding, foundfailed, foundoutdated, missings.keys()]

def _checker_prepare_dir(self, dir):
    olddir=os.getcwd()
    os.chdir(dir)
    shutil.rmtree(".osc")
    os.chdir(olddir)

def _checker_accept_request(self, opts, id, msg):
    code = 100
    query = { 'cmd': 'addreview', 'by_group':'autobuild-team' }
    url = makeurl(opts.apiurl, ['request', str(id)], query)
    if opts.verbose: print url
    try:
        r = http_POST(url, data="Please review sources")
    except urllib2.HTTPError, err:
        return 1
    code = ET.parse(r).getroot().attrib['code']
    if code == 100 or code == 'ok':
         self._checker_change_review_state(opts, id, 'accepted', by_group='factory-auto', message=msg)
         print "accepted " + msg
    return 0

def _checker_one_request(self, rq, cmd, opts):
    if (opts.verbose):
        ET.dump(rq)
        print(opts)
    id = int(rq.get('id'))
    act_id = 0
    actions = rq.findall('action')
    okay=True
    msg=""
    for act in actions:
        act_id += 1
        _type = act.get('type');
        if (_type == "submit"):
            pkg = act.find('source').get('package')
            prj = act.find('source').get('project')
            rev = act.find('source').get('rev')
            tprj = act.find('target').get('project')
            tpkg = act.find('target').get('package')

            src = { 'package': pkg, 'project': prj, 'rev':rev, 'error': None }
            e = []
            if not pkg:
                e.append('no source/package in request %d, action %d' % (id, act_id))
            if not prj:
                e.append('no source/project in request %d, action %d' % (id, act_id))
            if len(e): src.error = '; '.join(e)

            e = []
            if not tpkg:
                e.append('no target/package in request %d, action %d; ' % (id, act_id))
            if not prj:
                e.append('no target/project in request %d, action %d; ' % (id, act_id))
            # it is no error, if the target package dies not exist

            subm_id = "SUBMIT(%d):" % id
            print "\n%s %s/%s -> %s/%s" % (subm_id,
                prj,  pkg,
                tprj, tpkg)
            dpkg = self._checker_check_devel_package(opts, tprj, tpkg)
            self._devel_projects['network:messaging:amqp/'] = 'rabbitmq'
            if dpkg:
                [dprj, dpkg] = dpkg.split('/')
            else:
                dprj = None
            if dprj and (dprj != prj or dpkg != pkg):
                msg = "'%s/%s' is the devel package, submission is from '%s'" % (dprj, dpkg, prj)
                self._checker_change_review_state(opts, id, 'declined', by_group='factory-auto', message=msg)
                print "declined " + msg
                return
            if not dprj and not self._devel_projects.has_key(prj + "/"):
                msg = "'%s' is not a valid devel project of %s - please pick one of the existent" % (prj, tprj)
                self._checker_change_review_state(opts, id, 'declined', by_group='factory-auto', message=msg)
                print "declined " + msg
                return

            try:
                url = makeurl(opts.apiurl, ['status', "bsrequest?id=%d" % id])
                root = ET.parse(http_GET(url)).getroot()
            except urllib2.HTTPError:
                print "error"
                continue
            if root.attrib.has_key('code'):
                print ET.tostring(root)
                continue
            result = False
            goodrepo = None
            missings = {}
            alldisabled = True
	    foundbuilding = None
	    foundfailed = None
	    foundoutdated = None
            for repo in root.findall('repository'):
                [isgood, founddisabled, r_foundbuilding, r_foundfailed, r_foundoutdated, r_missings] = self._check_repo(repo)
		for p in r_missings:
		    missings[p] = 1
                if not founddisabled:
                    alldisabled = False
                if isgood:
                    if len(missings) == 0:
                        goodrepo = repo
                        result = True
		if r_foundbuilding:
		     foundbuilding = r_foundbuilding
		if r_foundfailed:
                     foundfailed = r_foundfailed
		if r_foundoutdated:
		     foundoutdated = r_foundoutdated

            if result == False:
                if len(missings.keys()):
	            smissing = []
		    missings.keys().sort()
                    for package in missings:
			request = self._checker_find_submit_request(opts, tprj, package)
			if request:
			   package = "%s(rq%s)" % (package, request) 
                        smissing.append(package)

                    msg = "please make sure to wait before these depencencies are in {0}: {1}".format(tprj, ', '.join(smissing))
                    self._checker_change_review_state(opts, id, 'new', by_group='factory-auto', message=msg)
                    print "updated " + msg
                    okay=False
                    continue
                if alldisabled:
                    msg = "the package is disabled or does not build against factory. Please fix and resubmit"
                    self._checker_change_review_state(opts, id, 'declined', by_group='factory-auto', message=msg)
                    print "declined " + msg
                    return
	        if foundoutdated:
		    msg = "the package sources were changed after submissions and the old sources never built. Please resubmit"
	            self._checker_change_review_state(opts, id, 'declined', by_group='factory-auto', message=msg)
		    print "declined " + msg
                    return
		if foundbuilding:	
		    msg = "the package is still building for repository {0}".format(foundbuilding)
		    self._checker_change_review_state(opts, id, 'new', by_group='factory-auto', message=msg)
                    print "updated " + msg
                    okay=False
                    continue
	        if foundfailed:
                    msg = "the package failed to build in repository {0} - not accepting".format(foundfailed)
                    self._checker_change_review_state(opts, id, 'new', by_group='factory-auto', message=msg)
		    print "updated " + msg
                    okay=False
                    continue

                print ET.tostring(root)
                continue

            dir = os.path.expanduser("~/co/%s" % str(id))
            if os.path.exists(dir):
                print "%s already exists" % dir
                continue
            os.mkdir(dir)
            os.chdir(dir)
            try:
                checkout_package(opts.apiurl, tprj, tpkg, pathname=dir,
                                 server_service_files=True, expand_link=True)
                self._checker_prepare_dir(tpkg)
                os.rename(tpkg, "_old")
            except urllib2.HTTPError:
		print "failed to checkout %s/%s" % (tprj, tpkg)
                pass
            checkout_package(opts.apiurl, prj, pkg, revision=rev,
                             pathname=dir, server_service_files=True, expand_link=True)
            os.rename(pkg, tpkg)
            self._checker_prepare_dir(tpkg)

  	    r=self._checker_fetch_rev_entry(opts.apiurl, prj, pkg, revision=rev)
	    if r.name != tpkg:
		msg = "A pkg submitted as %s has to build as 'Name: %s' - found Name '%s'" % (tpkg, tpkg, r.name)
                self._checker_change_review_state(opts, id, 'declined', by_group='factory-auto', message=msg)
                return

            civs = "LC_ALL=C perl /suse/coolo/checker/source-checker.pl _old %s 2>&1" % tpkg
            p = subprocess.Popen(civs, shell=True, stdout=subprocess.PIPE, close_fds=True)
            ret = os.waitpid(p.pid, 0)[1]
            checked = p.stdout.readlines()
            output = '  '.join(checked).translate(None, '\033')
            os.chdir("/tmp")
            
            if ret != 0:
                msg = "Output of check script:\n" + output
                self._checker_change_review_state(opts, id, 'declined', by_group='factory-auto', message=msg)
                print "declined " + msg
		shutil.rmtree(dir)
                return

	    firstarch=goodrepo.find('arch')
	    if not firstarch is None:
                url = makeurl(opts.apiurl, ['build', prj, goodrepo.attrib['name'], firstarch.attrib['arch'], pkg, "rpmlint.log"])
		try:
                   f = http_GET(url)
		   lines = f.readlines()
                except urllib2.HTTPError, err:
		   lines = []

		isdeclined = False
		for line in lines:
		    if re.search('W:.*invalid-license ', line):
	                msg = "Found rpmlint warning: \n" + line
                        msg += "Try the following patch\n"
                        civs = "export LC_ALL=C; for i in %s/%s/*.spec; do perl /suse/coolo/obs-service-format_spec_file/patch_license $i > $i.new || cp $i $i.new; diff -U0 $i $i.new; done 2>&1" % (dir, tpkg)
                        p = subprocess.Popen(civs, shell=True, stdout=subprocess.PIPE, close_fds=True)
                        ret = os.waitpid(p.pid, 0)[1]
                        checked = p.stdout.readlines()
                        output = '  '.join(checked).translate(None, '\033')
			msg += output
			print "declined " + msg
       		        self._checker_change_review_state(opts, id, 'new', by_group='factory-auto', message=msg)
			isdeclined = True
                        okay=False
			break
		if isdeclined: 
   		     shutil.rmtree(dir)
		     continue
 
	    shutil.rmtree(dir)
            msg = msg + "\n==================================\nBuilds for repo %s" % goodrepo.attrib['name']
            if len(checked):
                msg = msg + "\n\nOutput of check script (non-fatal):\n" + output
                
            if cmd == "list":
                pass
            elif cmd == "checker_checkout" or cmd == "co":
                opts.origin = opts.apiurl + '/request/' + str(id) + "\n";
                self._checker_checkout_add(prj, pkg, rev, opts)
            else:
                print "unknown command: %s" % cmd

        elif (_type == "delete"):
            print "DELETE(%d):" % id
            pass
        elif (_type == "change_devel"):
            print "CHANGE_DEVEL(%d):" % id
            pass
        elif (_type == "addrole"):
            print "ADD_ROLE(%d):" % id
            # we could allow roles like "bugowner" here ...
            msg = "Adding a role is not allow in Factory packages."
            self._checker_change_review_state(opts, id, 'declined', by_group='factory-auto', message=msg)
            print "declined " + msg
            return
        else:
            print "UNHANDLED TYPE(%d):" % id
            msg = "Action " + _type + " is not supported in Factory"
            self._checker_change_review_state(opts, id, 'declined', by_group='factory-auto', message=msg)
            print "declined " + msg
            return

    if okay:
        self._checker_accept_request(opts, id, msg)


def _checker_check_devel_package(self, opts, project, package):
    if not self._devel_projects.has_key(project):
        url = makeurl(opts.apiurl, ['search','package'], "match=[@project='%s']" % project)
        f = http_GET(url)
        root = ET.parse(f).getroot()
        for p in root.findall('package'):
            name = p.attrib['name']
            d = p.find('devel')
            if d != None:
                dprj = d.attrib['project']
                self._devel_projects["%s/%s" % (project, name)] = "%s/%s" % (dprj, d.attrib['package'])
                # for new packages to check
                self._devel_projects[dprj + "/"] = 1
            elif not name.startswith("_product"):
                print "NO DEVEL IN", name
            # mark we tried
            self._devel_projects[project] = 1
    try:
        return self._devel_projects["%s/%s" % (project, package)]
    except KeyError:
        return None

def _checker_check_dups(self, project, opts):
    url = makeurl(opts.apiurl, ['request'], "states=new,review&project=%s&view=collection" % project)
    f = http_GET(url)
    root = ET.parse(f).getroot()
    rqs = {}
    for rq in root.findall('request'):
        id = rq.attrib['id']
        for a in rq.findall('action'):
            source = a.find('source')
            target = a.find('target')
            type = a.attrib['type']
            assert target != None
            if target.attrib['project'] != project: continue
            package = target.attrib['package']
            if rqs.has_key(type + package):
                [oldid, oldsource] = rqs[type + package]
		print oldid, id
                assert oldid < id
                if source != None and oldsource != None:
                    if (source.attrib['project'] == oldsource.attrib['project'] and
                       source.attrib['package'] == oldsource.attrib['package']):
                        change_request_state(opts.apiurl, str(oldid), 'superseded',
                                     'superseded by %s' % id, id)
                        continue
                print "DUPS found:", id, oldid
            rqs[type + package] = [id, source]


def do_checker(self, subcmd, opts, *args):
    """${cmd_name}: checker review of submit requests.

    Usage:
      osc checker [OPT] [list] [FILTER|PACKAGE_SRC]
           Shows pending review requests and their current state.

    ${cmd_option_list}
    """

    if len(args) == 0:
        raise oscerr.WrongArgs("Please give a subcommand to 'osc checker' or try 'osc help checker'")

    self._devel_projects = {}
    opts.mode = ''
    opts.verbose = False
    if args[0] == 'auto':     opts.mode = 'auto'
    if args[0] == 'review':   opts.mode = 'both'
    if len(args) > 1 and args[0] in ('auto','manual') and args[1] in ('approve', 'reject'):
        args = args[1:]

    from pprint import pprint

    opts.apiurl = self.get_api_url()

    tmphome = None

    if args[0] == 'dups':
        for p in args[1:]:
            self._checker_check_dups(p, opts)
        return

    if args[0] == 'skip':
        for id in args[1:]:
           self._checker_accept_request(opts, id, "skip review")
        return
    ids = {}
    for a in args:
        if (re.match('\d+', a)):
            ids[a] = 1

    if (not len(ids)):
        # xpath query, using the -m, -r, -s options
        where = "@by_group='factory-auto'+and+@state='new'"

        url = makeurl(opts.apiurl, ['search','request'], "match=state/@name='review'+and+review["+where+"]")
        f = http_GET(url)
        root = ET.parse(f).getroot()
        for rq in root.findall('request'):
            tprj = rq.find('action/target').get('project')
            self._checker_one_request(rq, args[0], opts)
    else:
        # we have a list, use them.
        for id in ids.keys():
            url = makeurl(opts.apiurl, ['request', id])
            f = http_GET(url)
            xml = ET.parse(f)
            root = xml.getroot()
            self._checker_one_request(root, args[0], opts)

#Local Variables:
#mode: python
#py-indent-offset: 4
#tab-width: 8
#End: