diff --git a/configs/system/bz-make-components.cfg.erb b/configs/system/bz-make-components.cfg.erb new file mode 100644 index 0000000..0aec6f5 --- /dev/null +++ b/configs/system/bz-make-components.cfg.erb @@ -0,0 +1,16 @@ +## config file for pkgdb-sync-bugzilla +[global] +## Bugzilla config + +# bugzilla.url = https://bugdev.devel.redhat.com/bugzilla-cvs/xmlrpc.cgi +# bugzilla.url = https://bugzilla.redhat.com/xmlrpc.cgi +bugzilla.url = "https://bugzilla.redhat.com/xmlrpc.cgi" +bugzilla.username = "<%= bugzillaUser %>" +bugzilla.password = "<%= bugzillaPassword %>" + +bugzilla.component_api = "component.get" + +notify.email = "l10n-members@fedoraproject.org", + +## remove or comment the debug line when ready to run against the bz database +debug = False diff --git a/configs/system/bz-make-components.py b/configs/system/bz-make-components.py new file mode 100755 index 0000000..ed573b5 --- /dev/null +++ b/configs/system/bz-make-components.py @@ -0,0 +1,354 @@ +#!/usr/bin/python -tt +# -*- coding: utf-8 -*- +# +# Copyright © 2013 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, modify, +# copy, or redistribute it subject to the terms and conditions of the GNU +# General Public License v.2, or (at your option) any later version. This +# program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the GNU +# General Public License along with this program; if not, write to the Free +# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the source +# code or documentation are not subject to the GNU General Public License and +# may only be used or replicated with the express permission of Red Hat, Inc. +# +# Author(s): Elliot Lee +# Toshio Kuratomi +# Mike Watters +# +''' +sync information from the packagedb into bugzilla + +This short script takes information about package onwership and imports it +into bugzilla. +''' + +import sys +import os +import getopt +import itertools +import xmlrpclib +import codecs +import smtplib +import urllib2 +from collections import defaultdict +from email.Message import Message + +import bugzilla +from bunch import Bunch +from configobj import ConfigObj, flatten_errors +from kitchen.text.converters import to_bytes +from validate import Validator + +vldtr = Validator() +# configspec to set default values and validate types +configspec = ''' +[global] + bugzilla.url = string(default = 'https://bugdev.devel.redhat.com/bugzilla-cvs/xmlrpc.cgi') + bugzilla.username = string(default = '') + bugzilla.password = string(default = '') + bugzilla.component_api = string(default = 'component.get') + notify.email = force_list(default = list('')) + debug = boolean(default = 'False') +'''.splitlines() +cfg = ConfigObj('/etc/bz-make-components.cfg', configspec = configspec) +res = cfg.validate(vldtr, preserve_errors=True) + +for entry in flatten_errors(cfg, res): + section_list, key, error = entry + if error == False: + restore_default(key) + +BZSERVER = cfg['global']['bugzilla.url'] +BZUSER = cfg['global']['bugzilla.username'] +BZPASS = cfg['global']['bugzilla.password'] +BZCOMPAPI = cfg['global']['bugzilla.component_api'] +NOTIFYEMAIL = cfg['global']['notify.email'] +DRY_RUN = cfg['global']['debug'] + +# When querying for current info, take segments of 5000 packages a time +BZ_PKG_SEGMENT = 5000 + +class DataChangedError(Exception): + '''Raised when data we are manipulating changes while we're modifying it.''' + pass + +def segment(iterable, chunk, fill=None): + '''Collect data into `chunk` sized block''' + args = [iter(iterable)] * chunk + return itertools.izip_longest(*args, fillvalue=fill) + +class ProductCache(dict): + def __init__(self, bz, acls): + self.bz = bz + self.acls = acls + + # Ask bugzilla for a section of the pkglist. + # Save the information from the section that we want. + def __getitem__(self, key): + try: + return super(ProductCache, self).__getitem__(key) + except KeyError: + # We can only cache products we have pkgdb information for + if key not in self.acls: + raise + + if BZCOMPAPI == 'getcomponentsdetails': + # Old API -- in python-bugzilla. But with current server + # and a lot of records this gives ProxyError + products = self.server.getcomponentsdetails(key) + elif BZCOMPAPI == 'component.get': + # Way that's undocumented in the partner-bugzilla api but works + # currently + pkglist = acls[key].keys() + products = {} + for pkg_segment in segment(pkglist, BZ_PKG_SEGMENT): + # Format that bugzilla will understand. Strip None's that segment() pads + # out the final data segment() with + query = [dict(product=key, component=p) for p in pkg_segment if p is not None] + raw_data = self.bz._proxy.Component.get(dict(names=query)) + for package in raw_data['components']: + # Reformat data to be the same as what's returned from + # getcomponentsdetails + product = dict(initialowner=package['default_assignee'], + description=package['description'], + initialqacontact=package['default_qa_contact'], + initialcclist=package['default_cc']) + products[package['name'].lower()] = product + self[key] = products + + return super(ProductCache, self).__getitem__(key) + + +class Bugzilla(object): + + def __init__(self, bzServer, username, password, acls): + self.bzXmlRpcServer = bzServer + self.username = username + self.password = password + + self.server = bugzilla.Bugzilla(url=self.bzXmlRpcServer, user=self.username,password=self.password) + self.productCache = ProductCache(self.server, acls) + + def add_edit_component(self, package, collection, owner, description, + qacontact=None, cclist=None): + '''Add or update a component to have the values specified. + ''' + # Turn the cclist into something usable by bugzilla + if not cclist or 'people' not in cclist: + initialCCList = list() + else: + initialCCList = cclist['people'] + # Add owner to the cclist so comaintainers taking over a bug don't + # have to do this manually + if owner not in initialCCList: + initialCCList.append(owner) + + # Lookup product + try: + product = self.productCache[collection] + except xmlrpclib.Fault as e: + # Output something useful in args + e.args = (e.faultCode, e.faultString) + raise + except xmlrpclib.ProtocolError as e: + e.args = ('ProtocolError', e.errcode, e.errmsg) + raise + + pkgKey = package.lower() + if pkgKey in product: + # edit the package information + data = {} + + # Grab bugzilla email for things changable via xmlrpc + if not qacontact: + qacontact = 'extras-qa@fedoraproject.org' + + # Check for changes to the owner, qacontact, or description + if product[pkgKey]['initialowner'] != owner: + data['initialowner'] = owner + + if product[pkgKey]['description'] != description: + data['description'] = description + if product[pkgKey]['initialqacontact'] != qacontact and ( + qacontact or product[pkgKey]['initialqacontact']): + data['initialqacontact'] = qacontact + + if len(product[pkgKey]['initialcclist']) != len(initialCCList): + data['initialcclist'] = initialCCList + else: + for ccMember in product[pkgKey]['initialcclist']: + if ccMember not in initialCCList: + data['initialcclist'] = initialCCList + break + + if data: + ### FIXME: initialowner has been made mandatory for some + # reason. Asking dkl why. + data['initialowner'] = owner + + # Changes occurred. Submit a request to change via xmlrpc + data['product'] = collection + data['component'] = package + if DRY_RUN: + print '[EDITCOMP] Changing via editComponent(%s, %s, "xxxxx")' % ( + data, self.username) + print '[EDITCOMP] Former values: %s|%s|%s|%s' % ( + product[pkgKey]['initialowner'], + product[pkgKey]['description'], + product[pkgKey]['initialqacontact'], + product[pkgKey]['initialcclist']) + else: + try: + self.server.editcomponent(data) + except xmlrpclib.Fault, e: + # Output something useful in args + e.args = (data, e.faultCode, e.faultString) + raise + except xmlrpclib.ProtocolError, e: + e.args = ('ProtocolError', e.errcode, e.errmsg) + raise + else: + # Add component + if not qacontact: + qacontact = 'extras-qa@fedoraproject.org' + + data = {'product': collection, + 'component': package, + 'description': description, + 'initialowner': owner, + 'initialqacontact': qacontact} + if initialCCList: + data['initialcclist'] = initialCCList + + if DRY_RUN: + print '[ADDCOMP] Adding new component AddComponent:(%s, %s, "xxxxx")' % ( + data, self.username) + else: + try: + self.server.addcomponent(data) + except xmlrpclib.Fault, e: + # Output something useful in args + e.args = (data, e.faultCode, e.faultString) + raise + +def parseOwnerFile(url, warnings): + productInfo = defaultdict(Bunch) + ownerFile = urllib2.urlopen(url) + for line in ownerFile: + line = unicode(line, 'utf-8').strip() + if not line or line[0] == '#': + continue + pieces = line.split('|') + try: + product, component, summary, owner, qa = pieces[:5] + except: + warnings.append(to_bytes('%s: Invalid line %s' % (url, line))) + + owners = owner.split(',') + owner = owners[0] + cclist = owners[1:] or [] + if not owner: + warnings.append(to_bytes('%s: No owner in line %s' % (url, line))) + continue + + if len(pieces) > 5 and pieces[5].strip(): + for person in pieces[5].strip().split(','): + cclist.append(person.strip()) + + productInfo[product][component] = Bunch(owner=owner, summary=summary, qacontact=qa, cclist=Bunch(groups=[], people=cclist)) + + return productInfo + +def send_email(fromAddress, toAddress, subject, message): + '''Send an email if there's an error. + + This will be replaced by sending messages to a log later. + ''' + msg = Message() + msg.add_header('To', ','.join(toAddress)) + msg.add_header('From', fromAddress) + msg.add_header('Subject', subject) + msg.set_payload(message) + smtp = smtplib.SMTP('bastion') + smtp.sendmail(fromAddress, toAddress, msg.as_string()) + smtp.quit() + +if __name__ == '__main__': + sys.stdout = codecs.getwriter('utf-8')(sys.stdout) + + opts, args = getopt.getopt(sys.argv[1:], '', ('usage', 'help')) + if len(args) < 1 or ('--usage','') in opts or ('--help','') in opts: + print """Usage: bz-make-components.py URL1 [URL2]... + +This script takes URLs to files in owners.list format and makes changes in +bugzilla to reflect the ownership of bugzilla products that are listed there. +""" + sys.exit(1) + + # Non-fatal errors to alert people about + errors = [] + + # Iterate through the files in the argument list. Grab the owner + # information from each one and construct bugzilla information from it. + acls = Bunch() + for url in args: + data = parseOwnerFile(url, errors) + # Merge at the product level but overwrite at the component level + for product in data: + if product in acls: + acls[product].update(data[product]) + else: + acls[product] = data[product] + + + # Initialize the connection to bugzilla + bugzilla = Bugzilla(BZSERVER, BZUSER, BZPASS, acls) + + for product in acls.keys(): + if product != 'Fedora' and not product.startswith('Fedora '): + errors.append(to_bytes('%s: Invalid product %s in line %s' % + (url, product, line))) + continue + + for pkg in acls[product]: + pkgInfo = acls[product][pkg] + try: + bugzilla.add_edit_component(pkg, product, + pkgInfo['owner'], pkgInfo['summary'], + pkgInfo['qacontact'], pkgInfo['cclist']) + except ValueError, e: + # A username didn't have a bugzilla address + errors.append(str(e.args)) + except DataChangedError, e: + # A Package or Collection was returned via xmlrpc but wasn't + # present when we tried to change it + errors.append(str(e.args)) + except xmlrpclib.ProtocolError, e: + # Unrecoverable and likely means that nothing is going to + # succeed. + errors.append(str(e.args)) + break + except xmlrpclib.Error, e: + # An error occurred in the xmlrpc call. Shouldn't happen but + # we better see what it is + errors.append(str(e.args)) + + # Send notification of errors + if errors: + #print '[DEBUG]', '\n'.join(errors) + send_email('accounts@fedoraproject.org', + NOTIFYEMAIL, + 'Errors while syncing bugzilla with the PackageDB', +''' +The following errors were encountered while updating bugzilla with information +from the Package Database. Please have the problems taken care of: + +%s +''' % ('\n'.join(errors),)) + + sys.exit(0) diff --git a/configs/system/bz-make-components.py.erb b/configs/system/bz-make-components.py.erb deleted file mode 100755 index 67711ad..0000000 --- a/configs/system/bz-make-components.py.erb +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/python -tt -# -*- coding: utf-8 -*- -# -# Copyright © 2008 Red Hat, Inc. All rights reserved. -# -# This copyrighted material is made available to anyone wishing to use, modify, -# copy, or redistribute it subject to the terms and conditions of the GNU -# General Public License v.2. This program is distributed in the hope that it -# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the -# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. You should have -# received a copy of the GNU General Public License along with this program; -# if not, write to the Free Software Foundation, Inc., 51 Franklin Street, -# Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat trademarks that are -# incorporated in the source code or documentation are not subject to the GNU -# General Public License and may only be used or replicated with the express -# permission of Red Hat, Inc. -# -# Red Hat Author(s): Elliot Lee -# Toshio Kuratomi -# - -import sys -import urllib2 -import getopt -import xmlrpclib -from email.Message import Message -import smtplib - -from kitchen.text.converters import to_bytes - -# Set this to the production bugzilla account when we're ready to go live -#BZSERVER = 'https://bugdev.devel.redhat.com/bugzilla-cvs/xmlrpc.cgi' -BZSERVER = 'https://bugzilla.redhat.com/xmlrpc.cgi' -#BZSERVER = 'https://bzprx.vip.phx.redhat.com/xmlrpc.cgi' -BZUSER='<%= bugzillaUser %>' -BZPASS='<%= bugzillaPassword %>' - -DRY_RUN = False - -class Bugzilla(object): - def __init__(self, bzServer, username, password): - self.productCache = {} - self.bzXmlRpcServer = bzServer - self.username = username - self.password = password - - self.server = xmlrpclib.Server(bzServer) - - def add_edit_component(self, package, collection, owner, description, - qacontact=None, cclist=None): - '''Add or updatea component to have the values specified. - ''' - initialCCList = [p.lower() for p in cclist] or list() - - # Lookup product - try: - product = self.productCache[collection] - except KeyError: - product = {} - try: - components = self.server.bugzilla.getProdCompDetails(collection, - self.username, self.password) - except xmlrpclib.Fault, e: - # Output something useful in args - e.args = (e.faultCode, e.faultString) - raise - except xmlrpclib.ProtocolError, e: - e.args = ('ProtocolError', e.errcode, e.errmsg) - raise - - # This changes from the form: - # {'component': 'PackageName', - # 'initialowner': 'OwnerEmail', - # 'initialqacontact': 'QAContactEmail', - # 'description': 'One sentence summary'} - # to: - # product['packagename'] = {'component': 'PackageName', - # 'initialowner': 'OwnerEmail', - # 'initialqacontact': 'QAContactEmail', - # 'description': 'One sentenct summary'} - # This allows us to check in a case insensitive manner for the - # package. - for record in components: - record['component'] = unicode(record['component'], 'utf-8') - try: - record['description'] = unicode(record['description'], 'utf-8') - except TypeError: - try: - record['description'] = unicode(record['description'].data, 'utf-8') - except: - record['description'] = None - product[record['component'].lower()] = record - - self.productCache[collection] = product - - pkgKey = package.lower() - if pkgKey in product: - # edit the package information - data = {} - - # Grab bugzilla email for things changable via xmlrpc - owner = owner.lower() - if qacontact: - qacontact = qacontact.lower() - else: - qacontact = 'extras-qa@fedoraproject.org' - - # Check for changes to the owner, qacontact, or description - if product[pkgKey]['initialowner'] != owner: - data['initialowner'] = owner - - if product[pkgKey]['description'] != description: - data['description'] = description - if product[pkgKey]['initialqacontact'] != qacontact and ( - qacontact or product[pkgKey]['initialqacontact']): - data['initialqacontact'] = qacontact - - if len(product[pkgKey]['initialcclist']) != len(initialCCList): - data['initialcclist'] = initialCCList - else: - for ccMember in product[pkgKey]['initialcclist']: - if ccMember not in initialCCList: - data['initialcclist'] = initialCCList - break - - if data: - ### FIXME: initialowner has been made mandatory for some - # reason. Asking dkl why. - data['initialowner'] = owner - - # Changes occurred. Submit a request to change via xmlrpc - data['product'] = collection - data['component'] = product[pkgKey]['component'] - if DRY_RUN: - for key in data: - if isinstance(data[key], basestring): - data[key] = data[key].encode('ascii', 'replace') - print '[EDITCOMP] Changing via editComponent(%s, %s, "xxxxx")' % ( - data, self.username) - print '[EDITCOMP] Former values: %s|%s|%s' % ( - product[pkgKey]['initialowner'], - product[pkgKey]['description'].encode('ascii', 'replace'), - product[pkgKey]['initialqacontact']) - else: - try: - self.server.bugzilla.editComponent(data, self.username, - self.password) - except xmlrpclib.Fault, e: - # Output something useful in args - e.args = (data, e.faultCode, e.faultString) - raise - except xmlrpclib.ProtocolError, e: - e.args = ('ProtocolError', e.errcode, e.errmsg) - raise - else: - # Add component - owner = owner.lower() - if qacontact: - qacontact = qacontact - else: - qacontact = 'extras-qa@fedoraproject.org' - - data = {'product': collection, - 'component': package, - 'description': description, - 'initialowner': owner, - 'initialqacontact': qacontact} - if initialCCList: - data['initialcclist'] = initialCCList - - if DRY_RUN: - for key in data: - if isinstance(data[key], basestring): - data[key] = data[key].encode('ascii', 'replace') - print '[ADDCOMP] Adding new component AddComponent:(%s, %s, "xxxxx")' % ( - data, self.username) - else: - try: - self.server.bugzilla.addComponent(data, self.username, - self.password) - except xmlrpclib.Fault, e: - # Output something useful in args - e.args = (data, e.faultCode, e.faultString) - raise - -def parseOwnerFile(url, warnings): - pkgInfo = [] - ownerFile = urllib2.urlopen(url) - for line in ownerFile: - line = unicode(line, 'utf-8').strip() - if not line or line[0] == '#': - continue - pieces = line.split('|') - try: - product, component, summary, owner, qa = pieces[:5] - except: - warnings.append(to_bytes('%s: Invalid line %s' % (url, line))) - - owners = owner.split(',') - owner = owners[0] - cclist = owners[1:] or [] - if not owner: - warnings.append(to_bytes('%s: No owner in line %s' % (url, line))) - continue - - if len(pieces) > 5 and pieces[5].strip(): - for person in pieces[5].strip().split(','): - cclist.append(person.strip()) - - if product != 'Fedora' and not product.startswith('Fedora '): - warnings.append(to_bytes('%s: Invalid product %s in line %s' % - (url, product, line))) - continue - pkgInfo.append({'product': product, 'component': component, - 'owner': owner, 'summary': summary, 'qa': qa, 'cclist': cclist}) - - return pkgInfo - -def send_email(fromAddress, toAddress, subject, message): - '''Send an email if there's an error. - - This will be replaced by sending messages to a log later. - ''' - msg = Message() - msg.add_header('To', toAddress) - msg.add_header('From', fromAddress) - msg.add_header('Subject', subject) - msg.set_payload(message) - smtp = smtplib.SMTP('bastion') - smtp.sendmail(fromAddress, [toAddress], msg.as_string()) - smtp.quit() - -if __name__ == '__main__': - opts, args = getopt.getopt(sys.argv[1:], '', ('usage', 'help')) - if len(args) < 1 or ('--usage','') in opts or ('--help','') in opts: - print """Usage: bz-make-components.py URL1 [URL2]... - -This script takes URLs to files in owners.list format and makes changes in -bugzilla to reflect the ownership of bugzilla products that are listed there. -""" - sys.exit(1) - - # Initialize connection to bugzilla - bugzilla = Bugzilla(BZSERVER, BZUSER, BZPASS) - - warnings = [] - # Iterate through the files in the argument list. Grab the owner - # information from each one and construct bugzilla information from it. - pkgData = [] - for url in args: - pkgData.extend(parseOwnerFile(url, warnings)) - - for pkgInfo in pkgData: - try: - bugzilla.add_edit_component(pkgInfo['component'], - pkgInfo['product'], pkgInfo['owner'], - pkgInfo['summary'], pkgInfo['qa'], - pkgInfo['cclist']) - except ValueError, e: - # A username didn't have a bugzilla address - warnings.append(str(e.args)) - except xmlrpclib.ProtocolError, e: - # Unrecoverable and likely means that nothing is going to - # succeed. - warnings.append(str(e.args)) - break - except xmlrpclib.Error, e: - # An error occurred in the xmlrpc call. Shouldn't happen but - # we better see what it is - warnings.append(str(e.args)) - - if warnings: - #print '[DEBUG]', '\n'.join(warnings) - send_email('accounts@fedoraproject.org', 'l10n-admin-members@fedoraproject.org', - 'Errors while syncing bugzilla with owners.list', -''' -The following errors were encountered while updating bugzilla with information -from owners.list files. Please have the problem taken care of: - -%s -''' % ('\n\n'.join(warnings),)) - - sys.exit(0) diff --git a/manifests/services/bugzilla.pp b/manifests/services/bugzilla.pp index 54301bc..353feb9 100644 --- a/manifests/services/bugzilla.pp +++ b/manifests/services/bugzilla.pp @@ -1,18 +1,23 @@ class bugzilla-no-balance { $bugzillaUser='fedora-admin-xmlrpc@redhat.com' - templatefile { '/usr/local/bin/bz-make-components.py': - content => template('system/bz-make-components.py.erb'), + templatefile { '/etc/bz-make-components.cfg': + content => template('system/bz-make-components.cfg.erb'), owner => 48, group => 48, - # Presently contains passwords so it needs to be restricted. - # If passwords move to a config file later and this can be relaxed. - mode => '0750' + mode => '0600' } + + script { '/usr/local/bin/bz-make-components.py': + source => 'system/bz-make-components.py', + owner => 48, + group => 48, + mode => '0755' + } + cron { owners-bugzilla: command => "/usr/local/bin/bz-make-components.py 'https://git.fedorahosted.org/cgit/l10n/owners.git/plain/owners.list' 'https://git.fedorahosted.org/cgit/docs/owners.git/plain/owners.list' &> /dev/null", user => "apache", minute => 20, - ensure => present, - require => Package['fedora-packagedb'] + ensure => present } }