extras-buildsys/utils ExtrasMultiLib.py, NONE, 1.1 extras-push-new, 1.52, 1.53
Michael Schwendt (mschwendt)
fedora-extras-commits at redhat.com
Sun Sep 3 13:25:39 UTC 2006
Author: mschwendt
Update of /cvs/fedora/extras-buildsys/utils
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv27634
Modified Files:
extras-push-new
Added Files:
ExtrasMultiLib.py
Log Message:
* Previous stable script tagged push-new-production-6 and renamed to
"extras-push-new.last-stable", so it can be run in case this one breaks
unexpectedly.
* Code in the ExtrasMultiLib module implements black-list and white-list
(which obsoletes our copydict for wine'n'deps). Running the module as a
separate script tries to "fix" a multi-lib repository from scratch by
examining the basearch repos and copying multi-compat rpms as configured.
--- NEW FILE ExtrasMultiLib.py ---
#!/usr/bin/python -t
# -*- mode: Python; indent-tabs-mode: nil; -*-
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
import errno, os, sys
import glob, pickle, re
sys.path.append('/srv/extras-push/work/buildsys-utils')
from ExtrasPushUtils import *
import rpm, rpmUtils.arch, rpmUtils.transaction, rpmUtils.miscutils
import yum, yum.Errors
from yum.packages import YumLocalPackage
try:
from yum.packageSack import ListPackageSack
except: # yum < 2.9
from repomd.packageSack import ListPackageSack
sys.path.insert(0,'/srv/extras-push/work/extras-repoclosure')
from RepoSupport import RepoSupport
DEBUG = False
setdebug(DEBUG)
rundir = '/srv/rpmbuild/extras-push-rundir'
cachedir = rundir+'/mdcache'
multilibdists = [ 'development', '5', '4', '3' ]
# The following dict can be generated with rpmUtils.arch functions.
# target arch -> compatible archs
compatarchdict = { 'x86_64' : ['x86_64', 'athlon', 'i686', 'i586', 'i486', 'i386', 'noarch']
}
# map: target arch -> 'debug','rpms' -> list of new multicompat pkgs
# to be installed/linked/copied and resolved later in a second pass
missingdict = {}
# map: dist -> target arch -> list of pkg name regexp patterns
# We don't check virtual package names, unless it's a virtual -devel pkg.
# black-list takes precedence over white-list
blacklists = { 'development' : { 'x86_64' : [] },
}
whitelists = { 'development' : { 'x86_64' : [ 'wine' ] },
# 'development' : { 'x86_64' : [ '.*-devel', 'wine' ] },
'5' : { 'x86_64' : [ 'wine' ] },
'4' : { 'x86_64' : [ 'wine' ] },
'3' : { 'x86_64' : [ 'wine' ] },
}
def isWantedName(dist,targetarch,namelist):
"""Check a single package name against black-list and white-list. Since
a package can have virtual names, this takes a list of names as input.
Returns: True or False"""
for n in namelist:
if blacklists.has_key(dist) and blacklists[dist].has_key(targetarch):
for r in blacklists[dist][targetarch]:
if re.compile('^'+r+'$').search(n):
return False
if whitelists.has_key(dist) and whitelists[dist].has_key(targetarch):
for r in whitelists[dist][targetarch]:
if re.compile('^'+r+'$').search(n):
return True
return False # reject by default
def reset():
missingdict = {}
def evalPackage(ts,fname,dist, srpmlocdict):
# Examine the package, see whether we want it for any of the
# enabled multilib repositories. Then store the package names
# in a map, which is used in pass 2 (install+resolve deps).
if dist not in multilibdists:
return
for targetarch in archdict[dist]: # list of repo archs
if not compatarchdict.has_key(targetarch):
continue
pkg = YumLocalPackage(ts=ts,filename=fname)
srcrpm = pkg.tagByName('sourcerpm')
(n,a,e,v,r) = pkg.returnPackageTuple()
(sn,sv,sr,se,sa) = rpmUtils.miscutils.splitFilename(srcrpm)
debugrpm = '%s-debuginfo-%s-%s.%s.rpm' % (sn,sv,sr,a)
if a == targetarch or a == 'noarch': # installed by main push script
continue
if a not in compatarchdict[targetarch]: # incompatible pkg arch
continue
names = [n]
if not n.endswith('-devel'):
# packageObject in yum-2.6.0-1 is broken: no such attr 'prco'
#for (prov, flag, (prove, provv, provr)) in pkg.returnPrco('provides'):
hdr = rpmUtils.miscutils.hdrFromPackage(ts,fname)
for prov in hdr['provides']:
if prov.endswith('-devel'):
names.append(prov)
if not isWantedName(dist,targetarch,names):
continue
srpmloc = os.path.join(treedir,dist,'SRPMS',srcrpm)
if DEBUG and not os.path.isfile(srpmloc):
srpmloc = srpmlocdict[srcrpm]
try:
hdr = rpmUtils.miscutils.hdrFromPackage(ts,srpmloc)
excludearch = hdr['excludearch']
except rpmUtils.RpmUtilsError, e: # source rpm not there or bad
print 'ERROR: %s' % e
print srpmloc
print 'referenced by: %s' % fname
# TODO: We don't want such builds! Do something about it.
# Is this case possible at all? Or only during dep resolving
# and when running into old pkgs?
srcrpm = None
excludearch = []
if not srcrpm:
continue
if targetarch in excludearch:
print 'Multi-compat, but ExcludeArch %s' % excludearch
continue
# Save base file-names here.
pkgbasefn = os.path.basename(fname)
missingdict.setdefault(targetarch,{})
pkglist = missingdict[targetarch].setdefault('rpms',[])
if pkgbasefn not in pkglist:
pkglist.append(pkgbasefn)
missingdict[targetarch]['rpms'] = pkglist
print ' +', pkgbasefn
pkglist = missingdict[targetarch].setdefault('debug',[])
if debugrpm not in pkglist:
pkglist.append(debugrpm)
missingdict[targetarch]['debug'] = pkglist
print ' +', debugrpm
return
def savePendingPkgs(dist):
fn = os.path.join(rundir,'multilib.%s.pickle' % dist)
if DEBUG:
fn += '.DEBUG'
try:
f = file(fn,'w')
pickle.dump(missingdict,f)
f.close()
except:
sys.exit(1) # fatal for our needs
def loadPendingPkgs(dist):
fn = os.path.join(rundir,'multilib.%s.pickle' % dist)
if DEBUG:
fn += '.DEBUG'
try:
reset()
f = file(fn,'r')
missingdict = pickle.load(f)
f.close()
except:
# TODO: set a flag to locate all multi-compat packages from scratch?
pass
def evalFromTree(dist):
ts = rpmUtils.transaction.initReadOnlyTransaction()
for srcarch in archdict[dist]: # list of repo archs
srcdir = os.path.join(treedir,dist,srcarch)
print 'Scanning for multi-compat packages:', srcdir
files = glob.glob('%s/*.rpm' % srcdir)
for f in files:
evalPackage(ts,f,dist,{})
savePendingPkgs(dist)
return
# --------------------------------------------------------------------
class Resolver(yum.YumBase):
def __init__(self, arch = None, config = "/etc/yum.conf"):
yum.YumBase.__init__(self)
self.arch = arch
self.doConfigSetup(fn = config)
if hasattr(self.repos, 'sqlite'):
self.repos.sqlite = False
self.repos._selectSackType()
self.resolved = {}
self.seenbefore = []
self.needed = {}
self.needed['rpms'] = []
self.needed['debug'] = []
def readMetadata(self):
self.doRepoSetup()
self.doSackSetup(rpmUtils.arch.getArchList(self.arch))
for repo in self.repos.listEnabled():
self.repos.populateSack(which=[repo.id], with='filelists')
def addNeededPkg(self,pkg):
(n,a,e,v,r) = pkg.returnPackageTuple()
file = '%s-%s-%s.%s.rpm' % (n,v,r,a)
if file not in self.needed['rpms']:
self.needed['rpms'].append(file)
print ' +', file
binarch = a
srcrpm = pkg.returnSimple('sourcerpm')
(n,v,r,e,a) = rpmUtils.miscutils.splitFilename(srcrpm)
debugrpm = '%s-debuginfo-%s-%s.%s.rpm' % (n,v,r,binarch)
if debugrpm not in self.needed['debug']:
self.needed['debug'].append(debugrpm)
print ' +', debugrpm
def resolveRequires(self,pkg):
if pkg.returnNevraPrintable() in self.seenbefore:
#print " ! SEEN BEFORE"
return
self.seenbefore.append( pkg.returnNevraPrintable() )
# yum-2.6.0-1
# for (req,flags,(reqe,reqv,reqr)) in pkg.returnPrco('requires'):
# File "packageObject.py", line 224, in returnPrco
#AttributeError: YumLocalPackage instance has no attribute 'prco'
#for (req,flags,evr) in getRequires( ... ):
for dep in pkg.requiresList():
if dep.startswith('rpmlib'):
continue # ignore rpmlib deps
if self.resolved.has_key(dep):
continue # resolved before
results = []
try:
results = self.returnPackagesByDep(dep)
except:
pass
if len(results) < 1: # unresolved, most likely in Core
#print 'UNRESOLVED:', dep
continue
self.resolved[dep] = True
for p in results:
self.addNeededPkg(p)
self.resolveRequires(p)
def log(self, value, msg):
pass
def resolveMissing(dist,targetarch):
loadPendingPkgs(dist)
if not missingdict.has_key(targetarch):
return False
missingdict[targetarch].setdefault('rpms',[])
missingdict[targetarch].setdefault('debug',[])
rs = RepoSupport()
conf = rs.GenerateConfig([dist])
repoids = []
srcarch = rpmUtils.arch.getBaseArch( rpmUtils.arch.multilibArches[targetarch][0] )
# Only look at basearch repo.
a = srcarch
for r in rs.ReleaseRepoDict()[dist]:
if r.find('extras') < 0:
continue
repoid = '%s-%s-%s' % (r,dist,a)
repoids.append(repoid)
print repoids
# Only look at multicompat packages.
my = Resolver(arch = srcarch, config = conf)
for repo in my.repos.repos.values():
if repo.id not in repoids:
repo.disable()
else:
repo.enable()
my.repos.setCacheDir(cachedir)
try:
print 'Reading metadata...'
my.readMetadata()
except yum.Errors.RepoError, e:
print 'Yum error: %s' % e
print 'done.'
ts = rpmUtils.transaction.initReadOnlyTransaction()
compatrepodir = os.path.join(treedir,dist,srcarch)
for f in missingdict[targetarch]['rpms']:
print f
fpath = os.path.join(compatrepodir,f)
pkg = YumLocalPackage(ts=ts,filename=fpath)
my.resolveRequires(pkg)
print 'Installing needed packages for %s:' % targetarch
for f in my.needed['rpms']+missingdict[targetarch]['rpms']:
srcfile = os.path.join(treedir,dist,srcarch,f)
destfile = os.path.join(treedir,dist,targetarch,f)
#print srcfile
install_link_or_copy(srcfile,destfile)
for f in my.needed['debug']:
srcfile = os.path.join(treedir,dist,srcarch,'debug',f)
destfile = os.path.join(treedir,dist,targetarch,'debug',f)
#print srcfile
install_link_or_copy(srcfile,destfile)
n = len(my.needed['rpms'])+len(my.needed['debug'])+len(missingdict[targetarch]['rpms'])
return n>0
# --------------------------------------------------------------------
if __name__ == '__main__':
signer_gid_check()
os.umask(0002)
if len(sys.argv) < 2:
print 'SYNTAX: %s <dist> [dist]...' % sys.argv[0]
sys.exit(errno.EINVAL)
for dist in sys.argv[1:]:
evalFromTree(dist)
changed = False
for arch in archdict[dist]: # list of repo archs
changed |= resolveMissing(dist,arch)
if changed:
run_and_check('extras-repobuild.py %s' % dist)
sys.exit(0)
Index: extras-push-new
===================================================================
RCS file: /cvs/fedora/extras-buildsys/utils/extras-push-new,v
retrieving revision 1.52
retrieving revision 1.53
diff -u -r1.52 -r1.53
--- extras-push-new 20 Aug 2006 22:51:49 -0000 1.52
+++ extras-push-new 3 Sep 2006 13:25:36 -0000 1.53
@@ -15,6 +15,9 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+# TODO: We need log files instead of lame debug output!
+# Maybe use yum.logger everywhere.
+
import errno
import fcntl, fnmatch
import os, sys
@@ -24,9 +27,11 @@
import tempfile
import time
-#sys.path.append('/srv/extras-push/work/buildsys-utils')
+sys.path.append('/srv/extras-push/work/buildsys-utils')
from ExtrasPushUtils import *
+import ExtrasMultiLib
+
DEBUG = False
setdebug(DEBUG)
@@ -65,25 +70,6 @@
excludearch_for_noarch = True
upgradecheck = True
-# Note: *no* depsolving is done for these.
-copydict = { 'wine.i386': ['x86_64'],
- 'wine-arts.i386': ['x86_64'],
- 'wine-capi.i386': ['x86_64'],
- 'wine-cms.i386': ['x86_64'],
- 'wine-core.i386': ['x86_64'],
- 'wine-debuginfo.i386': ['x86_64'],
- 'wine-devel.i386': ['x86_64'],
- 'wine-esd.i386': ['x86_64'],
- 'wine-jack.i386': ['x86_64'],
- 'wine-ldap.i386': ['x86_64'],
- 'wine-nas.i386': ['x86_64'],
- 'wine-tools.i386': ['x86_64'],
- 'wine-twain.i386': ['x86_64'],
- # lcms for wine-cms in FE-3 and FE-4
- 'lcms.i386': ['x86_64'],
- 'lcms-debuginfo.i386': ['x86_64'],
- }
-
ts = rpmUtils.transaction.initReadOnlyTransaction()
@@ -147,7 +133,7 @@
# ====================================================================
def list_repo_root(repodir):
- """returns list of directory names in repository root directory"""
+ """returns list of package directory names in plague-results repository root directory, filtering out the repodata directory"""
d = os.listdir(repodir)
try:
d.remove('repodata')
@@ -157,7 +143,7 @@
def find_srcrpms(rootpath):
- """returns a list of source rpm path names in given tree"""
+ """returns list of source rpm path names in given tree"""
srcrpms = []
if not os.path.isdir(rootpath):
return srcrpms
@@ -166,6 +152,7 @@
srcrpms.append(os.path.join(root,f))
return srcrpms
+
def is_repo_changed(repodir):
"""Checks if the repository has changed and needs to be reindexed"""
ref_file = os.path.join(repodir, 'repodata', 'repomd.xml')
@@ -195,6 +182,7 @@
return True
return False
+
def mark_pkg_pushed(pkgrelroot):
fname = os.path.join(pkgrelroot,'PUSHED')
if os.path.isfile(fname):
@@ -288,36 +276,6 @@
(n,a,e,v,r) = (nextn,nexta,nexte,nextv,nextr)
-def debugcopy2(src,dest):
- """shutil.copy2 a file, but don't overwrite destination"""
- if os.path.exists(dest):
- debugprint('WARNING: %s already exists, ignoring new one' % dest)
- return
- debugprint('Copying %s to %s' % (src,dest))
- if not DEBUG:
- shutil.copy2(src,dest)
-
-
-def debugmove(src,dest):
- """shutil.move a file, but don't overwrite destination"""
- if os.path.exists(dest):
- debugprint('WARNING: %s already exists, ignoring new one' % dest)
- return
- debugprint('Moving %s to %s' % (src,dest))
- if not DEBUG:
- shutil.move(src,dest)
-
-
-def debuglink(src,dest):
- """os.link a file, but don't overwrite destination"""
- if os.path.exists(dest):
- debugprint('WARNING: %s already exists, ignoring new one' % dest)
- return
- debugprint('Linking %s to %s' % (src,dest))
- if not DEBUG:
- os.link(src,dest)
-
-
# get the path to where to look for the packages to be signed
# get the list of stuff there (logs, md5sums, specs, rpms)
# list the rpms
@@ -391,6 +349,24 @@
return (name, arch, epoch, ver, rel)
+def getexcludearch(pkgpath,reporoot):
+ """Returns list of excluded archs for a binary rpm.
+ Needs access to the repository root directory where it
+ examines the source rpm."""
+
+ global ts, opts, srpmlocdict
+ if not opts.excludearch_for_noarch:
+ return []
+ hdr = rpmUtils.miscutils.hdrFromPackage(ts,pkgpath)
+ source_rpm = hdr['sourcerpm']
+ if not DEBUG:
+ srpmloc = os.path.join(reporoot, 'SRPMS', source_rpm)
+ else: # in debug mode we cannot access signed/installed rpms
+ srpmloc = srpmlocdict[source_rpm]
+ hdr = rpmUtils.miscutils.hdrFromPackage(ts,srpmloc)
+ return hdr['excludearch']
+
+
def sign_pkgs(filelist):
"""gpg sign all the rpms"""
numfiles = len(filelist)
@@ -419,6 +395,11 @@
"""Copies, signs and moves packages for the given dist.
Returns (errcode, changed) where errcode is an error code and changed
is a boolean indicating whether the target repo was changed."""
+
+ # In the run-file we store the package names for the build report.
+ # Package names in that file also serve as an indication whether
+ # we need to update the repository metadata+stuff and sync it to
+ # the public master site.
runfilename = getrunfilename(dist)
try:
rundirfile = open(runfilename,'a')
@@ -426,6 +407,8 @@
print 'ERROR: %s: %s' % (strerr,runfilename)
sys.exit(err)
+ ExtrasMultiLib.reset()
+
distdir = '%s-%s-%s' % (distro, dist, project)
needsign = os.path.join(stagesdir, distdir)
destdir = os.path.join(treedir, dist)
@@ -490,7 +473,7 @@
if not DEBUG:
if not os.path.exists(destpath):
os.makedirs(destpath)
- debugcopy2(src,dst)
+ install_copy(src,dst)
except:
print 'ERROR: Creating temporary working copy failed.'
debugprint('Removing tmp tree %s' % signtmpdir)
@@ -508,8 +491,8 @@
if not DEBUG:
result = sign_pkgs(relocrpms)
if result:
- print "Error signing packages"
- debugprint('Removing tmp tree %s' % signtmpdir)
+ print "Error signing packages! Please be patient - cleaning up."
+ print('Removing tmp tree %s' % signtmpdir)
if not DEBUG:
shutil.rmtree(signtmpdir)
return result, changed
@@ -517,102 +500,68 @@
print "Copying packages into place:"
infolist = [] # list of src.rpm NEVR for build report
- srpmlocdict = {} # debug only
+ global srpmlocdict
+ srpmlocdict = {} # debug only
for package in files['srpm']:
(n,a,e,v,r) = naevr(package)
infolist.append('%s-%s-%s' % (n,v,r))
pkg_fn = os.path.basename(package)
srpmlocdict[pkg_fn] = package # debug only
destloc = os.path.join(destdir, 'SRPMS', pkg_fn)
- debugmove(signtmpdir+package,destloc)
+ install_move(signtmpdir+package,destloc)
# go through each package and move it to the right arch location
# if it is a noarch package, copy2 it to all arch locations
- # if it is listed in copydict, copy2 it around as appropriate
# if it is a debuginfo package, move it into the 'debug' dir for that arch
for package in files['rpm'] + files['debuginfo']:
# relocated path of signed rpm - always use this for copy2/move/link!
srcloc = signtmpdir+package
- pkg_fn = os.path.basename(package)
+ pkg_fn = pkg_destfn = os.path.basename(package)
+ if package in files['debuginfo']:
+ pkg_destfn = 'debug/' + pkg_fn
(n,a,e,v,r) = naevr(package)
if a == 'noarch':
- if opts.excludearch_for_noarch:
- hdr = rpmUtils.miscutils.hdrFromPackage(ts,package)
- source_rpm = hdr['sourcerpm']
- if not DEBUG:
- srpmloc = os.path.join(destdir, 'SRPMS', source_rpm)
- else: # in debug mode we cannot access signed/installed rpms
- srpmloc = srpmlocdict[source_rpm]
- hdr = rpmUtils.miscutils.hdrFromPackage(ts,srpmloc)
- excludearch = hdr['excludearch']
+ excludearch = getexcludearch(package,destdir)
linkloc = None
- for arch in archdict[dist]:
- if opts.excludearch_for_noarch and arch in excludearch:
- print 'EXCLUDEARCH: Not releasing %s for %s.' % (pkg_fn,arch)
- continue # with next arch
- if package in files['debuginfo']:
- arch = '%s/debug' % arch
- destloc = os.path.join(destdir, arch, pkg_fn)
- linked = False
- if opts.hardlink and linkloc:
- try:
- debuglink(linkloc,destloc)
- linked = True
- except OSError, e:
- print 'WARNING: Linking failed (%s), trying to copy...' % e
- if not linked:
- debugcopy2(srcloc,destloc)
+ for basearch in archdict[dist]:
+ if basearch in excludearch:
+ print 'EXCLUDEARCH: Not releasing %s for %s.' % (pkg_fn,basearch)
+ continue # with next basearch
+ destloc = os.path.join(destdir, basearch, pkg_destfn)
+ if linkloc:
+ install_link_or_copy(linkloc,destloc)
+ else:
+ install_copy(srcloc,destloc)
linkloc = destloc
continue # with next package
-
+
+ # rpmUtils.arch.getBaseArch(a)
elif a in ['i386', 'i486', 'i586', 'i686', 'athlon']:
- if package in files['debuginfo']:
- arch = 'i386/debug'
- else:
- arch = 'i386'
+ basearch = 'i386'
elif a in ['x86_64', 'ia32e', 'amd64']:
- if package in files['debuginfo']:
- arch = 'x86_64/debug'
- else:
- arch = 'x86_64'
+ basearch = 'x86_64'
elif a in ['ppc', 'ppc64', 'ppc32']:
- if package in files['debuginfo']:
- arch = 'ppc/debug'
- else:
- arch = 'ppc'
+ basearch = 'ppc'
else:
print 'Unknown arch %s' % a
continue # with next package
- destloc = os.path.join(destdir, arch, pkg_fn)
- debugmove(srcloc,destloc)
-
- # See whether we need to copy/link this rpm to other targets.
- srcloc = destloc # we copy/link inside destination tree
- copyarchs = copydict.get('%s.%s' % (n, a))
- if copyarchs:
- for arch in copyarchs:
- if not arch in archdict[dist]:
- continue
- if package in files['debuginfo']:
- arch = '%s/debug' % arch
- destloc = os.path.join(destdir, arch, pkg_fn)
- linked = False
- if opts.hardlink:
- try:
- debuglink(srcloc,destloc)
- linked = True
- except OSError, e:
- print 'WARNING: Linking failed (%s), trying to copy...' % e
- if not linked:
- debugcopy2(srcloc,destloc)
+ destloc = os.path.join(destdir, basearch, pkg_destfn)
+ install_move(srcloc,destloc)
+ # TODO:
+ if not (package in files['debuginfo']):
+ ExtrasMultiLib.evalPackage(ts,package,dist,srpmlocdict)
+
+ # TODO:
+ ExtrasMultiLib.savePendingPkgs(dist)
+
# Store list of new packages on disk, so it can be mailed later.
for add in infolist:
add += '\n'
@@ -681,6 +630,7 @@
s.sendmail(mail_from, [mail_to], msg.as_string())
s.close()
+ # Build report has been mailed, we can delete the run-files.
for dist in distlist:
emptyrunfile(dist)
@@ -752,6 +702,14 @@
for dist in changed:
run_and_check('extras-repobuild.py %s' % dist)
+ # TODO: multilib resolver hook
+ for dist in changed:
+ changedagain = False
+ for arch in archdict[dist]: # list of repo archs
+ changedagain |= resolveMissing(dist,arch)
+ if changedagain:
+ run_and_check('extras-repobuild.py %s' % dist)
+
if opts.doublesync:
run_and_check('extras-sync')
More information about the scm-commits
mailing list