extras-buildsys/utils extras-push-new,1.27,1.28
Michael Schwendt (mschwendt)
fedora-extras-commits at redhat.com
Fri Jun 9 09:10:24 UTC 2006
Author: mschwendt
Update of /cvs/fedora/extras-buildsys/utils
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv5272
Modified Files:
extras-push-new
Log Message:
merge from push-new-tmpdir_20060609 branch
Index: extras-push-new
===================================================================
RCS file: /cvs/fedora/extras-buildsys/utils/extras-push-new,v
retrieving revision 1.27
retrieving revision 1.28
diff -u -r1.27 -r1.28
--- extras-push-new 9 Jun 2006 08:47:32 -0000 1.27
+++ extras-push-new 9 Jun 2006 09:10:22 -0000 1.28
@@ -21,8 +21,9 @@
import rpmUtils
import shutil
import string
+import tempfile
-DEBUG = False
+DEBUG = True
signersgid = 100581 # signers group
signkeyname = 'extras at fedoraproject.org'
@@ -59,7 +60,6 @@
mail = True
doublesync = False
force = False
- hardlink = False
alldists = [ '5', '4', '3', 'development' ] # also the order in build reports
@@ -96,7 +96,7 @@
def run_and_check(cmd):
- debugprint(cmd)
+ debugprint('DEBUG: %s' % cmd)
if DEBUG:
return
result = os.system(cmd)
@@ -166,6 +166,16 @@
# ====================================================================
+def list_repo_root(repodir):
+ """returns list of directory names in repository root directory"""
+ d = os.listdir(repodir)
+ try:
+ d.remove('repodata')
+ except:
+ pass
+ return d
+
+
def find_srcrpms(rootpath):
"""returns a list of source rpm path names in given tree"""
srcrpms = []
@@ -177,15 +187,27 @@
return srcrpms
-def prune_needsign_tree(repodir):
- # The root dir contains a directory for every pushed package "name".
- pkgnames = os.listdir(repodir)
+def mark_pkg_pushed(pkgrelroot):
+ fname = os.path.join(pkgrelroot,'PUSHED')
+ if DEBUG:
+ print 'Creating %s' % fname
+ return
try:
- pkgnames.remove('repodata')
- except:
- pass
+ f = open(fname,'w')
+ except IOError, (err, strerr):
+ print 'ERROR: %s: %s' % (strerr,fname)
+ sys.exit(err) # fatal for our needs
+ f.close()
+
+
+def prune_needsign_tree(repodir):
+ # Currently, this only marks unneeded package build job directories
+ # as PUSHED.
+ #
+ # TODO: Make this remove sufficiently old PUSHED directories.
- for name in pkgnames:
+ # The root dir contains a directory for every pushed package "name".
+ for name in list_repo_root(repodir):
pkgroot = os.path.join(repodir,name)
if not os.path.isdir(pkgroot):
continue
@@ -215,37 +237,43 @@
(nextn,nexta,nexte,nextv,nextr) = naevr(nextsrcrpm)
# 1 means: e,v,r is higher than nexte,nextv,nextr
if rpmUtils.miscutils.compareEVR((e,v,r),(nexte,nextv,nextr)) == 1:
- debugprint('Removing: %s' % nextrelroot)
- if not DEBUG:
- shutil.rmtree(nextrelroot)
+ debugprint('Ignoring: %s' % nextrelroot)
+ mark_pkg_pushed(nextrelroot)
else:
- debugprint('Removing: %s' % relroot)
- if not DEBUG:
- shutil.rmtree(relroot)
+ debugprint('Ignoring: %s' % relroot)
+ mark_pkg_pushed(relroot)
# Make this the next newest package for ongoing comparison.
relroot = nextrelroot
(n,a,e,v,r) = (nextn,nexta,nexte,nextv,nextr)
-# ====================================================================
-# (previously in extras-sign-move.py)
+def debugcopy2(src,dest):
+ """shutil.copy2 a file, but don't overwrite destination"""
+ if os.path.exists(dest):
+ debugprint('WARNING: %s already exists, ignoring new one' % dest)
+ return
+ debugprint('Copying %s to %s' % (src,dest))
+ if not DEBUG:
+ shutil.copy2(src,dest)
+
+
+def debugmove(src,dest):
+ """shutil.move a file, but don't overwrite destination"""
+ if os.path.exists(dest):
+ debugprint('WARNING: %s already exists, ignoring new one' % dest)
+ return
+ debugprint('Copying %s to %s' % (src,dest))
+ if not DEBUG:
+ shutil.move(src,dest)
+
# get the path to where to look for the packages to be signed
# get the list of stuff there (logs, md5sums, specs, rpms)
-# list the rpms and pause
+# list the rpms
+# copy the rpms to a tmp directory
# sign the rpms
# collect list of srpms and get package n-v-r from them
-# move pkgs into repo
-
-def list_repo_root(repodir):
- """returns list of directory names in repository root directory"""
- d = os.listdir(repodir)
- try:
- d.remove('repodata')
- except:
- pass
- return d
-
+# move/copy pkgs into repo
def find_files(repodir):
"""returns a dict of filetypes and paths to those files"""
@@ -317,7 +345,7 @@
numfiles = len(filelist)
if numfiles < 1:
print "No packages to sign"
- return errno.EINVAL
+ return 0
while numfiles > 0:
if numfiles > 256:
@@ -336,7 +364,7 @@
return 0
-def sign_move(dist):
+def copy_sign_move(dist):
runfilename = getrunfilename(dist)
try:
rundirfile = open(runfilename,'a')
@@ -346,7 +374,14 @@
distdir = '%s-%s-%s' % (distro, dist, project)
needsign = os.path.join(stagesdir, distdir)
-
+ destdir = os.path.join(treedir, dist)
+ if not os.path.exists(destdir):
+ for arch in archdict[dist]:
+ if not DEBUG:
+ os.makedirs(destdir + '/' + arch + '/debug')
+ if not DEBUG:
+ os.makedirs(destdir + '/' + 'SRPMS')
+
repolockname = os.path.join(needsign,REPO_LOCKFILE_NAME)
repolock = LockFile(name=repolockname,blocking=True)
try:
@@ -360,52 +395,73 @@
prune_needsign_tree(needsign)
files = find_files(needsign)
+
+ repolock.unlock()
+
rpms = files['rpm'] + files['srpm'] + files['debuginfo']
+ print 'Total packages found: %d' % len(rpms)
+ if len(rpms) == 0:
+ print '%s: nothing to do' % dist
+ return 0
rpms.sort()
for item in rpms:
print ' %s' % os.path.basename(item)
- destdir = os.path.join(treedir, dist)
- if not os.path.exists(destdir):
- for arch in archdict[dist]:
+ print 'Copying packages to temporary working directory:'
+ if not DEBUG:
+ try:
+ signtmpdir = tempfile.mkdtemp('','.push',treedir)
+ if signtmpdir == treedir: # paranoid, should never happen
+ raise Exception
+ except:
+ return errno.EIO
+ else:
+ signtmpdir = os.path.join(treedir,'.push.tmp')
+ relocrpms = [] # just the list of target files for gpg signing
+ for src in rpms:
+ dst = signtmpdir + src
+ relocrpms.append(dst)
+ (destpath,destfile) = os.path.split(dst)
+ try:
if not DEBUG:
- os.makedirs(destdir + '/' + arch + '/debug')
- if not DEBUG:
- os.makedirs(destdir + '/' + 'SRPMS')
+ if not os.path.exists(destpath):
+ os.makedirs(destpath)
+ debugcopy2(src,dst)
+ except:
+ print 'ERROR: Creating temporary working copy failed.'
+ debugprint('Removing tree %s' % signtmpdir)
+ if not DEBUG:
+ shutil.rmtree(signtmpdir)
+ sys.exit(errno.EIO)
- rpms = files['rpm'] + files['srpm'] + files['debuginfo']
- if len(rpms) == 0:
- print '%s: nothing to sign' % dist
- repolock.unlock()
- return 0
+ # At this point, we have copies of all rpms below signtmpdir,
+ # including their full source path.
+ # We prefix every previously found file with signtmpdir, so
+ # that we work on the temporary copy and not in the original
+ # plague-results directory.
print "Signing Packages:"
if not DEBUG:
- result = sign_pkgs(rpms)
+ result = sign_pkgs(relocrpms)
if result:
print "Error signing packages"
- repolock.unlock()
+ debugprint('Removing tree %s' % signtmpdir)
+ if not DEBUG:
+ shutil.rmtree(signtmpdir)
return result
- print "Moving Packages into Place"
-
- infolist = []
+ print "Copying packages into place:"
+ infolist = [] # list of src.rpm NEVR for build report
+
for package in files['srpm']:
(n,a,e,v,r) = naevr(package)
+ infolist.append('%s-%s-%s' % (n,v,r))
pkg_fn = os.path.basename(package)
srpmloc = os.path.join(destdir, 'SRPMS', pkg_fn)
- debugprint('Moving %s to %s' % (package, srpmloc))
- if os.path.exists(srpmloc):
- debugprint('%s already exists, ignoring new one' % srpmloc)
- if not DEBUG:
- os.unlink(package)
- else:
- infolist.append('%s-%s-%s' % (n,v,r))
- if not DEBUG:
- shutil.move(package, srpmloc)
+ debugmove(signtmpdir+package,srpmloc)
- # go through each package and move it to the right arch location.
- # if it is a noarch package, copy2 it to all arch locations and unlink it
+ # go through each package and move it to the right arch location
+ # if it is a noarch package, copy2 it to all arch locations
# if it is listed in copydict, copy2 it around as appropriate
# if it is a debuginfo package, move it into the 'debug' dir for that arch
@@ -413,32 +469,22 @@
pkg_fn = os.path.basename(package)
(n,a,e,v,r) = naevr(package)
+ copyarchs = copydict.get('%s.%s' % (n, a))
+ if copyarchs:
+ for arch in copyarchs:
+ if not arch in archdict[dist]:
+ continue
+ if package in files['debuginfo']:
+ arch = '%s/debug' % arch
+ rpmloc = os.path.join(destdir, arch, pkg_fn)
+ debugcopy2(signtmpdir+package,rpmloc)
+
if a == 'noarch':
- prevloc = None
for arch in archdict[dist]:
if package in files['debuginfo']:
arch = '%s/debug' % arch
rpmloc = os.path.join(destdir, arch, pkg_fn)
- debugprint("Copying %s to %s" % (package, rpmloc))
- if os.path.exists(rpmloc):
- debugprint('%s already exists, ignoring new one' % rpmloc)
- else:
- linked = False
- if opts.hardlink and prevloc:
- debugprint("Linking %s to %s" % (prevloc, rpmloc))
- try:
- if not DEBUG:
- os.link(prevloc, rpmloc)
- linked = True
- except OSError, e:
- print "WARNING: Linking %s to %s failed (%s), trying to copy %s instead" % (prevloc, rpmloc, e, package)
- if not linked:
- if not DEBUG:
- shutil.copy2(package, rpmloc)
- prevloc = rpmloc
-
- if not DEBUG:
- os.unlink(package)
+ debugcopy2(signtmpdir+package,rpmloc)
continue
elif a in ['i386', 'i486', 'i586', 'i686', 'athlon']:
@@ -464,65 +510,21 @@
continue
rpmloc = os.path.join(destdir, arch, pkg_fn)
- debugprint('Moving %s to %s' % (package, rpmloc))
- if os.path.exists(rpmloc):
- debugprint('%s already exists, ignoring new one' % rpmloc)
- if not DEBUG:
- os.unlink(package)
- elif not DEBUG:
- shutil.move(package, rpmloc)
+ debugmove(signtmpdir+package,rpmloc)
- copyarchs = copydict.get('%s.%s' % (n, a))
- if copyarchs:
- for arch in copyarchs:
- if not arch in archdict[dist]:
- continue
- if package in files['debuginfo']:
- arch = '%s/debug' % arch
- copyloc = os.path.join(destdir, arch, pkg_fn)
- debugprint("Copying %s to %s" % (rpmloc, copyloc))
- if os.path.exists(copyloc):
- debugprint('%s already exists, ignoring new one' % copyloc)
- else:
- linked = False
- if opts.hardlink:
- debugprint("Linking %s to %s" % (rpmloc, copyloc))
- try:
- if not DEBUG:
- os.link(rpmloc, copyloc)
- linked = True
- except OSError, e:
- print "WARNING: Linking %s to %s failed (%s), copying instead" % (rpmloc, copyloc, e)
- if not linked and not DEBUG:
- shutil.copy2(rpmloc, copyloc)
-
- # Store list of move packages, so it can be mailed later.
+ # Store list of new packages on disk, so it can be mailed later.
for add in infolist:
add += '\n'
rundirfile.write( add )
rundirfile.close()
- # clean up the crap
- print "Cleaning Up"
- for file in files['log'] + files['md5sum'] + files['spec']:
- debugprint('Removing %s' % file)
- if not DEBUG:
- os.unlink(file)
-
- # clean up empty dirs, too.
- for root, dirs, files in os.walk(needsign,topdown=False):
- if 'repodata' in dirs:
- dirs.remove('repodata')
- for dir in dirs:
- path = os.path.join(root,dir)
- try:
- if not DEBUG:
- os.rmdir(path)
- debugprint('Removing directory %s' % path)
- except:
- pass
+ # Mark successfully signed packages as PUSHED.
+ for path in files['pkgrelroot']:
+ mark_pkg_pushed(path)
- repolock.unlock()
+ debugprint('Removing tree %s' % signtmpdir)
+ if not DEBUG:
+ shutil.rmtree(signtmpdir)
return(0)
# ====================================================================
@@ -619,10 +621,16 @@
print 'ERROR: script locked via lockfile %s - it seems to be running already' % lockfile
sys.exit(errno.EPERM)
+ changecount = {} # released build requests per dist
+ totalchanges = 0
for dist in diststopush:
- result = sign_move(dist)
+ result = copy_sign_move(dist)
if result:
sys.exit(result)
+ # len(getlinesfromrunfile(dist)) is the number of packages
+ # per dist which have been pushed.
+ changecount[dist] = len( getlinesfromrunfile(dist) )
+ totalchanges += changecount[dist]
# Option -f re-runs repobuild/repoview for all dists, even if
# no new packages have been pushed for a dist.
@@ -630,27 +638,25 @@
if opts.force:
changed = alldists
- # len(getlinesfromrunfile(dist)) is the number of packages
- # per dist which have been pushed.
-
for dist in changed:
- if opts.force or len( getlinesfromrunfile(dist) ):
+ if opts.force or changecount[dist]:
run_and_check('extras-repobuild.py %s' % dist)
- if opts.doublesync:
+ if opts.doublesync and totalchanges:
run_and_check('extras-sync')
for dist in changed:
- if opts.force or len( getlinesfromrunfile(dist) ):
+ if opts.force or changecount[dist]:
if opts.repoview:
run_and_check('extras-repoview.py %s' % dist)
- run_and_check('extras-sync')
+ if opts.force or totalchanges:
+ run_and_check('extras-sync')
if opts.mail:
email_list( diststopush )
- if opts.repoclosure:
+ if opts.repoclosure and (opts.force or totalchanges):
run_and_check('nohup rc-run-all.py &')
lock.unlock()
More information about the scm-commits
mailing list