#6006: fix builds in f21 where latest NVR is not latest NVR
by Fedora Release Engineering
#6006: fix builds in f21 where latest NVR is not latest NVR
-----------------------------+------------------------
Reporter: sharkcz | Owner: rel-eng@…
Type: task | Status: new
Milestone: Fedora 21 Final | Component: koji
Keywords: | Blocked By:
Blocking: |
-----------------------------+------------------------
There are 20+ builds in f21 tag where latest NVR is not latest NVR, see
output of "check-latest-build.py f21"
--
Ticket URL: <https://fedorahosted.org/rel-eng/ticket/6006>
Fedora Release Engineering <http://fedorahosted.org/rel-eng>
Release Engineering for the Fedora Project
9 years, 7 months
#6007: retire askbot fedora21
by Fedora Release Engineering
#6007: retire askbot fedora21
-----------------------------+------------------------
Reporter: sundaram | Owner: rel-eng@…
Type: task | Status: new
Milestone: Fedora 20 Final | Component: koji
Keywords: | Blocked By:
Blocking: |
-----------------------------+------------------------
python-django14 was retired from Fedora 21 breaking the dependency of
askbot
--
Ticket URL: <https://fedorahosted.org/rel-eng/ticket/6007>
Fedora Release Engineering <http://fedorahosted.org/rel-eng>
Release Engineering for the Fedora Project
9 years, 7 months
[releng] Updated tag rawhide-stable
by Dennis Gilmore
The lightweight tag 'rawhide-stable' was updated to point to:
cff1840... We need to use the full path to the script from where it wi
It previously pointed to:
b91f8da... escape the wildcard when checking srpms that are excluded f
NOTE: People pulling from the repository will not get the new tag.
For more information, please see:
http://live.gnome.org/Git/Help/TagUpdates
9 years, 7 months
[releng] We need to use the full path to the script from where it will be executed
by Dennis Gilmore
commit cff18408805bd4bd170b708fc87a3d83c669fbb2
Author: Dennis Gilmore <dennis(a)ausil.us>
Date: Mon Sep 29 14:57:27 2014 -0500
We need to use the full path to the script from where it will be executed
scripts/buildbranched | 2 +-
scripts/buildrawhide | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
---
diff --git a/scripts/buildbranched b/scripts/buildbranched
index 3875977..2647703 100755
--- a/scripts/buildbranched
+++ b/scripts/buildbranched
@@ -80,7 +80,7 @@ log "started"
send_fedmsg start start
log "blocking retired packages"
-./block_retired.py
+./scripts/block_retired.py
log "git clone of comps started"
pushd $TMPDIR
diff --git a/scripts/buildrawhide b/scripts/buildrawhide
index aef5b72..674ade9 100755
--- a/scripts/buildrawhide
+++ b/scripts/buildrawhide
@@ -80,7 +80,7 @@ log "started"
send_fedmsg start start
log "blocking retired packages"
-./block_retired.py
+./scripts/block_retired.py
log "git clone of comps started"
pushd $TMPDIR
9 years, 7 months
[releng] escape the wilecard in teh call to srpm-excluded-arch.py
by Dennis Gilmore
commit 459ac618d71c4840975d80504f5ca84f4ed9bf94
Author: Dennis Gilmore <dennis(a)ausil.us>
Date: Mon Sep 29 14:54:35 2014 -0500
escape the wilecard in teh call to srpm-excluded-arch.py
scripts/buildbranched | 2 +-
scripts/buildrawhide | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
---
diff --git a/scripts/buildbranched b/scripts/buildbranched
index e020a3e..3875977 100755
--- a/scripts/buildbranched
+++ b/scripts/buildbranched
@@ -221,7 +221,7 @@ do
elif [ "$koji" = "s390" ]; then
arches=s390,s390x
fi
- scripts/srpm-excluded-arch.py -a $arches --path /mnt/koji/mash/branched-$DATE/$BRANCHED$EXPANDARCH/source/SRPMS/*/ >$logdir/excludearch-$koji.log
+ scripts/srpm-excluded-arch.py -a $arches --path /mnt/koji/mash/branched-$DATE/$BRANCHED$EXPANDARCH/source/SRPMS/\*/ >$logdir/excludearch-$koji.log
done
send_fedmsg done complete
diff --git a/scripts/buildrawhide b/scripts/buildrawhide
index bbefad7..aef5b72 100755
--- a/scripts/buildrawhide
+++ b/scripts/buildrawhide
@@ -219,7 +219,7 @@ do
elif [ "$koji" = "s390" ]; then
arches=s390,s390x
fi
- scripts/srpm-excluded-arch.py -a $arches --path /mnt/koji/mash/rawhide-$DATE/$BRANCHED$EXPANDARCH/source/SRPMS/*/ >$logdir/excludearch-$koji.log
+ scripts/srpm-excluded-arch.py -a $arches --path /mnt/koji/mash/rawhide-$DATE/$BRANCHED$EXPANDARCH/source/SRPMS/\*/ >$logdir/excludearch-$koji.log
done
send_fedmsg done complete
9 years, 7 months
[releng] check-latest-build: PEP8 compliance/implement --fix
by Till Maas
commit 55d88e291c35640b4b560a2754d739d75393d237
Author: Till Maas <opensource(a)till.name>
Date: Mon Sep 29 21:32:44 2014 +0200
check-latest-build: PEP8 compliance/implement --fix
scripts/check-latest-build.py | 63 ++++++++++++++++++++++++----------------
1 files changed, 38 insertions(+), 25 deletions(-)
---
diff --git a/scripts/check-latest-build.py b/scripts/check-latest-build.py
index 6370939..422ee0c 100755
--- a/scripts/check-latest-build.py
+++ b/scripts/check-latest-build.py
@@ -9,33 +9,37 @@
# It can happen when bodhi pushes multiple builds of one package.
#
+import argparse
+import logging
+log = logging.getLogger(__name__)
+import subprocess
-import sys
-import os
import koji
-import time
-import string
-import rpm
-import shutil
-import argparse
+import rpm
# get parameters from command line
parser = argparse.ArgumentParser()
parser.add_argument("--arch", help="use when checking secondary arch koji")
-parser.add_argument("--fix", help="retag the highest NVR to became the latest (no-op now)", action="store_true")
-parser.add_argument("--verbose", help="be verbose during processing", action="store_true")
+parser.add_argument("--fix", help="retag the highest NVR to became the latest",
+ action="store_true")
+parser.add_argument("--verbose", help="be verbose during processing",
+ action="store_true")
+parser.add_argument("--debug", help="Enable debug output", action="store_true")
parser.add_argument("tag", help="tag to check")
parser.add_argument("package", nargs="*", help="packages to check")
args = parser.parse_args()
-#print("tag=%s arch=%s pkgs=%s" % (args.tag, args.arch, args.package))
+if args.debug:
+ log.setLevel(logging.DEBUG)
+log.debug("tag=%s arch=%s pkgs=%s", args.tag, args.arch, args.package)
if args.arch is None:
KOJIHUB = 'http://koji.fedoraproject.org/kojihub'
else:
KOJIHUB = 'http://%s.koji.fedoraproject.org/kojihub' % (args.arch)
-def _rpmvercmp ((e1, v1, r1), (e2, v2, r2)):
+
+def _rpmvercmp((e1, v1, r1), (e2, v2, r2)):
"""find out which build is newer"""
if e1 == "None":
e1 = "0"
@@ -43,30 +47,32 @@ def _rpmvercmp ((e1, v1, r1), (e2, v2, r2)):
e2 = "0"
rc = rpm.labelCompare((e1, v1, r1), (e2, v2, r2))
if rc == 1:
- #first evr wins
+ # first evr wins
return 1
elif rc == 0:
- #same evr
+ # same evr
return 0
else:
- #second evr wins
+ # second evr wins
return -1
kojisession = koji.ClientSession(KOJIHUB)
if args.package == []:
- latest_builds = sorted(kojisession.listTagged(args.tag, latest=True), key = lambda pkg: pkg['package_name'])
+ latest_builds = sorted(kojisession.listTagged(args.tag, latest=True),
+ key=lambda pkg: pkg['package_name'])
else:
-# print("pkgs=%s" % (args.package))
+ log.debug("pkgs=%s", args.package)
latest_builds = []
for p in args.package:
- latest_builds += kojisession.listTagged(args.tag, latest=True, package=p)
+ latest_builds += kojisession.listTagged(args.tag, latest=True,
+ package=p)
num = len(latest_builds)
-#print "latest builds=%d" % (num)
-#print("%s" % (latest_builds))
+log.debug("latest builds=%d", num)
+log.debug(str(latest_builds))
for build in latest_builds:
latest_evr = (str(build['epoch']), build['version'], build['release'])
@@ -75,9 +81,16 @@ for build in latest_builds:
builds = kojisession.listTagged(args.tag, package=build['package_name'])
for b in builds:
- evr = (str(b['epoch']), b['version'], b['release'])
- res = _rpmvercmp(latest_evr, evr)
- if res == -1:
- print("\tlatest is %s, but higher exists - %s" % (build['nvr'], b['nvr']))
- if args.fix is True:
- print("\t\tretagging (no-op)")
+ evr = (str(b['epoch']), b['version'], b['release'])
+ res = _rpmvercmp(latest_evr, evr)
+ if res == -1:
+ print("\tlatest is %s, but higher exists - %s" % (build['nvr'],
+ b['nvr']))
+ if args.fix is True:
+ cmd = ["koji", "untag-build", args.tag, b['nvr']]
+ print("running: " + " ".join(cmd))
+ subprocess.check_call(cmd)
+
+ cmd = ["koji", "tag-build", args.tag, b['nvr']]
+ print("running: " + " ".join(cmd))
+ subprocess.check_call(cmd)
9 years, 7 months
[releng] Sync buildrawhide -> buildbranched
by Till Maas
commit 50377fa0e6b3cd9569e72797db2933fed6e2f72a
Author: Till Maas <opensource(a)till.name>
Date: Mon Sep 29 21:05:28 2014 +0200
Sync buildrawhide -> buildbranched
scripts/buildbranched | 84 ++++++++++++++-----------------------------------
1 files changed, 24 insertions(+), 60 deletions(-)
---
diff --git a/scripts/buildbranched b/scripts/buildbranched
index 8373fa6..e020a3e 100755
--- a/scripts/buildbranched
+++ b/scripts/buildbranched
@@ -64,13 +64,20 @@ function log()
echo "$(date --utc) build${DIST}: ${message}" >> "${logfile}"
}
+function send_fedmsg()
+{
+ topic="${1}"
+ log="${2}"
+ # Emit a message using bodhi's cert (since we should be running as "masher").
+ echo "{\"log\": \"${log}\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
+ --cert-prefix bodhi \
+ --modname compose \
+ --topic "${DIST}.${topic}" \
+ --json-input
+}
+
log "started"
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"start\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic branched.start \
- --json-input
+send_fedmsg start start
log "blocking retired packages"
./block_retired.py
@@ -88,7 +95,9 @@ log "git clone of comps finished"
[ -f $logdir/comps-f$BRANCHED.xml ] || exit 1
+log "mock init"
mock -r $MOCKCONFIG --uniqueext=$DATE --init
+log "mock install base packages"
mock -r $MOCKCONFIG --uniqueext=$DATE --no-clean --install koji yum createrepo cvs make intltool findutils mash yum-utils rsync repoview hardlink
# until we move to bodhi lets not be strict about the gpg keys
mock -r $MOCKCONFIG --uniqueext=$DATE --chroot "sed -i -e 's|strict_keys = True|strict_keys = False|g' /etc/mash/branched.mash"
@@ -103,23 +112,13 @@ mock -r $MOCKCONFIG --uniqueext=$DATE --chroot "sed -i -e 's|strict_keys = True|
log "mock setup /etc/hosts"
mock -r $MOCKCONFIG --uniqueext=$DATE --copyin /etc/hosts /etc/hosts >/dev/null 2>&1 # this reports to fail, but actually works
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"start\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic branched.mash.start \
- --json-input
+send_fedmsg start mash.start
log "starting mash"
# Drop privs here so that we run as the masher UID
mock -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "mash $MASHOPTS -p $TREEPREFIX/development/$BRANCHED -o /mnt/koji/mash/branched-$DATE --compsfile $logdir/comps-f$BRANCHED.xml $BRANCHED$EXPANDARCH > $logdir/mash.log 2>&1" || exit 1
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"done\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic branched.mash.complete \
- --json-input
+send_fedmsg done mash.complete
log "finished mash"
log "starting hardlink"
@@ -134,12 +133,7 @@ log "finished repodiff"
mock -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "/usr/share/mash/spam-o-matic $DEPOPTS /mnt/koji/mash/branched-$DATE/$BRANCHED$EXPANDARCH >$logdir/depcheck" &
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"start\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic branched.pungify.start \
- --json-input
+send_fedmsg start pungify.start
log "starting critppath generation"
#only run critpath on primary arch
@@ -156,12 +150,7 @@ for basearch in armhfp i386 x86_64 ; do
wait
log "finished pungify"
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"done\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic branched.pungify.complete \
- --json-input
+send_fedmsg done pungify.complete
log "starting build_composeinfo"
echo "Running build_composeinfo"
@@ -184,12 +173,7 @@ echo "Compose finished at $(date --utc)" >> $logdir/finish
echo >> $logdir/finish
log "finished compose"
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"start\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic branched.rsync.start \
- --json-input
+send_fedmsg start rsync.start
# data
$RSYNCPREFIX /usr/bin/rsync $RSYNC_OPTS --link-dest=$TREEPREFIX/development/rawhide/ --exclude repodata/ /mnt/koji/mash/branched-$DATE/$BRANCHED$EXPANDARCH/ $DESTPATH
@@ -200,12 +184,7 @@ if [ "$?" = "0" ]; then
export mail=0
fi
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"done\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic branched.rsync.complete \
- --json-input
+send_fedmsg done rsync.complete
log "starting sending email report"
if [ "$mail" = "0" ]; then
@@ -214,12 +193,7 @@ fi
log "finished sending email report"
[ -z "$ARCH" ] && {
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"start\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic branched.image.start \
- --json-input
+send_fedmsg start image.start
log "started checking out spin-kickstarts"
pushd ../
@@ -235,12 +209,7 @@ log "finished starting building live/arm/cloud images"
popd
popd
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"done\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic branched.image.complete \
- --json-input
+send_fedmsg done image.complete
}
for koji in arm ppc s390
@@ -255,11 +224,6 @@ do
scripts/srpm-excluded-arch.py -a $arches --path /mnt/koji/mash/branched-$DATE/$BRANCHED$EXPANDARCH/source/SRPMS/*/ >$logdir/excludearch-$koji.log
done
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"done\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic branched.complete \
- --json-input
+send_fedmsg done complete
exit 0
9 years, 7 months
[releng] buildrawhide: Simplify fedmsg sending
by Till Maas
commit fe213ed64e17e47419417aa7371384fe95b74a04
Author: Till Maas <opensource(a)till.name>
Date: Mon Sep 29 21:02:18 2014 +0200
buildrawhide: Simplify fedmsg sending
scripts/buildrawhide | 82 +++++++++++++------------------------------------
1 files changed, 22 insertions(+), 60 deletions(-)
---
diff --git a/scripts/buildrawhide b/scripts/buildrawhide
index 79a1a94..bbefad7 100755
--- a/scripts/buildrawhide
+++ b/scripts/buildrawhide
@@ -64,13 +64,20 @@ function log()
echo "$(date --utc) build${DIST}: ${message}" >> "${logfile}"
}
+function send_fedmsg()
+{
+ topic="${1}"
+ log="${2}"
+ # Emit a message using bodhi's cert (since we should be running as "masher").
+ echo "{\"log\": \"${log}\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
+ --cert-prefix bodhi \
+ --modname compose \
+ --topic "${DIST}.${topic}" \
+ --json-input
+}
+
log "started"
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"start\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic rawhide.start \
- --json-input
+send_fedmsg start start
log "blocking retired packages"
./block_retired.py
@@ -96,23 +103,13 @@ mock -r $MOCKCONFIG --uniqueext=$DATE --no-clean --install koji yum createrepo c
log "mock setup /etc/hosts"
mock -r $MOCKCONFIG --uniqueext=$DATE --copyin /etc/hosts /etc/hosts >/dev/null 2>&1 # this reports to fail, but actually works
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"start\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic rawhide.mash.start \
- --json-input
+send_fedmsg start mash.start
log "starting mash"
# Drop privs here so that we run as the masher UID
mock -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "mash $MASHOPTS -p $TREEPREFIX/development/rawhide -o /mnt/koji/mash/rawhide-$DATE --compsfile $logdir/comps-rawhide.xml rawhide$EXPANDARCH > $logdir/mash.log 2>&1" || exit 1
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"done\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic rawhide.mash.complete \
- --json-input
+send_fedmsg done mash.complete
log "finished mash"
log "starting hardlink"
@@ -129,12 +126,7 @@ log "starting spam-o-matic"
mock -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "/usr/share/mash/spam-o-matic $DEPOPTS /mnt/koji/mash/rawhide-$DATE/rawhide$EXPANDARCH >$logdir/depcheck" &
log "finished spam-o-matic"
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"start\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic rawhide.pungify.start \
- --json-input
+send_fedmsg start pungify.start
log "starting critppath generation"
#only run critpath on primary arch
@@ -151,12 +143,7 @@ for basearch in armhfp i386 x86_64 ; do
wait
log "finished pungify"
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"done\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic rawhide.pungify.complete \
- --json-input
+send_fedmsg done pungify.complete
log "starting build_composeinfo"
echo "Running build_composeinfo"
@@ -180,12 +167,7 @@ echo "Compose finished at $(date --utc)" >> $logdir/finish
echo >> $logdir/finish
log "finished compose"
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"start\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic rawhide.rsync.start \
- --json-input
+send_fedmsg start rsync.start
log "started rawhide compose sync"
# data
@@ -198,12 +180,7 @@ if [ "$?" = "0" ]; then
export mail=0
fi
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"done\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic rawhide.rsync.complete \
- --json-input
+send_fedmsg done rsync.complete
log "starting sending email report"
if [ "$mail" = "0" ]; then
@@ -214,12 +191,7 @@ fi
log "finished sending email report"
[ -z "$ARCH" ] && {
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"start\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic rawhide.image.start \
- --json-input
+send_fedmsg start image.start
log "started checking out spin-kickstarts"
pushd ../
@@ -235,12 +207,7 @@ log "finished starting building live/arm/cloud images"
popd
popd
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"done\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic rawhide.image.complete \
- --json-input
+send_fedmsg done image.complete
}
for koji in arm ppc s390
@@ -255,11 +222,6 @@ do
scripts/srpm-excluded-arch.py -a $arches --path /mnt/koji/mash/rawhide-$DATE/$BRANCHED$EXPANDARCH/source/SRPMS/*/ >$logdir/excludearch-$koji.log
done
-# Emit a message using bodhi's cert (since we should be running as "masher").
-echo "{\"log\": \"done\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fedmsg-logger \
- --cert-prefix bodhi \
- --modname compose \
- --topic rawhide.complete \
- --json-input
+send_fedmsg done complete
exit 0
9 years, 7 months
[releng] run block_retired.py in buildbranched/buildrawhide
by Till Maas
commit 46a7ecfcc7280780c854b1df9998305e0d5091ae
Author: Till Maas <opensource(a)till.name>
Date: Mon Sep 29 20:48:10 2014 +0200
run block_retired.py in buildbranched/buildrawhide
scripts/buildbranched | 3 +++
scripts/buildrawhide | 3 +++
2 files changed, 6 insertions(+), 0 deletions(-)
---
diff --git a/scripts/buildbranched b/scripts/buildbranched
index 375f020..8373fa6 100755
--- a/scripts/buildbranched
+++ b/scripts/buildbranched
@@ -72,6 +72,9 @@ echo "{\"log\": \"start\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fe
--topic branched.start \
--json-input
+log "blocking retired packages"
+./block_retired.py
+
log "git clone of comps started"
pushd $TMPDIR
git clone git://git.fedorahosted.org/comps.git && {
diff --git a/scripts/buildrawhide b/scripts/buildrawhide
index e9124a9..79a1a94 100755
--- a/scripts/buildrawhide
+++ b/scripts/buildrawhide
@@ -72,6 +72,9 @@ echo "{\"log\": \"start\", \"branch\": \"$BRANCHED\", \"arch\": \"$ARCH\"}" | fe
--topic rawhide.start \
--json-input
+log "blocking retired packages"
+./block_retired.py
+
log "git clone of comps started"
pushd $TMPDIR
git clone git://git.fedorahosted.org/comps.git && {
9 years, 7 months
[releng] Add block_retired.py
by Till Maas
commit 3cc16616c7eabf920b68f3da800f700c1279fc2b
Author: Till Maas <opensource(a)till.name>
Date: Mon Sep 29 20:47:18 2014 +0200
Add block_retired.py
Helper script to block (and untag/unblock in build tag for EPEL) all
retired packages.
Reference: https://fedorahosted.org/rel-eng/ticket/5914
scripts/block_retired.py | 270 ++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 270 insertions(+), 0 deletions(-)
---
diff --git a/scripts/block_retired.py b/scripts/block_retired.py
new file mode 100755
index 0000000..2556980
--- /dev/null
+++ b/scripts/block_retired.py
@@ -0,0 +1,270 @@
+#!/usr/bin/python -tt
+# vim: fileencoding=utf8 foldmethod=marker
+# SPDX-License-Identifier: GPL-2.0+
+# {{{ License header: GPLv2+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+# }}}
+
+import argparse
+import datetime
+import getpass
+import logging
+import subprocess
+import time
+
+import koji
+import pkgdb2client
+
+
+log = logging.getLogger(__name__)
+RETIRING_BRANCHES = ["el5", "el6", "epel7", "f21", "master"]
+
+
+class ReleaseMapper(object):
+ BRANCHNAME = 0
+ KOJI_TAG = 1
+ EPEL_BUILD_TAG = 2
+
+ def __init__(self):
+
+ # git branchname, koji tag, pkgdb version
+ self.mapping = (
+ ("master", "f22", ""),
+ ("f21", "f21", ""),
+ ("f20", "f20", ""),
+ ("f19", "f19", ""),
+ ("f18", "f18", ""),
+ ("epel7", "epel7", "epel7-build"),
+ ("el6", "dist-6E-epel", "dist-6E-epel-build"),
+ ("el5", "dist-5E-epel", "dist-5E-epel-build"),
+ )
+
+ def branchname(self, key=""):
+ return self.lookup(key, self.BRANCHNAME)
+
+ def koji_tag(self, key=""):
+ return self.lookup(key, self.KOJI_TAG)
+
+ def epel_build_tag(self, key=""):
+ return self.lookup(key, self.EPEL_BUILD_TAG)
+
+ def lookup(self, key, column):
+ if key:
+ key = key.lower()
+ for row in self.mapping:
+ for c in row:
+ if c.lower() == key:
+ return row[column]
+ else:
+ return [row[column] for row in self.mapping]
+ return None
+
+
+def blocked_packages(branch="master"):
+ mapper = ReleaseMapper()
+ tag = mapper.koji_tag(branch)
+ kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub')
+ pkglist = kojisession.listPackages(tagID=tag, inherited=True)
+ blocked = [p["package_name"] for p in pkglist if p["blocked"]]
+ return blocked
+
+
+def get_retired_packages(branch="master"):
+ pkgdb = pkgdb2client.PkgDB()
+ retiredresponse = pkgdb.get_packages(
+ "", branches=branch, page="all", status="Retired")
+ retiredinfo = retiredresponse["packages"]
+ retiredpkgs = [p["name"] for p in retiredinfo]
+ return retiredpkgs
+
+
+def pkgdb_retirement_status(package, branch="master"):
+ """ Returns retirement info for `package` in `branch`
+
+ :returns: dict: retired: True - if retired, False if not, None if
+ there was an error, status_change: last status change as datetime object
+ """
+
+ pkgdb = pkgdb2client.PkgDB()
+ retired = None
+ status_change = None
+ try:
+ pkgdbresult = pkgdb.get_package(package, branches=branch)
+ if pkgdbresult["output"] == "ok":
+ for pkginfo in pkgdbresult["packages"]:
+ if pkginfo["package"]["name"] == package:
+ if pkginfo["status"] == "Retired":
+ retired = True
+ else:
+ retired = False
+ status_change = datetime.datetime.fromtimestamp(
+ pkginfo["status_change"])
+ break
+ except:
+ pass
+
+ return dict(retired=retired, status_change=status_change)
+
+
+def get_retirement_info(message):
+ """ Check whether a message is a retire message.
+
+ :param message: Message to check
+ :returns: (str, str, bool) or (None, None, None): package name and
+ branch the package was retired on, bool: True: package was retired, False:
+ package was unretired
+
+ """
+ if message['topic'] == \
+ u'org.fedoraproject.prod.pkgdb.package.update.status':
+ msg = message['msg']
+ pkgname = msg['package_listing']['package']['name']
+ branch = msg['package_listing']['collection']['branchname']
+ res = dict(name=pkgname, branch=branch)
+ if msg["prev_status"] != "Retired" and msg["status"] == "Retired":
+ res["retired"] = True
+ elif msg["prev_status"] == "Retired" and \
+ msg["status"] != "Retired":
+ res["retired"] = False
+ return res
+ return None
+
+
+def block_package(packages, branch="master"):
+ if isinstance(packages, basestring):
+ packages = [packages]
+
+ if len(packages) == 0:
+ return None
+
+ mapper = ReleaseMapper()
+ tag = mapper.koji_tag(branch)
+ cmd = ["koji", "block-pkg", tag] + packages
+ log.debug("Running: %s", " ".join(cmd))
+ subprocess.check_call(cmd)
+
+ epel_build_tag = mapper.epel_build_tag(branch)
+
+ if epel_build_tag:
+ cmd = ["koji", "untag-build", "--all", tag] + packages
+ log.debug("Running: %s", " ".join(cmd))
+ subprocess.check_call(cmd)
+
+ cmd = ["koji", "unblock-pkg", epel_build_tag] + packages
+ log.debug("Running: %s", " ".join(cmd))
+ subprocess.check_call(cmd)
+
+
+def handle_message(message, retiring_branches=RETIRING_BRANCHES):
+ messageinfo = get_retirement_info(message)
+ msg_id = message["msg_id"]
+ if messageinfo is None:
+ return None
+
+ if messageinfo["retired"] is False:
+ return False
+
+ branch = messageinfo["branch"]
+ if branch not in retiring_branches:
+ log.error("Message '%s' for the wrong branch '%s'", msg_id,
+ branch)
+ return None
+
+ package = messageinfo["name"]
+
+ pkgdbinfo = pkgdb_retirement_status(package, branch)
+
+ if pkgdbinfo["retired"] is not True:
+ log.error("Processing '%s', package '%s' not retired",
+ msg_id, package)
+
+ log.debug("'%s' retired on '%s'", package, pkgdbinfo["status_change"])
+ return block_package(package, branch)
+
+
+def block_all_retired(branches=RETIRING_BRANCHES):
+ for branch in branches:
+ log.debug("Processing branch %s", branch)
+ retired = get_retired_packages(branch)
+ blocked = blocked_packages(branch)
+
+ unblocked = []
+ for pkg in retired:
+ if pkg not in blocked:
+ unblocked.append(pkg)
+
+ if unblocked:
+ log.info("Blocked packages %s on %s", unblocked, branch)
+ block_package(unblocked, branch)
+
+
+class SubjectSMTPHandler(logging.handlers.SMTPHandler):
+ # Class copied from autosigner.py
+ subject_prefix = ""
+
+ def getSubject(self, record):
+ first_line = record.message.split("\n")[0]
+ fmt = self.subject_prefix + "{0.levelname}: {first_line}"
+
+ return fmt.format(record, first_line=first_line)
+
+
+def setup_logging(debug=False, mail=False):
+ if debug:
+ log.setLevel(logging.DEBUG)
+ else:
+ log.setLevel(logging.INFO)
+
+ formatter = logging.Formatter(
+ '%(asctime)s: %(levelname)s: %(message)s',
+ )
+ # Log in UTC
+ formatter.converter = time.gmtime
+
+ console_logger = logging.StreamHandler()
+ if debug:
+ console_logger.setLevel(logging.DEBUG)
+ else:
+ console_logger.setLevel(logging.INFO)
+ console_logger.setFormatter(formatter)
+ log.addHandler(console_logger)
+
+ if mail:
+ # FIXME: Make this a config option
+ fedora_user = getpass.getuser()
+ mail_logger = SubjectSMTPHandler(
+ "127.0.0.1", fedora_user, [fedora_user], "block_retired event")
+ if debug:
+ mail_logger.setLevel(logging.DEBUG)
+ mail_logger.setFormatter(formatter)
+ mail_logger.subject_prefix = "Package Blocker: "
+ log.addHandler(mail_logger)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser("Block retired packages")
+ parser.add_argument("--debug", default=False, action="store_true")
+ parser.add_argument("packages", nargs="*", metavar="package",
+ help="Packages to block, default all retired packages")
+ parser.add_argument(
+ "--branch", default="master",
+ help="Branch to retire specified packages on, default: %(default)s")
+ args = parser.parse_args()
+
+ setup_logging(args.debug)
+
+ if not args.packages:
+ block_all_retired()
+ else:
+ block_package(args.packages, args.branch)
9 years, 7 months
