Re: firewall rules on builders (iptables, firewalld, libvirt...)
by Paul W. Frields
On Tue, Oct 28, 2014 at 08:50:29AM -0600, Stephen John Smoogen wrote:
> On 28 October 2014 08:04, Matthew Miller <mattdm(a)fedoraproject.org> wrote:
>
> > It's my understanding (Dennis please correct if I'm wrong) that the
> > problem with cloud image creation was due to libvirt iptables rules
> > being lost when iptables was restarted. This is a fundamental known
> > issue (see last paragraph of <http://libvirt.org/firewall.html>), and
> > one of the things firewalld was meant to solve.
> >
> > Dennis says that there are lot of complicated rules on the builders
> > making switching to firewalld difficult. One possibility might be to
> > move those complicated rules from the builders to a network firewall,
> > and keep the host rules simple and functional. But that's probably a
> > big undertaking.
> >
> >
> It would be.. It would be creating a new network for these boxes, putting
> the hardware behind such a firewall, setting up routing for such devices
> etc etc. [Plus a budget needed for that hardware.]
>
>
> > In the meantime, any time iptables is restarted or reloaded, libvirt
> > needs a SIGHUP. (I suppose this means: ansible playbooks and also added
> > to any manual procedures.)
> >
> > That actually would be 'easier' to set up even if it is a cron job which
> checks to see if a marker is in iptables and if not sends a sighup to
> libvirt
The firewalld rich language is probably also worth looking into -- if
for no other reason than to determine whether it is capable of
handling these use cases. If not, we should file RFEs upstream
because we I'm betting we're not *that* special. :-)
--
Paul W. Frields http://paul.frields.org/
gpg fingerprint: 3DA6 A0AC 6D58 FEC4 0233 5906 ACDB C937 BD11 3717
http://redhat.com/ - - - - http://pfrields.fedorapeople.org/
The open source story continues to grow: http://opensource.com
8 years, 7 months
[releng] remove unneeded header infor for licensing, the SPDX header defines the license entirely
by Dennis Gilmore
commit cf18b3ebaa74c1c40af31b8e71112a692f82cb50
Author: Dennis Gilmore <dennis(a)ausil.us>
Date: Thu Oct 30 22:13:48 2014 -0500
remove unneeded header infor for licensing, the SPDX header defines the license entirely
scripts/autosigner.py | 14 --------------
scripts/block_retired.py | 14 --------------
scripts/check_sigul.py | 14 --------------
scripts/fedretire | 14 --------------
4 files changed, 0 insertions(+), 56 deletions(-)
---
diff --git a/scripts/autosigner.py b/scripts/autosigner.py
index d2cc120..7752f13 100755
--- a/scripts/autosigner.py
+++ b/scripts/autosigner.py
@@ -1,20 +1,6 @@
#!/usr/bin/python -tt
# vim: fileencoding=utf8 foldmethod=marker
# SPDX-License-Identifier: GPL-2.0+
-#{{{ License header: GPLv2+
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#}}}
import argparse
import datetime
diff --git a/scripts/block_retired.py b/scripts/block_retired.py
index 7685719..690ea7a 100755
--- a/scripts/block_retired.py
+++ b/scripts/block_retired.py
@@ -1,20 +1,6 @@
#!/usr/bin/python -tt
# vim: fileencoding=utf8 foldmethod=marker
# SPDX-License-Identifier: GPL-2.0+
-# {{{ License header: GPLv2+
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-# }}}
import argparse
import datetime
diff --git a/scripts/check_sigul.py b/scripts/check_sigul.py
index 63bbf2b..47568dd 100755
--- a/scripts/check_sigul.py
+++ b/scripts/check_sigul.py
@@ -1,20 +1,6 @@
#!/usr/bin/python -tt
# vim: fileencoding=utf8
# SPDX-License-Identifier: GPL-2.0+
-# {{{ License header: GPLv2+
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-# }}}
import getpass
import logging
diff --git a/scripts/fedretire b/scripts/fedretire
index c277f77..243c1f5 100755
--- a/scripts/fedretire
+++ b/scripts/fedretire
@@ -1,20 +1,6 @@
#!/usr/bin/python -tt
# vim: fileencoding=utf8
# SPDX-License-Identifier: GPL-2.0+
-# {{{ License header: GPLv2+
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-# }}}
import argparse
import os
8 years, 7 months
[releng] Updated tag rawhide-stable
by Dennis Gilmore
The lightweight tag 'rawhide-stable' was updated to point to:
46a1fd5... adjust the url based on if its a nightly compose or TC/RC o
It previously pointed to:
8703091... adjust the url based on if its a nightly compose or TC/RC o
NOTE: People pulling from the repository will not get the new tag.
For more information, please see:
http://live.gnome.org/Git/Help/TagUpdates
8 years, 7 months
[releng] adjust the url based on if its a nightly compose or TC/RC one
by Dennis Gilmore
commit 46a1fd5667b93bd936dd28e591f14578c8e213cd
Author: Dennis Gilmore <dennis(a)ausil.us>
Date: Thu Oct 30 02:17:41 2014 -0500
adjust the url based on if its a nightly compose or TC/RC one
scripts/build-cloud-images | 9 ++++++---
scripts/buildbranched | 2 +-
scripts/buildrawhide | 2 +-
3 files changed, 8 insertions(+), 5 deletions(-)
---
diff --git a/scripts/build-cloud-images b/scripts/build-cloud-images
index 01b6bc5..1a9582b 100755
--- a/scripts/build-cloud-images
+++ b/scripts/build-cloud-images
@@ -22,11 +22,16 @@ TARGET=f$VERSION-candidate
BRANCH=branched
fi
+if [ "$COMPOSE" == "nightly" ]; then
+ url=http://kojipkgs.fedoraproject.org/mash/$BRANCH-$BUILD/$VERSION/\$arch...
+else
+ url=http://compose-x86-02.phx2.fedoraproject.org/compose/$VERSION$COMPOSE...
+fi
+
for spin in Base
do
declare -l lspin
lspin=$spin
- url=http://compose-x86-02.phx2.fedoraproject.org/compose/$VERSION$COMPOSE...
kickstart=fedora-cloud-$lspin-$GITHASH.ks
ksflatten -c fedora-cloud-$lspin.ks -o $kickstart >& /dev/null
echo "url --url=$url"|sed -e 's|$arch|$basearch|g' >> $kickstart
@@ -38,7 +43,6 @@ for spin in Atomic
do
declare -l lspin
lspin=$spin
- url=http://compose-x86-02.phx2.fedoraproject.org/compose/$VERSION$COMPOSE...
kickstart=fedora-cloud-$lspin-$GITHASH.ks
ksflatten -c fedora-cloud-$lspin.ks -o $kickstart >& /dev/null
echo "url --url=$url"|sed -e 's|$arch|$basearch|g' >> $kickstart
@@ -50,7 +54,6 @@ for spin in Base
do
declare -l lspin
lspin=$spin
- url=http://compose-x86-02.phx2.fedoraproject.org/compose/$VERSION$COMPOSE...
kickstart=fedora-docker-$lspin-$GITHASH.ks
ksflatten -c fedora-docker-$lspin.ks -o $kickstart >& /dev/null
echo "url --url=$url"|sed -e 's|$arch|$basearch|g' >> $kickstart
diff --git a/scripts/buildbranched b/scripts/buildbranched
index fd6ef49..d34b3e6 100755
--- a/scripts/buildbranched
+++ b/scripts/buildbranched
@@ -214,7 +214,7 @@ log "finished checking out spin-kickstarts"
log "started building live/arm/cloud images"
../releng/scripts/build-livecds $BRANCHED $DATE $BRANCHED
../releng/scripts/build-arm-images $BRANCHED $DATE $BRANCHED
-../releng/scripts/build-cloud-images $BRANCHED $DATE $BRANCHED
+../releng/scripts/build-cloud-images $BRANCHED $DATE $BRANCHED nightly
log "finished starting building live/arm/cloud images"
popd
popd
diff --git a/scripts/buildrawhide b/scripts/buildrawhide
index 31c789f..0ab6ad3 100755
--- a/scripts/buildrawhide
+++ b/scripts/buildrawhide
@@ -206,7 +206,7 @@ log "finished checking out spin-kickstarts"
log "started building live/arm/cloud images"
../releng/scripts/build-livecds $BRANCHED $DATE $BRANCHED
../releng/scripts/build-arm-images $BRANCHED $DATE $BRANCHED
-../releng/scripts/build-cloud-images $BRANCHED $DATE $BRANCHED
+../releng/scripts/build-cloud-images $BRANCHED $DATE $BRANCHED nightly
log "finished starting building live/arm/cloud images"
popd
popd
8 years, 7 months
[releng] Updated tag rawhide-stable
by Dennis Gilmore
The lightweight tag 'rawhide-stable' was updated to point to:
8703091... adjust the url based on if its a nightly compose or TC/RC o
It previously pointed to:
b3c72ca... add sigul check script
NOTE: People pulling from the repository will not get the new tag.
For more information, please see:
http://live.gnome.org/Git/Help/TagUpdates
8 years, 7 months
[releng] check_sigul: Add debug info, fix it
by Till Maas
commit 4b4b26aab2f2fa39c877a39f5f8f26b76fd66457
Author: Till Maas <opensource(a)till.name>
Date: Wed Oct 29 20:45:03 2014 +0100
check_sigul: Add debug info, fix it
scripts/check_sigul.py | 12 ++++++++++--
1 files changed, 10 insertions(+), 2 deletions(-)
---
diff --git a/scripts/check_sigul.py b/scripts/check_sigul.py
index 73c3dd3..63bbf2b 100755
--- a/scripts/check_sigul.py
+++ b/scripts/check_sigul.py
@@ -44,7 +44,7 @@ if __name__ == "__main__":
mail_logger = SubjectSMTPHandler(
"127.0.0.1", fedora_user, [fedora_user], "Sigul check log event")
mail_logger.subject_prefix = "Sigul check: "
- mail_logger.setLevel(logging.DEBUG)
+ mail_logger.setLevel(logging.WARNING)
mail_logger.setFormatter(formatter)
log.addHandler(mail_logger)
@@ -68,7 +68,13 @@ if __name__ == "__main__":
for key, helper in helpers.items():
res = helper.get_public_key()
ret, pubkey, errors = res
- if status.setdefault(key, res) != res:
+ if ret != 0 or errors:
+ log.debug("Key '{}' not working {}:{}".format(
+ key, ret, errors))
+ else:
+ log.debug("Key '{}' working".format(key))
+
+ if status.get(key, res) != res:
if ret != 0 or errors:
log.error(
"Sigul for key '{}' stopped working: {}:{}".format(
@@ -77,4 +83,6 @@ if __name__ == "__main__":
log.warning(
"Sigul for key '{}' resumed working".format(key))
+ status[key] = res
+
time.sleep(600)
8 years, 7 months
[PATCH] Replace bare 'mock' with the full path.
by Ralph Bean
I hit issues on composer.stg where the $PATH was configured to have
/usr/sbin/mock precede /usr/bin/mock.
This patch makes it explicit.
---
scripts/buildbranched | 25 +++++++++++++------------
scripts/buildrawhide | 19 ++++++++++---------
2 files changed, 23 insertions(+), 21 deletions(-)
diff --git a/scripts/buildbranched b/scripts/buildbranched
index 450bf43..a5bfd00 100755
--- a/scripts/buildbranched
+++ b/scripts/buildbranched
@@ -20,6 +20,7 @@ MASHDIR="/mnt/koji/mash/${DIST}-$DATE"
EXPANDARCH=""
DEPOPTS="--treename F-$BRANCHED"
MASHOPTS=""
+MOCK="/usr/bin/mock"
MOCKCONFIG="fedora-${DIST}-compose-i386"
TOMAIL="devel(a)lists.fedoraproject.org test(a)lists.fedoraproject.org"
SUBJECT='F-'$BRANCHED' Branched report: '$DATE' changes'
@@ -109,43 +110,43 @@ log "git clone of comps finished"
[ -f "$logdir/${COMPSFILE}" ] || exit 1
log "mock init"
-mock -r $MOCKCONFIG --uniqueext=$DATE --init
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --init
log "mock install base packages"
-mock -r $MOCKCONFIG --uniqueext=$DATE --no-clean --install koji yum createrepo cvs make intltool findutils mash yum-utils rsync repoview hardlink
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --no-clean --install koji yum createrepo cvs make intltool findutils mash yum-utils rsync repoview hardlink
# until we move to bodhi lets not be strict about the gpg keys
-mock -r $MOCKCONFIG --uniqueext=$DATE --chroot "sed -i -e 's|strict_keys = True|strict_keys = False|g' /etc/mash/${DIST}.mash"
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --chroot "sed -i -e 's|strict_keys = True|strict_keys = False|g' /etc/mash/${DIST}.mash"
#disable delta close to release as we do not want them in the final trees
-mock -r $MOCKCONFIG --uniqueext=$DATE --chroot "sed -i -e 's|delta = True|delta = False|g' /etc/mash/${DIST}.mash"
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --chroot "sed -i -e 's|delta = True|delta = False|g' /etc/mash/${DIST}.mash"
# secondary arches are a bit harder to make sure everything is signed lets not be too strict, but actual release compsoes need to be.
[ -n "$ARCH" ] && {
-mock -r $MOCKCONFIG --uniqueext=$DATE --chroot "sed -i -e 's|strict_keys = True|strict_keys = False|g' /etc/mash/${DIST}.$ARCH.mash"
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --chroot "sed -i -e 's|strict_keys = True|strict_keys = False|g' /etc/mash/${DIST}.$ARCH.mash"
}
# Copy in the hosts file so that we get the right address for koji
log "mock setup /etc/hosts"
-mock -r $MOCKCONFIG --uniqueext=$DATE --copyin /etc/hosts /etc/hosts >/dev/null 2>&1 # this reports to fail, but actually works
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --copyin /etc/hosts /etc/hosts >/dev/null 2>&1 # this reports to fail, but actually works
send_fedmsg start mash.start
log "starting mash"
# Drop privs here so that we run as the masher UID
-mock -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "mash $MASHOPTS -p $TREEPREFIX/development/$BRANCHED -o ${MASHDIR} --compsfile $logdir/${COMPSFILE} $BRANCHED$EXPANDARCH > $logdir/mash.log 2>&1" || exit 1
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "mash $MASHOPTS -p $TREEPREFIX/development/$BRANCHED -o ${MASHDIR} --compsfile $logdir/${COMPSFILE} $BRANCHED$EXPANDARCH > $logdir/mash.log 2>&1" || exit 1
send_fedmsg done mash.complete
log "finished mash"
log "starting hardlink"
# hardlink the noarch deltarpms between x86_64 and i386
-mock -r $MOCKCONFIG --uniqueext=$DATE --chroot "hardlink -v -c ${MASHDIR}/$BRANCHED$EXPANDARCH"
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --chroot "hardlink -v -c ${MASHDIR}/$BRANCHED$EXPANDARCH"
log "finished hardlink"
log "starting repodiff"
-mock -r $MOCKCONFIG --uniqueext=$DATE --chroot "rm -f /var/lib/rpm/__db*"
-mock -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "/usr/bin/repodiff -s -q --new=file://${MASHDIR}/$BRANCHED$EXPANDARCH/source/SRPMS --old=file://$TREEPREFIX/development/$BRANCHED/source/SRPMS > $logdir/repodiff"
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --chroot "rm -f /var/lib/rpm/__db*"
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "/usr/bin/repodiff -s -q --new=file://${MASHDIR}/$BRANCHED$EXPANDARCH/source/SRPMS --old=file://$TREEPREFIX/development/$BRANCHED/source/SRPMS > $logdir/repodiff"
log "finished repodiff"
log "starting spam-o-matic"
-mock -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "/usr/share/mash/spam-o-matic $DEPOPTS ${MASHDIR}/$BRANCHED$EXPANDARCH >$logdir/depcheck" &
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "/usr/share/mash/spam-o-matic $DEPOPTS ${MASHDIR}/$BRANCHED$EXPANDARCH >$logdir/depcheck" &
log "finished spam-o-matic"
send_fedmsg start pungify.start
@@ -173,7 +174,7 @@ echo "Running build_composeinfo"
log "finished build_composeinfo"
log "starting mock clean"
-mock -r $MOCKCONFIG --uniqueext=$DATE --clean
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --clean
log "finished mock clean"
[ -n "$NOSYNC" ] && exit $rc
diff --git a/scripts/buildrawhide b/scripts/buildrawhide
index 96cb74a..d547a6e 100755
--- a/scripts/buildrawhide
+++ b/scripts/buildrawhide
@@ -20,6 +20,7 @@ MASHDIR="/mnt/koji/mash/${DIST}-$DATE"
EXPANDARCH=""
DEPOPTS=""
MASHOPTS=""
+MOCK="/usr/bin/mock"
MOCKCONFIG="fedora-${DIST}-compose-i386"
TOMAIL="devel(a)lists.fedoraproject.org test(a)lists.fedoraproject.org"
SUBJECT='rawhide report: '$DATE' changes'
@@ -109,34 +110,34 @@ log "git clone of comps finished"
[ -f "$logdir/${COMPSFILE}" ] || exit 1
log "mock init"
-mock -r $MOCKCONFIG --uniqueext=$DATE --init
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --init
log "mock install base packages"
-mock -r $MOCKCONFIG --uniqueext=$DATE --no-clean --install koji yum createrepo cvs make intltool findutils mash yum-utils rsync repoview hardlink
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --no-clean --install koji yum createrepo cvs make intltool findutils mash yum-utils rsync repoview hardlink
# Copy in the hosts file so that we get the right address for koji
log "mock setup /etc/hosts"
-mock -r $MOCKCONFIG --uniqueext=$DATE --copyin /etc/hosts /etc/hosts >/dev/null 2>&1 # this reports to fail, but actually works
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --copyin /etc/hosts /etc/hosts >/dev/null 2>&1 # this reports to fail, but actually works
send_fedmsg start mash.start
log "starting mash"
# Drop privs here so that we run as the masher UID
-mock -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "mash $MASHOPTS -p $TREEPREFIX/development/$BRANCHED -o ${MASHDIR} --compsfile $logdir/${COMPSFILE} $BRANCHED$EXPANDARCH > $logdir/mash.log 2>&1" || exit 1
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "mash $MASHOPTS -p $TREEPREFIX/development/$BRANCHED -o ${MASHDIR} --compsfile $logdir/${COMPSFILE} $BRANCHED$EXPANDARCH > $logdir/mash.log 2>&1" || exit 1
send_fedmsg done mash.complete
log "finished mash"
log "starting hardlink"
# hardlink the noarch deltarpms between x86_64 and i386
-mock -r $MOCKCONFIG --uniqueext=$DATE --chroot "hardlink -v -c ${MASHDIR}/$BRANCHED$EXPANDARCH/*/os/drpms/"
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --chroot "hardlink -v -c ${MASHDIR}/$BRANCHED$EXPANDARCH/*/os/drpms/"
log "finished hardlink"
log "starting repodiff"
-mock -r $MOCKCONFIG --uniqueext=$DATE --chroot "rm -f /var/lib/rpm/__db*"
-mock -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "/usr/bin/repodiff -s -q --new=file://${MASHDIR}/$BRANCHED$EXPANDARCH/source/SRPMS --old=file://$TREEPREFIX/development/$BRANCHED/source/SRPMS > $logdir/repodiff"
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --chroot "rm -f /var/lib/rpm/__db*"
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "/usr/bin/repodiff -s -q --new=file://${MASHDIR}/$BRANCHED$EXPANDARCH/source/SRPMS --old=file://$TREEPREFIX/development/$BRANCHED/source/SRPMS > $logdir/repodiff"
log "finished repodiff"
log "starting spam-o-matic"
-mock -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "/usr/share/mash/spam-o-matic $DEPOPTS ${MASHDIR}/$BRANCHED$EXPANDARCH >$logdir/depcheck" &
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --unpriv --chroot "/usr/share/mash/spam-o-matic $DEPOPTS ${MASHDIR}/$BRANCHED$EXPANDARCH >$logdir/depcheck" &
log "finished spam-o-matic"
send_fedmsg start pungify.start
@@ -165,7 +166,7 @@ echo "Running build_composeinfo"
wait
log "finished build_composeinfo"
log "starting mock clean"
-mock -r $MOCKCONFIG --uniqueext=$DATE --clean
+$MOCK -r $MOCKCONFIG --uniqueext=$DATE --clean
log "finished mock clean"
[ -n "$NOSYNC" ] && exit $rc
--
1.9.3
8 years, 7 months
[PATCH] Make compose scripts accept an environment argument.
by Ralph Bean
We can use this argument as a general switch to toggle behaviors in the
subprocesses. An example of how to use this with
scripts/block_retired.py is present, but we'll need to make more changes
like it before composes in staging are possible start-to-finish.
---
scripts/buildbranched | 12 +++++++++++-
scripts/buildrawhide | 12 +++++++++++-
2 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/scripts/buildbranched b/scripts/buildbranched
index fd6ef49..450bf43 100755
--- a/scripts/buildbranched
+++ b/scripts/buildbranched
@@ -7,6 +7,7 @@
DATE=$1
ARCH=$2
+ENVIRONMENT=$3
BRANCHED="21"
DIST=branched
@@ -39,6 +40,10 @@ RSYNCPREFIX=""
MOCKCONFIG="fedora-${DIST}-compose-ppc64"
}
+
+[ -n "$ENVIRONMENT" ] && {
+ENVIRONMENT='production';
+}
RSYNC_OPTS="-rlptDHhv --delay-updates"
DESTPATH="$TREEPREFIX/development/$BRANCHED/"
@@ -83,7 +88,12 @@ log "started"
send_fedmsg start start
log "blocking retired packages"
-./scripts/block_retired.py
+if [ "$ENVIRONMENT" == "production" ]; then
+ ./scripts/block_retired.py
+else
+ ./scripts/block_retired.py --staging
+fi
+
log "git clone of comps started"
pushd $TMPDIR
diff --git a/scripts/buildrawhide b/scripts/buildrawhide
index 31c789f..96cb74a 100755
--- a/scripts/buildrawhide
+++ b/scripts/buildrawhide
@@ -7,6 +7,7 @@
DATE=$1
ARCH=$2
+ENVIRONMENT=$3
BRANCHED=rawhide
DIST=rawhide
@@ -39,6 +40,10 @@ RSYNCPREFIX=""
MOCKCONFIG="fedora-${DIST}-compose-ppc64"
}
+
+[ -n "$ENVIRONMENT" ] && {
+ENVIRONMENT='production';
+}
RSYNC_OPTS="-rlptDHhv --delay-updates"
DESTPATH="$TREEPREFIX/development/$BRANCHED/"
@@ -83,7 +88,12 @@ log "started"
send_fedmsg start start
log "blocking retired packages"
-./scripts/block_retired.py
+if [ "$ENVIRONMENT" == "production" ]; then
+ ./scripts/block_retired.py
+else
+ ./scripts/block_retired.py --staging
+fi
+
log "git clone of comps started"
pushd $TMPDIR
--
1.9.3
8 years, 7 months
[PATCH] Update block_retired.py to work against staging.
by Ralph Bean
A --staging option is added that does 2 things:
- Points the script at staging services like koji.stg.fp.o and
admin.stg.fp.o/pkgdb.
- Limits the branches on which packages get retired in staging by
default. This is necessary only because, for instance, production
koji seems to have el5 defined while staging koji does not.
---
scripts/block_retired.py | 82 ++++++++++++++++++++++++++++++++----------------
1 file changed, 55 insertions(+), 27 deletions(-)
diff --git a/scripts/block_retired.py b/scripts/block_retired.py
index 2556980..2c87a0b 100755
--- a/scripts/block_retired.py
+++ b/scripts/block_retired.py
@@ -29,6 +29,13 @@ import pkgdb2client
log = logging.getLogger(__name__)
RETIRING_BRANCHES = ["el5", "el6", "epel7", "f21", "master"]
+PROD_ONLY_BRANCHES = ["el5", "el6", "epel7", "master"]
+
+PRODUCTION_PKGDB = "https://admin.fedoraproject.org/pkgdb"
+STAGING_PKGDB = "https://admin.stg.fedoraproject.org/pkgdb"
+
+PRODUCTION_KOJI = "https://koji.fedoraproject.org/kojihub"
+STAGING_KOJI = "https://koji.stg.fedoraproject.org/kojihub"
class ReleaseMapper(object):
@@ -36,7 +43,7 @@ class ReleaseMapper(object):
KOJI_TAG = 1
EPEL_BUILD_TAG = 2
- def __init__(self):
+ def __init__(self, staging=False):
# git branchname, koji tag, pkgdb version
self.mapping = (
@@ -45,10 +52,13 @@ class ReleaseMapper(object):
("f20", "f20", ""),
("f19", "f19", ""),
("f18", "f18", ""),
- ("epel7", "epel7", "epel7-build"),
- ("el6", "dist-6E-epel", "dist-6E-epel-build"),
- ("el5", "dist-5E-epel", "dist-5E-epel-build"),
)
+ if not staging:
+ self.mapping = self.mapping + (
+ ("epel7", "epel7", "epel7-build"),
+ ("el6", "dist-6E-epel", "dist-6E-epel-build"),
+ ("el5", "dist-5E-epel", "dist-5E-epel-build"),
+ )
def branchname(self, key=""):
return self.lookup(key, self.BRANCHNAME)
@@ -71,32 +81,42 @@ class ReleaseMapper(object):
return None
-def blocked_packages(branch="master"):
- mapper = ReleaseMapper()
+def blocked_packages(branch="master", staging=False):
+ mapper = ReleaseMapper(staging=staging)
tag = mapper.koji_tag(branch)
- kojisession = koji.ClientSession('https://koji.fedoraproject.org/kojihub')
+ url = PRODUCTION_KOJI if not staging else STAGING_KOJI
+ kojisession = koji.ClientSession(url)
pkglist = kojisession.listPackages(tagID=tag, inherited=True)
- blocked = [p["package_name"] for p in pkglist if p["blocked"]]
+ blocked = [p["package_name"] for p in pkglist if p.get("blocked")]
return blocked
-def get_retired_packages(branch="master"):
- pkgdb = pkgdb2client.PkgDB()
- retiredresponse = pkgdb.get_packages(
- "", branches=branch, page="all", status="Retired")
+def get_retired_packages(branch="master", staging=False):
+ url = PRODUCTION_PKGDB if not staging else STAGING_PKGDB
+ pkgdb = pkgdb2client.PkgDB(url)
+
+ try:
+ retiredresponse = pkgdb.get_packages(
+ "", branches=branch, page="all", status="Retired")
+ except pkgdb2client.PkgDBException as e:
+ if not "No packages found for these parameters" in str(e):
+ raise
+ return []
+
retiredinfo = retiredresponse["packages"]
retiredpkgs = [p["name"] for p in retiredinfo]
return retiredpkgs
-def pkgdb_retirement_status(package, branch="master"):
+def pkgdb_retirement_status(package, branch="master", staging=False):
""" Returns retirement info for `package` in `branch`
:returns: dict: retired: True - if retired, False if not, None if
there was an error, status_change: last status change as datetime object
"""
- pkgdb = pkgdb2client.PkgDB()
+ url = PRODUCTION_PKGDB if not staging else STAGING_PKGDB
+ pkgdb = pkgdb2client.PkgDB(url)
retired = None
status_change = None
try:
@@ -141,14 +161,16 @@ def get_retirement_info(message):
return None
-def block_package(packages, branch="master"):
+def block_package(packages, branch="master", staging=False):
if isinstance(packages, basestring):
packages = [packages]
if len(packages) == 0:
return None
- mapper = ReleaseMapper()
+ url = PRODUCTION_KOJI if not staging else STAGING_KOJI
+
+ mapper = ReleaseMapper(staging=staging)
tag = mapper.koji_tag(branch)
cmd = ["koji", "block-pkg", tag] + packages
log.debug("Running: %s", " ".join(cmd))
@@ -157,16 +179,16 @@ def block_package(packages, branch="master"):
epel_build_tag = mapper.epel_build_tag(branch)
if epel_build_tag:
- cmd = ["koji", "untag-build", "--all", tag] + packages
+ cmd = ["koji", "-s", url, "untag-build", "--all", tag] + packages
log.debug("Running: %s", " ".join(cmd))
subprocess.check_call(cmd)
- cmd = ["koji", "unblock-pkg", epel_build_tag] + packages
+ cmd = ["koji", "-s", url, "unblock-pkg", epel_build_tag] + packages
log.debug("Running: %s", " ".join(cmd))
subprocess.check_call(cmd)
-def handle_message(message, retiring_branches=RETIRING_BRANCHES):
+def handle_message(message, retiring_branches=RETIRING_BRANCHES, staging=False):
messageinfo = get_retirement_info(message)
msg_id = message["msg_id"]
if messageinfo is None:
@@ -183,21 +205,24 @@ def handle_message(message, retiring_branches=RETIRING_BRANCHES):
package = messageinfo["name"]
- pkgdbinfo = pkgdb_retirement_status(package, branch)
+ pkgdbinfo = pkgdb_retirement_status(package, branch, staging)
if pkgdbinfo["retired"] is not True:
log.error("Processing '%s', package '%s' not retired",
msg_id, package)
log.debug("'%s' retired on '%s'", package, pkgdbinfo["status_change"])
- return block_package(package, branch)
+ return block_package(package, branch, staging=staging)
-def block_all_retired(branches=RETIRING_BRANCHES):
+def block_all_retired(branches=RETIRING_BRANCHES, staging=False):
for branch in branches:
log.debug("Processing branch %s", branch)
- retired = get_retired_packages(branch)
- blocked = blocked_packages(branch)
+ if staging and branch in PROD_ONLY_BRANCHES:
+ log.warning('%s not handled in staging..' % branch)
+ continue
+ retired = get_retired_packages(branch, staging)
+ blocked = blocked_packages(branch, staging)
unblocked = []
for pkg in retired:
@@ -206,7 +231,7 @@ def block_all_retired(branches=RETIRING_BRANCHES):
if unblocked:
log.info("Blocked packages %s on %s", unblocked, branch)
- block_package(unblocked, branch)
+ block_package(unblocked, branch, staging=staging)
class SubjectSMTPHandler(logging.handlers.SMTPHandler):
@@ -260,11 +285,14 @@ if __name__ == "__main__":
parser.add_argument(
"--branch", default="master",
help="Branch to retire specified packages on, default: %(default)s")
+ parser.add_argument(
+ "--staging", default=False, action="store_true",
+ help="Talk to staging services (pkgdb), instead of production")
args = parser.parse_args()
setup_logging(args.debug)
if not args.packages:
- block_all_retired()
+ block_all_retired(staging=args.staging)
else:
- block_package(args.packages, args.branch)
+ block_package(args.packages, args.branch, staging=args.staging)
--
1.9.3
8 years, 7 months
