java/code/src/com/redhat/rhn/common/db/datasource/xml/Channel_queries.xml | 4 java/code/src/com/redhat/rhn/domain/errata/ErrataFactory.java | 355 +++-- java/code/src/com/redhat/rhn/domain/errata/test/ErrataFactoryTest.java | 7 java/code/src/com/redhat/rhn/frontend/xmlrpc/errata/ErrataHandler.java | 201 +-- java/code/src/com/redhat/rhn/manager/channel/ChannelEditor.java | 4 java/code/src/com/redhat/rhn/manager/channel/test/ChannelManagerTest.java | 8 java/code/src/com/redhat/rhn/manager/errata/test/ErrataManagerTest.java | 6 rel-eng/packages/spacewalk-utils | 2 utils/cloneByDate.py | 596 ++++++++++ utils/depsolver.py | 213 +++ utils/spacewalk-clone-by-date | 169 ++ utils/spacewalk-clone-by-date.sgml | 162 ++ utils/spacewalk-utils.spec | 7 13 files changed, 1454 insertions(+), 280 deletions(-)
New commits: commit d245bc796db28cb7b8c50aa9d64f2f5b078a68a7 Author: Justin Sherrill jsherril@redhat.com Date: Wed Feb 1 16:54:31 2012 -0500
Automatic commit of package [spacewalk-utils] release [1.7.4-1].
diff --git a/rel-eng/packages/spacewalk-utils b/rel-eng/packages/spacewalk-utils index 4cc0b5d..af9f3cd 100644 --- a/rel-eng/packages/spacewalk-utils +++ b/rel-eng/packages/spacewalk-utils @@ -1 +1 @@ -1.7.3-1 utils/ +1.7.4-1 utils/ diff --git a/utils/spacewalk-utils.spec b/utils/spacewalk-utils.spec index 618b02b..7eb87f3 100644 --- a/utils/spacewalk-utils.spec +++ b/utils/spacewalk-utils.spec @@ -1,7 +1,7 @@ %define rhnroot %{_prefix}/share/rhn
Name: spacewalk-utils -Version: 1.7.3 +Version: 1.7.4 Release: 1%{?dist} Summary: Utilities that may be run against a Spacewalk server.
@@ -76,6 +76,9 @@ rm -rf $RPM_BUILD_ROOT
%changelog +* Wed Feb 01 2012 Justin Sherrill jsherril@redhat.com 1.7.4-1 +- adding initial spacewalk-clone-by-date (jsherril@redhat.com) + * Thu Jan 05 2012 Michael Mraka michael.mraka@redhat.com 1.7.3-1 - removed map and filter from bad-function list
commit 1c18e496391340650f7327c1ffde323fa10bf65c Merge: 190a80a 73c9445 Author: Justin Sherrill jsherril@redhat.com Date: Wed Feb 1 16:49:28 2012 -0500
Merge branch 'errata-date-merge'
commit 73c9445568457cd7d8d3ec7728413f288d76fc88 Author: Justin Sherrill jsherril@redhat.com Date: Wed Feb 1 16:47:47 2012 -0500
spacewalk-utils - adding new requires (cherry picked from commit 5ed56e7be35109b7d2924e43c7a036cb2066cc04)
diff --git a/utils/spacewalk-utils.spec b/utils/spacewalk-utils.spec index 6059595..618b02b 100644 --- a/utils/spacewalk-utils.spec +++ b/utils/spacewalk-utils.spec @@ -34,6 +34,8 @@ Requires: spacewalk-admin Requires: spacewalk-certs-tools Requires: spacewalk-config Requires: spacewalk-setup +Requires: spacewalk-backend +Requires: yum-utils
%description Generic utilities that may be run against a Spacewalk server.
commit c0ccde760a8d4ee8af9b25f2a1065fd1c6d0376c Author: Justin Sherrill jsherril@redhat.com Date: Wed Feb 1 16:30:24 2012 -0500
errata date clone - adding repoclosure validation option
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index 50927cd..89f2e3f 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -19,12 +19,11 @@
import sys -import time -import copy import shutil import tempfile import xmlrpclib import pprint +import subprocess
from depsolver import DepSolver @@ -58,6 +57,24 @@ def confirm(txt, options): sys.exit(0)
+def validate(channel_labels): + tmp_dirs = {} + for label in channel_labels: + dir = "%s/rhn/repodata/%s" % ( CFG.REPOMD_CACHE_MOUNT_POINT, label) + tmp = tempfile.mkdtemp() + tmp_dirs[label] = tmp + shutil.copytree(dir, "%s/repodata/" % tmp) + + cmd = ["repoclosure"] + for label, path in tmp_dirs.items(): + cmd.append("--repofrompath=%s,%s" %(label, path)) + subprocess.call(cmd) + + for tmp in tmp_dirs.values(): + shutil.rmtree(tmp, True) + + + def main(options): xmlrpc = RemoteApi(options.server, options.username, options.password) db = DBApi() @@ -69,12 +86,25 @@ def main(options): log_debug(0, "Started spacewalk-clone-by-date") log_clean(0, pprint.pformat(cleansed))
+ + + + cloners = [] needed_channels = [] for channel_list in options.channels: tree_cloner = ChannelTreeCloner(channel_list, xmlrpc, db, options.to_date, options.blacklist) cloners.append(tree_cloner) needed_channels += tree_cloner.needing_create().values() + + + if options.validate: + if len(needed_channels) > 0: + raise UserError("Cannot validate channels that do not exist %s", str(needed_channels)) + for channel_list in options.channels: + validate(channel_list) + return +
if len(needed_channels) > 0: print "\nBy continuing the following channels will be created: " @@ -444,6 +474,8 @@ class ChannelCloner: self.remote_api.remove_packages(self.to_label, found_ids)
+ +
class RemoteApi: diff --git a/utils/spacewalk-clone-by-date b/utils/spacewalk-clone-by-date index 0f7056f..070a538 100755 --- a/utils/spacewalk-clone-by-date +++ b/utils/spacewalk-clone-by-date @@ -100,6 +100,7 @@ def parse_args(): parser.add_option("-d", "--to_date", dest="to_date", help="Clone errata to the specified date (YYYY-MM-DD)") parser.add_option("-y", "--assumeyes", dest='assumeyes', action='store_true', help="Assume yes for any prompts (unattended).") parser.add_option("-m", "--sample-config", dest='sample', action='store_true', help="Print a sample full configuration file and exit.") + parser.add_option("-v", "--validate", dest='validate', action='store_true', help="Run repoclosure on the set of specified repositories.")
(options, args) = parser.parse_args()
@@ -121,14 +122,17 @@ def parse_args(): if options.channels == None or len(options.channels) == 0: raise UserError("No channels specified. See --help for details.")
+ + if not options.validate: + options.to_date = parse_time(options.to_date) + if not options.password: options.password = getpass.getpass()
- options.to_date = parse_time(options.to_date) + return options
- def parse_time(time_str): """We need to use datetime, but python 2.4 does not support strptime(), so we have to parse ourselves""" try: @@ -154,8 +158,7 @@ def main(): except KeyboardInterrupt: systemExit(0, "\nUser interrupted process.") except UserError, error: - print error - return -1 + systemExit(-1, "\n%s" % error) return 0
commit 22c838cfc3ab4ee090ad3ab41cc3cb5308853122 Author: Justin Sherrill jsherril@redhat.com Date: Wed Feb 1 10:35:36 2012 -0500
checkstyle fixes
diff --git a/java/code/src/com/redhat/rhn/domain/errata/ErrataFactory.java b/java/code/src/com/redhat/rhn/domain/errata/ErrataFactory.java index b090228..8e81fe6 100644 --- a/java/code/src/com/redhat/rhn/domain/errata/ErrataFactory.java +++ b/java/code/src/com/redhat/rhn/domain/errata/ErrataFactory.java @@ -17,6 +17,24 @@ */ package com.redhat.rhn.domain.errata;
+import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.StringTokenizer; + +import org.apache.commons.collections.IteratorUtils; +import org.apache.log4j.Logger; +import org.hibernate.HibernateException; +import org.hibernate.Query; +import org.hibernate.Session; + import com.redhat.rhn.common.db.datasource.DataResult; import com.redhat.rhn.common.db.datasource.ModeFactory; import com.redhat.rhn.common.db.datasource.SelectMode; @@ -46,24 +64,6 @@ import com.redhat.rhn.manager.channel.ChannelManager; import com.redhat.rhn.manager.errata.ErrataManager; import com.redhat.rhn.manager.errata.cache.ErrataCacheManager;
-import org.apache.commons.collections.IteratorUtils; -import org.apache.log4j.Logger; -import org.hibernate.HibernateException; -import org.hibernate.Query; -import org.hibernate.Session; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.StringTokenizer; - /** * ErrataFactory - the singleton class used to fetch and store * com.redhat.rhn.domain.errata.Errata objects from the @@ -92,6 +92,7 @@ public class ErrataFactory extends HibernateFactory { * Get the Logger for the derived class so log messages * show up on the correct class */ + @Override protected Logger getLogger() { return log; } @@ -202,8 +203,8 @@ public class ErrataFactory extends HibernateFactory { if ((prefix.equals("CVE") || prefix.equals("CAN")) && identifier.length() > 7 && identifier.indexOf('-') == -1) { identifier = identifier.substring(0, 3) + "-" + - identifier.substring(3, 7) + "-" + - identifier.substring(7); + identifier.substring(3, 7) + "-" + + identifier.substring(7); } List erratas = ErrataFactory.lookupByCVE(identifier); retval.addAll(erratas); @@ -231,7 +232,7 @@ public class ErrataFactory extends HibernateFactory { if (unpublished.isCloned()) { published = new PublishedClonedErrata(); ((PublishedClonedErrata)published).setOriginal( - ((UnpublishedClonedErrata)unpublished).getOriginal()); + ((UnpublishedClonedErrata)unpublished).getOriginal()); } else { published = ErrataFactory.createPublishedErrata(); @@ -249,7 +250,7 @@ public class ErrataFactory extends HibernateFactory { } catch (HibernateException e) { throw new HibernateRuntimeException( - "Errors occurred while publishing errata", e); + "Errors occurred while publishing errata", e); }
//return the published errata @@ -262,131 +263,132 @@ public class ErrataFactory extends HibernateFactory { * all of the correct ErrataFile* entries. This method does push packages to * the appropriate channel. (Appropriate as defined as the channel previously * having a package with the same name). - * @param errata errata to publish + * @param errataList list of errata to publish * @param chan channel to publish it into. * @param user the user doing the pushing * @param inheritPackages include only original channel packages * @return the publsihed errata */ - public static List<Errata> publishToChannel(List<Errata> errataList, Channel chan, User user, - boolean inheritPackages) { - List<com.redhat.rhn.domain.errata.Errata> toReturn = new ArrayList<Errata>(); - for (Errata errata : errataList) { - if (!errata.isPublished()) { - errata = publish(errata); - } - errata.addChannel(chan); - errata.addChannelNotification(chan, new Date()); - - Set<Package> packagesToPush = new HashSet<Package>(); - DataResult<PackageOverview> packs; - if (inheritPackages) { - - if (!chan.isCloned()) { - throw new InvalidChannelException("Cloned channel expected: " + - chan.getLabel()); - } - Channel original = ((ClonedChannel) chan).getOriginal(); - packs = ErrataManager.listErrataChannelPacks(original, errata, user); - } - else { - packs = ErrataManager.lookupPacksFromErrataForChannel(chan, errata, user); - } - - for (PackageOverview packOver : packs) { - //lookup the Package object - Package pack = PackageFactory.lookupByIdAndUser( - packOver.getId().longValue(), user); - packagesToPush.add(pack); - } - - Errata e = publishErrataPackagesToChannel(errata, chan, user, packagesToPush); - toReturn.add(e); - } + public static List<Errata> publishToChannel(List<Errata> errataList, Channel chan, + User user, boolean inheritPackages) { + List<com.redhat.rhn.domain.errata.Errata> toReturn = new ArrayList<Errata>(); + for (Errata errata : errataList) { + if (!errata.isPublished()) { + errata = publish(errata); + } + errata.addChannel(chan); + errata.addChannelNotification(chan, new Date()); + + Set<Package> packagesToPush = new HashSet<Package>(); + DataResult<PackageOverview> packs; + if (inheritPackages) { + + if (!chan.isCloned()) { + throw new InvalidChannelException("Cloned channel expected: " + + chan.getLabel()); + } + Channel original = ((ClonedChannel) chan).getOriginal(); + packs = ErrataManager.listErrataChannelPacks(original, errata, user); + } + else { + packs = ErrataManager.lookupPacksFromErrataForChannel(chan, errata, user); + } + + for (PackageOverview packOver : packs) { + //lookup the Package object + Package pack = PackageFactory.lookupByIdAndUser( + packOver.getId().longValue(), user); + packagesToPush.add(pack); + } + + Errata e = publishErrataPackagesToChannel(errata, chan, user, packagesToPush); + toReturn.add(e); + } postPublishActions(chan, user); return toReturn; }
- - - /** - * Publish an errata to a channel but only push a small set of packages - * along with it - * - * @param errata errata to publish - * @param chan channel to publish it into. - * @param user the user doing the pushing - * @param packages the packages to push - * @return the published errata - */ - public static Errata publishToChannel(Errata errata, Channel chan, - User user, Set<Package> packages) { - if (!errata.isPublished()) { - errata = publish(errata); - } - errata.addChannel(chan); - errata = publishErrataPackagesToChannel(errata, chan, user, packages); - postPublishActions(chan, user); - return errata; - } - - - private static void postPublishActions(Channel chan, User user) { - ChannelManager.refreshWithNewestPackages(chan, "web.errata_push"); - ChannelManager.queueChannelChange(chan.getLabel(), - "java::publishErrataPackagesToChannel", user.getLogin()); - } - - + + + /** + * Publish an errata to a channel but only push a small set of packages + * along with it + * + * @param errata errata to publish + * @param chan channel to publish it into. + * @param user the user doing the pushing + * @param packages the packages to push + * @return the published errata + */ + public static Errata publishToChannel(Errata errata, Channel chan, + User user, Set<Package> packages) { + if (!errata.isPublished()) { + errata = publish(errata); + } + errata.addChannel(chan); + errata = publishErrataPackagesToChannel(errata, chan, user, packages); + postPublishActions(chan, user); + return errata; + } + + + private static void postPublishActions(Channel chan, User user) { + ChannelManager.refreshWithNewestPackages(chan, "web.errata_push"); + ChannelManager.queueChannelChange(chan.getLabel(), + "java::publishErrataPackagesToChannel", user.getLogin()); + } + + /** * Private helper method that pushes errata packages to a channel */ - private static Errata publishErrataPackagesToChannel(Errata errata, - Channel chan, User user, Set<Package> packages) { - // Much quicker to push all packages at once - List<Long> pids = new ArrayList<Long>(); - for (Package pack : packages) { - pids.add(pack.getId()); - } - ChannelManager.addPackages(chan, pids, user); - - for (Package pack : packages) { - List<ErrataFile> publishedFiles = ErrataFactory.lookupErrataFile( - errata, pack); - Map<String, ErrataFile> toAdd = new HashMap(); - if (publishedFiles.size() == 0) { - // Now create the appropriate ErrataFile object - ErrataFile publishedFile = ErrataFactory - .createPublishedErrataFile(ErrataFactory - .lookupErrataFileType("RPM"), pack - .getChecksum().getChecksum(), pack - .getNameEvra()); - publishedFile.addPackage(pack); - publishedFile.setErrata(errata); - publishedFile.setModified(new Date()); - ((PublishedErrataFile) publishedFile).addChannel(chan); - singleton.saveObject(publishedFile); - } else { - for (ErrataFile publishedFile : publishedFiles) { - String fileName = publishedFile.getFileName().substring( - publishedFile.getFileName().lastIndexOf("/") + 1); - if (!toAdd.containsKey(fileName)) { - toAdd.put(fileName, publishedFile); - ((PublishedErrataFile) publishedFile).addChannel(chan); - singleton.saveObject(publishedFile); - } - } - } - - } - ChannelFactory.save(chan); - List chanList = new ArrayList(); - chanList.add(chan.getId()); - - ErrataCacheManager.insertCacheForChannelErrataAsync(chanList, errata); - - return errata; - } + private static Errata publishErrataPackagesToChannel(Errata errata, + Channel chan, User user, Set<Package> packages) { + // Much quicker to push all packages at once + List<Long> pids = new ArrayList<Long>(); + for (Package pack : packages) { + pids.add(pack.getId()); + } + ChannelManager.addPackages(chan, pids, user); + + for (Package pack : packages) { + List<ErrataFile> publishedFiles = ErrataFactory.lookupErrataFile( + errata, pack); + Map<String, ErrataFile> toAdd = new HashMap(); + if (publishedFiles.size() == 0) { + // Now create the appropriate ErrataFile object + ErrataFile publishedFile = ErrataFactory + .createPublishedErrataFile(ErrataFactory + .lookupErrataFileType("RPM"), pack + .getChecksum().getChecksum(), pack + .getNameEvra()); + publishedFile.addPackage(pack); + publishedFile.setErrata(errata); + publishedFile.setModified(new Date()); + ((PublishedErrataFile) publishedFile).addChannel(chan); + singleton.saveObject(publishedFile); + } + else { + for (ErrataFile publishedFile : publishedFiles) { + String fileName = publishedFile.getFileName().substring( + publishedFile.getFileName().lastIndexOf("/") + 1); + if (!toAdd.containsKey(fileName)) { + toAdd.put(fileName, publishedFile); + ((PublishedErrataFile) publishedFile).addChannel(chan); + singleton.saveObject(publishedFile); + } + } + } + + } + ChannelFactory.save(chan); + List chanList = new ArrayList(); + chanList.add(chan.getId()); + + ErrataCacheManager.insertCacheForChannelErrataAsync(chanList, errata); + + return errata; + }
/** * @param org Org performing the cloning @@ -491,15 +493,15 @@ public class ErrataFactory extends HibernateFactory { Bug cloneB; if (copy.isPublished()) { //we want published bugs cloneB = ErrataManager.createNewPublishedBug(bugIn.getId(), - bugIn.getSummary(), - bugIn.getUrl()); + bugIn.getSummary(), + bugIn.getUrl()); } else { //we want unpublished bugs cloneB = ErrataManager.createNewUnpublishedBug(bugIn.getId(), - bugIn.getSummary(), - bugIn.getUrl()); + bugIn.getSummary(), + bugIn.getUrl()); } - copy.addBug(cloneB); + copy.addBug(cloneB); } }
@@ -557,8 +559,8 @@ public class ErrataFactory extends HibernateFactory { * @return new Unpublished Errata File */ public static ErrataFile createUnpublishedErrataFile(ErrataFileType ft, - String cs, - String name) { + String cs, + String name) { return createUnpublishedErrataFile(ft, cs, name, new HashSet()); }
@@ -571,9 +573,9 @@ public class ErrataFactory extends HibernateFactory { * @return new Unpublished Errata File */ public static ErrataFile createUnpublishedErrataFile(ErrataFileType ft, - String cs, - String name, - Set packages) { + String cs, + String name, + Set packages) { ErrataFile file = new UnpublishedErrataFile(); file.setFileType(ft); file.setChecksum(ChecksumFactory.safeCreate(cs, "md5")); @@ -590,8 +592,8 @@ public class ErrataFactory extends HibernateFactory { * @return new Published Errata File */ public static ErrataFile createPublishedErrataFile(ErrataFileType ft, - String cs, - String name) { + String cs, + String name) { return createPublishedErrataFile(ft, cs, name, new HashSet()); }
@@ -604,9 +606,9 @@ public class ErrataFactory extends HibernateFactory { * @return new Published Errata File */ public static ErrataFile createPublishedErrataFile(ErrataFileType ft, - String cs, - String name, - Set packages) { + String cs, + String name, + Set packages) { ErrataFile file = new PublishedErrataFile(); file.setFileType(ft); file.setChecksum(ChecksumFactory.safeCreate(cs, "md5")); @@ -626,7 +628,7 @@ public class ErrataFactory extends HibernateFactory { try { session = HibernateFactory.getSession(); retval = (ErrataFileType) session.getNamedQuery("ErrataFileType.findByLabel") - .setString("label", label).setCacheable(true).uniqueResult(); + .setString("label", label).setCacheable(true).uniqueResult(); } catch (HibernateException e) { throw new HibernateRuntimeException(e.getMessage(), e); @@ -695,14 +697,14 @@ public class ErrataFactory extends HibernateFactory { try { session = HibernateFactory.getSession(); retval = session.getNamedQuery("PublishedErrata.findByAdvisoryType") - .setString("type", advisoryType) - //Retrieve from cache if there - .setCacheable(true).list(); + .setString("type", advisoryType) + //Retrieve from cache if there + .setCacheable(true).list(); } catch (HibernateException he) { log.error("Error loading ActionArchTypes from DB", he); throw new - HibernateRuntimeException("Error loading ActionArchTypes from db"); + HibernateRuntimeException("Error loading ActionArchTypes from db"); } return retval; } @@ -718,15 +720,15 @@ public class ErrataFactory extends HibernateFactory { try { session = HibernateFactory.getSession(); retval = session.getNamedQuery("PublishedErrata.findSecurityBySynopsis") - .setString("type", ERRATA_TYPE_SECURITY) - .setString("synopsis", synopsis) - //Retrieve from cache if there - .setCacheable(true).list(); + .setString("type", ERRATA_TYPE_SECURITY) + .setString("synopsis", synopsis) + //Retrieve from cache if there + .setCacheable(true).list(); } catch (HibernateException he) { log.error("Error loading ActionArchTypes from DB", he); throw new - HibernateRuntimeException("Error loading ActionArchTypes from db"); + HibernateRuntimeException("Error loading ActionArchTypes from db"); } return retval; } @@ -742,12 +744,12 @@ public class ErrataFactory extends HibernateFactory { try { session = HibernateFactory.getSession(); retval = (Errata) session.getNamedQuery("PublishedErrata.findById") - .setLong("id", id.longValue()).uniqueResult(); + .setLong("id", id.longValue()).uniqueResult(); } catch (HibernateException he) { log.error("Error loading ActionArchTypes from DB", he); throw new - HibernateRuntimeException("Error loading ActionArchTypes from db"); + HibernateRuntimeException("Error loading ActionArchTypes from db"); } return retval; } @@ -761,24 +763,24 @@ public class ErrataFactory extends HibernateFactory { public static Errata lookupByAdvisory(String advisory) { Session session = null; Errata retval = null; - // try { - //look for a published errata first - session = HibernateFactory.getSession(); - retval = (Errata) session.getNamedQuery("PublishedErrata.findByAdvisoryName") - .setString("advisory", advisory) - .uniqueResult(); - //if nothing was found, check the unpublished errata table - if (retval == null) { - retval = (Errata) - session.getNamedQuery("UnpublishedErrata.findByAdvisoryName") - .setString("advisory", advisory) - .uniqueResult(); - } - // } - // catch (HibernateException e) { - // throw new - // HibernateRuntimeException("Error looking up errata by advisory name"); - // } + // try { + //look for a published errata first + session = HibernateFactory.getSession(); + retval = (Errata) session.getNamedQuery("PublishedErrata.findByAdvisoryName") + .setString("advisory", advisory) + .uniqueResult(); + //if nothing was found, check the unpublished errata table + if (retval == null) { + retval = (Errata) + session.getNamedQuery("UnpublishedErrata.findByAdvisoryName") + .setString("advisory", advisory) + .uniqueResult(); + } + // } + // catch (HibernateException e) { + // throw new + // HibernateRuntimeException("Error looking up errata by advisory name"); + // } return retval; }
@@ -794,20 +796,20 @@ public class ErrataFactory extends HibernateFactory { //look for a published errata first session = HibernateFactory.getSession(); retval = (Errata) session.getNamedQuery("PublishedErrata.findByAdvisory") - .setString("advisory", advisoryId) - .uniqueResult(); + .setString("advisory", advisoryId) + .uniqueResult();
if (retval == null) { retval = (Errata) - session.getNamedQuery("UnpublishedErrata.findByAdvisory") - .setString("advisory", advisoryId) - .uniqueResult(); + session.getNamedQuery("UnpublishedErrata.findByAdvisory") + .setString("advisory", advisoryId) + .uniqueResult(); } } catch (HibernateException e) {
throw new - HibernateRuntimeException("Error looking up errata by advisory name"); + HibernateRuntimeException("Error looking up errata by advisory name"); } return retval; } @@ -845,9 +847,9 @@ public class ErrataFactory extends HibernateFactory { try { session = HibernateFactory.getSession(); retval = session. - getNamedQuery("UnpublishedClonedErrata.findByOriginal") - .setParameter("original", original) - .setParameter("org", org).list(); + getNamedQuery("UnpublishedClonedErrata.findByOriginal") + .setParameter("original", original) + .setParameter("org", org).list();
if (retval == null) { retval = lookupPublishedByOriginal(org, original); @@ -856,7 +858,7 @@ public class ErrataFactory extends HibernateFactory { } catch (HibernateException e) { throw new - HibernateRuntimeException("Error looking up errata by original errata"); + HibernateRuntimeException("Error looking up errata by original errata"); } return retval; } @@ -874,12 +876,12 @@ public class ErrataFactory extends HibernateFactory { try { session = HibernateFactory.getSession(); retval = session.getNamedQuery("PublishedClonedErrata.findByOriginal") - .setParameter("original", original) - .setParameter("org", org).list(); + .setParameter("original", original) + .setParameter("org", org).list(); } catch (HibernateException e) { throw new - HibernateRuntimeException("Error looking up errata by original errata"); + HibernateRuntimeException("Error looking up errata by original errata"); } return retval; } @@ -899,12 +901,12 @@ public class ErrataFactory extends HibernateFactory { try { session = HibernateFactory.getSession(); retval = session.getNamedQuery("PublishedErrata.findSameInChannels") - .setParameter("channel_from", channelFrom) - .setParameter("channel_to", channelTo).list(); + .setParameter("channel_from", channelFrom) + .setParameter("channel_to", channelTo).list(); } catch (HibernateException e) { throw new - HibernateRuntimeException("Error looking up errata by original errata"); + HibernateRuntimeException("Error looking up errata by original errata"); } return retval; } @@ -925,12 +927,12 @@ public class ErrataFactory extends HibernateFactory { try { session = HibernateFactory.getSession(); retval = session.getNamedQuery("PublishedClonedErrata.findBrothersInChannel") - .setParameter("channel_from", channelFrom) - .setParameter("channel_to", channelTo).list(); + .setParameter("channel_from", channelFrom) + .setParameter("channel_to", channelTo).list(); } catch (HibernateException e) { throw new - HibernateRuntimeException("Error looking up errata by original errata"); + HibernateRuntimeException("Error looking up errata by original errata"); } return retval; } @@ -950,13 +952,13 @@ public class ErrataFactory extends HibernateFactory { try { session = HibernateFactory.getSession(); retval = session.getNamedQuery("PublishedErrata.findClonesInChannel") - .setParameter("channel_from", channelFrom) - .setParameter("channel_to", channelTo) - .list(); + .setParameter("channel_from", channelFrom) + .setParameter("channel_to", channelTo) + .list(); } catch (HibernateException e) { throw new - HibernateRuntimeException("Error looking up errata by original errata"); + HibernateRuntimeException("Error looking up errata by original errata"); } return retval; } @@ -1005,10 +1007,10 @@ public class ErrataFactory extends HibernateFactory { public static List lookupByChannelSorted(Org org, Channel channel) {
return HibernateFactory.getSession(). - getNamedQuery("PublishedErrata.lookupSortedByChannel") - .setParameter("org", org) - .setParameter("channel", channel) - .list(); + getNamedQuery("PublishedErrata.lookupSortedByChannel") + .setParameter("org", org) + .setParameter("channel", channel) + .list(); }
/** @@ -1025,12 +1027,12 @@ public class ErrataFactory extends HibernateFactory { String startDate, String endDate) {
return HibernateFactory.getSession(). - getNamedQuery("PublishedErrata.lookupByChannelBetweenDates") - .setParameter("org", org) - .setParameter("channel", channel) - .setParameter("start_date", startDate) - .setParameter("end_date", endDate) - .list(); + getNamedQuery("PublishedErrata.lookupByChannelBetweenDates") + .setParameter("org", org) + .setParameter("channel", channel) + .setParameter("start_date", startDate) + .setParameter("end_date", endDate) + .list(); }
/** @@ -1045,7 +1047,7 @@ public class ErrataFactory extends HibernateFactory { params.put("uid", user.getId()); params.put("set", set); return singleton.listObjectsByNamedQuery( - "PublishedErrata.lookupFromSet", params); + "PublishedErrata.lookupFromSet", params); }
diff --git a/java/code/src/com/redhat/rhn/domain/errata/test/ErrataFactoryTest.java b/java/code/src/com/redhat/rhn/domain/errata/test/ErrataFactoryTest.java index d91f5ec..e3f5b20 100644 --- a/java/code/src/com/redhat/rhn/domain/errata/test/ErrataFactoryTest.java +++ b/java/code/src/com/redhat/rhn/domain/errata/test/ErrataFactoryTest.java @@ -141,7 +141,8 @@ public class ErrataFactoryTest extends BaseTestCaseWithUser {
List<Errata> errataList = new ArrayList<Errata>(); errataList.add(e); - List<Errata> publishedList = ErrataFactory.publishToChannel(errataList, channel, user, false); + List<Errata> publishedList = ErrataFactory.publishToChannel(errataList, + channel, user, false); Errata published = publishedList.get(0); assertTrue(channel.getPackages().contains(errataPack)); List<PublishedErrataFile> errataFile = diff --git a/java/code/src/com/redhat/rhn/frontend/xmlrpc/errata/ErrataHandler.java b/java/code/src/com/redhat/rhn/frontend/xmlrpc/errata/ErrataHandler.java index 93d63e3..3eea6c6 100644 --- a/java/code/src/com/redhat/rhn/frontend/xmlrpc/errata/ErrataHandler.java +++ b/java/code/src/com/redhat/rhn/frontend/xmlrpc/errata/ErrataHandler.java @@ -17,6 +17,19 @@ */ package com.redhat.rhn.frontend.xmlrpc.errata;
+import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.collections.IteratorUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + import com.redhat.rhn.FaultException; import com.redhat.rhn.common.db.datasource.DataResult; import com.redhat.rhn.common.hibernate.HibernateFactory; @@ -57,20 +70,6 @@ import com.redhat.rhn.manager.errata.cache.ErrataCacheManager; import com.redhat.rhn.manager.rhnpackage.PackageManager; import com.redhat.rhn.manager.user.UserManager;
-import org.apache.commons.collections.IteratorUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; - -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -
/** * ErrataHandler - provides methods to access errata information. @@ -181,7 +180,7 @@ public class ErrataHandler extends BaseHandler { } return retval; } - */ + */
/** * GetDetails - Retrieves the details for a given errata. @@ -225,13 +224,13 @@ public class ErrataHandler extends BaseHandler {
if (errata.getIssueDate() != null) { errataMap.put("issue_date", - LocalizationService.getInstance() - .formatShortDate(errata.getIssueDate())); + LocalizationService.getInstance() + .formatShortDate(errata.getIssueDate())); } if (errata.getUpdateDate() != null) { errataMap.put("update_date", - LocalizationService.getInstance() - .formatShortDate(errata.getUpdateDate())); + LocalizationService.getInstance() + .formatShortDate(errata.getUpdateDate())); } if (errata.getLastModified() != null) { errataMap.put("last_modified_date", errata.getLastModified().toString()); @@ -246,17 +245,17 @@ public class ErrataHandler extends BaseHandler { errataMap.put("solution", StringUtils.defaultString(errata.getSolution())); errataMap.put("description", - StringUtils.defaultString(errata.getDescription())); + StringUtils.defaultString(errata.getDescription())); errataMap.put("synopsis", - StringUtils.defaultString(errata.getSynopsis())); + StringUtils.defaultString(errata.getSynopsis())); errataMap.put("topic", - StringUtils.defaultString(errata.getTopic())); + StringUtils.defaultString(errata.getTopic())); errataMap.put("references", - StringUtils.defaultString(errata.getRefersTo())); + StringUtils.defaultString(errata.getRefersTo())); errataMap.put("notes", - StringUtils.defaultString(errata.getNotes())); + StringUtils.defaultString(errata.getNotes())); errataMap.put("type", - StringUtils.defaultString(errata.getAdvisoryType())); + StringUtils.defaultString(errata.getAdvisoryType()));
return errataMap; @@ -334,7 +333,7 @@ public class ErrataHandler extends BaseHandler { validKeys.add("url"); if (details.containsKey("bugs")) { for (Map<String, Object> bugMap : - (ArrayList<Map<String, Object>>) details.get("bugs")) { + (ArrayList<Map<String, Object>>) details.get("bugs")) {
validateMap(validKeys, bugMap); } @@ -394,12 +393,12 @@ public class ErrataHandler extends BaseHandler { }
for (Map<String, Object> bugMap : - (ArrayList<Map<String, Object>>) details.get("bugs")) { + (ArrayList<Map<String, Object>>) details.get("bugs")) {
if (bugMap.containsKey("id") && bugMap.containsKey("summary")) { String url = ""; if (bugMap.containsKey("url")) { - url = (String) bugMap.get("url"); + url = (String) bugMap.get("url"); }
Bug bug = ErrataFactory.createPublishedBug( @@ -463,7 +462,7 @@ public class ErrataHandler extends BaseHandler { * #array_end() */ public Object[] listAffectedSystems(String sessionKey, String advisoryName) - throws FaultException { + throws FaultException {
// Get the logged in user User loggedInUser = getLoggedInUser(sessionKey); @@ -496,7 +495,7 @@ public class ErrataHandler extends BaseHandler { * #struct_end() */ public Map bugzillaFixes(String sessionKey, String advisoryName) - throws FaultException { + throws FaultException {
// Get the logged in user User loggedInUser = getLoggedInUser(sessionKey); @@ -534,7 +533,7 @@ public class ErrataHandler extends BaseHandler {
*/ public Object[] listKeywords(String sessionKey, String advisoryName) - throws FaultException { + throws FaultException {
// Get the logged in user User loggedInUser = getLoggedInUser(sessionKey); @@ -575,14 +574,14 @@ public class ErrataHandler extends BaseHandler { * #array_end() */ public Object[] applicableToChannels(String sessionKey, String advisoryName) - throws FaultException { + throws FaultException {
// Get the logged in user User loggedInUser = getLoggedInUser(sessionKey); Errata errata = lookupErrata(advisoryName, loggedInUser.getOrg());
return ErrataManager.applicableChannels(errata.getId(), - loggedInUser.getOrg().getId(), null, Map.class).toArray(); + loggedInUser.getOrg().getId(), null, Map.class).toArray(); }
/** @@ -697,7 +696,7 @@ public class ErrataHandler extends BaseHandler { * #array_end() */ public List<Map> listPackages(String sessionKey, String advisoryName) - throws FaultException { + throws FaultException { // Get the logged in user User loggedInUser = getLoggedInUser(sessionKey); Errata errata = lookupErrata(advisoryName, loggedInUser.getOrg()); @@ -755,7 +754,7 @@ public class ErrataHandler extends BaseHandler {
//Update Errata Cache if ((packagesAdded > 0) && errata.isPublished() && - (errata.getChannels() != null)) { + (errata.getChannels() != null)) { ErrataCacheManager.updateCacheForChannelsAsync( errata.getChannels()); } @@ -839,7 +838,7 @@ public class ErrataHandler extends BaseHandler { */ if (errata == null) { throw new FaultException(-208, "no_such_errata", - "The errata " + advisoryName + " cannot be found."); + "The errata " + advisoryName + " cannot be found."); } /** * errata with org_id of null are public, but ones with an org id of !null are not @@ -875,18 +874,18 @@ public class ErrataHandler extends BaseHandler { */ public Object[] clone(String sessionKey, String channelLabel, List advisoryNames) throws InvalidChannelRoleException { - return clone(sessionKey, channelLabel, advisoryNames, false); + return clone(sessionKey, channelLabel, advisoryNames, false); }
- + private Object[] clone(String sessionKey, String channelLabel, - List<String> advisoryNames, boolean inheritAllPackages){ + List<String> advisoryNames, boolean inheritAllPackages) { User loggedInUser = getLoggedInUser(sessionKey); - + Logger log = Logger.getLogger(ErrataFactory.class); - + Channel channel = ChannelFactory.lookupByLabelAndUser(channelLabel, - loggedInUser); + loggedInUser);
if (channel == null) { throw new NoSuchChannelException(); @@ -894,26 +893,26 @@ public class ErrataHandler extends BaseHandler {
if (!channel.isCloned()) { throw new InvalidChannelException("Cloned channel expected: " + - channel.getLabel()); + channel.getLabel()); }
Channel original = ChannelFactory.lookupOriginalChannel(channel);
if (original == null) { throw new InvalidChannelException("Cannot access original " + - "of the channel: " + channel.getLabel()); + "of the channel: " + channel.getLabel()); }
// check access to the original if (ChannelFactory.lookupByIdAndUser(original.getId(), loggedInUser) == null) { throw new LookupException("User " + loggedInUser.getLogin() + - " does not have access to channel " + original.getLabel()); + " does not have access to channel " + original.getLabel()); }
if (!UserManager.verifyChannelAdmin(loggedInUser, channel)) { throw new PermissionCheckFailureException(); } - + List<Errata> errataToClone = new ArrayList<Errata>(); List<Errata> errataToPublish = new ArrayList<Errata>(); List<Errata> toReturn = new ArrayList<Errata>(); @@ -923,30 +922,31 @@ public class ErrataHandler extends BaseHandler { Errata toClone = lookupErrata(advisory, loggedInUser.getOrg()); errataToClone.add(toClone); } - + //For each errata look up existing clones, or manually clone it - for (Errata toClone : errataToClone) { - List<Errata> clones = ErrataManager.lookupPublishedByOriginal( + for (Errata toClone : errataToClone) { + List<Errata> clones = ErrataManager.lookupPublishedByOriginal( loggedInUser, toClone); - if (clones.isEmpty()) { - errataToPublish.add(PublishErrataHelper.cloneErrataFast(toClone, loggedInUser.getOrg())); - } - else { - errataToPublish.add(clones.get(0)); - } - } - + if (clones.isEmpty()) { + errataToPublish.add(PublishErrataHelper.cloneErrataFast(toClone, + loggedInUser.getOrg())); + } + else { + errataToPublish.add(clones.get(0)); + } + } + //Now publish them all to the channel in a single shot List<Errata> published = ErrataFactory.publishToChannel(errataToPublish, channel, loggedInUser, true); for (Errata e : published) { - ErrataFactory.save(e); + ErrataFactory.save(e); } - - return toReturn.toArray(); + + return toReturn.toArray(); } - - + + /** * Clones a list of errata into a specified cloned channel * according the original erratas @@ -971,12 +971,12 @@ public class ErrataHandler extends BaseHandler { */ public Object[] cloneAsOriginal(String sessionKey, String channelLabel, List<String> advisoryNames) throws InvalidChannelRoleException { - return clone(sessionKey, channelLabel, advisoryNames, true); + return clone(sessionKey, channelLabel, advisoryNames, true); }
- - + + private Object getRequiredAttribute(Map map, String attribute) { Object value = map.get(attribute); if (value == null || StringUtils.isEmpty(value.toString())) { @@ -1083,7 +1083,7 @@ public class ErrataHandler extends BaseHandler { //so check first before creating anything List channels = null; if (publish) { - channels = verifyChannelList(channelLabels, loggedInUser); + channels = verifyChannelList(channelLabels, loggedInUser); }
String synopsis = (String) getRequiredAttribute(errataInfo, "synopsis"); @@ -1192,7 +1192,7 @@ public class ErrataHandler extends BaseHandler { * @xmlrpc.returntype #return_int_success() */ public Integer delete(String sessionKey, String advisoryName) - throws FaultException { + throws FaultException { User loggedInUser = getLoggedInUser(sessionKey); Errata errata = lookupErrata(advisoryName, loggedInUser.getOrg());
@@ -1223,7 +1223,7 @@ public class ErrataHandler extends BaseHandler { * $ErrataSerializer */ public Errata publish(String sessionKey, String advisory, List channelLabels) - throws InvalidChannelRoleException { + throws InvalidChannelRoleException { User loggedInUser = getLoggedInUser(sessionKey); List channels = verifyChannelList(channelLabels, loggedInUser); Errata toPublish = lookupErrata(advisory, loggedInUser.getOrg()); @@ -1324,9 +1324,9 @@ public class ErrataHandler extends BaseHandler { for (Channel chan : channels) { List<Errata> list = new ArrayList<Errata>(); list.add(published); - published = ErrataFactory.publishToChannel(list, chan, user, + published = ErrataFactory.publishToChannel(list, chan, user, inheritPackages).get(0); - + } return published; } @@ -1348,6 +1348,7 @@ public class ErrataHandler extends BaseHandler { * $ErrataSerializer * #array_end() */ + @Deprecated public List listByDate(String sessionKey, String channelLabel) { User loggedInUser = getLoggedInUser(sessionKey); Channel channel = ChannelFactory.lookupByLabel(loggedInUser.getOrg(), diff --git a/java/code/src/com/redhat/rhn/manager/channel/test/ChannelManagerTest.java b/java/code/src/com/redhat/rhn/manager/channel/test/ChannelManagerTest.java index 4c92285..e58c813 100644 --- a/java/code/src/com/redhat/rhn/manager/channel/test/ChannelManagerTest.java +++ b/java/code/src/com/redhat/rhn/manager/channel/test/ChannelManagerTest.java @@ -293,7 +293,7 @@ public class ChannelManagerTest extends BaseTestCaseWithUser { Channel c = ChannelFactoryTest.createTestChannel(user); Errata e = ErrataFactoryTest.createTestErrata(user.getOrg().getId()); List<Errata> errataList = new ArrayList<Errata>(); - errataList.add(e); + errataList.add(e); ErrataFactory.publishToChannel(errataList, c, user, false);
e = (Errata) TestUtils.saveAndReload(e); diff --git a/java/code/src/com/redhat/rhn/manager/errata/test/ErrataManagerTest.java b/java/code/src/com/redhat/rhn/manager/errata/test/ErrataManagerTest.java index e35ff11..95f4f73 100644 --- a/java/code/src/com/redhat/rhn/manager/errata/test/ErrataManagerTest.java +++ b/java/code/src/com/redhat/rhn/manager/errata/test/ErrataManagerTest.java @@ -175,7 +175,8 @@ public class ErrataManagerTest extends RhnBaseTestCase { Channel baseChannel = ChannelTestUtils.createBaseChannel(user); List<Errata> errataList = new ArrayList<Errata>(); errataList.add(e); - List<Errata> publishedList = ErrataFactory.publishToChannel(errataList, baseChannel, user, false); + List<Errata> publishedList = ErrataFactory.publishToChannel(errataList, + baseChannel, user, false); Errata publish = publishedList.get(0); assertTrue(publish instanceof PublishedErrata);
commit b7baf0795687a808492b07be90821f002ba9b85e Author: Justin Sherrill jsherril@redhat.com Date: Tue Jan 31 13:30:48 2012 -0500
errata date clone - adding proper logging
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index 25eddf8..50927cd 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -24,12 +24,15 @@ import copy import shutil import tempfile import xmlrpclib +import pprint
from depsolver import DepSolver
try: from spacewalk.common.rhnConfig import CFG, initCFG + from spacewalk.common import rhnLog + from spacewalk.common.rhnLog import log_debug, log_clean from spacewalk.satellite_tools.progress_bar import ProgressBar from spacewalk.server import rhnSQL except: @@ -37,10 +40,14 @@ except: if _LIBPATH not in sys.path: sys.path.append(_LIBPATH) from server import rhnSQL - from common import CFG, initCFG + from common import rhnLog + from common.rhnLog import log_debug, log_clean + from common.rhnConfig import CFG, initCFG from satellite_tools.progress_bar import ProgressBar
+LOG_LOCATION = '/var/log/rhn/errata-clone.log' + def confirm(txt, options): if not options.assumeyes: confirm = raw_input(txt) @@ -55,7 +62,12 @@ def main(options): xmlrpc = RemoteApi(options.server, options.username, options.password) db = DBApi() initCFG('server') - + rhnLog.initLOG(LOG_LOCATION) + + cleansed = vars(options) + cleansed["password"] = "*****" + log_debug(0, "Started spacewalk-clone-by-date") + log_clean(0, pprint.pformat(cleansed))
cloners = [] needed_channels = [] @@ -144,7 +156,7 @@ class ChannelTreeCloner: nvreas = []
#clone the destination parent if it doesn't exist - if dest_parent in to_create.values(): + if dest_parent in to_create.values(): self.remote_api.clone_channel(self.src_parent, dest_parent, None) del to_create[self.src_parent] cloner = self.find_cloner(self.src_parent) @@ -207,7 +219,11 @@ class ChannelTreeCloner: added_pkgs = [] for cloner in self.cloners: cloner.process() - added_pkgs += cloner.pkg_diff() + pkg_diff = cloner.pkg_diff() + added_pkgs += pkg_diff + log_clean(0, "") + log_clean(0, "%i packages were added to %s as a result of clone:" % (len(pkg_diff), cloner.dest_label())) + log_clean(0, "\n".join([pkg['nvrea'] for pkg in pkg_diff])) self.dep_solve([pkg['nvrea'] for pkg in added_pkgs])
@@ -321,15 +337,21 @@ class ChannelCloner:
def process_deps(self, needed_pkgs): needed_ids = [] + needed_names = [] unsolved_deps = [] for pkg in needed_pkgs: found = self.src_pkg_exist([pkg]) if found: needed_ids.append(found['id']) + needed_names.append(found['nvrea']) else: unsolved_deps.append(pkg)
- if len(needed_ids) > 0: + if len(needed_ids) > 0: + log_clean(0, "") + log_clean(0, "Adding %i needed dependencies to %l" % (len(needed_ids), self.to_label)) + for name in needed_names: + log_clean(0, name) self.remote_api.add_packages(self.to_label, needed_ids)
@@ -375,7 +397,13 @@ class ChannelCloner: if len(errata_ids) == 0: return
- print 'Cloning Errata into %s (%i):' % (self.to_label, len(errata_ids)) + msg = 'Cloning Errata into %s (%i):' % (self.to_label, len(errata_ids)) + print msg + log_clean(0, "") + log_clean(0, msg) + for e in self.errata_to_clone: + log_clean(0, "%s - %s" % (e['advisory_name'], e['synopsis'])) + pb = ProgressBar(prompt="", endTag=' - complete', finalSize=len(errata_ids), finalBarLength=40, stream=sys.stdout) pb.printAll(1); @@ -401,9 +429,16 @@ class ChannelCloner:
def remove_blacklisted(self, pkg_names): found_ids = [] + found_names = [] for pkg in self.reset_new_pkgs().values(): if pkg['name'] in pkg_names: - found_ids.append(pkg['id']) + found_ids.append(pkg['id']) + found_names.append(pkg['nvrea']) + + log_clean(0, "") + log_clean(0, "Removing %i packages from %s." (len(found_ids), self.to_label)) + log_clean(0, "\n".join(found_names)) + if len(found_ids) > 0: print "Removing %i packages from %s" % (len(found_ids), self.to_label) self.remote_api.remove_packages(self.to_label, found_ids) @@ -466,7 +501,7 @@ class RemoteApi: del package_ids[:20] self.client.channel.software.addPackages(self.auth_token, label, set)
- def remove_packages(self, label, package_ids): + def remove_packages(self, label, package_ids): while(len(package_ids) > 0): set = package_ids[:20] del package_ids[:20] @@ -475,8 +510,12 @@ class RemoteApi: def clone_channel(self, original_label, new_label, parent): details = {'name': new_label, 'label':new_label, 'summary': new_label} if parent and parent != '': - details['parent_label'] = parent - print "Cloning %s to %s with original package set." % (original_label, new_label) + details['parent_label'] = parent + + msg = "Cloning %s to %s with original package set." % (original_label, new_label) + log_clean(0, "") + log_clean(0, msg) + print(msg) self.client.channel.software.clone(self.auth_token, original_label, details, True)
@@ -496,7 +535,7 @@ class DBApi: """list of errata that is applicable to be cloned, used db because we need to exclude cloned errata too""" h = rhnSQL.prepare(""" - select e.id, e.advisory_name, e.advisory_type, e.issue_date + select e.id, e.advisory_name, e.advisory_type, e.issue_date, e.synopsis from rhnErrata e inner join rhnChannelErrata ce on e.id = ce.errata_id inner join rhnChannel c on c.id = ce.channel_id
commit a700123e605c7862e6dbe7c8e015b4641a1385b5 Author: Justin Sherrill jsherril@redhat.com Date: Tue Jan 31 10:47:06 2012 -0500
errata date clone - improving use on terminal with a smaller width
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index d3af3ff..25eddf8 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -224,8 +224,9 @@ class ChannelTreeCloner: def process_deps(self, deps): needed_list = dict((label, []) for label in self.channel_map.values()) unsolved_deps = [] - - pb = ProgressBar(prompt="Processing Dependencies: ", endTag=' - complete', + + print('Processing Dependencies:') + pb = ProgressBar(prompt="", endTag=' - complete', finalSize=len(deps), finalBarLength=40, stream=sys.stdout) pb.printAll(1);
@@ -374,8 +375,8 @@ class ChannelCloner: if len(errata_ids) == 0: return
- msg = 'Cloning Errata into %s (%i): ' % (self.to_label, len(errata_ids)) - pb = ProgressBar(prompt=msg, endTag=' - complete', + print 'Cloning Errata into %s (%i):' % (self.to_label, len(errata_ids)) + pb = ProgressBar(prompt="", endTag=' - complete', finalSize=len(errata_ids), finalBarLength=40, stream=sys.stdout) pb.printAll(1); while(len(errata_ids) > 0): diff --git a/utils/depsolver.py b/utils/depsolver.py index 7de4bed..0a1d3cb 100644 --- a/utils/depsolver.py +++ b/utils/depsolver.py @@ -124,7 +124,8 @@ class DepSolver: results = {} regex_filename_match = re.compile('[/*?]|[[^]]*/[^]]*]').match
- pb = ProgressBar(prompt="Solving Dependencies (%i): " % len(pkgs), endTag=' - complete', + print("Solving Dependencies (%i): " % len(pkgs)) + pb = ProgressBar(prompt='', endTag=' - complete', finalSize=len(pkgs), finalBarLength=40, stream=sys.stdout) pb.printAll(1);
commit 83ed6ed3287e5301f0242fb0d20c87b73452cadb Author: Justin Sherrill jsherril@redhat.com Date: Tue Jan 31 10:33:59 2012 -0500
errata date clone - prompting for password if not supplied
diff --git a/utils/spacewalk-clone-by-date b/utils/spacewalk-clone-by-date index e6ef745..0f7056f 100755 --- a/utils/spacewalk-clone-by-date +++ b/utils/spacewalk-clone-by-date @@ -20,6 +20,7 @@
import sys import datetime +import getpass from optparse import OptionParser import simplejson as json
@@ -114,10 +115,15 @@ def parse_args():
options = merge_config(options)
+ if not options.username: + raise UserError("Username not specified")
if options.channels == None or len(options.channels) == 0: raise UserError("No channels specified. See --help for details.")
+ if not options.password: + options.password = getpass.getpass() + options.to_date = parse_time(options.to_date) return options
commit 026d77602bacdbb6e7aab930edb30fefb4c5e26a Author: Justin Sherrill jsherril@redhat.com Date: Tue Jan 31 10:01:49 2012 -0500
adding man page for spacewalk-clone-by-date
diff --git a/utils/spacewalk-clone-by-date.sgml b/utils/spacewalk-clone-by-date.sgml new file mode 100644 index 0000000..b179451 --- /dev/null +++ b/utils/spacewalk-clone-by-date.sgml @@ -0,0 +1,162 @@ +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook V3.1//EN" [ +<!ENTITY RHNSAT "RHN Management Satellite Server" > +<!ENTITY RHNSAT "RHN Satellite system Migration Tool" > +]> +<refentry> + +<RefMeta> +<RefEntryTitle>spacewalk-clone-by-date</RefEntryTitle><manvolnum>8</manvolnum> +<RefMiscInfo>Version 1.0</RefMiscInfo> +</RefMeta> + +<RefNameDiv> +<RefName><command>spacewalk-clone-by-date</command></RefName> +<RefPurpose> +Script to clone software channels and errata up to specific dates ensuring any added packages have their +dependencies satisifed. Any destination channels that do not exist will be created. + +By specifying channels on the command line, only a single channel tree (a base channel and its children) +can be cloned with a single command. If you would like to specify multiple trees within a single command, +simply use a configuration file. See --sample-config for a sample. + +All options can either be specified in the configuration file or via command line. Any option specified via +command line will override a configuration file value with the exception of channels. If a configuration file is +specified, --channels is not a valid command line argument. + +</RefPurpose> +</RefNameDiv> + +<RefSynopsisDiv> +<Synopsis> + <cmdsynopsis> + <command>spacewalk-clone-by-date</command> + <arg>options <replaceable>...</replaceable></arg> + </cmdsynopsis> + <cmdsynopsis> + <arg> -c <replaceable>CONFIGFILE</replaceable></arg> + <arg> --config=<replaceable>CONFIGFILE</replaceable></arg> + </cmdsynopsis> + <cmdsynopsis> + <arg> -m </arg><arg> --sample-config</arg> + </cmdsynopsis> + <cmdsynopsis> + <arg>-u<replaceable>USERNAME</replaceable></arg> + <arg>--username=<replaceable>USERNAME</replaceable></arg> + </cmdsynopsis> + <cmdsynopsis> + <arg>-p<replaceable>PASSWORD</replaceable></arg> + <arg>--password=<replaceable>PASSWORD</replaceable></arg> + </cmdsynopsis> + <cmdsynopsis> + <arg>-c<replaceable>SRC DEST</replaceable></arg> + <arg>--channels=<replaceable>SRC DEST</replaceable></arg> + </cmdsynopsis> + <cmdsynopsis> + <arg> -d=<replaceable>YYYY-MM-DD</replaceable></arg> + <arg> --to_date=<replaceable>YYYY-MM-DD</replaceable></arg> + </cmdsynopsis> + <cmdsynopsis> + <arg>-b<replaceable>PKG1,PKG2,PKG3</replaceable></arg> + <arg>--blacklist=<replaceable>PKG1,PKG2,PKG3</replaceable></arg> + </cmdsynopsis> + <cmdsynopsis> + <arg>-y</arg><arg> --assumeyes </arg> + </cmdsynopsis> + <cmdsynopsis> + <arg>-h</arg><arg>--help</arg> + </cmdsynopsis> +</Synopsis> +</RefSynopsisDiv> + +<RefSect1><Title>Description</Title> + <para> + <emphasis>spacewalk-clone-by-date</emphasis> clones a channel with errata to a specific date. + </para> +</RefSect1> + +<RefSect1><Title>Options</Title> +<variablelist> + <varlistentry> + <term>-h, --help</term> + <listitem> + <para>Display the help screen with a list of options.</para> + </listitem> + </varlistentry> + <varlistentry> + <term>-c <replaceable>FILE</replaceable> + --config=<replaceable>FILE</replaceable></term> + <listitem> + <para>Configuration file holding parameters, see --sample-config for an example. + Any commandline parameters override those in specified config file.</para> + </listitem> + </varlistentry> + <varlistentry> + <term>-m --sample-config</term> + <listitem> + <para>Generate a sample configuration file.</para> + </listitem> + </varlistentry> + <varlistentry> + <term>-u<replaceable>USERNAME</replaceable> + --username=<replaceable>USERNAME</replaceable></term> + <listitem> + <para>username of user that has administrative access.</para> + </listitem> + </varlistentry> + <varlistentry> + <term>-p<replaceable>PASSWORD</replaceable> + --password=<replaceable>PASSWORD</replaceable></term> + <listitem> + <para>password of user that has administrative access.</para> + </listitem> + </varlistentry> + <varlistentry> + <term> -c <replaceable>SRC_LABEL DEST_LABEL</replaceable> + --channels=<replaceable>SRC_LABEL DEST_LABEL</replaceable></term> + <listitem> + <para>Space seperated list of source channel and destination channel. Can be + specified multiple times to provide base channel and child channel pairs of a + single channel tree. To specify more than one channel tree, specify a config file.</para> + </listitem> + </varlistentry> + <varlistentry> + <term> -y --assumeyes + <listitem> + <para>Instead of asking for confirmation before cloning a channel or errata, + continue uninterrupted.</para> + </listitem> + </varlistentry> + <varlistentry> + <term> -b <replaceable>PKG1,PKG2,PKG3</replaceable> + --blacklist=<replaceable>PKG1,PKG2,PKG3</replaceable> </term> + <listitem> + <para>Comma seperated list of package names to be removed after cloning. + Dependency resolution is not ensured on resulting repository.</para> + </listitem> + </varlistentry> +</variablelist> +</RefSect1> + + +<RefSect1><Title>Examples</Title> +<example> + <title>Clone a base channel and child channel to 2008-12-20 with a small blacklist.</title> + spacewalk-clone-by-date --channel=rhel-x86_64-server-5 clone-rhel --channel=rhn-tools-rhel-x86_64-server-5 clone-tools --username admin --password redhat --to_date=2008-12-20 --blacklist=sendmail,squid +</example> +<example> + <title>Clone with options completely from a config file.</title> + spacewalk-clone-by-date --config=/etc/clone.conf +</example> +<example> + <title>Clone while overriding some options from the commandline.</title> + spacewalk-clone-by-date --config=/etc/clone.conf --username rocky --password squirrel --to_date=2010-10-09 +</example> +</RefSect1> + +<RefSect1><Title>Authors</Title> +<simplelist> + <member>Justin Sherrill <email>jsherrill@redhat.com</email></member> +</simplelist> +</RefSect1> +</RefEntry> +
commit 97a9f02838622a1dc2f3c6c7c7e8e1648e225d2c Author: Justin Sherrill jsherril@redhat.com Date: Tue Jan 31 10:01:34 2012 -0500
errata date clone - a few fixes
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index 9434a9c..d3af3ff 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -86,7 +86,7 @@ def main(options):
confirm("\nContinue with clone (y/n)?", options) for cloner in cloners: - cloner.clone() + cloner.clone() cloner.remove_blacklisted()
@@ -257,9 +257,10 @@ class ChannelTreeCloner: if len(needed) > 0: cloner.process_deps(needed)
- def remove_blacklisted(self): - for cloner in self.cloners: - cloner.remove_blacklisted(self.blacklist) + def remove_blacklisted(self): + if self.blacklist: + for cloner in self.cloners: + cloner.remove_blacklisted(self.blacklist)
def repodata(self, label): return "%s/rhn/repodata/%s" % ( CFG.REPOMD_CACHE_MOUNT_POINT, label) diff --git a/utils/spacewalk-clone-by-date b/utils/spacewalk-clone-by-date index f369b44..e6ef745 100755 --- a/utils/spacewalk-clone-by-date +++ b/utils/spacewalk-clone-by-date @@ -36,6 +36,7 @@ SAMPLE_CONFIG = """ "password":"redhat", "assumeyes":true, "to_date": "2011-10-01", + "blacklist": ["foo", "bar"], "channels":[ { "rhel-x86_64-server-5":"my-rhel5-x86_64-clone", @@ -50,9 +51,12 @@ SAMPLE_CONFIG = """
def merge_config(options): - if not options.config: + if options.channels: options.channels = transform_arg_channels(options.channels) return options + elif not options.config: + return options + try: config = json.load(open(options.config)) except: @@ -90,7 +94,7 @@ def parse_args(): parser.add_option("-u", "--username", dest="username", help="Username") parser.add_option("-p", "--password", dest="password", help="Password") parser.add_option("-s", "--server", dest="server", help="Server URL to use for api connections (defaults to https://localhost/rpc/api)", default="https://localhost/rpc/api") - parser.add_option("-l", "--channels", dest="channels", nargs=2, action="append", help="Original channel and clone channel labels space seperated (e.g. --channels=rhel-i386-server-5 myclone)") + parser.add_option("-l", "--channels", dest="channels", nargs=2, action="append", help="Original channel and clone channel labels space seperated (e.g. --channels=rhel-i386-server-5 myclone). Can be specified multiple times.") parser.add_option("-b", "--blacklist", dest="blacklist", help="Comman separated list of package names") parser.add_option("-d", "--to_date", dest="to_date", help="Clone errata to the specified date (YYYY-MM-DD)") parser.add_option("-y", "--assumeyes", dest='assumeyes', action='store_true', help="Assume yes for any prompts (unattended).") @@ -109,10 +113,12 @@ def parse_args(): options.blacklist = options.blacklist.split(",")
options = merge_config(options) - options.to_date = parse_time(options.to_date) +
if options.channels == None or len(options.channels) == 0: - raise UserError("No channels specified.") + raise UserError("No channels specified. See --help for details.") + + options.to_date = parse_time(options.to_date) return options
commit e17c4364b267c4549ed77335ccea1384ba52bdff Author: Justin Sherrill jsherril@redhat.com Date: Mon Jan 30 16:38:24 2012 -0500
errata date clone - some general cleanup
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index 90a0590..9434a9c 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -23,14 +23,10 @@ import time import copy import shutil import tempfile -from depsolver import DepSolver +import xmlrpclib
-try: - import json -except ImportError: - import simplejson as json
-import xmlrpclib +from depsolver import DepSolver
try: from spacewalk.common.rhnConfig import CFG, initCFG @@ -58,6 +54,8 @@ def confirm(txt, options): def main(options): xmlrpc = RemoteApi(options.server, options.username, options.password) db = DBApi() + initCFG('server') +
cloners = [] needed_channels = [] @@ -96,7 +94,7 @@ def main(options):
class ChannelTreeCloner: """Usage: - a = ChannelTreeCloner(channel_hash, xmlrpc, db, to_date) + a = ChannelTreeCloner(channel_hash, xmlrpc, db, to_date, blacklist) a.create_channels() a.prepare() a.clone() @@ -107,14 +105,15 @@ class ChannelTreeCloner: self.channel_map = channels self.to_date = to_date self.cloners = [] - self.blacklist = blacklist + self.blacklist = blacklist
- self.validate_source_channels() - + self.validate_source_channels() for from_label in self.ordered_labels(): to_label = self.channel_map[from_label] cloner = ChannelCloner(from_label, to_label, self.to_date, self.remote_api, self.db_api) - self.cloners.append(cloner) + self.cloners.append(cloner) + +
#returns a trimmed down version of channel_map where the value needs creating def needing_create(self): @@ -212,7 +211,6 @@ class ChannelTreeCloner: self.dep_solve([pkg['nvrea'] for pkg in added_pkgs])
- def dep_solve(self, nvrea_list, labels=None): if not labels: labels = self.channel_map.keys() @@ -264,14 +262,8 @@ class ChannelTreeCloner: cloner.remove_blacklisted(self.blacklist)
def repodata(self, label): - repo_dir = "/var/cache/rhn/repodata/%s" % label - tmp_dir = tempfile.mkdtemp(suffix="clone-by-date") - try: - shutil.copytree(repo_dir, tmp_dir + "/repodata/") - except: - raise UserError("Could not find repodata for %s in %s" % (label, repo_dir)) - return tmp_dir - + return "%s/rhn/repodata/%s" % ( CFG.REPOMD_CACHE_MOUNT_POINT, label) +
@@ -340,10 +332,8 @@ class ChannelCloner:
def list_to_hash(self, pkg_list, key): - pkg_hash = {} - for pkg in pkg_list: - pkg_hash[pkg[key]] = pkg - return pkg_hash + return dict((pkg[key], pkg) for pkg in pkg_list) +
def src_pkg_exist(self, needed_list): if not self.from_pkg_hash: @@ -379,7 +369,7 @@ class ChannelCloner:
def clone(self): bunch_size = 10 - errata_ids = self.collect(self.errata_to_clone, "advisory_name") + errata_ids = [ e["advisory_name"] for e in self.errata_to_clone] if len(errata_ids) == 0: return
@@ -394,19 +384,7 @@ class ChannelCloner: pb.addTo(bunch_size) pb.printIncrement() pb.printComplete() - - def collect(self, items, attribute): - to_ret = [] - for item in items: - to_ret.append(item[attribute]) - return to_ret - - def repodata(self, label): - repo_dir = "/var/cache/rhn/repodata/%s" % label - tmp_dir = tempfile.mkdtemp(suffix="clone-by-date") - shutil.copytree(repo_dir, tmp_dir + "/repodata/") - return tmp_dir - + def get_errata(self): """ Returns tuple of all available for cloning, and what falls in the date range""" available_errata = self.db_api.applicable_errata(self.from_label, self.to_label) diff --git a/utils/spacewalk-clone-by-date b/utils/spacewalk-clone-by-date index b40d94b..f369b44 100755 --- a/utils/spacewalk-clone-by-date +++ b/utils/spacewalk-clone-by-date @@ -53,7 +53,10 @@ def merge_config(options): if not options.config: options.channels = transform_arg_channels(options.channels) return options - config = json.load(open(options.config)) + try: + config = json.load(open(options.config)) + except: + raise UserError("Configuration file is invalid, please check syntax.")
#if soemthing is in the config and not passed in as an argument # add it to options
commit f72e4d7e0549723da9c648a08d539d297219f153 Author: Justin Sherrill jsherril@redhat.com Date: Fri Jan 20 09:59:21 2012 -0500
improving speed of errata cloning within the spacewalk api
diff --git a/java/code/src/com/redhat/rhn/common/db/datasource/xml/Channel_queries.xml b/java/code/src/com/redhat/rhn/common/db/datasource/xml/Channel_queries.xml index d04f5f5..ef3e251 100644 --- a/java/code/src/com/redhat/rhn/common/db/datasource/xml/Channel_queries.xml +++ b/java/code/src/com/redhat/rhn/common/db/datasource/xml/Channel_queries.xml @@ -20,7 +20,9 @@ DELETE <write-mode name="add_channel_packages"> <query params="cid"> INSERT INTO rhnChannelPackage (channel_id, package_id) - select :cid, P.id from rhnPackage P where P.id in (%s) + select :cid, P.id from rhnPackage P + where P.id in (%s) and + P.id not in (select package_id from rhnChannelPackage where channel_id = :cid) </query> </write-mode>
diff --git a/java/code/src/com/redhat/rhn/domain/errata/ErrataFactory.java b/java/code/src/com/redhat/rhn/domain/errata/ErrataFactory.java index 555c0a2..b090228 100644 --- a/java/code/src/com/redhat/rhn/domain/errata/ErrataFactory.java +++ b/java/code/src/com/redhat/rhn/domain/errata/ErrataFactory.java @@ -268,106 +268,125 @@ public class ErrataFactory extends HibernateFactory { * @param inheritPackages include only original channel packages * @return the publsihed errata */ - public static Errata publishToChannel(Errata errata, Channel chan, User user, + public static List<Errata> publishToChannel(List<Errata> errataList, Channel chan, User user, boolean inheritPackages) { - if (!errata.isPublished()) { - errata = publish(errata); - } - errata.addChannel(chan); - errata.addChannelNotification(chan, new Date()); - - Set<Package> packagesToPush = new HashSet<Package>(); - DataResult<PackageOverview> packs; - if (inheritPackages) { - if (!chan.isCloned()) { - throw new InvalidChannelException("Cloned channel expected: " + - chan.getLabel()); - } - Channel original = ((ClonedChannel) chan).getOriginal(); - packs = ErrataManager.listErrataChannelPacks(original, errata, user); - } - else { - packs = ErrataManager.lookupPacksFromErrataForChannel(chan, errata, user); - } - for (PackageOverview packOver : packs) { - //lookup the Package object - Package pack = PackageFactory.lookupByIdAndUser( - packOver.getId().longValue(), user); - packagesToPush.add(pack); - } - return publishErrataPackagesToChannel(errata, chan, user, packagesToPush); - } - - /** - * Publish an errata to a channel but only push a small set of packages along with it - * @param errata errata to publish - * @param chan channel to publish it into. - * @param user the user doing the pushing - * @param packages the packages to push - * @return the published errata - */ - public static Errata publishToChannel(Errata errata, Channel chan, User user, - Set<Package> packages) { - if (!errata.isPublished()) { - errata = publish(errata); - } - errata.addChannel(chan); - return publishErrataPackagesToChannel(errata, chan, user, packages); + List<com.redhat.rhn.domain.errata.Errata> toReturn = new ArrayList<Errata>(); + for (Errata errata : errataList) { + if (!errata.isPublished()) { + errata = publish(errata); + } + errata.addChannel(chan); + errata.addChannelNotification(chan, new Date()); + + Set<Package> packagesToPush = new HashSet<Package>(); + DataResult<PackageOverview> packs; + if (inheritPackages) { + + if (!chan.isCloned()) { + throw new InvalidChannelException("Cloned channel expected: " + + chan.getLabel()); + } + Channel original = ((ClonedChannel) chan).getOriginal(); + packs = ErrataManager.listErrataChannelPacks(original, errata, user); + } + else { + packs = ErrataManager.lookupPacksFromErrataForChannel(chan, errata, user); + } + + for (PackageOverview packOver : packs) { + //lookup the Package object + Package pack = PackageFactory.lookupByIdAndUser( + packOver.getId().longValue(), user); + packagesToPush.add(pack); + } + + Errata e = publishErrataPackagesToChannel(errata, chan, user, packagesToPush); + toReturn.add(e); + } + postPublishActions(chan, user); + return toReturn; }
- + + + /** + * Publish an errata to a channel but only push a small set of packages + * along with it + * + * @param errata errata to publish + * @param chan channel to publish it into. + * @param user the user doing the pushing + * @param packages the packages to push + * @return the published errata + */ + public static Errata publishToChannel(Errata errata, Channel chan, + User user, Set<Package> packages) { + if (!errata.isPublished()) { + errata = publish(errata); + } + errata.addChannel(chan); + errata = publishErrataPackagesToChannel(errata, chan, user, packages); + postPublishActions(chan, user); + return errata; + } + + + private static void postPublishActions(Channel chan, User user) { + ChannelManager.refreshWithNewestPackages(chan, "web.errata_push"); + ChannelManager.queueChannelChange(chan.getLabel(), + "java::publishErrataPackagesToChannel", user.getLogin()); + } + + /** * Private helper method that pushes errata packages to a channel */ - private static Errata publishErrataPackagesToChannel(Errata errata, Channel chan, - User user, Set<Package> packages) { - for (Package pack : packages) { - - //push the package to the approrpiate channel - chan.addPackage(pack); - - List<ErrataFile> publishedFiles = ErrataFactory.lookupErrataFile(errata, pack); - Map<String, ErrataFile> toAdd = new HashMap(); - if (publishedFiles.size() == 0) { - //Now create the appropriate ErrataFile object - ErrataFile publishedFile = ErrataFactory.createPublishedErrataFile( - ErrataFactory.lookupErrataFileType("RPM"), - pack.getChecksum().getChecksum(), pack.getNameEvra()); - publishedFile.addPackage(pack); - publishedFile.setErrata(errata); - publishedFile.setModified(new Date()); - ((PublishedErrataFile) publishedFile).addChannel(chan); - singleton.saveObject(publishedFile); - } - else { - for (ErrataFile publishedFile : publishedFiles) { - String fileName = publishedFile.getFileName().substring( - publishedFile.getFileName().lastIndexOf("/") + 1); - if (!toAdd.containsKey(fileName)) { - toAdd.put(fileName, publishedFile); - ((PublishedErrataFile) publishedFile).addChannel(chan); - singleton.saveObject(publishedFile); - } - } - } - - } - - ChannelFactory.save(chan); - - List chanList = new ArrayList(); - chanList.add(chan.getId()); - //ErrataCacheManager.updateErrataCacheForChannelsAsync(chanList, user.getOrg()); - ErrataCacheManager.insertCacheForChannelErrataAsync(chanList, errata); - ChannelManager.refreshWithNewestPackages(chan, "web.errata_push"); - - // Mark the affected channel to have it's metadata evaluated, where necessary - // (RHEL5+, mostly) - ChannelManager.queueChannelChange(chan.getLabel(), - "java::publishErrataPackagesToChannel", user.getLogin()); - - return errata; - } + private static Errata publishErrataPackagesToChannel(Errata errata, + Channel chan, User user, Set<Package> packages) { + // Much quicker to push all packages at once + List<Long> pids = new ArrayList<Long>(); + for (Package pack : packages) { + pids.add(pack.getId()); + } + ChannelManager.addPackages(chan, pids, user); + + for (Package pack : packages) { + List<ErrataFile> publishedFiles = ErrataFactory.lookupErrataFile( + errata, pack); + Map<String, ErrataFile> toAdd = new HashMap(); + if (publishedFiles.size() == 0) { + // Now create the appropriate ErrataFile object + ErrataFile publishedFile = ErrataFactory + .createPublishedErrataFile(ErrataFactory + .lookupErrataFileType("RPM"), pack + .getChecksum().getChecksum(), pack + .getNameEvra()); + publishedFile.addPackage(pack); + publishedFile.setErrata(errata); + publishedFile.setModified(new Date()); + ((PublishedErrataFile) publishedFile).addChannel(chan); + singleton.saveObject(publishedFile); + } else { + for (ErrataFile publishedFile : publishedFiles) { + String fileName = publishedFile.getFileName().substring( + publishedFile.getFileName().lastIndexOf("/") + 1); + if (!toAdd.containsKey(fileName)) { + toAdd.put(fileName, publishedFile); + ((PublishedErrataFile) publishedFile).addChannel(chan); + singleton.saveObject(publishedFile); + } + } + } + + } + ChannelFactory.save(chan); + List chanList = new ArrayList(); + chanList.add(chan.getId()); + + ErrataCacheManager.insertCacheForChannelErrataAsync(chanList, errata); + + return errata; + }
/** * @param org Org performing the cloning diff --git a/java/code/src/com/redhat/rhn/domain/errata/test/ErrataFactoryTest.java b/java/code/src/com/redhat/rhn/domain/errata/test/ErrataFactoryTest.java index deaa3d4..d91f5ec 100644 --- a/java/code/src/com/redhat/rhn/domain/errata/test/ErrataFactoryTest.java +++ b/java/code/src/com/redhat/rhn/domain/errata/test/ErrataFactoryTest.java @@ -42,6 +42,7 @@ import com.redhat.rhn.testing.ChannelTestUtils; import com.redhat.rhn.testing.TestUtils; import com.redhat.rhn.testing.UserTestUtils;
+import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -138,7 +139,10 @@ public class ErrataFactoryTest extends BaseTestCaseWithUser { channel.addPackage(chanPack); e.addPackage(errataPack);
- Errata published = ErrataFactory.publishToChannel(e, channel, user, false); + List<Errata> errataList = new ArrayList<Errata>(); + errataList.add(e); + List<Errata> publishedList = ErrataFactory.publishToChannel(errataList, channel, user, false); + Errata published = publishedList.get(0); assertTrue(channel.getPackages().contains(errataPack)); List<PublishedErrataFile> errataFile = ErrataFactory.lookupErrataFilesByErrataAndFileType(published.getId(), "RPM"); diff --git a/java/code/src/com/redhat/rhn/frontend/xmlrpc/errata/ErrataHandler.java b/java/code/src/com/redhat/rhn/frontend/xmlrpc/errata/ErrataHandler.java index e9e8c72..93d63e3 100644 --- a/java/code/src/com/redhat/rhn/frontend/xmlrpc/errata/ErrataHandler.java +++ b/java/code/src/com/redhat/rhn/frontend/xmlrpc/errata/ErrataHandler.java @@ -36,6 +36,7 @@ import com.redhat.rhn.domain.org.Org; import com.redhat.rhn.domain.rhnpackage.Package; import com.redhat.rhn.domain.rhnpackage.PackageFactory; import com.redhat.rhn.domain.user.User; +import com.redhat.rhn.frontend.action.channel.manage.PublishErrataHelper; import com.redhat.rhn.frontend.dto.CVE; import com.redhat.rhn.frontend.dto.PackageDto; import com.redhat.rhn.frontend.xmlrpc.BaseHandler; @@ -58,8 +59,10 @@ import com.redhat.rhn.manager.user.UserManager;
import org.apache.commons.collections.IteratorUtils; import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger;
import java.util.ArrayList; +import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -68,6 +71,7 @@ import java.util.List; import java.util.Map; import java.util.Set;
+ /** * ErrataHandler - provides methods to access errata information. * @version $Rev$ @@ -871,8 +875,16 @@ public class ErrataHandler extends BaseHandler { */ public Object[] clone(String sessionKey, String channelLabel, List advisoryNames) throws InvalidChannelRoleException { - User loggedInUser = getLoggedInUser(sessionKey); + return clone(sessionKey, channelLabel, advisoryNames, false); + }
+ + private Object[] clone(String sessionKey, String channelLabel, + List<String> advisoryNames, boolean inheritAllPackages){ + User loggedInUser = getLoggedInUser(sessionKey); + + Logger log = Logger.getLogger(ErrataFactory.class); + Channel channel = ChannelFactory.lookupByLabelAndUser(channelLabel, loggedInUser);
@@ -880,32 +892,61 @@ public class ErrataHandler extends BaseHandler { throw new NoSuchChannelException(); }
- if (!UserManager.verifyChannelAdmin(loggedInUser, channel)) { - throw new PermissionCheckFailureException(); + if (!channel.isCloned()) { + throw new InvalidChannelException("Cloned channel expected: " + + channel.getLabel()); }
- List errataToClone = new ArrayList(); - List toReturn = new ArrayList(); + Channel original = ChannelFactory.lookupOriginalChannel(channel);
- //We loop through once, making sure all the errata exist - for (Iterator itr = advisoryNames.iterator(); itr.hasNext();) { - Errata toClone = lookupErrata((String)itr.next(), loggedInUser.getOrg()); - errataToClone.add(toClone); + if (original == null) { + throw new InvalidChannelException("Cannot access original " + + "of the channel: " + channel.getLabel()); } - //now that we know its all valid, we clone everything. - for (Iterator itr = errataToClone.iterator(); itr.hasNext();) { - Errata cloned = ErrataManager.createClone(loggedInUser, (Errata)itr.next()); - Errata publishedClone = ErrataManager.publish(cloned);
- publishedClone = ErrataFactory.publishToChannel(publishedClone, channel, - loggedInUser, false); - ErrataFactory.save(publishedClone); + // check access to the original + if (ChannelFactory.lookupByIdAndUser(original.getId(), loggedInUser) == null) { + throw new LookupException("User " + loggedInUser.getLogin() + + " does not have access to channel " + original.getLabel()); + }
- toReturn.add(publishedClone); + if (!UserManager.verifyChannelAdmin(loggedInUser, channel)) { + throw new PermissionCheckFailureException(); } - return toReturn.toArray(); - } + + List<Errata> errataToClone = new ArrayList<Errata>(); + List<Errata> errataToPublish = new ArrayList<Errata>(); + List<Errata> toReturn = new ArrayList<Errata>();
+ //We loop through once, making sure all the errata exist + for (String advisory : advisoryNames) { + Errata toClone = lookupErrata(advisory, loggedInUser.getOrg()); + errataToClone.add(toClone); + } + + //For each errata look up existing clones, or manually clone it + for (Errata toClone : errataToClone) { + List<Errata> clones = ErrataManager.lookupPublishedByOriginal( + loggedInUser, toClone); + if (clones.isEmpty()) { + errataToPublish.add(PublishErrataHelper.cloneErrataFast(toClone, loggedInUser.getOrg())); + } + else { + errataToPublish.add(clones.get(0)); + } + } + + //Now publish them all to the channel in a single shot + List<Errata> published = ErrataFactory.publishToChannel(errataToPublish, channel, + loggedInUser, true); + for (Errata e : published) { + ErrataFactory.save(e); + } + + return toReturn.toArray(); + } + + /** * Clones a list of errata into a specified cloned channel * according the original erratas @@ -929,61 +970,13 @@ public class ErrataHandler extends BaseHandler { * #array_end() */ public Object[] cloneAsOriginal(String sessionKey, String channelLabel, - List advisoryNames) throws InvalidChannelRoleException { - User loggedInUser = getLoggedInUser(sessionKey); - - Channel channel = ChannelFactory.lookupByLabelAndUser(channelLabel, - loggedInUser); - - if (channel == null) { - throw new NoSuchChannelException(); - } - - if (!channel.isCloned()) { - throw new InvalidChannelException("Cloned channel expected: " + - channel.getLabel()); - } - - Channel original = ChannelFactory.lookupOriginalChannel(channel); - - if (original == null) { - throw new InvalidChannelException("Cannot access original " + - "of the channel: " + channel.getLabel()); - } - - // check access to the original - if (ChannelFactory.lookupByIdAndUser(original.getId(), loggedInUser) == null) { - throw new LookupException("User " + loggedInUser.getLogin() + - " does not have access to channel " + original.getLabel()); - } - - if (!UserManager.verifyChannelAdmin(loggedInUser, channel)) { - throw new PermissionCheckFailureException(); - } - - List errataToClone = new ArrayList(); - List toReturn = new ArrayList(); - - //We loop through once, making sure all the errata exist - for (Iterator itr = advisoryNames.iterator(); itr.hasNext();) { - Errata toClone = lookupErrata((String)itr.next(), loggedInUser.getOrg()); - errataToClone.add(toClone); - } - //now that we know its all valid, we clone everything. - for (Iterator itr = errataToClone.iterator(); itr.hasNext();) { - Errata cloned = ErrataManager.createClone(loggedInUser, (Errata)itr.next()); - Errata publishedClone = ErrataManager.publish(cloned); - - publishedClone = ErrataFactory.publishToChannel(publishedClone, channel, - loggedInUser, true); - ErrataFactory.save(publishedClone); - - toReturn.add(publishedClone); - } - return toReturn.toArray(); + List<String> advisoryNames) throws InvalidChannelRoleException { + return clone(sessionKey, channelLabel, advisoryNames, true); }
+ + private Object getRequiredAttribute(Map map, String attribute) { Object value = map.get(attribute); if (value == null || StringUtils.isEmpty(value.toString())) { @@ -1329,8 +1322,11 @@ public class ErrataHandler extends BaseHandler { boolean inheritPackages) { Errata published = ErrataFactory.publish(errata); for (Channel chan : channels) { - published = ErrataFactory.publishToChannel(published, chan, user, - inheritPackages); + List<Errata> list = new ArrayList<Errata>(); + list.add(published); + published = ErrataFactory.publishToChannel(list, chan, user, + inheritPackages).get(0); + } return published; } diff --git a/java/code/src/com/redhat/rhn/manager/channel/ChannelEditor.java b/java/code/src/com/redhat/rhn/manager/channel/ChannelEditor.java index ef14bbe..748145d 100644 --- a/java/code/src/com/redhat/rhn/manager/channel/ChannelEditor.java +++ b/java/code/src/com/redhat/rhn/manager/channel/ChannelEditor.java @@ -100,10 +100,6 @@ public class ChannelEditor { longPackageIds.add(new Long(((Number)it.next()).longValue())); }
- List<Long> existingPids = ChannelFactory.getPackageIds(channel.getId()); - if (add) { - longPackageIds.removeAll(existingPids); - }
PackageManager.verifyPackagesChannelArchCompatAndOrgAccess(user, channel, longPackageIds, add); diff --git a/java/code/src/com/redhat/rhn/manager/channel/test/ChannelManagerTest.java b/java/code/src/com/redhat/rhn/manager/channel/test/ChannelManagerTest.java index 3ffc957..4c92285 100644 --- a/java/code/src/com/redhat/rhn/manager/channel/test/ChannelManagerTest.java +++ b/java/code/src/com/redhat/rhn/manager/channel/test/ChannelManagerTest.java @@ -292,7 +292,9 @@ public class ChannelManagerTest extends BaseTestCaseWithUser { public void testListErrata() throws Exception { Channel c = ChannelFactoryTest.createTestChannel(user); Errata e = ErrataFactoryTest.createTestErrata(user.getOrg().getId()); - ErrataFactory.publishToChannel(e, c, user, false); + List<Errata> errataList = new ArrayList<Errata>(); + errataList.add(e); + ErrataFactory.publishToChannel(errataList, c, user, false);
e = (Errata) TestUtils.saveAndReload(e);
@@ -828,8 +830,10 @@ public class ChannelManagerTest extends BaseTestCaseWithUser {
public void testRemoveErrata() throws Exception { Channel c = ChannelFactoryTest.createTestChannel(user); + List<Errata> errataList = new ArrayList<Errata>(); Errata e = ErrataFactoryTest.createTestErrata(user.getOrg().getId()); - ErrataFactory.publishToChannel(e, c, user, false); + errataList.add(e); + ErrataFactory.publishToChannel(errataList, c, user, false);
e = (Errata) TestUtils.saveAndReload(e);
diff --git a/java/code/src/com/redhat/rhn/manager/errata/test/ErrataManagerTest.java b/java/code/src/com/redhat/rhn/manager/errata/test/ErrataManagerTest.java index 959d930..e35ff11 100644 --- a/java/code/src/com/redhat/rhn/manager/errata/test/ErrataManagerTest.java +++ b/java/code/src/com/redhat/rhn/manager/errata/test/ErrataManagerTest.java @@ -173,7 +173,10 @@ public class ErrataManagerTest extends RhnBaseTestCase { e.addPackage(p);
Channel baseChannel = ChannelTestUtils.createBaseChannel(user); - Errata publish = ErrataFactory.publishToChannel(e, baseChannel, user, false); + List<Errata> errataList = new ArrayList<Errata>(); + errataList.add(e); + List<Errata> publishedList = ErrataFactory.publishToChannel(errataList, baseChannel, user, false); + Errata publish = publishedList.get(0); assertTrue(publish instanceof PublishedErrata);
List eids = new ArrayList();
commit 32c971c5ef2987f1579b0ff6e1f00235bc37500b Author: Justin Sherrill jsherril@redhat.com Date: Mon Jan 30 14:48:44 2012 -0500
errata date clone - adding additional progress bars and cleaning up some output
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index 1891cd7..90a0590 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -93,14 +93,14 @@ def main(options):
-## -# Usage: -# a = ChannelTreeCloner(channel_hash, xmlrpc, db, to_date) -# a.create_channels() -# a.prepare() -# a.clone() -# + class ChannelTreeCloner: + """Usage: + a = ChannelTreeCloner(channel_hash, xmlrpc, db, to_date) + a.create_channels() + a.prepare() + a.clone() + """ def __init__(self, channels, remote_api, db_api, to_date, blacklist): self.remote_api = remote_api self.db_api = db_api @@ -218,20 +218,16 @@ class ChannelTreeCloner: labels = self.channel_map.keys() repos = [{"id":label, "relative_path":self.repodata(label)} for label in labels]
- print "Solving deps" - solver = DepSolver(repos, nvrea_list) dep_results = solver.processResults(solver.getDependencylist()) - - print "Processing" + self.process_deps(dep_results) -
def process_deps(self, deps): needed_list = dict((label, []) for label in self.channel_map.values()) unsolved_deps = []
- pb = ProgressBar(prompt="Proccessing Depedencies: ", endTag=' - complete\n', + pb = ProgressBar(prompt="Processing Dependencies: ", endTag=' - complete', finalSize=len(deps), finalBarLength=40, stream=sys.stdout) pb.printAll(1);
@@ -251,20 +247,19 @@ class ChannelTreeCloner: found = True if not found: unsolved_deps.append((pkg)) - pb.complete() + pb.printComplete()
- print "Unsolved deps: %i" % len(unsolved_deps) - print "Needed deps: " - for label in needed_list.keys(): - print "%s: %i" % (label, len(needed_list[label])) +# print "Unsolved deps: %i" % len(unsolved_deps) +# print "Needed deps: " +# for label in needed_list.keys(): +# print "%s: %i" % (label, len(needed_list[label]))
for cloner in self.cloners: needed = needed_list[cloner.dest_label()] if len(needed) > 0: cloner.process_deps(needed)
- def remove_blacklisted(self): - print self.blacklist + def remove_blacklisted(self): for cloner in self.cloners: cloner.remove_blacklisted(self.blacklist)
@@ -323,12 +318,12 @@ class ChannelCloner: return len(self.errata_to_clone)
def pre_summary(self): - print "%s -> %s (%i/%i Errata)" %(self.from_label, self.to_label, len(self.errata_to_clone), len(self.available_errata)) + print " %s -> %s (%i/%i Errata)" %(self.from_label, self.to_label, len(self.errata_to_clone), len(self.available_errata))
def process(self): self.clone(); self.reset_new_pkgs() - print "New packages added: %i" % (len(self.new_pkg_hash) - len(self.old_pkg_hash)) + #print "New packages added: %i" % (len(self.new_pkg_hash) - len(self.old_pkg_hash))
def process_deps(self, needed_pkgs): needed_ids = [] @@ -340,8 +335,7 @@ class ChannelCloner: else: unsolved_deps.append(pkg)
- if len(needed_ids) > 0: - print "Adding dependencies: %i" % len(needed_ids) + if len(needed_ids) > 0: self.remote_api.add_packages(self.to_label, needed_ids)
@@ -386,6 +380,9 @@ class ChannelCloner: def clone(self): bunch_size = 10 errata_ids = self.collect(self.errata_to_clone, "advisory_name") + if len(errata_ids) == 0: + return + msg = 'Cloning Errata into %s (%i): ' % (self.to_label, len(errata_ids)) pb = ProgressBar(prompt=msg, endTag=' - complete', finalSize=len(errata_ids), finalBarLength=40, stream=sys.stdout) @@ -419,7 +416,8 @@ class ChannelCloner: if err['issue_date'] <= self.to_date: to_clone.append(err)
- return (to_clone, available_errata) + return (to_clone, available_errata) +
def remove_blacklisted(self, pkg_names): found_ids = [] diff --git a/utils/depsolver.py b/utils/depsolver.py index b1ee2e2..7de4bed 100644 --- a/utils/depsolver.py +++ b/utils/depsolver.py @@ -27,6 +27,15 @@ from yum.packageSack import ListPackageSack from yum.packages import parsePackages from yum.repos import RepoStorage
+try: + from spacewalk.satellite_tools.progress_bar import ProgressBar +except: + _LIBPATH = "/usr/share/rhn" + if _LIBPATH not in sys.path: + sys.path.append(_LIBPATH) + from satellite_tools.progress_bar import ProgressBar + + log = logging.getLogger(__name__)
CACHE_DIR="/tmp/cache/yum" @@ -71,7 +80,8 @@ class DepSolver: The package name format could be any of the following: name, name.arch, name-ver-rel.arch, name-ver, name-ver-rel, epoch:name-ver-rel.arch, name-epoch:ver-rel.arch - """ + """ + ematch, match, unmatch = parsePackages(self._repostore.pkgSack, self.pkgs) pkgs = [] for po in ematch + match: @@ -91,7 +101,8 @@ class DepSolver: """ solved = [] to_solve = self.pkgs - all_results = {} + all_results = {} + while to_solve: log.debug("Solving %s \n\n" % to_solve) results = self.getDependencylist() @@ -112,7 +123,14 @@ class DepSolver: def __locateDeps(self, pkgs): results = {} regex_filename_match = re.compile('[/*?]|[[^]]*/[^]]*]').match + + pb = ProgressBar(prompt="Solving Dependencies (%i): " % len(pkgs), endTag=' - complete', + finalSize=len(pkgs), finalBarLength=40, stream=sys.stdout) + pb.printAll(1); + for pkg in pkgs: + pb.addTo(1) + pb.printIncrement() results[pkg] = {} reqs = pkg.requires reqs.sort() @@ -129,6 +147,7 @@ class DepSolver: po.checkPrco('provides', (r, f, v)): satisfiers.append(po) pkgresults[req] = satisfiers + pb.printComplete() return results
def __whatProvides(self, name, flags, version):
commit 72978488be770327bfc992164612443b21b26681 Author: Pradeep Kilambi pkilambi@redhat.com Date: Mon Jan 30 13:22:06 2012 -0500
fix to improve dep solver performance
diff --git a/utils/depsolver.py b/utils/depsolver.py index 04c6bac..b1ee2e2 100644 --- a/utils/depsolver.py +++ b/utils/depsolver.py @@ -24,6 +24,7 @@ import sys import yum from yum.misc import prco_tuple_to_string from yum.packageSack import ListPackageSack +from yum.packages import parsePackages from yum.repos import RepoStorage
log = logging.getLogger(__name__) @@ -71,7 +72,7 @@ class DepSolver: name, name.arch, name-ver-rel.arch, name-ver, name-ver-rel, epoch:name-ver-rel.arch, name-epoch:ver-rel.arch """ - ematch, match, unmatch = self._repostore.pkgSack.matchPackageNames(self.pkgs) + ematch, match, unmatch = parsePackages(self._repostore.pkgSack, self.pkgs) pkgs = [] for po in ematch + match: pkgs.append(po)
commit ebbe962c3d98f0ad04e97394cc8c3f6d9a9ccaf2 Author: Justin Sherrill jsherril@redhat.com Date: Mon Jan 30 11:27:27 2012 -0500
errata date clone - adding package blacklist removal
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index 106be52..1891cd7 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -62,7 +62,7 @@ def main(options): cloners = [] needed_channels = [] for channel_list in options.channels: - tree_cloner = ChannelTreeCloner(channel_list, xmlrpc, db, options.to_date) + tree_cloner = ChannelTreeCloner(channel_list, xmlrpc, db, options.to_date, options.blacklist) cloners.append(tree_cloner) needed_channels += tree_cloner.needing_create().values()
@@ -81,7 +81,7 @@ def main(options): for cloner in cloners: cloner.pre_summary() total += cloner.pending() - + if total == 0: print ("\nNothing to do.") sys.exit(0) @@ -89,6 +89,7 @@ def main(options): confirm("\nContinue with clone (y/n)?", options) for cloner in cloners: cloner.clone() + cloner.remove_blacklisted()
@@ -100,14 +101,16 @@ def main(options): # a.clone() # class ChannelTreeCloner: - def __init__(self, channels, remote_api, db_api, to_date): + def __init__(self, channels, remote_api, db_api, to_date, blacklist): self.remote_api = remote_api self.db_api = db_api self.channel_map = channels self.to_date = to_date self.cloners = [] - self.validate_source_channels() + self.blacklist = blacklist
+ self.validate_source_channels() + for from_label in self.ordered_labels(): to_label = self.channel_map[from_label] cloner = ChannelCloner(from_label, to_label, self.to_date, self.remote_api, self.db_api) @@ -227,9 +230,16 @@ class ChannelTreeCloner: def process_deps(self, deps): needed_list = dict((label, []) for label in self.channel_map.values()) unsolved_deps = [] + + pb = ProgressBar(prompt="Proccessing Depedencies: ", endTag=' - complete\n', + finalSize=len(deps), finalBarLength=40, stream=sys.stdout) + pb.printAll(1); + #loop through all the deps and find any that don't exist in the # destination channels - for pkg in deps: + for pkg in deps: + pb.addTo(1) + pb.printIncrement() for dep, solved_list in pkg.items(): found = False for cloner in self.cloners: @@ -241,19 +251,23 @@ class ChannelTreeCloner: found = True if not found: unsolved_deps.append((pkg)) + pb.complete()
print "Unsolved deps: %i" % len(unsolved_deps) print "Needed deps: " for label in needed_list.keys(): print "%s: %i" % (label, len(needed_list[label])) - - #import pdb; pdb.set_trace() + for cloner in self.cloners: needed = needed_list[cloner.dest_label()] if len(needed) > 0: cloner.process_deps(needed)
- + def remove_blacklisted(self): + print self.blacklist + for cloner in self.cloners: + cloner.remove_blacklisted(self.blacklist) + def repodata(self, label): repo_dir = "/var/cache/rhn/repodata/%s" % label tmp_dir = tempfile.mkdtemp(suffix="clone-by-date") @@ -272,9 +286,7 @@ class ChannelCloner: self.db_api = db_api self.from_label = from_label self.to_label = to_label - self.to_date = to_date - - + self.to_date = to_date self.from_pkg_hash = None
self.new_pkg_hash = {} @@ -331,11 +343,8 @@ class ChannelCloner: if len(needed_ids) > 0: print "Adding dependencies: %i" % len(needed_ids) self.remote_api.add_packages(self.to_label, needed_ids) - if len(unsolved_deps) > 0: - print "Unresolved dependencies: %i" % len(unsolved_deps)
- - + def list_to_hash(self, pkg_list, key): pkg_hash = {} for pkg in pkg_list: @@ -411,6 +420,18 @@ class ChannelCloner: to_clone.append(err)
return (to_clone, available_errata) + + def remove_blacklisted(self, pkg_names): + found_ids = [] + for pkg in self.reset_new_pkgs().values(): + if pkg['name'] in pkg_names: + found_ids.append(pkg['id']) + if len(found_ids) > 0: + print "Removing %i packages from %s" % (len(found_ids), self.to_label) + self.remote_api.remove_packages(self.to_label, found_ids) + + +
class RemoteApi: """ Class for connecting to the XMLRPC spacewalk interface""" @@ -466,7 +487,13 @@ class RemoteApi: set = package_ids[:20] del package_ids[:20] self.client.channel.software.addPackages(self.auth_token, label, set) - + + def remove_packages(self, label, package_ids): + while(len(package_ids) > 0): + set = package_ids[:20] + del package_ids[:20] + self.client.channel.software.removePackages(self.auth_token, label, set) + def clone_channel(self, original_label, new_label, parent): details = {'name': new_label, 'label':new_label, 'summary': new_label} if parent and parent != '': diff --git a/utils/spacewalk-clone-by-date b/utils/spacewalk-clone-by-date index 3fad22a..b40d94b 100755 --- a/utils/spacewalk-clone-by-date +++ b/utils/spacewalk-clone-by-date @@ -87,8 +87,8 @@ def parse_args(): parser.add_option("-u", "--username", dest="username", help="Username") parser.add_option("-p", "--password", dest="password", help="Password") parser.add_option("-s", "--server", dest="server", help="Server URL to use for api connections (defaults to https://localhost/rpc/api)", default="https://localhost/rpc/api") - parser.add_option("-l", "--channels", dest="channels", nargs=2, action="append", help="Origianl channel and clone channel labels space seperated (e.g. --channels=rhel-i386-server-5 myclone)") - parser.add_option("-b", "--blacklist", dest="blacklist", nargs="*", help="Space seperated list of package names") + parser.add_option("-l", "--channels", dest="channels", nargs=2, action="append", help="Original channel and clone channel labels space seperated (e.g. --channels=rhel-i386-server-5 myclone)") + parser.add_option("-b", "--blacklist", dest="blacklist", help="Comman separated list of package names") parser.add_option("-d", "--to_date", dest="to_date", help="Clone errata to the specified date (YYYY-MM-DD)") parser.add_option("-y", "--assumeyes", dest='assumeyes', action='store_true', help="Assume yes for any prompts (unattended).") parser.add_option("-m", "--sample-config", dest='sample', action='store_true', help="Print a sample full configuration file and exit.") @@ -101,7 +101,10 @@ def parse_args():
if options.config and options.channels: raise UserError("Cannot specify both --channels and --config.") - + + if options.blacklist: + options.blacklist = options.blacklist.split(",") + options = merge_config(options) options.to_date = parse_time(options.to_date)
commit ed4d17a4427a44f0e52619fc2c4ca666dc28fbf7 Author: Justin Sherrill jsherril@redhat.com Date: Mon Jan 30 10:06:17 2012 -0500
moving dep solving to channel tree instead of individual channels
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index d39438d..106be52 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -64,7 +64,7 @@ def main(options): for channel_list in options.channels: tree_cloner = ChannelTreeCloner(channel_list, xmlrpc, db, options.to_date) cloners.append(tree_cloner) - needed_channels += tree_cloner.needing_create() + needed_channels += tree_cloner.needing_create().values()
if len(needed_channels) > 0: print "\nBy continuing the following channels will be created: " @@ -113,13 +113,14 @@ class ChannelTreeCloner: cloner = ChannelCloner(from_label, to_label, self.to_date, self.remote_api, self.db_api) self.cloners.append(cloner)
+ #returns a trimmed down version of channel_map where the value needs creating def needing_create(self): - to_ret = [] + to_create = {} existing = self.remote_api.list_channel_labels() - for label in self.channel_map.values(): - if existing.count(label) == 0: - to_ret.append(label) - return to_ret + for src, dest in self.channel_map.items(): + if dest not in existing: + to_create[src] = dest + return to_create
def pending(self): total = 0 @@ -127,23 +128,33 @@ class ChannelTreeCloner: total += cloner.pending() return total
+ def find_cloner(self, src_label): + for cloner in self.cloners: + if cloner.src_label() == src_label: + return cloner + def create_channels(self): to_create = self.needing_create() + if len(to_create) == 0: return dest_parent = self.channel_map[self.src_parent] nvreas = [] - if to_create.count(dest_parent) > 0: + + #clone the destination parent if it doesn't exist + if dest_parent in to_create.values(): self.remote_api.clone_channel(self.src_parent, dest_parent, None) - to_create.remove(dest_parent) - cloner = self.cloners[0] + del to_create[self.src_parent] + cloner = self.find_cloner(self.src_parent) nvreas += [ pkg['nvrea'] for pkg in cloner.reset_new_pkgs().values()] - + #clone the children for cloner in self.cloners: - if cloner.dest_label() in to_create: + if cloner.dest_label() in to_create.values(): self.remote_api.clone_channel(cloner.src_label(), cloner.dest_label(), dest_parent) nvreas += [ pkg['nvrea'] for pkg in cloner.reset_new_pkgs().values() ] - self.dep_solve(nvreas, labels=to_create) + + #dep solve all added packages with the parent channel + self.dep_solve(nvreas, labels=(to_create.keys() + [self.src_parent]))
def validate_source_channels(self): @@ -195,17 +206,17 @@ class ChannelTreeCloner: for cloner in self.cloners: cloner.process() added_pkgs += cloner.pkg_diff() - self.dep_solve(added_pkgs) + self.dep_solve([pkg['nvrea'] for pkg in added_pkgs])
- def dep_solve(self, nvrea_list, labels=None): + def dep_solve(self, nvrea_list, labels=None): if not labels: labels = self.channel_map.keys() repos = [{"id":label, "relative_path":self.repodata(label)} for label in labels]
print "Solving deps" - + solver = DepSolver(repos, nvrea_list) dep_results = solver.processResults(solver.getDependencylist())
@@ -222,13 +233,16 @@ class ChannelTreeCloner: for dep, solved_list in pkg.items(): found = False for cloner in self.cloners: - if cloner.src_pkg_exist(solved_list) and not cloner.dest_pkg_exist(solved_list): + exists_from = cloner.src_pkg_exist(solved_list) + exists_to = cloner.dest_pkg_exist(solved_list) + if exists_from and not exists_to: needed_list[cloner.dest_label()].append(solved_list[0]) #grab oldest package - found = True + elif exists_from: + found = True if not found: unsolved_deps.append((pkg))
- print "Unsolved deps: %i" % len(unsolved_deps) + print "Unsolved deps: %i" % len(unsolved_deps) print "Needed deps: " for label in needed_list.keys(): print "%s: %i" % (label, len(needed_list[label])) @@ -243,7 +257,10 @@ class ChannelTreeCloner: def repodata(self, label): repo_dir = "/var/cache/rhn/repodata/%s" % label tmp_dir = tempfile.mkdtemp(suffix="clone-by-date") - shutil.copytree(repo_dir, tmp_dir + "/repodata/") + try: + shutil.copytree(repo_dir, tmp_dir + "/repodata/") + except: + raise UserError("Could not find repodata for %s in %s" % (label, repo_dir)) return tmp_dir
@@ -299,7 +316,7 @@ class ChannelCloner: def process(self): self.clone(); self.reset_new_pkgs() - print "New packages added: %i" % len(len(self.new_pkg_hash) - len(self.old_pkg_hash)) + print "New packages added: %i" % (len(self.new_pkg_hash) - len(self.old_pkg_hash))
def process_deps(self, needed_pkgs): needed_ids = []
commit a0448d0fa7500f79b47c647539ba31068c6a1f7b Author: Justin Sherrill jsherril@redhat.com Date: Mon Jan 30 10:04:58 2012 -0500
errata date clone - adding sample configuration option
diff --git a/utils/spacewalk-clone-by-date b/utils/spacewalk-clone-by-date index e44728a..3fad22a 100755 --- a/utils/spacewalk-clone-by-date +++ b/utils/spacewalk-clone-by-date @@ -21,6 +21,7 @@ import sys import datetime from optparse import OptionParser +import simplejson as json
#try: import cloneByDate @@ -29,6 +30,24 @@ from cloneByDate import UserError # from utils import cloneByDate
+SAMPLE_CONFIG = """ +{ + "username":"admin", + "password":"redhat", + "assumeyes":true, + "to_date": "2011-10-01", + "channels":[ + { + "rhel-x86_64-server-5":"my-rhel5-x86_64-clone", + "rhn-tools-rhel-x86_64-server-5": "my-tools-5-x86_64-clone" + }, + { + "rhel-i386-server-5": "my-rhel5-i386-clone" + } + ] +} +""" +
def merge_config(options): if not options.config: @@ -38,7 +57,7 @@ def merge_config(options):
#if soemthing is in the config and not passed in as an argument # add it to options - overwrite = ["username", "password", "blacklist", "channels", "server", "assumeyes"] + overwrite = ["username", "password", "blacklist", "channels", "server", "assumeyes", "to_date"] for key in overwrite: if config.has_key(key) and not getattr(options, key): setattr(options, key, config[key]) @@ -72,12 +91,20 @@ def parse_args(): parser.add_option("-b", "--blacklist", dest="blacklist", nargs="*", help="Space seperated list of package names") parser.add_option("-d", "--to_date", dest="to_date", help="Clone errata to the specified date (YYYY-MM-DD)") parser.add_option("-y", "--assumeyes", dest='assumeyes', action='store_true', help="Assume yes for any prompts (unattended).") + parser.add_option("-m", "--sample-config", dest='sample', action='store_true', help="Print a sample full configuration file and exit.")
(options, args) = parser.parse_args()
- options.to_date = parse_time(options.to_date) + if options.sample: + print SAMPLE_CONFIG + sys.exit(0) + if options.config and options.channels: raise UserError("Cannot specify both --channels and --config.") + + options = merge_config(options) + options.to_date = parse_time(options.to_date) + if options.channels == None or len(options.channels) == 0: raise UserError("No channels specified.") return options @@ -104,9 +131,8 @@ def systemExit(code, msgs=None):
def main(): try: - args = parse_args(); - options = merge_config(args) - return cloneByDate.main(options) + args = parse_args(); + return cloneByDate.main(args) except KeyboardInterrupt: systemExit(0, "\nUser interrupted process.") except UserError, error:
commit c0caebe95a29451ecccf92f0ca45d849da347e08 Author: Justin Sherrill jsherril@redhat.com Date: Fri Jan 27 16:31:14 2012 -0500
moving dep solver to channel tree from individual channel
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index 16bf870..d39438d 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -17,7 +17,7 @@ # in this software or its documentation. #
-import pdb + import sys import time import copy @@ -94,9 +94,8 @@ def main(options):
## # Usage: -# a = ChannelTreeCloner(channel_hash, xmlrpc, db, to_date) -# if len(a.needing_channels()) > 0: -# a.create_channels() +# a = ChannelTreeCloner(channel_hash, xmlrpc, db, to_date) +# a.create_channels() # a.prepare() # a.clone() # @@ -128,25 +127,23 @@ class ChannelTreeCloner: total += cloner.pending() return total
- def find_cloner(self, label): - for cloner in self.cloners: - if cloner.dest_label() == label: - return cloner - def create_channels(self): to_create = self.needing_create() if len(to_create) == 0: return dest_parent = self.channel_map[self.src_parent] + nvreas = [] if to_create.count(dest_parent) > 0: self.remote_api.clone_channel(self.src_parent, dest_parent, None) to_create.remove(dest_parent) - self.find_cloner(dest_parent).after_create() - for from_label, to_label in self.channel_map.items(): - if to_create.count(to_label) > 0: - self.remote_api.clone_channel(from_label, to_label, dest_parent) - self.find_cloner(to_label).after_create() - + cloner = self.cloners[0] + nvreas += [ pkg['nvrea'] for pkg in cloner.reset_new_pkgs().values()] + + for cloner in self.cloners: + if cloner.dest_label() in to_create: + self.remote_api.clone_channel(cloner.src_label(), cloner.dest_label(), dest_parent) + nvreas += [ pkg['nvrea'] for pkg in cloner.reset_new_pkgs().values() ] + self.dep_solve(nvreas, labels=to_create)
def validate_source_channels(self): @@ -194,10 +191,60 @@ class ChannelTreeCloner: cloner.pre_summary();
def clone(self): + added_pkgs = [] for cloner in self.cloners: cloner.process() + added_pkgs += cloner.pkg_diff() + self.dep_solve(added_pkgs)
+ + + def dep_solve(self, nvrea_list, labels=None): + if not labels: + labels = self.channel_map.keys() + repos = [{"id":label, "relative_path":self.repodata(label)} for label in labels] + + print "Solving deps" + + solver = DepSolver(repos, nvrea_list) + dep_results = solver.processResults(solver.getDependencylist()) + + print "Processing" + self.process_deps(dep_results) + + + def process_deps(self, deps): + needed_list = dict((label, []) for label in self.channel_map.values()) + unsolved_deps = [] + #loop through all the deps and find any that don't exist in the + # destination channels + for pkg in deps: + for dep, solved_list in pkg.items(): + found = False + for cloner in self.cloners: + if cloner.src_pkg_exist(solved_list) and not cloner.dest_pkg_exist(solved_list): + needed_list[cloner.dest_label()].append(solved_list[0]) #grab oldest package + found = True + if not found: + unsolved_deps.append((pkg)) + + print "Unsolved deps: %i" % len(unsolved_deps) + print "Needed deps: " + for label in needed_list.keys(): + print "%s: %i" % (label, len(needed_list[label])) + + #import pdb; pdb.set_trace() + for cloner in self.cloners: + needed = needed_list[cloner.dest_label()] + if len(needed) > 0: + cloner.process_deps(needed) +
+ def repodata(self, label): + repo_dir = "/var/cache/rhn/repodata/%s" % label + tmp_dir = tempfile.mkdtemp(suffix="clone-by-date") + shutil.copytree(repo_dir, tmp_dir + "/repodata/") + return tmp_dir
@@ -209,15 +256,39 @@ class ChannelCloner: self.from_label = from_label self.to_label = to_label self.to_date = to_date - self.original_packages = None + + + self.from_pkg_hash = None + + self.new_pkg_hash = {} + self.old_pkg_hash = {}
def dest_label(self): return self.to_label + + def src_label(self): + return self.from_label + + def pkg_diff(self): + return self.diff_packages(self.old_pkg_hash.values(), self.new_pkg_hash.values()) +
- def prepare(self): - self.original_packages = self.remote_api.list_packages(self.to_label) - self.errata_to_clone, self.available_errata = self.get_errata() + def reset_original_pkgs(self): + self.old_pkg_hash = self.list_to_hash(self.remote_api.list_packages(self.to_label), 'nvrea') + return self.old_pkg_hash + + def reset_new_pkgs(self): + self.new_pkg_hash = self.list_to_hash(self.remote_api.list_packages(self.to_label), 'nvrea') + return self.new_pkg_hash + + def reset_from_pkgs(self): + self.from_pkg_hash = self.list_to_hash(self.remote_api.list_packages(self.from_label), 'nvrea') + + def prepare(self): + self.reset_original_pkgs() + self.errata_to_clone, self.available_errata = self.get_errata() +
def pending(self): return len(self.errata_to_clone) @@ -227,51 +298,14 @@ class ChannelCloner:
def process(self): self.clone(); - new_packages = self.remote_api.list_packages(self.to_label) - pkg_diff = self.diff_packages(self.original_packages, new_packages) - print "New packages added: %i" % len(pkg_diff) - self.dep_solve(pkg_diff, new_packages) - - def after_create(self): - new_packages = self.remote_api.list_packages(self.to_label) - self.dep_solve(new_packages, new_packages) - - # pkg_list - list of packages to solve deps for - # trim_list - list of packages that already exist - def dep_solve(self, pkg_list, trim_list): - repo_dir = self.repodata(self.from_label) - print "Solving deps" - nvreas = [pkg['nvrea'] for pkg in pkg_list] - solver = DepSolver([{"id":self.from_label, "relative_path":repo_dir}], nvreas) - dep_results = solver.processResults(solver.getDependencylist()) - - print "Processing" - self.process_deps(trim_list, dep_results) - - - def process_deps(self, new_packages, deps): - to_pkg_hash = self.list_to_hash(new_packages, 'nvrea') - - needed_list = [] - - for pkg in deps: - for dep, solved_list in pkg.items(): -# if dep == "qffmpeg-libs = 0.4.9-0.16.20080908.el5_5": -# import pdb; pdb.set_trace(); - if not self.pkg_exists(solved_list, to_pkg_hash): - if len(solved_list) > 0: - needed_list.append(solved_list[0]) #grab oldest package - else: - print "No packages %s" % dep - - print "Unsolved deps: %i" % len(needed_list) - - from_pkg_hash = self.list_to_hash(self.remote_api.list_packages(self.from_label), 'nvrea') - + self.reset_new_pkgs() + print "New packages added: %i" % len(len(self.new_pkg_hash) - len(self.old_pkg_hash)) + + def process_deps(self, needed_pkgs): needed_ids = [] unsolved_deps = [] - for pkg in needed_list: - found = self.pkg_exists([pkg], from_pkg_hash) + for pkg in needed_pkgs: + found = self.src_pkg_exist([pkg]) if found: needed_ids.append(found['id']) else: @@ -291,14 +325,21 @@ class ChannelCloner: pkg_hash[pkg[key]] = pkg return pkg_hash
+ def src_pkg_exist(self, needed_list): + if not self.from_pkg_hash: + self.reset_from_pkgs() + return self.pkg_exists(needed_list, self.from_pkg_hash) + + def dest_pkg_exist(self, needed_list): + return self.pkg_exists(needed_list, self.new_pkg_hash)
- def pkg_exists(self, needed_list, pkg_hash): + def pkg_exists(self, needed_list, pkg_list): """Given a list of packages in [N, V, E, R, A] format, do any of them exist in the pkg_hash with key of N-V-R.A format""" for i in needed_list: key = "%s-%s-%s.%s" % (i[0], i[1], i[3], i[4]) - if pkg_hash.has_key(key): - return pkg_hash[key] + if pkg_list.has_key(key): + return pkg_list[key] return False
def diff_packages(self, old, new): @@ -347,6 +388,7 @@ class ChannelCloner: """ Returns tuple of all available for cloning, and what falls in the date range""" available_errata = self.db_api.applicable_errata(self.from_label, self.to_label) to_clone = [] + for err in available_errata: if err['issue_date'] <= self.to_date: to_clone.append(err) @@ -402,11 +444,11 @@ class RemoteApi: except xmlrpclib.Fault, e: raise UserError(e.faultString + ": " + label)
- def add_packages(self, label, package_ids): + def add_packages(self, label, package_ids): while(len(package_ids) > 0): set = package_ids[:20] del package_ids[:20] - self.client.channel.software.addPackages(self.auth_token, label, package_ids) + self.client.channel.software.addPackages(self.auth_token, label, set)
def clone_channel(self, original_label, new_label, parent): details = {'name': new_label, 'label':new_label, 'summary': new_label} @@ -451,7 +493,8 @@ class DBApi: where c2.label = :to_label) """) h.execute(from_label=from_label, to_label=to_label) - return h.fetchall_dict() + to_ret = h.fetchall_dict() or [] + return to_ret
class UserError(Exception): def __init__(self, msg):
commit 7127884dbed67250a8ef0f214169a4617e1367f7 Author: Justin Sherrill jsherril@redhat.com Date: Wed Jan 25 09:47:14 2012 -0500
errata date clone - locating and adding deps from dep solver
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index e97fe38..16bf870 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -61,8 +61,8 @@ def main(options):
cloners = [] needed_channels = [] - for list in options.channels: - tree_cloner = ChannelTreeCloner(list, xmlrpc, db, options.to_date) + for channel_list in options.channels: + tree_cloner = ChannelTreeCloner(channel_list, xmlrpc, db, options.to_date) cloners.append(tree_cloner) needed_channels += tree_cloner.needing_create()
@@ -73,23 +73,33 @@ def main(options): for cloner in cloners: cloner.create_channels()
- - for tree_cloner in cloners: tree_cloner.prepare();
print "\nBy continuing the following will be cloned:" + total = 0 for cloner in cloners: cloner.pre_summary() - print "\n" - confirm("Continue with clone (y/n)?", options) + total += cloner.pending()
+ if total == 0: + print ("\nNothing to do.") + sys.exit(0) + + confirm("\nContinue with clone (y/n)?", options) for cloner in cloners: cloner.clone()
- +## +# Usage: +# a = ChannelTreeCloner(channel_hash, xmlrpc, db, to_date) +# if len(a.needing_channels()) > 0: +# a.create_channels() +# a.prepare() +# a.clone() +# class ChannelTreeCloner: def __init__(self, channels, remote_api, db_api, to_date): self.remote_api = remote_api @@ -98,6 +108,11 @@ class ChannelTreeCloner: self.to_date = to_date self.cloners = [] self.validate_source_channels() + + for from_label in self.ordered_labels(): + to_label = self.channel_map[from_label] + cloner = ChannelCloner(from_label, to_label, self.to_date, self.remote_api, self.db_api) + self.cloners.append(cloner)
def needing_create(self): to_ret = [] @@ -107,6 +122,16 @@ class ChannelTreeCloner: to_ret.append(label) return to_ret
+ def pending(self): + total = 0 + for cloner in self.cloners: + total += cloner.pending() + return total + + def find_cloner(self, label): + for cloner in self.cloners: + if cloner.dest_label() == label: + return cloner
def create_channels(self): to_create = self.needing_create() @@ -116,10 +141,13 @@ class ChannelTreeCloner: if to_create.count(dest_parent) > 0: self.remote_api.clone_channel(self.src_parent, dest_parent, None) to_create.remove(dest_parent) + self.find_cloner(dest_parent).after_create() for from_label, to_label in self.channel_map.items(): if to_create.count(to_label) > 0: self.remote_api.clone_channel(from_label, to_label, dest_parent) - + self.find_cloner(to_label).after_create() + +
def validate_source_channels(self): self.channel_details = self.remote_api.channel_details(self.channel_map, values=False) @@ -151,17 +179,14 @@ class ChannelTreeCloner:
def ordered_labels(self): """Return list of labels with parent first""" - list = self.channel_map.keys() - list.remove(self.src_parent) - list.insert(0,self.src_parent) - return list + labels = self.channel_map.keys() + labels.remove(self.src_parent) + labels.insert(0,self.src_parent) + return labels
- def prepare(self): - for from_label in self.ordered_labels(): - to_label = self.channel_map[from_label] - self.validate_dest_channels() - cloner = ChannelCloner(from_label, to_label, self.to_date, self.remote_api, self.db_api) - self.cloners.append(cloner) + def prepare(self): + self.validate_dest_channels() + for cloner in self.cloners: cloner.prepare()
def pre_summary(self): @@ -169,7 +194,7 @@ class ChannelTreeCloner: cloner.pre_summary();
def clone(self): - for cloner in self.cloners: + for cloner in self.cloners: cloner.process()
@@ -186,58 +211,95 @@ class ChannelCloner: self.to_date = to_date self.original_packages = None
+ + def dest_label(self): + return self.to_label + def prepare(self): self.original_packages = self.remote_api.list_packages(self.to_label) self.errata_to_clone, self.available_errata = self.get_errata()
+ def pending(self): + return len(self.errata_to_clone) + def pre_summary(self): print "%s -> %s (%i/%i Errata)" %(self.from_label, self.to_label, len(self.errata_to_clone), len(self.available_errata))
def process(self): self.clone(); new_packages = self.remote_api.list_packages(self.to_label) - pkg_diff = self.diff_packages(self.original_packages, new_packages) - + pkg_diff = self.diff_packages(self.original_packages, new_packages) print "New packages added: %i" % len(pkg_diff) + self.dep_solve(pkg_diff, new_packages)
+ def after_create(self): + new_packages = self.remote_api.list_packages(self.to_label) + self.dep_solve(new_packages, new_packages) + + # pkg_list - list of packages to solve deps for + # trim_list - list of packages that already exist + def dep_solve(self, pkg_list, trim_list): repo_dir = self.repodata(self.from_label) - print "Solving deps" - nvreas = [pkg['nvrea'] for pkg in pkg_diff] + nvreas = [pkg['nvrea'] for pkg in pkg_list] solver = DepSolver([{"id":self.from_label, "relative_path":repo_dir}], nvreas) dep_results = solver.processResults(solver.getDependencylist())
- #import pdb; pdb.set_trace() print "Processing" - self.process_deps(new_packages, dep_results) - - #self.remote_api.add_packages() + self.process_deps(trim_list, dep_results) + + + def process_deps(self, new_packages, deps): + to_pkg_hash = self.list_to_hash(new_packages, 'nvrea')
- def process_deps(self, pkg_list, deps): - pkg_hash = {} - for pkg in pkg_list: - pkg_hash[pkg['nvrea']] = pkg - needed_list = [] - for pkg in deps: - + + for pkg in deps: for dep, solved_list in pkg.items(): - if not self.pkg_exists(solved_list, pkg_hash): - needed_list.append(solved_list) - print len(needed_list) - +# if dep == "qffmpeg-libs = 0.4.9-0.16.20080908.el5_5": +# import pdb; pdb.set_trace(); + if not self.pkg_exists(solved_list, to_pkg_hash): + if len(solved_list) > 0: + needed_list.append(solved_list[0]) #grab oldest package + else: + print "No packages %s" % dep + + print "Unsolved deps: %i" % len(needed_list) + + from_pkg_hash = self.list_to_hash(self.remote_api.list_packages(self.from_label), 'nvrea') + + needed_ids = [] + unsolved_deps = [] + for pkg in needed_list: + found = self.pkg_exists([pkg], from_pkg_hash) + if found: + needed_ids.append(found['id']) + else: + unsolved_deps.append(pkg) + + if len(needed_ids) > 0: + print "Adding dependencies: %i" % len(needed_ids) + self.remote_api.add_packages(self.to_label, needed_ids) + if len(unsolved_deps) > 0: + print "Unresolved dependencies: %i" % len(unsolved_deps) + + + + def list_to_hash(self, pkg_list, key): + pkg_hash = {} + for pkg in pkg_list: + pkg_hash[pkg[key]] = pkg + return pkg_hash +
- def pkg_exists(self, list, pkg_hash): + def pkg_exists(self, needed_list, pkg_hash): """Given a list of packages in [N, V, E, R, A] format, do any of them exist - in the pkg_hash with key of N-V-R.A format""" - for i in list: + in the pkg_hash with key of N-V-R.A format""" + for i in needed_list: key = "%s-%s-%s.%s" % (i[0], i[1], i[3], i[4]) if pkg_hash.has_key(key): - return True + return pkg_hash[key] return False - - -
def diff_packages(self, old, new): old_hash = {} @@ -250,7 +312,7 @@ class ChannelCloner: new_hash[pkg["id"]] = pkg id_diff = set(new_hash.keys()) - set(old_hash.keys()) for id in id_diff: - to_ret.append(new_hash[id]) + to_ret.append(new_hash[id]) return to_ret
@@ -266,6 +328,7 @@ class ChannelCloner: del errata_ids[:bunch_size] self.remote_api.clone_errata(self.to_label, errata_set) pb.addTo(bunch_size) + pb.printIncrement() pb.printComplete()
def collect(self, items, attribute): @@ -281,7 +344,7 @@ class ChannelCloner: return tmp_dir
def get_errata(self): - """ Returns tuple of all available for cloning, and what falls in teh date range""" + """ Returns tuple of all available for cloning, and what falls in the date range""" available_errata = self.db_api.applicable_errata(self.from_label, self.to_label) to_clone = [] for err in available_errata: @@ -301,16 +364,15 @@ class RemoteApi: self.auth_token = self.client.auth.login(username, password) except xmlrpclib.Fault, e: raise UserError(e.faultString) - - + def list_channel_labels(self): key = "chan_labels" if self.cache.has_key(key): return self.cache[key]
- list = self.client.channel.listAllChannels(self.auth_token) + chan_list = self.client.channel.listAllChannels(self.auth_token) to_ret = [] - for item in list: + for item in chan_list: to_ret.append(item["label"]) self.cache[key] = to_ret return to_ret @@ -367,7 +429,8 @@ class DBApi:
def applicable_errata(self, from_label, to_label): - """list of errata that is applicable to be cloned, used db because we need to exclude cloned errata too""" + """list of errata that is applicable to be cloned, used db because we + need to exclude cloned errata too""" h = rhnSQL.prepare(""" select e.id, e.advisory_name, e.advisory_type, e.issue_date from rhnErrata e inner join
commit 990ebf0c9c3595c3a366bf796473d60046c9cd36 Author: Pradeep Kilambi pkilambi@redhat.com Date: Fri Jan 27 10:08:11 2012 -0500
adding regex match to include file based deps
diff --git a/utils/depsolver.py b/utils/depsolver.py index cfb78ec..04c6bac 100644 --- a/utils/depsolver.py +++ b/utils/depsolver.py @@ -18,6 +18,7 @@ # in this software or its documentation
import logging +import re import shutil import sys import yum @@ -109,6 +110,7 @@ class DepSolver:
def __locateDeps(self, pkgs): results = {} + regex_filename_match = re.compile('[/*?]|[[^]]*/[^]]*]').match for pkg in pkgs: results[pkg] = {} reqs = pkg.requires @@ -122,7 +124,8 @@ class DepSolver: for po in self.__whatProvides(r, f, v): # verify this po indeed provides the dep, # el5 version could give some false positives - if po.checkPrco('provides', (r, f, v)): + if regex_filename_match(r) or \ + po.checkPrco('provides', (r, f, v)): satisfiers.append(po) pkgresults[req] = satisfiers return results
commit 60fa3a42d732a09a545b788011bf7a52c2c90944 Author: Pradeep Kilambi pkilambi@redhat.com Date: Wed Jan 25 17:29:14 2012 -0500
Adding a CheckProc to verify if the po is indeed providing the dep
diff --git a/utils/depsolver.py b/utils/depsolver.py index f0eb4ba..cfb78ec 100644 --- a/utils/depsolver.py +++ b/utils/depsolver.py @@ -120,7 +120,10 @@ class DepSolver: continue satisfiers = [] for po in self.__whatProvides(r, f, v): - satisfiers.append(po) + # verify this po indeed provides the dep, + # el5 version could give some false positives + if po.checkPrco('provides', (r, f, v)): + satisfiers.append(po) pkgresults[req] = satisfiers return results
commit 60016a3dcd5b3753b8342e7bc6a9520adbee8891 Author: Justin Sherrill jsherril@redhat.com Date: Mon Jan 23 16:28:18 2012 -0500
errata date clone - adding progress bar for errata cloning
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index 54bfa25..e97fe38 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -34,6 +34,7 @@ import xmlrpclib
try: from spacewalk.common.rhnConfig import CFG, initCFG + from spacewalk.satellite_tools.progress_bar import ProgressBar from spacewalk.server import rhnSQL except: _LIBPATH = "/usr/share/rhn" @@ -41,6 +42,7 @@ except: sys.path.append(_LIBPATH) from server import rhnSQL from common import CFG, initCFG + from satellite_tools.progress_bar import ProgressBar
def confirm(txt, options): @@ -253,13 +255,18 @@ class ChannelCloner:
def clone(self): + bunch_size = 10 errata_ids = self.collect(self.errata_to_clone, "advisory_name") + msg = 'Cloning Errata into %s (%i): ' % (self.to_label, len(errata_ids)) + pb = ProgressBar(prompt=msg, endTag=' - complete', + finalSize=len(errata_ids), finalBarLength=40, stream=sys.stdout) + pb.printAll(1); while(len(errata_ids) > 0): - set = errata_ids[:10] - del errata_ids[:10] - print "Cloning set:" - print set - self.remote_api.clone_errata(self.to_label, set) + errata_set = errata_ids[:bunch_size] + del errata_ids[:bunch_size] + self.remote_api.clone_errata(self.to_label, errata_set) + pb.addTo(bunch_size) + pb.printComplete()
def collect(self, items, attribute): to_ret = []
commit 2b8b2922660340c9815b96e62df07456910b4c92 Author: Justin Sherrill jsherril@redhat.com Date: Mon Jan 23 16:13:45 2012 -0500
errata date clone - adding dep solving integration, not adding results yet though
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index 1189e47..54bfa25 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -194,17 +194,46 @@ class ChannelCloner: def process(self): self.clone(); new_packages = self.remote_api.list_packages(self.to_label) - pkg_idiff = self.diff_packages(self.original_packages, new_packages) + pkg_diff = self.diff_packages(self.original_packages, new_packages)
- print "New packages added: %i" % len(pkg_idiff) + print "New packages added: %i" % len(pkg_diff) + repo_dir = self.repodata(self.from_label) + + print "Solving deps" + nvreas = [pkg['nvrea'] for pkg in pkg_diff] + solver = DepSolver([{"id":self.from_label, "relative_path":repo_dir}], nvreas) + dep_results = solver.processResults(solver.getDependencylist())
- deps = []### dep solve on diff - dep_package_ids = [] + #import pdb; pdb.set_trace() + print "Processing" + self.process_deps(new_packages, dep_results) + + #self.remote_api.add_packages()
- #solver = DepSolver([{"id":self.from_label, "relative_path":repo_dir}]) + def process_deps(self, pkg_list, deps): + pkg_hash = {} + for pkg in pkg_list: + pkg_hash[pkg['nvrea']] = pkg + + needed_list = [] + for pkg in deps: + + for dep, solved_list in pkg.items(): + if not self.pkg_exists(solved_list, pkg_hash): + needed_list.append(solved_list) + print len(needed_list) + + + def pkg_exists(self, list, pkg_hash): + """Given a list of packages in [N, V, E, R, A] format, do any of them exist + in the pkg_hash with key of N-V-R.A format""" + for i in list: + key = "%s-%s-%s.%s" % (i[0], i[1], i[3], i[4]) + if pkg_hash.has_key(key): + return True + return False
- #self.remote_api.add_packages()
@@ -240,8 +269,8 @@ class ChannelCloner:
def repodata(self, label): repo_dir = "/var/cache/rhn/repodata/%s" % label - tmp_dir = tempfile.mkdtemp(suffix="clone-by-date") + "/repo/" - shutil.copytree(repo_dir, tmp_dir) + tmp_dir = tempfile.mkdtemp(suffix="clone-by-date") + shutil.copytree(repo_dir, tmp_dir + "/repodata/") return tmp_dir
def get_errata(self):
commit 7dcc26ec1a6c4774219c708533a6cddb19241381 Author: Justin Sherrill jsherril@redhat.com Date: Mon Jan 23 13:34:50 2012 -0500
errata clone by datete - adding channel cloning
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index 2327f6e..1189e47 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -23,6 +23,7 @@ import time import copy import shutil import tempfile +from depsolver import DepSolver
try: import json @@ -42,29 +43,44 @@ except: from common import CFG, initCFG
+def confirm(txt, options): + if not options.assumeyes: + confirm = raw_input(txt) + while ['y', 'n'].count(confirm.lower()) == 0: + confirm = raw_input(txt) + if confirm.lower() == "n": + print "Cancelling" + sys.exit(0) + + def main(options): xmlrpc = RemoteApi(options.server, options.username, options.password) db = DBApi()
cloners = [] + needed_channels = [] for list in options.channels: tree_cloner = ChannelTreeCloner(list, xmlrpc, db, options.to_date) cloners.append(tree_cloner) + needed_channels += tree_cloner.needing_create() + + if len(needed_channels) > 0: + print "\nBy continuing the following channels will be created: " + print ", ".join(needed_channels) + confirm("\nContinue with channel creation (y/n)?", options) + for cloner in cloners: + cloner.create_channels() + + + + for tree_cloner in cloners: tree_cloner.prepare();
print "\nBy continuing the following will be cloned:" for cloner in cloners: cloner.pre_summary() print "\n" - - if not options.assumeyes: - txt = "Continue with clone (y/n)?" - confirm = raw_input(txt) - while ['y', 'n'].count(confirm.lower()) == 0: - confirm = raw_input(txt) - if confirm.lower() == "n": - print "Cancelling" - sys.exit(0) + confirm("Continue with clone (y/n)?", options)
for cloner in cloners: cloner.clone() @@ -79,14 +95,39 @@ class ChannelTreeCloner: self.channel_map = channels self.to_date = to_date self.cloners = [] + self.validate_source_channels()
- def validate_channels(self): - self.channel_details = self.remote_api.channel_details(self.channel_map) + def needing_create(self): + to_ret = [] + existing = self.remote_api.list_channel_labels() + for label in self.channel_map.values(): + if existing.count(label) == 0: + to_ret.append(label) + return to_ret + + + def create_channels(self): + to_create = self.needing_create() + if len(to_create) == 0: + return + dest_parent = self.channel_map[self.src_parent] + if to_create.count(dest_parent) > 0: + self.remote_api.clone_channel(self.src_parent, dest_parent, None) + to_create.remove(dest_parent) + for from_label, to_label in self.channel_map.items(): + if to_create.count(to_label) > 0: + self.remote_api.clone_channel(from_label, to_label, dest_parent) + + + def validate_source_channels(self): + self.channel_details = self.remote_api.channel_details(self.channel_map, values=False) self.src_parent = self.find_parent(self.channel_map.keys()) - self.validate_children(self.src_parent, self.channel_map.keys()) - self.dest_parent = self.find_parent(self.channel_map.values()) - self.validate_children(self.dest_parent, self.channel_map.values()) - return self.channel_details + self.validate_children(self.src_parent, self.channel_map.keys()) + + def validate_dest_channels(self): + self.channel_details = self.remote_api.channel_details(self.channel_map) + self.dest_parent = self.find_parent(self.channel_map.values()) + self.validate_children(self.dest_parent, self.channel_map.values())
def validate_children(self, parent, label_list): """ Make sure all children are children of the parent""" @@ -113,11 +154,10 @@ class ChannelTreeCloner: list.insert(0,self.src_parent) return list
- - def prepare(self): - self.validate_channels() + def prepare(self): for from_label in self.ordered_labels(): to_label = self.channel_map[from_label] + self.validate_dest_channels() cloner = ChannelCloner(from_label, to_label, self.to_date, self.remote_api, self.db_api) self.cloners.append(cloner) cloner.prepare() @@ -161,6 +201,9 @@ class ChannelCloner:
deps = []### dep solve on diff dep_package_ids = [] + + #solver = DepSolver([{"id":self.from_label, "relative_path":repo_dir}]) + #self.remote_api.add_packages()
@@ -214,6 +257,8 @@ class ChannelCloner: class RemoteApi: """ Class for connecting to the XMLRPC spacewalk interface"""
+ cache = {} + def __init__(self, server_url, username, password): self.client = xmlrpclib.Server(server_url) try: @@ -222,18 +267,33 @@ class RemoteApi: raise UserError(e.faultString)
- def list_channels(self): - "" + def list_channel_labels(self): + key = "chan_labels" + if self.cache.has_key(key): + return self.cache[key] + + list = self.client.channel.listAllChannels(self.auth_token) + to_ret = [] + for item in list: + to_ret.append(item["label"]) + self.cache[key] = to_ret + return to_ret
- def channel_details(self, label_hash): + def channel_details(self, label_hash, keys=True, values=True): to_ret = {} - for src, dst in label_hash.items(): - to_ret[src] = self.get_details(src) - to_ret[dst] = self.get_details(dst) + for src, dst in label_hash.items(): + if keys: + to_ret[src] = self.get_details(src) + if values: + to_ret[dst] = self.get_details(dst) return to_ret
def list_packages(self, label): - return self.client.channel.software.listAllPackages(self.auth_token, label) + list = self.client.channel.software.listAllPackages(self.auth_token, label) + #name-ver-rel.arch, + for pkg in list: + pkg['nvrea'] = "%s-%s-%s.%s" % (pkg['name'], pkg['version'], pkg['release'], pkg['arch_label']) + return list
def clone_errata(self, to_label, errata_list): self.client.errata.cloneAsOriginal(self.auth_token, to_label, errata_list) @@ -249,6 +309,14 @@ class RemoteApi: set = package_ids[:20] del package_ids[:20] self.client.channel.software.addPackages(self.auth_token, label, package_ids) + + def clone_channel(self, original_label, new_label, parent): + details = {'name': new_label, 'label':new_label, 'summary': new_label} + if parent and parent != '': + details['parent_label'] = parent + print "Cloning %s to %s with original package set." % (original_label, new_label) + self.client.channel.software.clone(self.auth_token, original_label, details, True) +
class DBApi:
commit d493306485fd39d8e11346ba9d5689e3c5e31cde Merge: df25ba7 5d472e3 Author: Justin Sherrill jsherril@redhat.com Date: Mon Jan 23 11:08:31 2012 -0500
Merge branch 'errata-date' of ssh://git.fedorahosted.org/git/spacewalk into errata-date
commit 5d472e3138f75532d592775fe6c9274b9620414f Author: Pradeep Kilambi pkilambi@redhat.com Date: Mon Jan 23 11:07:42 2012 -0500
dependency lookup for errata clone-by-date
diff --git a/utils/depsolver.py b/utils/depsolver.py new file mode 100644 index 0000000..f0eb4ba --- /dev/null +++ b/utils/depsolver.py @@ -0,0 +1,186 @@ +#!/usr/bin/python +# +# -*- coding: utf-8 -*- +# +# Copyright (c) 2012 Red Hat, Inc. +# +# Lookup package dependencies in a yum repository +# +# This software is licensed to you under the GNU General Public License, +# version 2 (GPLv2). There is NO WARRANTY for this software, express or +# implied, including the implied warranties of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 +# along with this software; if not, see +# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. +# +# Red Hat trademarks are not licensed under GPLv2. No permission is +# granted to use or replicate Red Hat trademarks that are incorporated +# in this software or its documentation + +import logging +import shutil +import sys +import yum +from yum.misc import prco_tuple_to_string +from yum.packageSack import ListPackageSack +from yum.repos import RepoStorage + +log = logging.getLogger(__name__) + +CACHE_DIR="/tmp/cache/yum" + +class DepSolver: + def __init__(self, repos, pkgs=[]): + self.pkgs = pkgs + self.repos = repos + self._repostore = RepoStorage(self) + self.setup() + self.loadPackages() + + def setup(self): + """ + Load the repos into repostore to query package dependencies + """ + for repo in self.repos: + self.yrepo = yum.yumRepo.YumRepository(repo['id']) + self.yrepo.baseurl = ["file://%s/" % str(repo['relative_path'])] + self.yrepo.basecachedir = CACHE_DIR + self._repostore.add(self.yrepo) + + def loadPackages(self): + """ + populate the repostore with packages + """ + self._repostore._setup = True + self._repostore.populateSack(which='all') + + def cleanup(self): + """ + clean up the repo metadata cache from /tmp/cache/yum + """ + for repo in self._repostore.repos: + cachedir = "%s/%s" % (constants.CACHE_DIR, repo) + shutil.rmtree(cachedir) + + def getDependencylist(self): + """ + Get dependency list and suggested packages for package names provided. + The dependency lookup is only one level in this case. + The package name format could be any of the following: + name, name.arch, name-ver-rel.arch, name-ver, name-ver-rel, + epoch:name-ver-rel.arch, name-epoch:ver-rel.arch + """ + ematch, match, unmatch = self._repostore.pkgSack.matchPackageNames(self.pkgs) + pkgs = [] + for po in ematch + match: + pkgs.append(po) + results = self.__locateDeps(pkgs) + return results + + def getRecursiveDepList(self): + """ + Get dependency list and suggested packages for package names provided. + The dependency lookup is recursive. All available packages in the repo + are returned matching whatprovides. + The package name format could be any of the following: + name, name.arch, name-ver-rel.arch, name-ver, name-ver-rel, + epoch:name-ver-rel.arch, name-epoch:ver-rel.arch + returns a dictionary of {'n-v-r.a' : [n,v,e,r,a],...} + """ + solved = [] + to_solve = self.pkgs + all_results = {} + while to_solve: + log.debug("Solving %s \n\n" % to_solve) + results = self.getDependencylist() + all_results.update(results) + found = self.processResults(results)[0] + solved += to_solve + to_solve = [] + for dep, pkgs in found.items(): + for pkg in pkgs: + name, version, epoch, release, arch = pkg + ndep = "%s-%s-%s.%s" % (name, version, release, arch) + solved = list(set(solved)) + if ndep not in solved: + to_solve.append(ndep) + self.pkgs = to_solve + return all_results + + def __locateDeps(self, pkgs): + results = {} + for pkg in pkgs: + results[pkg] = {} + reqs = pkg.requires + reqs.sort() + pkgresults = results[pkg] + for req in reqs: + (r, f, v) = req + if r.startswith('rpmlib('): + continue + satisfiers = [] + for po in self.__whatProvides(r, f, v): + satisfiers.append(po) + pkgresults[req] = satisfiers + return results + + def __whatProvides(self, name, flags, version): + try: + return ListPackageSack(self._repostore.pkgSack.searchProvides((name, flags, version))) + except: + #perhaps we're on older version of yum try old style + return ListPackageSack(self._repostore.pkgSack.searchProvides(name)) + + def processResults(self, results): + reqlist = {} + notfound = {} + for pkg in results: + if len(results[pkg]) == 0: + continue + for req in results[pkg]: + rlist = results[pkg][req] + if not rlist: + # Unsatisfied dependency + notfound[prco_tuple_to_string(req)] = [] + continue + reqlist[prco_tuple_to_string(req)] = rlist + found = {} + for req, rlist in reqlist.items(): + found[req] = [] + for r in rlist: + dep = [r.name, r.version, r.epoch, r.release, r.arch] + if dep not in found[req]: + found[req].append(dep) + return found, notfound + + def printable_result(self, results): + print_doc_str = "" + for pkg in results: + if len(results[pkg]) == 0: + continue + for req in results[pkg]: + rlist = results[pkg][req] + print_doc_str += "\n dependency: %s \n" % prco_tuple_to_string(req) + if not rlist: + # Unsatisfied dependency + print_doc_str += " Unsatisfied dependency \n" + continue + + for po in rlist: + print_doc_str += " provider: %s\n" % po.compactPrint() + return print_doc_str + + +if __name__ == '__main__': + if len(sys.argv) < 3: + print "USAGE: python depsolver.py <repoid> <repodata_path> <pkgname1> <pkgname2> ....<pkgnameN>" + sys.exit(0) + import pprint + repo = {'id' : sys.argv[1], + 'relative_path' : sys.argv[2],} #path to where repodata is located + pkgs = sys.argv[3:] + dsolve = DepSolver([repo], pkgs) + deplist = dsolve.getDependencylist() + result_set = dsolve.processResults(deplist) + print result_set + print "Printable dependency Results: \n\n %s" % dsolve.printable_result(deplist)
commit df25ba75641b426d75dff9a78bab5bd989d2003a Author: Justin Sherrill jsherril@redhat.com Date: Mon Jan 23 10:56:34 2012 -0500
errata date clone - adding package diffing to know what changed
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index 2100614..2327f6e 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -21,6 +21,8 @@ import pdb import sys import time import copy +import shutil +import tempfile
try: import json @@ -78,8 +80,8 @@ class ChannelTreeCloner: self.to_date = to_date self.cloners = []
- def validate_channels(self): - self.channel_details = self.remote_api.list_channels(self.channel_map) + def validate_channels(self): + self.channel_details = self.remote_api.channel_details(self.channel_map) self.src_parent = self.find_parent(self.channel_map.keys()) self.validate_children(self.src_parent, self.channel_map.keys()) self.dest_parent = self.find_parent(self.channel_map.values()) @@ -126,7 +128,7 @@ class ChannelTreeCloner:
def clone(self): for cloner in self.cloners: - cloner.clone() + cloner.process()
@@ -151,18 +153,38 @@ class ChannelCloner:
def process(self): self.clone(); - self.new_packages = self.remote_api.list_packages(self.to_label) - print "%i, %i" % (len(self.original_packages), len(self.new_packages)) - ### diff lists - ### dep solve on diff - ### added found deps + new_packages = self.remote_api.list_packages(self.to_label) + pkg_idiff = self.diff_packages(self.original_packages, new_packages) + + print "New packages added: %i" % len(pkg_idiff) + repo_dir = self.repodata(self.from_label) + + deps = []### dep solve on diff + dep_package_ids = [] + #self.remote_api.add_packages() + + + + def diff_packages(self, old, new): + old_hash = {} + new_hash = {} + to_ret = [] + + for pkg in old: + old_hash[pkg["id"]] = pkg + for pkg in new: + new_hash[pkg["id"]] = pkg + id_diff = set(new_hash.keys()) - set(old_hash.keys()) + for id in id_diff: + to_ret.append(new_hash[id]) + return to_ret
def clone(self): errata_ids = self.collect(self.errata_to_clone, "advisory_name") while(len(errata_ids) > 0): - set = errata_ids[:5] - del errata_ids[:5] + set = errata_ids[:10] + del errata_ids[:10] print "Cloning set:" print set self.remote_api.clone_errata(self.to_label, set) @@ -172,6 +194,12 @@ class ChannelCloner: for item in items: to_ret.append(item[attribute]) return to_ret + + def repodata(self, label): + repo_dir = "/var/cache/rhn/repodata/%s" % label + tmp_dir = tempfile.mkdtemp(suffix="clone-by-date") + "/repo/" + shutil.copytree(repo_dir, tmp_dir) + return tmp_dir
def get_errata(self): """ Returns tuple of all available for cloning, and what falls in teh date range""" @@ -193,7 +221,11 @@ class RemoteApi: except xmlrpclib.Fault, e: raise UserError(e.faultString)
- def list_channels(self, label_hash): + + def list_channels(self): + "" + + def channel_details(self, label_hash): to_ret = {} for src, dst in label_hash.items(): to_ret[src] = self.get_details(src) @@ -211,6 +243,12 @@ class RemoteApi: return self.client.channel.software.getDetails(self.auth_token, label) except xmlrpclib.Fault, e: raise UserError(e.faultString + ": " + label) + + def add_packages(self, label, package_ids): + while(len(package_ids) > 0): + set = package_ids[:20] + del package_ids[:20] + self.client.channel.software.addPackages(self.auth_token, label, package_ids)
class DBApi: @@ -222,6 +260,8 @@ class DBApi: db_string = CFG.DEFAULT_DB #"rhnsat/rhnsat@rhnsat" rhnSQL.initDB(db_string)
+ + def applicable_errata(self, from_label, to_label): """list of errata that is applicable to be cloned, used db because we need to exclude cloned errata too""" h = rhnSQL.prepare("""
commit 4cb3e8e7ae98f737ac6dd7110d95adbf94814243 Author: Justin Sherrill jsherril@redhat.com Date: Fri Jan 20 10:23:50 2012 -0500
got errata cloning working in clone-by-date
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py index ac24d72..2100614 100644 --- a/utils/cloneByDate.py +++ b/utils/cloneByDate.py @@ -18,58 +18,236 @@ #
import pdb -import json -from optparse import OptionParser +import sys +import time +import copy
+try: + import json +except ImportError: + import simplejson as json
+import xmlrpclib
-def main(): - options = merge_config(parse_args()); +try: + from spacewalk.common.rhnConfig import CFG, initCFG + from spacewalk.server import rhnSQL +except: + _LIBPATH = "/usr/share/rhn" + if _LIBPATH not in sys.path: + sys.path.append(_LIBPATH) + from server import rhnSQL + from common import CFG, initCFG + + +def main(options): + xmlrpc = RemoteApi(options.server, options.username, options.password) + db = DBApi()
- -def merge_config(options): - if not options.config: - options.channels = transform_arg_channels(options.channels) - return options - config = json.load(open(options.config)) + cloners = [] + for list in options.channels: + tree_cloner = ChannelTreeCloner(list, xmlrpc, db, options.to_date) + cloners.append(tree_cloner) + tree_cloner.prepare(); + + print "\nBy continuing the following will be cloned:" + for cloner in cloners: + cloner.pre_summary() + print "\n"
- #if soemthing is in the config and not passed in as an argument - # add it to options - overwrite = ["username", "password", "blacklist", "channels"] - for key in overwrite: - if config.has_key(key) and not getattr(options, key): - setattr(options, key, config[key]) + if not options.assumeyes: + txt = "Continue with clone (y/n)?" + confirm = raw_input(txt) + while ['y', 'n'].count(confirm.lower()) == 0: + confirm = raw_input(txt) + if confirm.lower() == "n": + print "Cancelling" + sys.exit(0)
- if type(options.channels) == dict: - options.channels = [options.channels] - return options - - -# Using --channels as an argument only supports a single channel 'tree' -# So we need to convert a 2-tuple list of channel labels into an array with a hash -# ex: [ ("rhel-i386-servr-5", "my-rhel-clone"), ('rhel-child', 'clone-child')] -# should become -# [{ -# "rhel-i386-servr-5" : "my-rhel-clone", -# 'rhel-child': 'clone-child' -# }] -def transform_arg_channels(chan_list): - to_ret = {} - for src, dest in chan_list: - to_ret[src] = dest - return [to_ret] + for cloner in cloners: + cloner.clone() + + + + +class ChannelTreeCloner: + def __init__(self, channels, remote_api, db_api, to_date): + self.remote_api = remote_api + self.db_api = db_api + self.channel_map = channels + self.to_date = to_date + self.cloners = [] + + def validate_channels(self): + self.channel_details = self.remote_api.list_channels(self.channel_map) + self.src_parent = self.find_parent(self.channel_map.keys()) + self.validate_children(self.src_parent, self.channel_map.keys()) + self.dest_parent = self.find_parent(self.channel_map.values()) + self.validate_children(self.dest_parent, self.channel_map.values()) + return self.channel_details + + def validate_children(self, parent, label_list): + """ Make sure all children are children of the parent""" + for label in label_list: + if label != parent: + if self.channel_details[label]['parent_channel_label'] != parent: + raise UserError("Child channel '%s' is not a child of parent channel '%s'" % (label, parent)) + + def find_parent(self, label_list): + found_list = [] + for label in label_list: + if self.channel_details[label]['parent_channel_label'] == '': + found_list.append(label) + if len(found_list) == 0: + UserError("Parent Channel not specified.") + if len(found_list) > 1: + UserError("Multiple parent channels specified within the same channel tree.") + return found_list[0] + + def ordered_labels(self): + """Return list of labels with parent first""" + list = self.channel_map.keys() + list.remove(self.src_parent) + list.insert(0,self.src_parent) + return list + + + def prepare(self): + self.validate_channels() + for from_label in self.ordered_labels(): + to_label = self.channel_map[from_label] + cloner = ChannelCloner(from_label, to_label, self.to_date, self.remote_api, self.db_api) + self.cloners.append(cloner) + cloner.prepare() + + def pre_summary(self): + for cloner in self.cloners: + cloner.pre_summary(); + + def clone(self): + for cloner in self.cloners: + cloner.clone() + + + + + + +class ChannelCloner: + def __init__(self, from_label, to_label, to_date, remote_api, db_api): + self.remote_api = remote_api + self.db_api = db_api + self.from_label = from_label + self.to_label = to_label + self.to_date = to_date + self.original_packages = None + + def prepare(self): + self.original_packages = self.remote_api.list_packages(self.to_label) + self.errata_to_clone, self.available_errata = self.get_errata()
-def parse_args(): - parser = OptionParser() - parser.add_option("-c", "--config", dest="config", help="Config file specifying options") - parser.add_option("-u", "--username", dest="username", help="Username") - parser.add_option("-p", "--password", dest="password", help="Password") - parser.add_option("-l", "--channels", dest="channels", nargs=2, action="append", help="Origianl channel and clone channel labels space seperated (e.g. --channels=rhel-i386-server-5 myclone)") - parser.add_option("-b", "--blacklist", dest="blacklist", nargs="*", help="Space seperated list of package names") - (options, args) = parser.parse_args() - - if options.config and options.channels: - raise Exception("Cannot specify both --channels and --config.") + def pre_summary(self): + print "%s -> %s (%i/%i Errata)" %(self.from_label, self.to_label, len(self.errata_to_clone), len(self.available_errata)) + + def process(self): + self.clone(); + self.new_packages = self.remote_api.list_packages(self.to_label) + print "%i, %i" % (len(self.original_packages), len(self.new_packages)) + ### diff lists + ### dep solve on diff + ### added found deps + + + def clone(self): + errata_ids = self.collect(self.errata_to_clone, "advisory_name") + while(len(errata_ids) > 0): + set = errata_ids[:5] + del errata_ids[:5] + print "Cloning set:" + print set + self.remote_api.clone_errata(self.to_label, set) + + def collect(self, items, attribute): + to_ret = [] + for item in items: + to_ret.append(item[attribute]) + return to_ret + + def get_errata(self): + """ Returns tuple of all available for cloning, and what falls in teh date range""" + available_errata = self.db_api.applicable_errata(self.from_label, self.to_label) + to_clone = [] + for err in available_errata: + if err['issue_date'] <= self.to_date: + to_clone.append(err) + + return (to_clone, available_errata) + +class RemoteApi: + """ Class for connecting to the XMLRPC spacewalk interface""" + + def __init__(self, server_url, username, password): + self.client = xmlrpclib.Server(server_url) + try: + self.auth_token = self.client.auth.login(username, password) + except xmlrpclib.Fault, e: + raise UserError(e.faultString)
- return options - \ No newline at end of file + def list_channels(self, label_hash): + to_ret = {} + for src, dst in label_hash.items(): + to_ret[src] = self.get_details(src) + to_ret[dst] = self.get_details(dst) + return to_ret + + def list_packages(self, label): + return self.client.channel.software.listAllPackages(self.auth_token, label) + + def clone_errata(self, to_label, errata_list): + self.client.errata.cloneAsOriginal(self.auth_token, to_label, errata_list) + + def get_details(self, label): + try: + return self.client.channel.software.getDetails(self.auth_token, label) + except xmlrpclib.Fault, e: + raise UserError(e.faultString + ": " + label) + + +class DBApi: + """Class for connecting to the spacewalk DB""" + + + def __init__(self): + initCFG('server') + db_string = CFG.DEFAULT_DB #"rhnsat/rhnsat@rhnsat" + rhnSQL.initDB(db_string) + + def applicable_errata(self, from_label, to_label): + """list of errata that is applicable to be cloned, used db because we need to exclude cloned errata too""" + h = rhnSQL.prepare(""" + select e.id, e.advisory_name, e.advisory_type, e.issue_date + from rhnErrata e inner join + rhnChannelErrata ce on e.id = ce.errata_id inner join + rhnChannel c on c.id = ce.channel_id + where C.label = :from_label and + e.id not in (select e2.id + from rhnErrata e2 inner join + rhnChannelErrata ce2 on ce2.errata_id = e2.id inner join + rhnChannel c2 on c2.id = ce2.channel_id + where c2.label = :to_label + UNION + select cloned.original_id + from rhnErrata e2 inner join + rhnErrataCloned cloned on cloned.id = e2.id inner join + rhnChannelErrata ce2 on ce2.errata_id = e2.id inner join + rhnChannel c2 on c2.id = ce2.channel_id + where c2.label = :to_label) + """) + h.execute(from_label=from_label, to_label=to_label) + return h.fetchall_dict() + +class UserError(Exception): + def __init__(self, msg): + self.msg = msg + def __str__(self): + return self.msg \ No newline at end of file diff --git a/utils/spacewalk-clone-by-date b/utils/spacewalk-clone-by-date index f953986..e44728a 100755 --- a/utils/spacewalk-clone-by-date +++ b/utils/spacewalk-clone-by-date @@ -19,13 +19,79 @@
import sys +import datetime +from optparse import OptionParser
-try: - import cloneByDate -except: - from utils import cloneByDate +#try: +import cloneByDate +from cloneByDate import UserError +#except: +# from utils import cloneByDate
+ +def merge_config(options): + if not options.config: + options.channels = transform_arg_channels(options.channels) + return options + config = json.load(open(options.config)) + + #if soemthing is in the config and not passed in as an argument + # add it to options + overwrite = ["username", "password", "blacklist", "channels", "server", "assumeyes"] + for key in overwrite: + if config.has_key(key) and not getattr(options, key): + setattr(options, key, config[key]) + + if type(options.channels) == dict: + options.channels = [options.channels] + return options + + +# Using --channels as an argument only supports a single channel 'tree' +# So we need to convert a 2-tuple list of channel labels into an array with a hash +# ex: [ ("rhel-i386-servr-5", "my-rhel-clone"), ('rhel-child', 'clone-child')] +# should become +# [{ +# "rhel-i386-servr-5" : "my-rhel-clone", +# 'rhel-child': 'clone-child' +# }] +def transform_arg_channels(chan_list): + to_ret = {} + for src, dest in chan_list: + to_ret[src] = dest + return [to_ret] + +def parse_args(): + parser = OptionParser() + parser.add_option("-c", "--config", dest="config", help="Config file specifying options") + parser.add_option("-u", "--username", dest="username", help="Username") + parser.add_option("-p", "--password", dest="password", help="Password") + parser.add_option("-s", "--server", dest="server", help="Server URL to use for api connections (defaults to https://localhost/rpc/api)", default="https://localhost/rpc/api") + parser.add_option("-l", "--channels", dest="channels", nargs=2, action="append", help="Origianl channel and clone channel labels space seperated (e.g. --channels=rhel-i386-server-5 myclone)") + parser.add_option("-b", "--blacklist", dest="blacklist", nargs="*", help="Space seperated list of package names") + parser.add_option("-d", "--to_date", dest="to_date", help="Clone errata to the specified date (YYYY-MM-DD)") + parser.add_option("-y", "--assumeyes", dest='assumeyes', action='store_true', help="Assume yes for any prompts (unattended).") + + (options, args) = parser.parse_args() + + options.to_date = parse_time(options.to_date) + if options.config and options.channels: + raise UserError("Cannot specify both --channels and --config.") + if options.channels == None or len(options.channels) == 0: + raise UserError("No channels specified.") + return options + + + +def parse_time(time_str): + """We need to use datetime, but python 2.4 does not support strptime(), so we have to parse ourselves""" + try: + split = time_str.split("-") + return datetime.datetime(int(split[0]), int(split[1]), int(split[2])) + except: + raise UserError("Invalid date format (%s), expected YYYY-MM-DD" % time_str) + def systemExit(code, msgs=None): "Exit with a code and optional message(s). Saved a few lines of code."
@@ -36,14 +102,16 @@ def systemExit(code, msgs=None): sys.stderr.write(str(msg)+'\n') sys.exit(code)
-def main(): - # execute +def main(): try: - return cloneByDate.main() + args = parse_args(); + options = merge_config(args) + return cloneByDate.main(options) except KeyboardInterrupt: systemExit(0, "\nUser interrupted process.") - #except Exception as e: - # print e.message + except UserError, error: + print error + return -1 return 0
commit 4dacab98d08e76fe4e6a6d23af46d1c6780fe37e Author: Justin Sherrill jsherril@redhat.com Date: Wed Jan 18 15:00:01 2012 -0500
adding initial spacewalk-clone-by-date
diff --git a/utils/cloneByDate.py b/utils/cloneByDate.py new file mode 100644 index 0000000..ac24d72 --- /dev/null +++ b/utils/cloneByDate.py @@ -0,0 +1,75 @@ +#!/usr/bin/python +# +# Clonse channels by a particular date +# +# Copyright (c) 2008 Red Hat, Inc. +# +# +# This software is licensed to you under the GNU General Public License, +# version 2 (GPLv2). There is NO WARRANTY for this software, express or +# implied, including the implied warranties of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 +# along with this software; if not, see +# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. +# +# Red Hat trademarks are not licensed under GPLv2. No permission is +# granted to use or replicate Red Hat trademarks that are incorporated +# in this software or its documentation. +# + +import pdb +import json +from optparse import OptionParser + + + +def main(): + options = merge_config(parse_args()); + + +def merge_config(options): + if not options.config: + options.channels = transform_arg_channels(options.channels) + return options + config = json.load(open(options.config)) + + #if soemthing is in the config and not passed in as an argument + # add it to options + overwrite = ["username", "password", "blacklist", "channels"] + for key in overwrite: + if config.has_key(key) and not getattr(options, key): + setattr(options, key, config[key]) + + if type(options.channels) == dict: + options.channels = [options.channels] + return options + + +# Using --channels as an argument only supports a single channel 'tree' +# So we need to convert a 2-tuple list of channel labels into an array with a hash +# ex: [ ("rhel-i386-servr-5", "my-rhel-clone"), ('rhel-child', 'clone-child')] +# should become +# [{ +# "rhel-i386-servr-5" : "my-rhel-clone", +# 'rhel-child': 'clone-child' +# }] +def transform_arg_channels(chan_list): + to_ret = {} + for src, dest in chan_list: + to_ret[src] = dest + return [to_ret] + +def parse_args(): + parser = OptionParser() + parser.add_option("-c", "--config", dest="config", help="Config file specifying options") + parser.add_option("-u", "--username", dest="username", help="Username") + parser.add_option("-p", "--password", dest="password", help="Password") + parser.add_option("-l", "--channels", dest="channels", nargs=2, action="append", help="Origianl channel and clone channel labels space seperated (e.g. --channels=rhel-i386-server-5 myclone)") + parser.add_option("-b", "--blacklist", dest="blacklist", nargs="*", help="Space seperated list of package names") + (options, args) = parser.parse_args() + + if options.config and options.channels: + raise Exception("Cannot specify both --channels and --config.") + + return options + \ No newline at end of file diff --git a/utils/spacewalk-clone-by-date b/utils/spacewalk-clone-by-date new file mode 100755 index 0000000..f953986 --- /dev/null +++ b/utils/spacewalk-clone-by-date @@ -0,0 +1,54 @@ +#!/usr/bin/python +# +# Clonse channels by a particular date +# +# Copyright (c) 2008 Red Hat, Inc. +# +# +# This software is licensed to you under the GNU General Public License, +# version 2 (GPLv2). There is NO WARRANTY for this software, express or +# implied, including the implied warranties of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2 +# along with this software; if not, see +# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. +# +# Red Hat trademarks are not licensed under GPLv2. No permission is +# granted to use or replicate Red Hat trademarks that are incorporated +# in this software or its documentation. +# + + +import sys + +try: + import cloneByDate +except: + from utils import cloneByDate + + +def systemExit(code, msgs=None): + "Exit with a code and optional message(s). Saved a few lines of code." + + if msgs: + if type(msgs) not in [type([]), type(())]: + msgs = (msgs, ) + for msg in msgs: + sys.stderr.write(str(msg)+'\n') + sys.exit(code) + +def main(): + # execute + try: + return cloneByDate.main() + except KeyboardInterrupt: + systemExit(0, "\nUser interrupted process.") + #except Exception as e: + # print e.message + return 0 + + +if __name__ == '__main__': + try: + sys.exit(abs(main() or 0)) + except KeyboardInterrupt: + systemExit(0, "\nUser interrupted process.")
spacewalk-commits@lists.fedorahosted.org