backend/satellite_tools/repo_plugins/yum_src.py | 78 ++++++++++++++++++++----
backend/satellite_tools/reposync.py | 30 ++++++++-
2 files changed, 92 insertions(+), 16 deletions(-)
New commits:
commit 952a2d726114507b2c7e984c1cd46345463ebb73
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Thu Sep 1 13:51:56 2011 +0200
added timing information
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py
index 58d80fa..c1ec24b 100644
--- a/backend/satellite_tools/reposync.py
+++ b/backend/satellite_tools/reposync.py
@@ -114,6 +114,7 @@ class RepoSync:
print "Channel does not exist or is not custom"
sys.exit(1)
+ start_time = datetime.now()
for url in self.urls:
plugin = self.load_plugin()(url, self.channel_label)
self.import_packages(plugin, url)
@@ -123,8 +124,10 @@ class RepoSync:
[self.channel_label], [], "server.app.yumreposync")
taskomatic.add_to_erratacache_queue(self.channel_label)
self.update_date()
- rhnSQL.commit()
- self.print_msg("Sync complete")
+ rhnSQL.commit()
+ total_time = datetime.now() - start_time
+ self.print_msg("Sync completed.")
+ self.print_msg("Total time: %s" %
str(total_time).split('.')[0])
def update_date(self):
commit a4c57f37ebdb4a2986c9a1fef881ca98b7d4fbec
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Thu Sep 1 12:13:06 2011 +0200
made package numbers more understandable
diff --git a/backend/satellite_tools/repo_plugins/yum_src.py
b/backend/satellite_tools/repo_plugins/yum_src.py
index 810016e..2d422ce 100644
--- a/backend/satellite_tools/repo_plugins/yum_src.py
+++ b/backend/satellite_tools/repo_plugins/yum_src.py
@@ -108,6 +108,8 @@ class ContentSource:
warnings.restore()
repo.setup(False)
+ self.num_packages = 0
+ self.num_excluded = 0
def list_packages(self, filters):
@@ -115,9 +117,11 @@ class ContentSource:
sack = self.repo.getPackageSack()
sack.populate(self.repo, 'metadata', None, 0)
list = sack.returnPackages()
+ self.num_packages = len(list)
if filters:
list = self._filter_packages(list, filters)
list = self._get_package_dependencies(sack, list)
+ self.num_excluded = self.num_packages - len(list)
to_return = []
for pack in list:
if pack.arch == 'src':
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py
index 9a0696e..58d80fa 100644
--- a/backend/satellite_tools/reposync.py
+++ b/backend/satellite_tools/reposync.py
@@ -321,7 +321,11 @@ class RepoSync:
def import_packages(self, plug, url):
packages = plug.list_packages(self.filters)
to_process = []
- self.print_msg("Repo " + url + " has " + str(len(packages)) +
" packages.")
+ num_passed = len(packages)
+ self.print_msg("Repo URL: %s" % url)
+ self.print_msg("Packages in repo: %5d" %
plug.num_packages)
+ if plug.num_excluded:
+ self.print_msg("Packages passed filter rules: %5d" % num_passed)
for pack in packages:
db_pack = rhnPackage.get_info_for_package(
[pack.name, pack.version, pack.release, pack.epoch, pack.arch],
@@ -349,6 +353,10 @@ class RepoSync:
if num_to_process == 0:
self.print_msg("No new packages to sync.")
return
+ else:
+ self.print_msg("Packages already synced: %5d" %
+ (num_passed - num_to_process))
+ self.print_msg("Packages to sync: %5d" %
num_to_process)
self.regen=True
is_non_local_repo = (url.find("file://") < 0)
commit 284b635df86f63180fd0e4b2f684c742c7819c7b
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Thu Sep 1 11:45:16 2011 +0200
add package dependencies into list of packages
diff --git a/backend/satellite_tools/repo_plugins/yum_src.py
b/backend/satellite_tools/repo_plugins/yum_src.py
index 53e3390..810016e 100644
--- a/backend/satellite_tools/repo_plugins/yum_src.py
+++ b/backend/satellite_tools/repo_plugins/yum_src.py
@@ -117,6 +117,7 @@ class ContentSource:
list = sack.returnPackages()
if filters:
list = self._filter_packages(list, filters)
+ list = self._get_package_dependencies(sack, list)
to_return = []
for pack in list:
if pack.arch == 'src':
@@ -173,6 +174,15 @@ class ContentSource:
raise UpdateNoticeException
return selected
+ def _get_package_dependencies(self, sack, packages):
+ yumbase = yum.YumBase()
+ yumbase.pkgSack = sack
+ resolved_deps = yumbase.findDeps(packages)
+ for (pkg,deps) in resolved_deps.items():
+ for (dep,dep_packages) in deps.items():
+ packages.extend(dep_packages)
+ return yum.misc.unique(packages)
+
def get_package(self, package):
""" get package """
check = (self.verify_pkg, (package.unique_id ,1), {})
commit 2191b0231f750464795469503b02bdf317927e05
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Thu Sep 1 11:42:22 2011 +0200
implemented include/exclude package filtering
diff --git a/backend/satellite_tools/repo_plugins/yum_src.py
b/backend/satellite_tools/repo_plugins/yum_src.py
index 8a48b5c..53e3390 100644
--- a/backend/satellite_tools/repo_plugins/yum_src.py
+++ b/backend/satellite_tools/repo_plugins/yum_src.py
@@ -115,6 +115,8 @@ class ContentSource:
sack = self.repo.getPackageSack()
sack.populate(self.repo, 'metadata', None, 0)
list = sack.returnPackages()
+ if filters:
+ list = self._filter_packages(list, filters)
to_return = []
for pack in list:
if pack.arch == 'src':
@@ -130,6 +132,47 @@ class ContentSource:
to_return.append(new_pack)
return to_return
+ def _filter_packages(self, packages, filters):
+ """ implement include / exclude logic
+ filters are: [ ('+', includelist1), ('-', excludelist1),
+ ('+', includelist2), ... ]
+ """
+ if filters is None:
+ return
+
+ selected = []
+ excluded = []
+ if filters[0][0] == '-':
+ # first filter is exclude, start with full package list
+ # and then exclude from it
+ selected = packages
+ else:
+ excluded = packages
+
+ for filter in filters:
+ sense, pkg_list = filter
+ if sense == '+':
+ # include
+ exactmatch, matched, unmatched = yum.packages.parsePackages(
+ excluded, pkg_list)
+ allmatched = yum.misc.unique(exactmatch + matched)
+ selected = yum.misc.unique(selected + allmatched)
+ for pkg in allmatched:
+ if pkg in excluded:
+ excluded.remove(pkg)
+ elif sense == '-':
+ # exclude
+ exactmatch, matched, unmatched = yum.packages.parsePackages(
+ selected, pkg_list)
+ allmatched = yum.misc.unique(exactmatch + matched)
+ for pkg in allmatched:
+ if pkg in selected:
+ selected.remove(pkg)
+ excluded = yum.misc.unique(excluded + allmatched)
+ else:
+ raise UpdateNoticeException
+ return selected
+
def get_package(self, package):
""" get package """
check = (self.verify_pkg, (package.unique_id ,1), {})
commit 44feb7be9915925cf355cf6a80cf57abe3887d93
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Thu Sep 1 11:38:30 2011 +0200
implemented options for package filtering
diff --git a/backend/satellite_tools/repo_plugins/yum_src.py
b/backend/satellite_tools/repo_plugins/yum_src.py
index eb70438..8a48b5c 100644
--- a/backend/satellite_tools/repo_plugins/yum_src.py
+++ b/backend/satellite_tools/repo_plugins/yum_src.py
@@ -109,7 +109,8 @@ class ContentSource:
repo.setup(False)
- def list_packages(self):
+
+ def list_packages(self, filters):
""" list packages"""
sack = self.repo.getPackageSack()
sack.populate(self.repo, 'metadata', None, 0)
diff --git a/backend/satellite_tools/reposync.py b/backend/satellite_tools/reposync.py
index 6834adb..9a0696e 100644
--- a/backend/satellite_tools/reposync.py
+++ b/backend/satellite_tools/reposync.py
@@ -15,6 +15,7 @@
#
import sys, os, time
import hashlib
+import re
from datetime import datetime
import traceback
from optparse import OptionParser
@@ -34,6 +35,11 @@ from spacewalk.server import taskomatic
default_log_location = '/var/log/rhn/reposync/'
default_hash = 'sha256'
+def set_filter_opt(option, opt_str, value, parser):
+ if opt_str in [ '--include', '-i']: f_type = '+'
+ else: f_type = '-'
+ parser.values.filters.append((f_type, re.split('[,\s]+', value)))
+
class RepoSync:
parser = None
@@ -44,6 +50,7 @@ class RepoSync:
fail = False
quiet = False
regen = False
+ filters = []
def main(self):
initCFG('server')
@@ -89,6 +96,7 @@ class RepoSync:
quit = True
self.error_msg("--channel must be specified")
+ self.filters = options.filters
self.log_msg("\nSync started: %s" % (time.asctime(time.localtime())))
self.log_msg(str(sys.argv))
@@ -132,6 +140,9 @@ class RepoSync:
self.parser.add_option('-t', '--type', action='store',
dest='type', help='The type of repo, currently only "yum" is
supported', default='yum')
self.parser.add_option('-f', '--fail',
action='store_true', dest='fail', default=False , help="If a package
import fails, fail the entire operation")
self.parser.add_option('-q', '--quiet',
action='store_true', dest='quiet', default=False, help="Print no
output, still logs output")
+ self.parser.add_option('-i', '--include',
action='callback', callback=set_filter_opt, type='str', nargs=1,
dest='filters', default=[], help="List of included packages")
+ self.parser.add_option('-e', '--exclude',
action='callback', callback=set_filter_opt, type='str', nargs=1,
dest='filters', default=[], help="List of excluded packages")
+
return self.parser.parse_args()
def load_plugin(self):
@@ -308,7 +319,7 @@ class RepoSync:
self.regen = True
def import_packages(self, plug, url):
- packages = plug.list_packages()
+ packages = plug.list_packages(self.filters)
to_process = []
self.print_msg("Repo " + url + " has " + str(len(packages)) +
" packages.")
for pack in packages:
commit 41fb79e669361907bc22e4747f669e7f7383e400
Author: Michael Mraka <michael.mraka(a)redhat.com>
Date: Wed Aug 31 13:50:50 2011 +0200
moved repo init from list_packages() to __init__()
diff --git a/backend/satellite_tools/repo_plugins/yum_src.py
b/backend/satellite_tools/repo_plugins/yum_src.py
index bfc6161..eb70438 100644
--- a/backend/satellite_tools/repo_plugins/yum_src.py
+++ b/backend/satellite_tools/repo_plugins/yum_src.py
@@ -75,13 +75,9 @@ class YumUpdateMetadata(UpdateMetadata):
no.add(un)
class ContentSource:
- url = None
- name = None
repo = None
cache_dir = '/var/cache/rhn/reposync/'
def __init__(self, url, name):
- self.url = url
- self.name = name
self._clean_cache(self.cache_dir + name)
# read the proxy configuration in /etc/rhn/rhn.conf
@@ -96,15 +92,12 @@ class ContentSource:
self.proxy_url = "http://%s" %(self.proxy_addr)
else:
self.proxy_url = None
-
- def list_packages(self):
- """ list packages"""
- repo = yum.yumRepo.YumRepository(self.name)
+ repo = yum.yumRepo.YumRepository(name)
self.repo = repo
repo.cache = 0
repo.metadata_expire = 0
- repo.mirrorlist = self.url
- repo.baseurl = [self.url]
+ repo.mirrorlist = url
+ repo.baseurl = [url]
repo.basecachedir = self.cache_dir
if self.proxy_url is not None:
repo.proxy = self.proxy_url
@@ -115,8 +108,11 @@ class ContentSource:
warnings.restore()
repo.setup(False)
- sack = repo.getPackageSack()
- sack.populate(repo, 'metadata', None, 0)
+
+ def list_packages(self):
+ """ list packages"""
+ sack = self.repo.getPackageSack()
+ sack.populate(self.repo, 'metadata', None, 0)
list = sack.returnPackages()
to_return = []
for pack in list: