4:44 p.m.
On Friday, June 26, 2015 04:00:57 AM Daniel Mach wrote:
---
cli/koji | 137
++++++++++++------------------------------------------- docs/profiles |
57 +++++++++++++++++++++++
koji/__init__.py | 130 ++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 215 insertions(+), 109 deletions(-)
create mode 100644 docs/profiles
diff --git a/cli/koji b/cli/koji
index f521d6d..29f26bb 100755
--- a/cli/koji
+++ b/cli/koji
@@ -186,88 +186,9 @@ def get_options():
list_commands()
parser.error('Unknown command: %s' % args[0])
assert False
- # load local config
- defaults = {
- 'server' : 'http://localhost/kojihub',
- 'weburl' : 'http://localhost/koji',
- 'topurl' : None,
- 'pkgurl' : None,
- 'topdir' : '/mnt/koji',
- 'max_retries' : None,
- 'retry_interval': None,
- 'anon_retry' : None,
- 'offline_retry' : None,
- 'offline_retry_interval' : None,
- 'keepalive' : True,
- 'timeout' : None,
- 'use_fast_upload': False,
- 'poll_interval': 5,
- 'krbservice': 'host',
- 'cert': '~/.koji/client.crt',
- 'ca': '~/.koji/clientca.crt',
- 'serverca': '~/.koji/serverca.crt',
- 'authtype': None
- }
- #note: later config files override earlier ones
- configs = koji.config_directory_contents('/etc/koji.conf.d')
- if os.access('/etc/koji.conf', os.F_OK):
- configs.append('/etc/koji.conf')
- if options.configFile:
- fn = os.path.expanduser(options.configFile)
- if os.path.isdir(fn):
- contents = koji.config_directory_contents(fn)
- if not contents:
- parser.error("No config files found in directory: %s" % fn)
- configs.extend(contents)
- else:
- if not os.access(fn, os.F_OK):
- parser.error("No such file: %s" % fn)
- configs.append(fn)
- else:
- user_config_dir = os.path.expanduser("~/.koji/config.d")
- configs.extend(koji.config_directory_contents(user_config_dir))
- fn = os.path.expanduser("~/.koji/config")
- if os.access(fn, os.F_OK):
- configs.append(fn)
- got_conf = False
- for configFile in configs:
- f = open(configFile)
- config = ConfigParser.ConfigParser()
- config.readfp(f)
- f.close()
- if config.has_section(options.profile):
- got_conf = True
- for name, value in config.items(options.profile):
- #note the defaults dictionary also serves to indicate which
- #options *can* be set via the config file. Such options
should - #not have a default value set in the option parser.
- if defaults.has_key(name):
- if name in ('anon_retry', 'offline_retry',
'keepalive',
'use_fast_upload'): - defaults[name] =
config.getboolean(options.profile, name) - elif name in
('max_retries', 'retry_interval', -
'offline_retry_interval', 'poll_interval', 'timeout'): -
try:
- defaults[name] = int(value)
- except ValueError:
- parser.error("value for %s config option must
be a valid integer" % name) - assert False
- else:
- defaults[name] = value
- if configs and not got_conf:
- warn("Warning: no configuration for profile name: %s" %
options.profile) - for name, value in defaults.iteritems():
- if getattr(options, name, None) is None:
- setattr(options, name, value)
- dir_opts = ('topdir', 'cert', 'ca', 'serverca')
- for name in dir_opts:
- # expand paths here, so we don't have to worry about it later
- value = os.path.expanduser(getattr(options, name))
- setattr(options, name, value)
-
- #honor topdir
- if options.topdir:
- koji.BASEDIR = options.topdir
- koji.pathinfo.topdir = options.topdir
+
+ defaults = koji.read_config(options.profile,
user_config=options.configFile) +
options._update_loose(defaults.__dict__)
#pkgurl is obsolete
if options.pkgurl:
@@ -1503,11 +1424,11 @@ def linked_upload(localfile, path, name=None):
try:
if name is None:
name = os.path.basename(localfile)
- dest_dir = os.path.join(koji.pathinfo.work(), path)
+ dest_dir = os.path.join(koji_profile_module.pathinfo.work(), path)
dst = os.path.join(dest_dir, name)
koji.ensuredir(dest_dir)
# fix uid/gid to keep httpd happy
- st = os.stat(koji.pathinfo.work())
+ st = os.stat(koji_profile_module.pathinfo.work())
os.chown(dest_dir, st.st_uid, st.st_gid)
print "Linking rpm to: %s" % dst
os.link(localfile, dst)
@@ -2018,7 +1939,7 @@ def handle_prune_signed_copies(options, session,
args): for sig in sigs:
sigkey = sig['sigkey']
by_sig.setdefault(sigkey, []).append(rpminfo)
- builddir = koji.pathinfo.build(binfo)
+ builddir = koji_profile_module.pathinfo.build(binfo)
build_files = 0
build_space = 0
if not by_sig and options.debug:
@@ -2028,7 +1949,7 @@ def handle_prune_signed_copies(options, session,
args): archdirs = {}
sigdirs = {}
for rpminfo in rpms:
- signedpath = "%s/%s" % (builddir,
koji.pathinfo.signed(rpminfo, sigkey)) + signedpath =
"%s/%s" % (builddir, koji_profile_module.pathinfo.signed(rpminfo, sigkey))
try:
st = os.lstat(signedpath)
except OSError:
@@ -2285,8 +2206,8 @@ def handle_list_signed(options, session, args):
binfo = session.getBuild(rinfo['build_id'])
build_idx[binfo['id']] = binfo
binfo['name'] = binfo['package_name']
- builddir = koji.pathinfo.build(binfo)
- signedpath = "%s/%s" % (builddir, koji.pathinfo.signed(rinfo,
sigkey)) + builddir = koji_profile_module.pathinfo.build(binfo)
+ signedpath = "%s/%s" % (builddir,
koji_profile_module.pathinfo.signed(rinfo, sigkey)) if not
os.path.exists(signedpath):
if options.debug:
print "No copy: %s" % signedpath
@@ -2492,7 +2413,6 @@ def anon_handle_latest_build(options, session, args):
if len(args) < 2:
parser.error(_("A tag name and package name must be
specified")) assert False
- pathinfo = koji.PathInfo()
for pkg in args[1:]:
if options.arch:
@@ -2503,7 +2423,7 @@ def anon_handle_latest_build(options, session, args):
for x in data:
z = x.copy()
x['name'] =
builds_hash[x['build_id']]['package_name']
- x['path'] = os.path.join(pathinfo.build(x),
pathinfo.rpm(z)) + x['path'] =
os.path.join(koji_profile_module.pathinfo.build(x),
koji_profile_module.pathinfo.rpm(z)) fmt = "%(path)s"
else:
fmt = "%(name)s-%(version)s-%(release)s.%(arch)s"
@@ -2515,11 +2435,11 @@ def anon_handle_latest_build(options, session,
args): if options.paths:
if options.type == 'maven':
for x in data:
- x['path'] = pathinfo.mavenbuild(x)
+ x['path'] =
koji_profile_module.pathinfo.mavenbuild(x) fmt = "%(path)-40s
%(tag_name)-20s %(maven_group_id)-20s %(maven_artifact_id)-20s
%(owner_name)s" else:
for x in data:
- x['path'] = pathinfo.build(x)
+ x['path'] = koji_profile_module.pathinfo.build(x)
fmt = "%(path)-40s %(tag_name)-20s %(owner_name)s"
else:
if options.type == 'maven':
@@ -2597,7 +2517,6 @@ def anon_handle_list_tagged(options, session, args):
parser.error(_("Only one package name may be specified"))
assert False
activate_session(session)
- pathinfo = koji.PathInfo()
package = None
if len(args) > 1:
package = args[1]
@@ -2630,14 +2549,14 @@ def anon_handle_list_tagged(options, session, args):
build_idx = dict([(b['id'],b) for b in builds])
for rinfo in data:
build = build_idx[rinfo['build_id']]
- builddir = pathinfo.build(build)
+ builddir = koji_profile_module.pathinfo.build(build)
if options.sigs:
sigkey = rinfo['sigkey']
- signedpath = os.path.join(builddir,
pathinfo.signed(rinfo, sigkey)) + signedpath =
os.path.join(builddir, koji_profile_module.pathinfo.signed(rinfo, sigkey))
if os.path.exists(signedpath):
rinfo['path'] = signedpath
else:
- rinfo['path'] = os.path.join(builddir,
pathinfo.rpm(rinfo)) + rinfo['path'] =
os.path.join(builddir, koji_profile_module.pathinfo.rpm(rinfo)) fmt =
"%(path)s"
data = [x for x in data if x.has_key('path')]
else:
@@ -2649,11 +2568,11 @@ def anon_handle_list_tagged(options, session, args):
if options.paths:
if options.type == 'maven':
for x in data:
- x['path'] = pathinfo.mavenbuild(x)
+ x['path'] = koji_profile_module.pathinfo.mavenbuild(x)
fmt = "%(path)-40s %(tag_name)-20s %(maven_group_id)-20s
%(maven_artifact_id)-20s %(owner_name)s" else:
for x in data:
- x['path'] = pathinfo.build(x)
+ x['path'] = koji_profile_module.pathinfo.build(x)
fmt = "%(path)-40s %(tag_name)-20s %(owner_name)s"
else:
if options.type == 'maven':
@@ -2721,7 +2640,6 @@ def anon_handle_list_untagged(options, session, args):
opts = {}
if package:
opts['name'] = package
- pathinfo = koji.PathInfo()
data = session.untaggedBuilds(**opts)
if options.show_references:
@@ -2741,7 +2659,7 @@ def anon_handle_list_untagged(options, session, args):
#data = [x for x in data if not refs.has_key(x['id'])]
if options.paths:
for x in data:
- x['path'] = pathinfo.build(x)
+ x['path'] = koji_profile_module.pathinfo.build(x)
fmt = "%(path)s"
else:
fmt = "%(name)s-%(version)s-%(release)s"
@@ -3053,9 +2971,9 @@ def anon_handle_rpminfo(options, session, args):
print "External Repository: %(name)s [%(id)i]" % repo
print "External Repository url: %(url)s" % repo
else:
- print "RPM Path: %s" %
os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(info)) +
print "RPM Path: %s" %
os.path.join(koji_profile_module.pathinfo.build(buildinfo),
koji_profile_module.pathinfo.rpm(info)) print "SRPM:
%(epoch)s%(name)s-%(version)s-%(release)s [%(id)d]" % buildinfo -
print "SRPM Path: %s" % os.path.join(koji.pathinfo.build(buildinfo),
koji.pathinfo.rpm(buildinfo)) + print "SRPM Path: %s" %
os.path.join(koji_profile_module.pathinfo.build(buildinfo),
koji_profile_module.pathinfo.rpm(buildinfo)) print "Built: %s" %
time.strftime('%a, %d %b %Y %H:%M:%S %Z',
time.localtime(info['buildtime'])) print "Payload: %(payloadhash)s"
%info
print "Size: %(size)s" %info
@@ -3127,23 +3045,23 @@ def anon_handle_buildinfo(options, session, args):
if maven_archives:
print "Maven archives:"
for archive in maven_archives:
- print os.path.join(koji.pathinfo.mavenbuild(info),
koji.pathinfo.mavenfile(archive)) + print
os.path.join(koji_profile_module.pathinfo.mavenbuild(info),
koji_profile_module.pathinfo.mavenfile(archive)) win_archives =
session.listArchives(buildID=info['id'], type='win') if win_archives:
print "Windows archives:"
for archive in win_archives:
- print os.path.join(koji.pathinfo.winbuild(info),
koji.pathinfo.winfile(archive)) + print
os.path.join(koji_profile_module.pathinfo.winbuild(info),
koji_profile_module.pathinfo.winfile(archive)) rpms =
session.listRPMs(buildID=info['id'])
image_info = session.getImageBuild(info['id'])
img_archives = session.listArchives(buildID=info['id'],
type='image') if img_archives:
print 'Image archives:'
for archive in img_archives:
- print os.path.join(koji.pathinfo.imagebuild(info),
archive['filename']) + print
os.path.join(koji_profile_module.pathinfo.imagebuild(info),
archive['filename']) if rpms:
print "RPMs:"
for rpm in rpms:
- print os.path.join(koji.pathinfo.build(info),
koji.pathinfo.rpm(rpm)) + print
os.path.join(koji_profile_module.pathinfo.build(info),
koji_profile_module.pathinfo.rpm(rpm)) if options.changelog:
changelog = session.getChangelogEntries(info['id'])
if changelog:
@@ -4261,7 +4179,7 @@ def _printTaskInfo(session, task_id, level=0,
recurse=True, verbose=True): files = session.listTaskOutput(task_id)
logs = [filename for filename in files if filename.endswith('.log')]
output = [filename for filename in files if not
filename.endswith('.log')] - files_dir = '%s/%s' %
(koji.pathinfo.work(), koji.pathinfo.taskrelpath(task_id)) + files_dir =
'%s/%s' % (koji_profile_module.pathinfo.work(),
koji_profile_module.pathinfo.taskrelpath(task_id))
owner = session.getUser(info['owner'])['name']
@@ -6124,9 +6042,8 @@ def anon_handle_download_build(options, session,
args): if not suboptions.topurl:
print "You must specify --topurl to download images"
return 1
- pi = koji.PathInfo(topdir=suboptions.topurl)
for archive in archives:
- url = '%s/%s' % (pi.imagebuild(info),
archive['filename'])
+ url = '%s/%s' % (pathinfo.imagebuild(info),
archive['filename']) urls.append((url, archive['filename']))
else:
# can't happen
@@ -6628,7 +6545,9 @@ def activate_session(session):
if __name__ == "__main__":
global options
+ global koji_profile_module
options, command, args = get_options()
+ koji_profile_module = koji.get_profile_module(options.profile)
logger = logging.getLogger("koji")
handler = logging.StreamHandler(sys.stderr)
diff --git a/docs/profiles b/docs/profiles
new file mode 100644
index 0000000..2f21723
--- /dev/null
+++ b/docs/profiles
@@ -0,0 +1,57 @@
+=============
+Koji Profiles
+=============
+This document describes how to work with koji profiles.
+
+
+Command Line Interface
+======================
+Koji client allows connecting to multiple koji instances from CLI
+by using profiles. The default profile is given by executable file name,
+which is 'koji'.
+
+To change koji profile, you can:
+* run koji with --profile=$profile_name argument
+* change executable file name by symlinking $profile_name -> koji
+
+
+Configuration Files
+===================
+Configuration files are located in following locations:
+* /etc/koji.conf
+* /etc/koji.conf.d/*.conf
+* ~/.koji/config.d/*.conf
+* user-specified config
+Koji reads them all, looking for [$profile_name] sections.
+
+
+Using Koji Profiles in Python
+=============================
+Instead of using koji module directly,
+get profile specific module by calling:
+>>> mod = koji.get_profile_module($profile_name)
+
+This module is clone of koji module with additional
+profile specific tweaks.
+
+Profile configuration is available via:
+>>> mod.config
+
+
+Example
+-------
+import koji
+
+fedora_koji = koji.get_profile_module("koji")
+ppc_koji = koji.get_profile_module("ppc-koji")
+
+for i in (fedora_koji, ppc_koji):
+ print "PROFILE: %s" % i.config.profile
+ for key, value in sorted(i.config.__dict__.items()):
+ print " %s = %s" % (key, value)
+ print
+
+
+TODO
+====
+* consider using pyxdg for user config locations
diff --git a/koji/__init__.py b/koji/__init__.py
index c0d5a30..dae52fc 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -28,13 +28,16 @@ except ImportError:
sys.stderr.write("Warning: Could not install krbV module. Kerberos
support will be disabled.\n") sys.stderr.flush()
import base64
+import ConfigParser
import datetime
import errno
from fnmatch import fnmatch
import httplib
+import imp
import logging
import logging.handlers
from koji.util import md5_constructor
+import optparse
import os
import os.path
import pwd
@@ -1457,6 +1460,133 @@ def config_directory_contents(dir_name):
return configs
+CONFIG_DEFAULTS = {
+ 'server': 'http://localhost/kojihub',
+ 'weburl': 'http://localhost/koji',
+ 'topurl': None,
+ 'pkgurl': None,
+ 'topdir': '/mnt/koji',
+ 'max_retries': None,
+ 'retry_interval': None,
+ 'anon_retry': None,
+ 'offline_retry': None,
+ 'offline_retry_interval': None,
+ 'keepalive': True,
+ 'timeout': None,
+ 'use_fast_upload': False,
+ 'poll_interval': 5,
+ 'krbservice': 'host',
+ 'cert': '~/.koji/client.crt',
+ 'ca': '~/.koji/clientca.crt',
+ 'serverca': '~/.koji/serverca.crt',
+ 'authtype': None
+}
+
+
+INT_OPTIONS = ['max_retries', 'retry_interval',
'offline_retry_interval',
'poll_interval', 'timeout'] +BOOL_OPTIONS = ['anon_retry',
'offline_retry',
'keepalive', 'use_fast_upload'] +PATH_OPTIONS = ['topdir',
'cert', 'ca',
'serverca']
+
+
+def config_directory_contents(dir_name):
+ configs = []
+ try:
+ conf_dir_contents = os.listdir(dir_name)
+ except OSError, exception:
+ if exception.errno != errno.ENOENT:
+ raise
+ else:
+ for name in sorted(conf_dir_contents):
+ if not name.endswith('.conf'):
+ continue
+ config_full_name = os.path.join(dir_name, name)
+ configs.append(config_full_name)
+ return configs
+
+
+def read_config(profile_name, user_config=None):
+ result = CONFIG_DEFAULTS.copy()
+ for option in CONFIG_DEFAULTS:
+ if option in PATH_OPTIONS:
+ result[option] = os.path.expanduser(result[option])
+
+ configs = []
+
+ # main config
+ configs.append("/etc/koji.conf")
+
+ # conf.d
+ configs.extend(config_directory_contents("/etc/koji.conf.d"))
+
+ # user config
+ configs.append(os.path.expanduser("~/.koji/config"))
+
+ # user conf.d
+
configs.extend(config_directory_contents(os.path.expanduser("~/.koji/conf.d
"))) +
+ # TODO: read configs via xdg.BaseDirectory.load_config_path("koji")
+
+ # user config specified in runtime
+ if user_config is not None:
+ configs.append(user_config)
+
+ # read configs in particular order, use the last value found
+ for config_path in configs:
+ if not os.access(config_path, os.F_OK):
+ continue
+ config = ConfigParser.SafeConfigParser()
+ config.readfp(open(config_path, "r"))
+
+ if profile_name not in config.sections():
+ continue
+
+ # check for invalid options
+ invalid_options = []
+ for option in config.options(profile_name):
+ if option not in result:
+ invalid_options.append(option)
+
+ if invalid_options:
+ raise ValueError("Invalid options: %s" % ",
".join(invalid_options)) +
+ for option in config.options(profile_name):
+ if option in BOOL_OPTIONS:
+ result[option] = config.getboolean(profile_name, option)
+ elif option in INT_OPTIONS:
+ result[option] = config.getint(profile_name, option)
+ else:
+ result[option] = config.get(profile_name, option)
+ if option in PATH_OPTIONS:
+ result[option] = os.path.expanduser(result[option])
+
+ result["profile"] = profile_name
+
+ # convert dict to optparse Values
+ options = optparse.Values(result)
+ return options
+
+
+config = read_config("koji")
+
+
+def get_profile_module(profile_name, config=None):
+ """
+ Create module for a koji instance.
+ Override profile specific module attributes:
+ * BASEDIR
+ * config
+ * pathinfo
+ """
+ if config is None:
+ config = read_config(profile_name)
+ mod = imp.new_module("_koji__%s" % profile_name)
+ mod.__dict__.update(globals())
+ mod.config = config
+ mod.BASEDIR = config.topdir
+ mod.pathinfo = PathInfo(topdir=config.topdir)
+ return mod
+
+
class PathInfo(object):
# ASCII numbers and upper- and lower-case letter for use in tmpdir()
ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) +
range(97, 123)]
with pungi I hit
2015-07-07 20:52:15 [CRITICAL] Compose failed:
/mnt/koji/compose/rawhide/Fedora-23-20150707.n.4
Traceback (most recent call last):
File "/bin/pungi-koji", line 326, in <module>
main()
File "/bin/pungi-koji", line 180, in main
run_compose(compose)
File "/bin/pungi-koji", line 254, in run_compose
buildinstall_phase.stop()
File "/usr/lib/python2.7/site-packages/pungi/phases/base.py", line 68, in
stop
self.pool.stop()
File "/usr/lib/python2.7/site-packages/kobo/threads.py", line 65, in run
self.process(item, num)
File "/usr/lib/python2.7/site-packages/pungi/phases/buildinstall.py", line
340, in process
koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
File "/usr/lib/python2.7/site-packages/kobo/conf.py", line 90, in
__getitem__
return dict.__getitem__(self, name)
KeyError: 'koji_profile'
is this patch needed to make things work?
Dennis