[PATCH] Improved support for koji profiles.
Dennis Gilmore
dennis at ausil.us
Tue Jul 7 21:44:16 UTC 2015
On Friday, June 26, 2015 04:00:57 AM Daniel Mach wrote:
> ---
> cli/koji | 137
> ++++++++++++------------------------------------------- docs/profiles |
> 57 +++++++++++++++++++++++
> koji/__init__.py | 130 ++++++++++++++++++++++++++++++++++++++++++++++++++++
> 3 files changed, 215 insertions(+), 109 deletions(-)
> create mode 100644 docs/profiles
>
> diff --git a/cli/koji b/cli/koji
> index f521d6d..29f26bb 100755
> --- a/cli/koji
> +++ b/cli/koji
> @@ -186,88 +186,9 @@ def get_options():
> list_commands()
> parser.error('Unknown command: %s' % args[0])
> assert False
> - # load local config
> - defaults = {
> - 'server' : 'http://localhost/kojihub',
> - 'weburl' : 'http://localhost/koji',
> - 'topurl' : None,
> - 'pkgurl' : None,
> - 'topdir' : '/mnt/koji',
> - 'max_retries' : None,
> - 'retry_interval': None,
> - 'anon_retry' : None,
> - 'offline_retry' : None,
> - 'offline_retry_interval' : None,
> - 'keepalive' : True,
> - 'timeout' : None,
> - 'use_fast_upload': False,
> - 'poll_interval': 5,
> - 'krbservice': 'host',
> - 'cert': '~/.koji/client.crt',
> - 'ca': '~/.koji/clientca.crt',
> - 'serverca': '~/.koji/serverca.crt',
> - 'authtype': None
> - }
> - #note: later config files override earlier ones
> - configs = koji.config_directory_contents('/etc/koji.conf.d')
> - if os.access('/etc/koji.conf', os.F_OK):
> - configs.append('/etc/koji.conf')
> - if options.configFile:
> - fn = os.path.expanduser(options.configFile)
> - if os.path.isdir(fn):
> - contents = koji.config_directory_contents(fn)
> - if not contents:
> - parser.error("No config files found in directory: %s" % fn)
> - configs.extend(contents)
> - else:
> - if not os.access(fn, os.F_OK):
> - parser.error("No such file: %s" % fn)
> - configs.append(fn)
> - else:
> - user_config_dir = os.path.expanduser("~/.koji/config.d")
> - configs.extend(koji.config_directory_contents(user_config_dir))
> - fn = os.path.expanduser("~/.koji/config")
> - if os.access(fn, os.F_OK):
> - configs.append(fn)
> - got_conf = False
> - for configFile in configs:
> - f = open(configFile)
> - config = ConfigParser.ConfigParser()
> - config.readfp(f)
> - f.close()
> - if config.has_section(options.profile):
> - got_conf = True
> - for name, value in config.items(options.profile):
> - #note the defaults dictionary also serves to indicate which
> - #options *can* be set via the config file. Such options
> should - #not have a default value set in the option parser.
> - if defaults.has_key(name):
> - if name in ('anon_retry', 'offline_retry', 'keepalive',
> 'use_fast_upload'): - defaults[name] =
> config.getboolean(options.profile, name) - elif name in
> ('max_retries', 'retry_interval', -
> 'offline_retry_interval', 'poll_interval', 'timeout'): -
> try:
> - defaults[name] = int(value)
> - except ValueError:
> - parser.error("value for %s config option must
> be a valid integer" % name) - assert False
> - else:
> - defaults[name] = value
> - if configs and not got_conf:
> - warn("Warning: no configuration for profile name: %s" %
> options.profile) - for name, value in defaults.iteritems():
> - if getattr(options, name, None) is None:
> - setattr(options, name, value)
> - dir_opts = ('topdir', 'cert', 'ca', 'serverca')
> - for name in dir_opts:
> - # expand paths here, so we don't have to worry about it later
> - value = os.path.expanduser(getattr(options, name))
> - setattr(options, name, value)
> -
> - #honor topdir
> - if options.topdir:
> - koji.BASEDIR = options.topdir
> - koji.pathinfo.topdir = options.topdir
> +
> + defaults = koji.read_config(options.profile,
> user_config=options.configFile) +
> options._update_loose(defaults.__dict__)
>
> #pkgurl is obsolete
> if options.pkgurl:
> @@ -1503,11 +1424,11 @@ def linked_upload(localfile, path, name=None):
> try:
> if name is None:
> name = os.path.basename(localfile)
> - dest_dir = os.path.join(koji.pathinfo.work(), path)
> + dest_dir = os.path.join(koji_profile_module.pathinfo.work(), path)
> dst = os.path.join(dest_dir, name)
> koji.ensuredir(dest_dir)
> # fix uid/gid to keep httpd happy
> - st = os.stat(koji.pathinfo.work())
> + st = os.stat(koji_profile_module.pathinfo.work())
> os.chown(dest_dir, st.st_uid, st.st_gid)
> print "Linking rpm to: %s" % dst
> os.link(localfile, dst)
> @@ -2018,7 +1939,7 @@ def handle_prune_signed_copies(options, session,
> args): for sig in sigs:
> sigkey = sig['sigkey']
> by_sig.setdefault(sigkey, []).append(rpminfo)
> - builddir = koji.pathinfo.build(binfo)
> + builddir = koji_profile_module.pathinfo.build(binfo)
> build_files = 0
> build_space = 0
> if not by_sig and options.debug:
> @@ -2028,7 +1949,7 @@ def handle_prune_signed_copies(options, session,
> args): archdirs = {}
> sigdirs = {}
> for rpminfo in rpms:
> - signedpath = "%s/%s" % (builddir,
> koji.pathinfo.signed(rpminfo, sigkey)) + signedpath =
> "%s/%s" % (builddir, koji_profile_module.pathinfo.signed(rpminfo, sigkey))
> try:
> st = os.lstat(signedpath)
> except OSError:
> @@ -2285,8 +2206,8 @@ def handle_list_signed(options, session, args):
> binfo = session.getBuild(rinfo['build_id'])
> build_idx[binfo['id']] = binfo
> binfo['name'] = binfo['package_name']
> - builddir = koji.pathinfo.build(binfo)
> - signedpath = "%s/%s" % (builddir, koji.pathinfo.signed(rinfo,
> sigkey)) + builddir = koji_profile_module.pathinfo.build(binfo)
> + signedpath = "%s/%s" % (builddir,
> koji_profile_module.pathinfo.signed(rinfo, sigkey)) if not
> os.path.exists(signedpath):
> if options.debug:
> print "No copy: %s" % signedpath
> @@ -2492,7 +2413,6 @@ def anon_handle_latest_build(options, session, args):
> if len(args) < 2:
> parser.error(_("A tag name and package name must be
> specified")) assert False
> - pathinfo = koji.PathInfo()
>
> for pkg in args[1:]:
> if options.arch:
> @@ -2503,7 +2423,7 @@ def anon_handle_latest_build(options, session, args):
> for x in data:
> z = x.copy()
> x['name'] = builds_hash[x['build_id']]['package_name']
> - x['path'] = os.path.join(pathinfo.build(x),
> pathinfo.rpm(z)) + x['path'] =
> os.path.join(koji_profile_module.pathinfo.build(x),
> koji_profile_module.pathinfo.rpm(z)) fmt = "%(path)s"
> else:
> fmt = "%(name)s-%(version)s-%(release)s.%(arch)s"
> @@ -2515,11 +2435,11 @@ def anon_handle_latest_build(options, session,
> args): if options.paths:
> if options.type == 'maven':
> for x in data:
> - x['path'] = pathinfo.mavenbuild(x)
> + x['path'] =
> koji_profile_module.pathinfo.mavenbuild(x) fmt = "%(path)-40s
> %(tag_name)-20s %(maven_group_id)-20s %(maven_artifact_id)-20s
> %(owner_name)s" else:
> for x in data:
> - x['path'] = pathinfo.build(x)
> + x['path'] = koji_profile_module.pathinfo.build(x)
> fmt = "%(path)-40s %(tag_name)-20s %(owner_name)s"
> else:
> if options.type == 'maven':
> @@ -2597,7 +2517,6 @@ def anon_handle_list_tagged(options, session, args):
> parser.error(_("Only one package name may be specified"))
> assert False
> activate_session(session)
> - pathinfo = koji.PathInfo()
> package = None
> if len(args) > 1:
> package = args[1]
> @@ -2630,14 +2549,14 @@ def anon_handle_list_tagged(options, session, args):
> build_idx = dict([(b['id'],b) for b in builds])
> for rinfo in data:
> build = build_idx[rinfo['build_id']]
> - builddir = pathinfo.build(build)
> + builddir = koji_profile_module.pathinfo.build(build)
> if options.sigs:
> sigkey = rinfo['sigkey']
> - signedpath = os.path.join(builddir,
> pathinfo.signed(rinfo, sigkey)) + signedpath =
> os.path.join(builddir, koji_profile_module.pathinfo.signed(rinfo, sigkey))
> if os.path.exists(signedpath):
> rinfo['path'] = signedpath
> else:
> - rinfo['path'] = os.path.join(builddir,
> pathinfo.rpm(rinfo)) + rinfo['path'] =
> os.path.join(builddir, koji_profile_module.pathinfo.rpm(rinfo)) fmt =
> "%(path)s"
> data = [x for x in data if x.has_key('path')]
> else:
> @@ -2649,11 +2568,11 @@ def anon_handle_list_tagged(options, session, args):
> if options.paths:
> if options.type == 'maven':
> for x in data:
> - x['path'] = pathinfo.mavenbuild(x)
> + x['path'] = koji_profile_module.pathinfo.mavenbuild(x)
> fmt = "%(path)-40s %(tag_name)-20s %(maven_group_id)-20s
> %(maven_artifact_id)-20s %(owner_name)s" else:
> for x in data:
> - x['path'] = pathinfo.build(x)
> + x['path'] = koji_profile_module.pathinfo.build(x)
> fmt = "%(path)-40s %(tag_name)-20s %(owner_name)s"
> else:
> if options.type == 'maven':
> @@ -2721,7 +2640,6 @@ def anon_handle_list_untagged(options, session, args):
> opts = {}
> if package:
> opts['name'] = package
> - pathinfo = koji.PathInfo()
>
> data = session.untaggedBuilds(**opts)
> if options.show_references:
> @@ -2741,7 +2659,7 @@ def anon_handle_list_untagged(options, session, args):
> #data = [x for x in data if not refs.has_key(x['id'])]
> if options.paths:
> for x in data:
> - x['path'] = pathinfo.build(x)
> + x['path'] = koji_profile_module.pathinfo.build(x)
> fmt = "%(path)s"
> else:
> fmt = "%(name)s-%(version)s-%(release)s"
> @@ -3053,9 +2971,9 @@ def anon_handle_rpminfo(options, session, args):
> print "External Repository: %(name)s [%(id)i]" % repo
> print "External Repository url: %(url)s" % repo
> else:
> - print "RPM Path: %s" %
> os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(info)) +
> print "RPM Path: %s" %
> os.path.join(koji_profile_module.pathinfo.build(buildinfo),
> koji_profile_module.pathinfo.rpm(info)) print "SRPM:
> %(epoch)s%(name)s-%(version)s-%(release)s [%(id)d]" % buildinfo -
> print "SRPM Path: %s" % os.path.join(koji.pathinfo.build(buildinfo),
> koji.pathinfo.rpm(buildinfo)) + print "SRPM Path: %s" %
> os.path.join(koji_profile_module.pathinfo.build(buildinfo),
> koji_profile_module.pathinfo.rpm(buildinfo)) print "Built: %s" %
> time.strftime('%a, %d %b %Y %H:%M:%S %Z',
> time.localtime(info['buildtime'])) print "Payload: %(payloadhash)s" %info
> print "Size: %(size)s" %info
> @@ -3127,23 +3045,23 @@ def anon_handle_buildinfo(options, session, args):
> if maven_archives:
> print "Maven archives:"
> for archive in maven_archives:
> - print os.path.join(koji.pathinfo.mavenbuild(info),
> koji.pathinfo.mavenfile(archive)) + print
> os.path.join(koji_profile_module.pathinfo.mavenbuild(info),
> koji_profile_module.pathinfo.mavenfile(archive)) win_archives =
> session.listArchives(buildID=info['id'], type='win') if win_archives:
> print "Windows archives:"
> for archive in win_archives:
> - print os.path.join(koji.pathinfo.winbuild(info),
> koji.pathinfo.winfile(archive)) + print
> os.path.join(koji_profile_module.pathinfo.winbuild(info),
> koji_profile_module.pathinfo.winfile(archive)) rpms =
> session.listRPMs(buildID=info['id'])
> image_info = session.getImageBuild(info['id'])
> img_archives = session.listArchives(buildID=info['id'],
> type='image') if img_archives:
> print 'Image archives:'
> for archive in img_archives:
> - print os.path.join(koji.pathinfo.imagebuild(info),
> archive['filename']) + print
> os.path.join(koji_profile_module.pathinfo.imagebuild(info),
> archive['filename']) if rpms:
> print "RPMs:"
> for rpm in rpms:
> - print os.path.join(koji.pathinfo.build(info),
> koji.pathinfo.rpm(rpm)) + print
> os.path.join(koji_profile_module.pathinfo.build(info),
> koji_profile_module.pathinfo.rpm(rpm)) if options.changelog:
> changelog = session.getChangelogEntries(info['id'])
> if changelog:
> @@ -4261,7 +4179,7 @@ def _printTaskInfo(session, task_id, level=0,
> recurse=True, verbose=True): files = session.listTaskOutput(task_id)
> logs = [filename for filename in files if filename.endswith('.log')]
> output = [filename for filename in files if not
> filename.endswith('.log')] - files_dir = '%s/%s' %
> (koji.pathinfo.work(), koji.pathinfo.taskrelpath(task_id)) + files_dir =
> '%s/%s' % (koji_profile_module.pathinfo.work(),
> koji_profile_module.pathinfo.taskrelpath(task_id))
>
> owner = session.getUser(info['owner'])['name']
>
> @@ -6124,9 +6042,8 @@ def anon_handle_download_build(options, session,
> args): if not suboptions.topurl:
> print "You must specify --topurl to download images"
> return 1
> - pi = koji.PathInfo(topdir=suboptions.topurl)
> for archive in archives:
> - url = '%s/%s' % (pi.imagebuild(info), archive['filename'])
> + url = '%s/%s' % (pathinfo.imagebuild(info),
> archive['filename']) urls.append((url, archive['filename']))
> else:
> # can't happen
> @@ -6628,7 +6545,9 @@ def activate_session(session):
>
> if __name__ == "__main__":
> global options
> + global koji_profile_module
> options, command, args = get_options()
> + koji_profile_module = koji.get_profile_module(options.profile)
>
> logger = logging.getLogger("koji")
> handler = logging.StreamHandler(sys.stderr)
> diff --git a/docs/profiles b/docs/profiles
> new file mode 100644
> index 0000000..2f21723
> --- /dev/null
> +++ b/docs/profiles
> @@ -0,0 +1,57 @@
> +=============
> +Koji Profiles
> +=============
> +This document describes how to work with koji profiles.
> +
> +
> +Command Line Interface
> +======================
> +Koji client allows connecting to multiple koji instances from CLI
> +by using profiles. The default profile is given by executable file name,
> +which is 'koji'.
> +
> +To change koji profile, you can:
> +* run koji with --profile=$profile_name argument
> +* change executable file name by symlinking $profile_name -> koji
> +
> +
> +Configuration Files
> +===================
> +Configuration files are located in following locations:
> +* /etc/koji.conf
> +* /etc/koji.conf.d/*.conf
> +* ~/.koji/config.d/*.conf
> +* user-specified config
> +Koji reads them all, looking for [$profile_name] sections.
> +
> +
> +Using Koji Profiles in Python
> +=============================
> +Instead of using koji module directly,
> +get profile specific module by calling:
> +>>> mod = koji.get_profile_module($profile_name)
> +
> +This module is clone of koji module with additional
> +profile specific tweaks.
> +
> +Profile configuration is available via:
> +>>> mod.config
> +
> +
> +Example
> +-------
> +import koji
> +
> +fedora_koji = koji.get_profile_module("koji")
> +ppc_koji = koji.get_profile_module("ppc-koji")
> +
> +for i in (fedora_koji, ppc_koji):
> + print "PROFILE: %s" % i.config.profile
> + for key, value in sorted(i.config.__dict__.items()):
> + print " %s = %s" % (key, value)
> + print
> +
> +
> +TODO
> +====
> +* consider using pyxdg for user config locations
> diff --git a/koji/__init__.py b/koji/__init__.py
> index c0d5a30..dae52fc 100644
> --- a/koji/__init__.py
> +++ b/koji/__init__.py
> @@ -28,13 +28,16 @@ except ImportError:
> sys.stderr.write("Warning: Could not install krbV module. Kerberos
> support will be disabled.\n") sys.stderr.flush()
> import base64
> +import ConfigParser
> import datetime
> import errno
> from fnmatch import fnmatch
> import httplib
> +import imp
> import logging
> import logging.handlers
> from koji.util import md5_constructor
> +import optparse
> import os
> import os.path
> import pwd
> @@ -1457,6 +1460,133 @@ def config_directory_contents(dir_name):
> return configs
>
>
> +CONFIG_DEFAULTS = {
> + 'server': 'http://localhost/kojihub',
> + 'weburl': 'http://localhost/koji',
> + 'topurl': None,
> + 'pkgurl': None,
> + 'topdir': '/mnt/koji',
> + 'max_retries': None,
> + 'retry_interval': None,
> + 'anon_retry': None,
> + 'offline_retry': None,
> + 'offline_retry_interval': None,
> + 'keepalive': True,
> + 'timeout': None,
> + 'use_fast_upload': False,
> + 'poll_interval': 5,
> + 'krbservice': 'host',
> + 'cert': '~/.koji/client.crt',
> + 'ca': '~/.koji/clientca.crt',
> + 'serverca': '~/.koji/serverca.crt',
> + 'authtype': None
> +}
> +
> +
> +INT_OPTIONS = ['max_retries', 'retry_interval', 'offline_retry_interval',
> 'poll_interval', 'timeout'] +BOOL_OPTIONS = ['anon_retry', 'offline_retry',
> 'keepalive', 'use_fast_upload'] +PATH_OPTIONS = ['topdir', 'cert', 'ca',
> 'serverca']
> +
> +
> +def config_directory_contents(dir_name):
> + configs = []
> + try:
> + conf_dir_contents = os.listdir(dir_name)
> + except OSError, exception:
> + if exception.errno != errno.ENOENT:
> + raise
> + else:
> + for name in sorted(conf_dir_contents):
> + if not name.endswith('.conf'):
> + continue
> + config_full_name = os.path.join(dir_name, name)
> + configs.append(config_full_name)
> + return configs
> +
> +
> +def read_config(profile_name, user_config=None):
> + result = CONFIG_DEFAULTS.copy()
> + for option in CONFIG_DEFAULTS:
> + if option in PATH_OPTIONS:
> + result[option] = os.path.expanduser(result[option])
> +
> + configs = []
> +
> + # main config
> + configs.append("/etc/koji.conf")
> +
> + # conf.d
> + configs.extend(config_directory_contents("/etc/koji.conf.d"))
> +
> + # user config
> + configs.append(os.path.expanduser("~/.koji/config"))
> +
> + # user conf.d
> +
> configs.extend(config_directory_contents(os.path.expanduser("~/.koji/conf.d
> "))) +
> + # TODO: read configs via xdg.BaseDirectory.load_config_path("koji")
> +
> + # user config specified in runtime
> + if user_config is not None:
> + configs.append(user_config)
> +
> + # read configs in particular order, use the last value found
> + for config_path in configs:
> + if not os.access(config_path, os.F_OK):
> + continue
> + config = ConfigParser.SafeConfigParser()
> + config.readfp(open(config_path, "r"))
> +
> + if profile_name not in config.sections():
> + continue
> +
> + # check for invalid options
> + invalid_options = []
> + for option in config.options(profile_name):
> + if option not in result:
> + invalid_options.append(option)
> +
> + if invalid_options:
> + raise ValueError("Invalid options: %s" % ",
> ".join(invalid_options)) +
> + for option in config.options(profile_name):
> + if option in BOOL_OPTIONS:
> + result[option] = config.getboolean(profile_name, option)
> + elif option in INT_OPTIONS:
> + result[option] = config.getint(profile_name, option)
> + else:
> + result[option] = config.get(profile_name, option)
> + if option in PATH_OPTIONS:
> + result[option] = os.path.expanduser(result[option])
> +
> + result["profile"] = profile_name
> +
> + # convert dict to optparse Values
> + options = optparse.Values(result)
> + return options
> +
> +
> +config = read_config("koji")
> +
> +
> +def get_profile_module(profile_name, config=None):
> + """
> + Create module for a koji instance.
> + Override profile specific module attributes:
> + * BASEDIR
> + * config
> + * pathinfo
> + """
> + if config is None:
> + config = read_config(profile_name)
> + mod = imp.new_module("_koji__%s" % profile_name)
> + mod.__dict__.update(globals())
> + mod.config = config
> + mod.BASEDIR = config.topdir
> + mod.pathinfo = PathInfo(topdir=config.topdir)
> + return mod
> +
> +
> class PathInfo(object):
> # ASCII numbers and upper- and lower-case letter for use in tmpdir()
> ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) +
> range(97, 123)]
with pungi I hit
2015-07-07 20:52:15 [CRITICAL] Compose failed:
/mnt/koji/compose/rawhide/Fedora-23-20150707.n.4
Traceback (most recent call last):
File "/bin/pungi-koji", line 326, in <module>
main()
File "/bin/pungi-koji", line 180, in main
run_compose(compose)
File "/bin/pungi-koji", line 254, in run_compose
buildinstall_phase.stop()
File "/usr/lib/python2.7/site-packages/pungi/phases/base.py", line 68, in
stop
self.pool.stop()
File "/usr/lib/python2.7/site-packages/kobo/threads.py", line 65, in run
self.process(item, num)
File "/usr/lib/python2.7/site-packages/pungi/phases/buildinstall.py", line
340, in process
koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
File "/usr/lib/python2.7/site-packages/kobo/conf.py", line 90, in
__getitem__
return dict.__getitem__(self, name)
KeyError: 'koji_profile'
is this patch needed to make things work?
Dennis
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 819 bytes
Desc: This is a digitally signed message part.
URL: <http://lists.fedoraproject.org/pipermail/buildsys/attachments/20150707/eae323f5/attachment.sig>
More information about the buildsys
mailing list