[PATCH] Improved support for koji profiles.

[Daniel Mach]

---
 cli/koji         | 137 ++++++++++++-------------------------------------------
 docs/profiles    |  57 +++++++++++++++++++++++
 koji/__init__.py | 130 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 215 insertions(+), 109 deletions(-)
 create mode 100644 docs/profiles

diff --git a/cli/koji b/cli/koji
index f521d6d..29f26bb 100755
--- a/cli/koji
+++ b/cli/koji
@@ -186,88 +186,9 @@ def get_options():
         list_commands()
         parser.error('Unknown command: %s' % args[0])
         assert False
-    # load local config
-    defaults = {
-        'server' : 'http://localhost/kojihub',
-        'weburl' : 'http://localhost/koji',
-        'topurl' : None,
-        'pkgurl' : None,
-        'topdir' : '/mnt/koji',
-        'max_retries' : None,
-        'retry_interval': None,
-        'anon_retry' : None,
-        'offline_retry' : None,
-        'offline_retry_interval' : None,
-        'keepalive' : True,
-        'timeout' : None,
-        'use_fast_upload': False,
-        'poll_interval': 5,
-        'krbservice': 'host',
-        'cert': '~/.koji/client.crt',
-        'ca': '~/.koji/clientca.crt',
-        'serverca': '~/.koji/serverca.crt',
-        'authtype': None
-        }
-    #note: later config files override earlier ones
-    configs = koji.config_directory_contents('/etc/koji.conf.d')
-    if os.access('/etc/koji.conf', os.F_OK):
-        configs.append('/etc/koji.conf')
-    if options.configFile:
-        fn = os.path.expanduser(options.configFile)
-        if os.path.isdir(fn):
-            contents = koji.config_directory_contents(fn)
-            if not contents:
-                parser.error("No config files found in directory: %s" % fn)
-            configs.extend(contents)
-        else:
-            if not os.access(fn, os.F_OK):
-                parser.error("No such file: %s" % fn)
-            configs.append(fn)
-    else:
-        user_config_dir = os.path.expanduser("~/.koji/config.d")
-        configs.extend(koji.config_directory_contents(user_config_dir))
-        fn = os.path.expanduser("~/.koji/config")
-        if os.access(fn, os.F_OK):
-            configs.append(fn)
-    got_conf = False
-    for configFile in configs:
-        f = open(configFile)
-        config = ConfigParser.ConfigParser()
-        config.readfp(f)
-        f.close()
-        if config.has_section(options.profile):
-            got_conf = True
-            for name, value in config.items(options.profile):
-                #note the defaults dictionary also serves to indicate which
-                #options *can* be set via the config file. Such options should
-                #not have a default value set in the option parser.
-                if defaults.has_key(name):
-                    if name in ('anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload'):
-                        defaults[name] = config.getboolean(options.profile, name)
-                    elif name in ('max_retries', 'retry_interval',
-                                  'offline_retry_interval', 'poll_interval', 'timeout'):
-                        try:
-                            defaults[name] = int(value)
-                        except ValueError:
-                            parser.error("value for %s config option must be a valid integer" % name)
-                            assert False
-                    else:
-                        defaults[name] = value
-    if configs and not got_conf:
-        warn("Warning: no configuration for profile name: %s" % options.profile)
-    for name, value in defaults.iteritems():
-        if getattr(options, name, None) is None:
-            setattr(options, name, value)
-    dir_opts = ('topdir', 'cert', 'ca', 'serverca')
-    for name in dir_opts:
-        # expand paths here, so we don't have to worry about it later
-        value = os.path.expanduser(getattr(options, name))
-        setattr(options, name, value)
-
-    #honor topdir
-    if options.topdir:
-        koji.BASEDIR = options.topdir
-        koji.pathinfo.topdir = options.topdir
+
+    defaults = koji.read_config(options.profile, user_config=options.configFile)
+    options._update_loose(defaults.__dict__)

#pkgurl is obsolete
     if options.pkgurl:
@@ -1503,11 +1424,11 @@ def linked_upload(localfile, path, name=None):
     try:
         if name is None:
             name = os.path.basename(localfile)
-        dest_dir = os.path.join(koji.pathinfo.work(), path)
+        dest_dir = os.path.join(koji_profile_module.pathinfo.work(), path)
         dst = os.path.join(dest_dir, name)
         koji.ensuredir(dest_dir)
         # fix uid/gid to keep httpd happy
-        st = os.stat(koji.pathinfo.work())
+        st = os.stat(koji_profile_module.pathinfo.work())
         os.chown(dest_dir, st.st_uid, st.st_gid)
         print "Linking rpm to: %s" % dst
         os.link(localfile, dst)
@@ -2018,7 +1939,7 @@ def handle_prune_signed_copies(options, session, args):
             for sig in sigs:
                 sigkey = sig['sigkey']
                 by_sig.setdefault(sigkey, []).append(rpminfo)
-        builddir = koji.pathinfo.build(binfo)
+        builddir = koji_profile_module.pathinfo.build(binfo)
         build_files = 0
         build_space = 0
         if not by_sig and options.debug:
@@ -2028,7 +1949,7 @@ def handle_prune_signed_copies(options, session, args):
             archdirs = {}
             sigdirs = {}
             for rpminfo in rpms:
-                signedpath = "%s/%s" % (builddir, koji.pathinfo.signed(rpminfo, sigkey))
+                signedpath = "%s/%s" % (builddir, koji_profile_module.pathinfo.signed(rpminfo, sigkey))
                 try:
                     st = os.lstat(signedpath)
                 except OSError:
@@ -2285,8 +2206,8 @@ def handle_list_signed(options, session, args):
             binfo = session.getBuild(rinfo['build_id'])
             build_idx[binfo['id']] = binfo
         binfo['name'] = binfo['package_name']
-        builddir = koji.pathinfo.build(binfo)
-        signedpath = "%s/%s" % (builddir, koji.pathinfo.signed(rinfo, sigkey))
+        builddir = koji_profile_module.pathinfo.build(binfo)
+        signedpath = "%s/%s" % (builddir, koji_profile_module.pathinfo.signed(rinfo, sigkey))
         if not os.path.exists(signedpath):
             if options.debug:
                 print "No copy: %s" % signedpath
@@ -2492,7 +2413,6 @@ def anon_handle_latest_build(options, session, args):
         if len(args) < 2:
             parser.error(_("A tag name and package name must be specified"))
             assert False
-    pathinfo = koji.PathInfo()

for pkg in args[1:]:
         if options.arch:
@@ -2503,7 +2423,7 @@ def anon_handle_latest_build(options, session, args):
                 for x in data:
                     z = x.copy()
                     x['name'] = builds_hash[x['build_id']]['package_name']
-                    x['path'] = os.path.join(pathinfo.build(x), pathinfo.rpm(z))
+                    x['path'] = os.path.join(koji_profile_module.pathinfo.build(x), koji_profile_module.pathinfo.rpm(z))
                 fmt = "%(path)s"
             else:
                 fmt = "%(name)s-%(version)s-%(release)s.%(arch)s"
@@ -2515,11 +2435,11 @@ def anon_handle_latest_build(options, session, args):
             if options.paths:
                 if options.type == 'maven':
                     for x in data:
-                        x['path'] = pathinfo.mavenbuild(x)
+                        x['path'] = koji_profile_module.pathinfo.mavenbuild(x)
                     fmt = "%(path)-40s  %(tag_name)-20s  %(maven_group_id)-20s  %(maven_artifact_id)-20s  %(owner_name)s"
                 else:
                     for x in data:
-                        x['path'] = pathinfo.build(x)
+                        x['path'] = koji_profile_module.pathinfo.build(x)
                     fmt = "%(path)-40s  %(tag_name)-20s  %(owner_name)s"
             else:
                 if options.type == 'maven':
@@ -2597,7 +2517,6 @@ def anon_handle_list_tagged(options, session, args):
         parser.error(_("Only one package name may be specified"))
         assert False
     activate_session(session)
-    pathinfo = koji.PathInfo()
     package = None
     if len(args) > 1:
         package = args[1]
@@ -2630,14 +2549,14 @@ def anon_handle_list_tagged(options, session, args):
             build_idx = dict([(b['id'],b) for b in builds])
             for rinfo in data:
                 build = build_idx[rinfo['build_id']]
-                builddir = pathinfo.build(build)
+                builddir = koji_profile_module.pathinfo.build(build)
                 if options.sigs:
                     sigkey = rinfo['sigkey']
-                    signedpath = os.path.join(builddir, pathinfo.signed(rinfo, sigkey))
+                    signedpath = os.path.join(builddir, koji_profile_module.pathinfo.signed(rinfo, sigkey))
                     if os.path.exists(signedpath):
                         rinfo['path'] = signedpath
                 else:
-                    rinfo['path'] = os.path.join(builddir, pathinfo.rpm(rinfo))
+                    rinfo['path'] = os.path.join(builddir, koji_profile_module.pathinfo.rpm(rinfo))
             fmt = "%(path)s"
             data = [x for x in data if x.has_key('path')]
         else:
@@ -2649,11 +2568,11 @@ def anon_handle_list_tagged(options, session, args):
         if options.paths:
             if options.type == 'maven':
                 for x in data:
-                    x['path'] = pathinfo.mavenbuild(x)
+                    x['path'] = koji_profile_module.pathinfo.mavenbuild(x)
                 fmt = "%(path)-40s  %(tag_name)-20s  %(maven_group_id)-20s  %(maven_artifact_id)-20s  %(owner_name)s"
             else:
                 for x in data:
-                    x['path'] = pathinfo.build(x)
+                    x['path'] = koji_profile_module.pathinfo.build(x)
                 fmt = "%(path)-40s  %(tag_name)-20s  %(owner_name)s"
         else:
             if options.type == 'maven':
@@ -2721,7 +2640,6 @@ def anon_handle_list_untagged(options, session, args):
     opts = {}
     if package:
         opts['name'] = package
-    pathinfo = koji.PathInfo()

data = session.untaggedBuilds(**opts)
     if options.show_references:
@@ -2741,7 +2659,7 @@ def anon_handle_list_untagged(options, session, args):
         #data = [x for x in data if not refs.has_key(x['id'])]
     if options.paths:
         for x in data:
-            x['path'] = pathinfo.build(x)
+            x['path'] = koji_profile_module.pathinfo.build(x)
         fmt = "%(path)s"
     else:
         fmt = "%(name)s-%(version)s-%(release)s"
@@ -3053,9 +2971,9 @@ def anon_handle_rpminfo(options, session, args):
             print "External Repository: %(name)s [%(id)i]" % repo
             print "External Repository url: %(url)s" % repo
         else:
-            print "RPM Path: %s" % os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(info))
+            print "RPM Path: %s" % os.path.join(koji_profile_module.pathinfo.build(buildinfo), koji_profile_module.pathinfo.rpm(info))
             print "SRPM: %(epoch)s%(name)s-%(version)s-%(release)s [%(id)d]" % buildinfo
-            print "SRPM Path: %s" % os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(buildinfo))
+            print "SRPM Path: %s" % os.path.join(koji_profile_module.pathinfo.build(buildinfo), koji_profile_module.pathinfo.rpm(buildinfo))
             print "Built: %s" % time.strftime('%a, %d %b %Y %H:%M:%S %Z', time.localtime(info['buildtime']))
         print "Payload: %(payloadhash)s" %info
         print "Size: %(size)s" %info
@@ -3127,23 +3045,23 @@ def anon_handle_buildinfo(options, session, args):
         if maven_archives:
             print "Maven archives:"
             for archive in maven_archives:
-                print os.path.join(koji.pathinfo.mavenbuild(info), koji.pathinfo.mavenfile(archive))
+                print os.path.join(koji_profile_module.pathinfo.mavenbuild(info), koji_profile_module.pathinfo.mavenfile(archive))
         win_archives = session.listArchives(buildID=info['id'], type='win')
         if win_archives:
             print "Windows archives:"
             for archive in win_archives:
-                print os.path.join(koji.pathinfo.winbuild(info), koji.pathinfo.winfile(archive))
+                print os.path.join(koji_profile_module.pathinfo.winbuild(info), koji_profile_module.pathinfo.winfile(archive))
         rpms = session.listRPMs(buildID=info['id'])
         image_info = session.getImageBuild(info['id'])
         img_archives = session.listArchives(buildID=info['id'], type='image')
         if img_archives:
             print 'Image archives:'
             for archive in img_archives:
-                print os.path.join(koji.pathinfo.imagebuild(info), archive['filename'])
+                print os.path.join(koji_profile_module.pathinfo.imagebuild(info), archive['filename'])
         if rpms:
             print "RPMs:"
             for rpm in rpms:
-                print os.path.join(koji.pathinfo.build(info), koji.pathinfo.rpm(rpm))
+                print os.path.join(koji_profile_module.pathinfo.build(info), koji_profile_module.pathinfo.rpm(rpm))
         if options.changelog:
             changelog = session.getChangelogEntries(info['id'])
             if changelog:
@@ -4261,7 +4179,7 @@ def _printTaskInfo(session, task_id, level=0, recurse=True, verbose=True):
     files = session.listTaskOutput(task_id)
     logs = [filename for filename in files if filename.endswith('.log')]
     output = [filename for filename in files if not filename.endswith('.log')]
-    files_dir = '%s/%s' % (koji.pathinfo.work(), koji.pathinfo.taskrelpath(task_id))
+    files_dir = '%s/%s' % (koji_profile_module.pathinfo.work(), koji_profile_module.pathinfo.taskrelpath(task_id))

owner = session.getUser(info['owner'])['name']

@@ -6124,9 +6042,8 @@ def anon_handle_download_build(options, session, args):
             if not suboptions.topurl:
                 print "You must specify --topurl to download images"
                 return 1
-            pi = koji.PathInfo(topdir=suboptions.topurl)
             for archive in archives:
-                url = '%s/%s' % (pi.imagebuild(info), archive['filename'])
+                url = '%s/%s' % (pathinfo.imagebuild(info), archive['filename'])
                 urls.append((url, archive['filename']))
         else:
             # can't happen
@@ -6628,7 +6545,9 @@ def activate_session(session):

if __name__ == "__main__":
     global options
+    global koji_profile_module
     options, command, args = get_options()
+    koji_profile_module = koji.get_profile_module(options.profile)

logger = logging.getLogger("koji")
     handler = logging.StreamHandler(sys.stderr)
diff --git a/docs/profiles b/docs/profiles
new file mode 100644
index 0000000..2f21723
--- /dev/null
+++ b/docs/profiles
@@ -0,0 +1,57 @@
+=============
+Koji Profiles
+=============
+This document describes how to work with koji profiles.
+
+
+Command Line Interface
+======================
+Koji client allows connecting to multiple koji instances from CLI
+by using profiles. The default profile is given by executable file name,
+which is 'koji'.
+
+To change koji profile, you can:
+* run koji with --profile=$profile_name argument
+* change executable file name by symlinking $profile_name -> koji
+
+
+Configuration Files
+===================
+Configuration files are located in following locations:
+* /etc/koji.conf
+* /etc/koji.conf.d/*.conf
+* ~/.koji/config.d/*.conf
+* user-specified config
+Koji reads them all, looking for [$profile_name] sections.
+
+
+Using Koji Profiles in Python
+=============================
+Instead of using koji module directly,
+get profile specific module by calling:
+>>> mod = koji.get_profile_module($profile_name)
+
+This module is clone of koji module with additional
+profile specific tweaks.
+
+Profile configuration is available via:
+>>> mod.config
+
+
+Example
+-------
+import koji
+
+fedora_koji = koji.get_profile_module("koji")
+ppc_koji = koji.get_profile_module("ppc-koji")
+
+for i in (fedora_koji, ppc_koji):
+    print "PROFILE: %s" % i.config.profile
+    for key, value in sorted(i.config.__dict__.items()):
+        print "    %s = %s" % (key, value)
+    print
+
+
+TODO
+====
+* consider using pyxdg for user config locations
diff --git a/koji/__init__.py b/koji/__init__.py
index c0d5a30..dae52fc 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -28,13 +28,16 @@ except ImportError:
     sys.stderr.write("Warning: Could not install krbV module. Kerberos support will be disabled.\n")
     sys.stderr.flush()
 import base64
+import ConfigParser
 import datetime
 import errno
 from fnmatch import fnmatch
 import httplib
+import imp
 import logging
 import logging.handlers
 from koji.util import md5_constructor
+import optparse
 import os
 import os.path
 import pwd
@@ -1457,6 +1460,133 @@ def config_directory_contents(dir_name):
     return configs

+CONFIG_DEFAULTS = {
+    'server': 'http://localhost/kojihub',
+    'weburl': 'http://localhost/koji',
+    'topurl': None,
+    'pkgurl': None,
+    'topdir': '/mnt/koji',
+    'max_retries': None,
+    'retry_interval': None,
+    'anon_retry': None,
+    'offline_retry': None,
+    'offline_retry_interval': None,
+    'keepalive': True,
+    'timeout': None,
+    'use_fast_upload': False,
+    'poll_interval': 5,
+    'krbservice': 'host',
+    'cert': '~/.koji/client.crt',
+    'ca': '~/.koji/clientca.crt',
+    'serverca': '~/.koji/serverca.crt',
+    'authtype': None
+}
+
+
+INT_OPTIONS = ['max_retries', 'retry_interval', 'offline_retry_interval', 'poll_interval', 'timeout']
+BOOL_OPTIONS = ['anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload']
+PATH_OPTIONS = ['topdir', 'cert', 'ca', 'serverca']
+
+
+def config_directory_contents(dir_name):
+    configs = []
+    try:
+        conf_dir_contents = os.listdir(dir_name)
+    except OSError, exception:
+        if exception.errno != errno.ENOENT:
+            raise
+    else:
+        for name in sorted(conf_dir_contents):
+            if not name.endswith('.conf'):
+                continue
+            config_full_name = os.path.join(dir_name, name)
+            configs.append(config_full_name)
+    return configs
+
+
+def read_config(profile_name, user_config=None):
+    result = CONFIG_DEFAULTS.copy()
+    for option in CONFIG_DEFAULTS:
+        if option in PATH_OPTIONS:
+            result[option] = os.path.expanduser(result[option])
+
+    configs = []
+
+    # main config
+    configs.append("/etc/koji.conf")
+
+    # conf.d
+    configs.extend(config_directory_contents("/etc/koji.conf.d"))
+
+    # user config
+    configs.append(os.path.expanduser("~/.koji/config"))
+
+    # user conf.d
+    configs.extend(config_directory_contents(os.path.expanduser("~/.koji/conf.d")))
+
+    # TODO: read configs via xdg.BaseDirectory.load_config_path("koji")
+
+    # user config specified in runtime
+    if user_config is not None:
+        configs.append(user_config)
+
+    # read configs in particular order, use the last value found
+    for config_path in configs:
+        if not os.access(config_path, os.F_OK):
+            continue
+        config = ConfigParser.SafeConfigParser()
+        config.readfp(open(config_path, "r"))
+
+        if profile_name not in config.sections():
+            continue
+
+        # check for invalid options
+        invalid_options = []
+        for option in config.options(profile_name):
+            if option not in result:
+                invalid_options.append(option)
+
+        if invalid_options:
+            raise ValueError("Invalid options: %s" % ", ".join(invalid_options))
+
+        for option in config.options(profile_name):
+            if option in BOOL_OPTIONS:
+                result[option] = config.getboolean(profile_name, option)
+            elif option in INT_OPTIONS:
+                result[option] = config.getint(profile_name, option)
+            else:
+                result[option] = config.get(profile_name, option)
+                if option in PATH_OPTIONS:
+                    result[option] = os.path.expanduser(result[option])
+
+    result["profile"] = profile_name
+
+    # convert dict to optparse Values
+    options = optparse.Values(result)
+    return options
+
+
+config = read_config("koji")
+
+
+def get_profile_module(profile_name, config=None):
+    """
+    Create module for a koji instance.
+    Override profile specific module attributes:
+     * BASEDIR
+     * config
+     * pathinfo
+    """
+    if config is None:
+        config = read_config(profile_name)
+    mod = imp.new_module("_koji__%s" % profile_name)
+    mod.__dict__.update(globals())
+    mod.config = config
+    mod.BASEDIR = config.topdir
+    mod.pathinfo = PathInfo(topdir=config.topdir)
+    return mod
+
+
 class PathInfo(object):
     # ASCII numbers and upper- and lower-case letter for use in tmpdir()
     ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) + range(97, 123)]

[Dennis Gilmore]

On Friday, June 26, 2015 04:00:57 AM Daniel Mach wrote:
> cli/koji         | 137
> ++++++++++++------------------------------------------- docs/profiles    | 
> 57 +++++++++++++++++++++++
>  koji/__init__.py | 130 ++++++++++++++++++++++++++++++++++++++++++++++++++++
> 3 files changed, 215 insertions(+), 109 deletions(-)
>  create mode 100644 docs/profiles
>
> diff --git a/cli/koji b/cli/koji
> index f521d6d..29f26bb 100755
> --- a/cli/koji
> +++ b/cli/koji
> @@ -186,88 +186,9 @@ def get_options():
>          list_commands()
>          parser.error('Unknown command: %s' % args[0])
>          assert False
> # load local config
> defaults = {
>    'server' : 'http://localhost/kojihub',
>    'weburl' : 'http://localhost/koji',
>    'topurl' : None,
>    'pkgurl' : None,
>    'topdir' : '/mnt/koji',
>    'max_retries' : None,
>    'retry_interval': None,
>    'anon_retry' : None,
>    'offline_retry' : None,
>    'offline_retry_interval' : None,
>    'keepalive' : True,
>    'timeout' : None,
>    'use_fast_upload': False,
>    'poll_interval': 5,
>    'krbservice': 'host',
>    'cert': '~/.koji/client.crt',
>    'ca': '~/.koji/clientca.crt',
>    'serverca': '~/.koji/serverca.crt',
>    'authtype': None
>    }
> #note: later config files override earlier ones
> configs = koji.config_directory_contents('/etc/koji.conf.d')
> if os.access('/etc/koji.conf', os.F_OK):
>    configs.append('/etc/koji.conf')
> if options.configFile:
>    fn = os.path.expanduser(options.configFile)
>    if os.path.isdir(fn):
>        contents = koji.config_directory_contents(fn)
>        if not contents:
>            parser.error("No config files found in directory: %s" % fn)
>        configs.extend(contents)
>    else:
>        if not os.access(fn, os.F_OK):
>            parser.error("No such file: %s" % fn)
>        configs.append(fn)
> else:
>    user_config_dir = os.path.expanduser("~/.koji/config.d")
>    configs.extend(koji.config_directory_contents(user_config_dir))
>    fn = os.path.expanduser("~/.koji/config")
>    if os.access(fn, os.F_OK):
>        configs.append(fn)
> got_conf = False
> for configFile in configs:
>    f = open(configFile)
>    config = ConfigParser.ConfigParser()
>    config.readfp(f)
>    f.close()
>    if config.has_section(options.profile):
>        got_conf = True
>        for name, value in config.items(options.profile):
>            #note the defaults dictionary also serves to indicate which
>            #options *can* be set via the config file. Such options
>
> should -                #not have a default value set in the option parser.
>            if defaults.has_key(name):
>                if name in ('anon_retry', 'offline_retry', 'keepalive',
>
> 'use_fast_upload'): -                        defaults[name] =
> config.getboolean(options.profile, name) -                    elif name in
> ('max_retries', 'retry_interval', -                                 
> 'offline_retry_interval', 'poll_interval', 'timeout'): -                   
>     try:
>                        defaults[name] = int(value)
>                    except ValueError:
>                        parser.error("value for %s config option must
>
> be a valid integer" % name) -                            assert False
>                else:
>                    defaults[name] = value
> if configs and not got_conf:
>    warn("Warning: no configuration for profile name: %s" %
>
> options.profile) -    for name, value in defaults.iteritems():
>    if getattr(options, name, None) is None:
>        setattr(options, name, value)
> dir_opts = ('topdir', 'cert', 'ca', 'serverca')
> for name in dir_opts:
>    # expand paths here, so we don't have to worry about it later
>    value = os.path.expanduser(getattr(options, name))
>    setattr(options, name, value)
> #honor topdir
> if options.topdir:
>    koji.BASEDIR = options.topdir
>    koji.pathinfo.topdir = options.topdir
> defaults = koji.read_config(options.profile,
>
> user_config=options.configFile) +   
> options._update_loose(defaults.__dict__)
>  #pkgurl is obsolete
>  if options.pkgurl:
>
> @@ -1503,11 +1424,11 @@ def linked_upload(localfile, path, name=None):
>      try:
>          if name is None:
>              name = os.path.basename(localfile)
>    dest_dir = os.path.join(koji.pathinfo.work(), path)
>    dest_dir = os.path.join(koji_profile_module.pathinfo.work(), path)
>    dst = os.path.join(dest_dir, name)
>    koji.ensuredir(dest_dir)
>    # fix uid/gid to keep httpd happy
>    st = os.stat(koji.pathinfo.work())
>    st = os.stat(koji_profile_module.pathinfo.work())
>    os.chown(dest_dir, st.st_uid, st.st_gid)
>    print "Linking rpm to: %s" % dst
>    os.link(localfile, dst)
>
> @@ -2018,7 +1939,7 @@ def handle_prune_signed_copies(options, session,
> args): for sig in sigs:
>                  sigkey = sig['sigkey']
>                  by_sig.setdefault(sigkey, []).append(rpminfo)
>    builddir = koji.pathinfo.build(binfo)
>    builddir = koji_profile_module.pathinfo.build(binfo)
>    build_files = 0
>    build_space = 0
>    if not by_sig and options.debug:
>
> @@ -2028,7 +1949,7 @@ def handle_prune_signed_copies(options, session,
> args): archdirs = {}
>              sigdirs = {}
>              for rpminfo in rpms:
>            signedpath = "%s/%s" % (builddir,
>
> koji.pathinfo.signed(rpminfo, sigkey)) +                signedpath =
> "%s/%s" % (builddir, koji_profile_module.pathinfo.signed(rpminfo, sigkey))
> try:
>                      st = os.lstat(signedpath)
>                  except OSError:
> @@ -2285,8 +2206,8 @@ def handle_list_signed(options, session, args):
>              binfo = session.getBuild(rinfo['build_id'])
>              build_idx[binfo['id']] = binfo
>          binfo['name'] = binfo['package_name']
>    builddir = koji.pathinfo.build(binfo)
>    signedpath = "%s/%s" % (builddir, koji.pathinfo.signed(rinfo,
>
> sigkey)) +        builddir = koji_profile_module.pathinfo.build(binfo)
>    signedpath = "%s/%s" % (builddir,
>
> koji_profile_module.pathinfo.signed(rinfo, sigkey)) if not
> os.path.exists(signedpath):
>              if options.debug:
>                  print "No copy: %s" % signedpath
> @@ -2492,7 +2413,6 @@ def anon_handle_latest_build(options, session, args):
>          if len(args) < 2:
>              parser.error(_("A tag name and package name must be
> specified")) assert False
>
> pathinfo = koji.PathInfo()
>
> for pkg in args[1:]:
>     if options.arch:
>
> @@ -2503,7 +2423,7 @@ def anon_handle_latest_build(options, session, args):
>                  for x in data:
>                      z = x.copy()
>                      x['name'] = builds_hash[x['build_id']]['package_name']
>                x['path'] = os.path.join(pathinfo.build(x),
>
> pathinfo.rpm(z)) +                    x['path'] =
> os.path.join(koji_profile_module.pathinfo.build(x),
> koji_profile_module.pathinfo.rpm(z)) fmt = "%(path)s"
>              else:
>                  fmt = "%(name)s-%(version)s-%(release)s.%(arch)s"
> @@ -2515,11 +2435,11 @@ def anon_handle_latest_build(options, session,
> args): if options.paths:
>                  if options.type == 'maven':
>                      for x in data:
>                    x['path'] = pathinfo.mavenbuild(x)
>                    x['path'] =
>
> koji_profile_module.pathinfo.mavenbuild(x) fmt = "%(path)-40s 
> %(tag_name)-20s  %(maven_group_id)-20s  %(maven_artifact_id)-20s 
> %(owner_name)s" else:
>                      for x in data:
>                    x['path'] = pathinfo.build(x)
>                    x['path'] = koji_profile_module.pathinfo.build(x)
>                fmt = "%(path)-40s  %(tag_name)-20s  %(owner_name)s"
>        else:
>            if options.type == 'maven':
>
> @@ -2597,7 +2517,6 @@ def anon_handle_list_tagged(options, session, args):
>          parser.error(_("Only one package name may be specified"))
>          assert False
>      activate_session(session)
> pathinfo = koji.PathInfo()
> package = None
> if len(args) > 1:
>     package = args[1]
>
> @@ -2630,14 +2549,14 @@ def anon_handle_list_tagged(options, session, args):
> build_idx = dict([(b['id'],b) for b in builds])
>              for rinfo in data:
>                  build = build_idx[rinfo['build_id']]
>            builddir = pathinfo.build(build)
>            builddir = koji_profile_module.pathinfo.build(build)
>            if options.sigs:
>                sigkey = rinfo['sigkey']
>                signedpath = os.path.join(builddir,
>
> pathinfo.signed(rinfo, sigkey)) +                    signedpath =
> os.path.join(builddir, koji_profile_module.pathinfo.signed(rinfo, sigkey))
> if os.path.exists(signedpath):
>                          rinfo['path'] = signedpath
>                  else:
>                rinfo['path'] = os.path.join(builddir,
>
> pathinfo.rpm(rinfo)) +                    rinfo['path'] =
> os.path.join(builddir, koji_profile_module.pathinfo.rpm(rinfo)) fmt =
> "%(path)s"
>              data = [x for x in data if x.has_key('path')]
>          else:
> @@ -2649,11 +2568,11 @@ def anon_handle_list_tagged(options, session, args):
> if options.paths:
>              if options.type == 'maven':
>                  for x in data:
>                x['path'] = pathinfo.mavenbuild(x)
>                x['path'] = koji_profile_module.pathinfo.mavenbuild(x)
>            fmt = "%(path)-40s  %(tag_name)-20s  %(maven_group_id)-20s 
>
> %(maven_artifact_id)-20s  %(owner_name)s" else:
>                  for x in data:
>                x['path'] = pathinfo.build(x)
>                x['path'] = koji_profile_module.pathinfo.build(x)
>            fmt = "%(path)-40s  %(tag_name)-20s  %(owner_name)s"
>    else:
>        if options.type == 'maven':
>
> @@ -2721,7 +2640,6 @@ def anon_handle_list_untagged(options, session, args):
> opts = {}
>      if package:
>          opts['name'] = package
>
> pathinfo = koji.PathInfo()
>
> data = session.untaggedBuilds(**opts)
> if options.show_references:
>
> @@ -2741,7 +2659,7 @@ def anon_handle_list_untagged(options, session, args):
> #data = [x for x in data if not refs.has_key(x['id'])]
>      if options.paths:
>          for x in data:
>        x['path'] = pathinfo.build(x)
>        x['path'] = koji_profile_module.pathinfo.build(x)
>    fmt = "%(path)s"
> else:
>        fmt = "%(name)s-%(version)s-%(release)s"
>
> @@ -3053,9 +2971,9 @@ def anon_handle_rpminfo(options, session, args):
>              print "External Repository: %(name)s [%(id)i]" % repo
>              print "External Repository url: %(url)s" % repo
>          else:
>        print "RPM Path: %s" %
>
> os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(info)) +    
>        print "RPM Path: %s" %
> os.path.join(koji_profile_module.pathinfo.build(buildinfo),
> koji_profile_module.pathinfo.rpm(info)) print "SRPM:
> %(epoch)s%(name)s-%(version)s-%(release)s [%(id)d]" % buildinfo -          
>  print "SRPM Path: %s" % os.path.join(koji.pathinfo.build(buildinfo),
> koji.pathinfo.rpm(buildinfo)) +            print "SRPM Path: %s" %
> os.path.join(koji_profile_module.pathinfo.build(buildinfo),
> koji_profile_module.pathinfo.rpm(buildinfo)) print "Built: %s" %
> time.strftime('%a, %d %b %Y %H:%M:%S %Z',
> time.localtime(info['buildtime'])) print "Payload: %(payloadhash)s" %info
>          print "Size: %(size)s" %info
> @@ -3127,23 +3045,23 @@ def anon_handle_buildinfo(options, session, args):
>          if maven_archives:
>              print "Maven archives:"
>              for archive in maven_archives:
>            print os.path.join(koji.pathinfo.mavenbuild(info),
>
> koji.pathinfo.mavenfile(archive)) +                print
> os.path.join(koji_profile_module.pathinfo.mavenbuild(info),
> koji_profile_module.pathinfo.mavenfile(archive)) win_archives =
> session.listArchives(buildID=info['id'], type='win') if win_archives:
>              print "Windows archives:"
>              for archive in win_archives:
>            print os.path.join(koji.pathinfo.winbuild(info),
>
> koji.pathinfo.winfile(archive)) +                print
> os.path.join(koji_profile_module.pathinfo.winbuild(info),
> koji_profile_module.pathinfo.winfile(archive)) rpms =
> session.listRPMs(buildID=info['id'])
>          image_info = session.getImageBuild(info['id'])
>          img_archives = session.listArchives(buildID=info['id'],
> type='image') if img_archives:
>              print 'Image archives:'
>              for archive in img_archives:
>            print os.path.join(koji.pathinfo.imagebuild(info),
>
> archive['filename']) +                print
> os.path.join(koji_profile_module.pathinfo.imagebuild(info),
> archive['filename']) if rpms:
>              print "RPMs:"
>              for rpm in rpms:
>            print os.path.join(koji.pathinfo.build(info),
>
> koji.pathinfo.rpm(rpm)) +                print
> os.path.join(koji_profile_module.pathinfo.build(info),
> koji_profile_module.pathinfo.rpm(rpm)) if options.changelog:
>              changelog = session.getChangelogEntries(info['id'])
>              if changelog:
> @@ -4261,7 +4179,7 @@ def _printTaskInfo(session, task_id, level=0,
> recurse=True, verbose=True): files = session.listTaskOutput(task_id)
>      logs = [filename for filename in files if filename.endswith('.log')]
>      output = [filename for filename in files if not
> filename.endswith('.log')] -    files_dir = '%s/%s' %
> (koji.pathinfo.work(), koji.pathinfo.taskrelpath(task_id)) +    files_dir =
> '%s/%s' % (koji_profile_module.pathinfo.work(),
> koji_profile_module.pathinfo.taskrelpath(task_id))
>  owner = session.getUser(info['owner'])['name']
>
> @@ -6124,9 +6042,8 @@ def anon_handle_download_build(options, session,
> args): if not suboptions.topurl:
>                  print "You must specify --topurl to download images"
>                  return 1
>        pi = koji.PathInfo(topdir=suboptions.topurl)
>        for archive in archives:
>            url = '%s/%s' % (pi.imagebuild(info), archive['filename'])
>            url = '%s/%s' % (pathinfo.imagebuild(info),
>
> archive['filename']) urls.append((url, archive['filename']))
>          else:
>              # can't happen
> @@ -6628,7 +6545,9 @@ def activate_session(session):
>
> if __name__ == "__main__":
>      global options
>
> global koji_profile_module
> options, command, args = get_options()
>
> koji_profile_module = koji.get_profile_module(options.profile)
>
> logger = logging.getLogger("koji")
> handler = logging.StreamHandler(sys.stderr)
>
> diff --git a/docs/profiles b/docs/profiles
> new file mode 100644
> index 0000000..2f21723
> --- /dev/null
> +++ b/docs/profiles
> @@ -0,0 +1,57 @@
> +=============
> +Koji Profiles
> +=============
> +This document describes how to work with koji profiles.
>
> +Command Line Interface
> +======================
> +Koji client allows connecting to multiple koji instances from CLI
> +by using profiles. The default profile is given by executable file name,
> +which is 'koji'.
>
> +To change koji profile, you can:
> +* run koji with --profile=$profile_name argument
> +* change executable file name by symlinking $profile_name -> koji
>
> +Configuration Files
> +===================
> +Configuration files are located in following locations:
> +* /etc/koji.conf
> +* /etc/koji.conf.d/*.conf
> +* ~/.koji/config.d/*.conf
> +* user-specified config
> +Koji reads them all, looking for [$profile_name] sections.
>
> +Using Koji Profiles in Python
> +=============================
> +Instead of using koji module directly,
> +get profile specific module by calling:
> +>>> mod = koji.get_profile_module($profile_name)
>
> +This module is clone of koji module with additional
> +profile specific tweaks.
>
> +Profile configuration is available via:
> +>>> mod.config
>
> +Example
> +-------
> +import koji
>
> +fedora_koji = koji.get_profile_module("koji")
> +ppc_koji = koji.get_profile_module("ppc-koji")
>
> +for i in (fedora_koji, ppc_koji):
> print "PROFILE: %s" % i.config.profile
> for key, value in sorted(i.config.__dict__.items()):
>    print "    %s = %s" % (key, value)
> print
>
> +TODO
> +====
> +* consider using pyxdg for user config locations
> diff --git a/koji/__init__.py b/koji/__init__.py
> index c0d5a30..dae52fc 100644
> --- a/koji/__init__.py
> +++ b/koji/__init__.py
> @@ -28,13 +28,16 @@ except ImportError:
>      sys.stderr.write("Warning: Could not install krbV module. Kerberos
> support will be disabled.\n") sys.stderr.flush()
>  import base64
> +import ConfigParser
>  import datetime
>  import errno
>  from fnmatch import fnmatch
>  import httplib
> +import imp
>  import logging
>  import logging.handlers
>  from koji.util import md5_constructor
> +import optparse
>  import os
>  import os.path
>  import pwd
> @@ -1457,6 +1460,133 @@ def config_directory_contents(dir_name):
>      return configs
>
> +CONFIG_DEFAULTS = {
> 'server': 'http://localhost/kojihub',
> 'weburl': 'http://localhost/koji',
> 'topurl': None,
> 'pkgurl': None,
> 'topdir': '/mnt/koji',
> 'max_retries': None,
> 'retry_interval': None,
> 'anon_retry': None,
> 'offline_retry': None,
> 'offline_retry_interval': None,
> 'keepalive': True,
> 'timeout': None,
> 'use_fast_upload': False,
> 'poll_interval': 5,
> 'krbservice': 'host',
> 'cert': '~/.koji/client.crt',
> 'ca': '~/.koji/clientca.crt',
> 'serverca': '~/.koji/serverca.crt',
> 'authtype': None
>
> +}
>
> +INT_OPTIONS = ['max_retries', 'retry_interval', 'offline_retry_interval',
> 'poll_interval', 'timeout'] +BOOL_OPTIONS = ['anon_retry', 'offline_retry',
> 'keepalive', 'use_fast_upload'] +PATH_OPTIONS = ['topdir', 'cert', 'ca',
> 'serverca']
>
> +def config_directory_contents(dir_name):
> configs = []
> try:
>    conf_dir_contents = os.listdir(dir_name)
> except OSError, exception:
>    if exception.errno != errno.ENOENT:
>        raise
> else:
>    for name in sorted(conf_dir_contents):
>        if not name.endswith('.conf'):
>            continue
>        config_full_name = os.path.join(dir_name, name)
>        configs.append(config_full_name)
> return configs
>
> +def read_config(profile_name, user_config=None):
> result = CONFIG_DEFAULTS.copy()
> for option in CONFIG_DEFAULTS:
>    if option in PATH_OPTIONS:
>        result[option] = os.path.expanduser(result[option])
> configs = []
> # main config
> configs.append("/etc/koji.conf")
> # conf.d
> configs.extend(config_directory_contents("/etc/koji.conf.d"))
> # user config
> configs.append(os.path.expanduser("~/.koji/config"))
> # user conf.d
>
> configs.extend(config_directory_contents(os.path.expanduser("~/.koji/conf.d
> "))) +
> # TODO: read configs via xdg.BaseDirectory.load_config_path("koji")
> # user config specified in runtime
> if user_config is not None:
>    configs.append(user_config)
> # read configs in particular order, use the last value found
> for config_path in configs:
>    if not os.access(config_path, os.F_OK):
>        continue
>    config = ConfigParser.SafeConfigParser()
>    config.readfp(open(config_path, "r"))
>    if profile_name not in config.sections():
>        continue
>    # check for invalid options
>    invalid_options = []
>    for option in config.options(profile_name):
>        if option not in result:
>            invalid_options.append(option)
>    if invalid_options:
>        raise ValueError("Invalid options: %s" % ",
>
> ".join(invalid_options)) +
>    for option in config.options(profile_name):
>        if option in BOOL_OPTIONS:
>            result[option] = config.getboolean(profile_name, option)
>        elif option in INT_OPTIONS:
>            result[option] = config.getint(profile_name, option)
>        else:
>            result[option] = config.get(profile_name, option)
>            if option in PATH_OPTIONS:
>                result[option] = os.path.expanduser(result[option])
> result["profile"] = profile_name
> # convert dict to optparse Values
> options = optparse.Values(result)
> return options
>
> +config = read_config("koji")
>
> +def get_profile_module(profile_name, config=None):
> """
> Create module for a koji instance.
> Override profile specific module attributes:
> * BASEDIR
> * config
> * pathinfo
> """
> if config is None:
>    config = read_config(profile_name)
> mod = imp.new_module("_koji__%s" % profile_name)
> mod.__dict__.update(globals())
> mod.config = config
> mod.BASEDIR = config.topdir
> mod.pathinfo = PathInfo(topdir=config.topdir)
> return mod
>
> class PathInfo(object):
>      # ASCII numbers and upper- and lower-case letter for use in tmpdir()
>      ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) +
> range(97, 123)]

with pungi I hit 
2015-07-07 20:52:15 [CRITICAL] Compose failed: 
/mnt/koji/compose/rawhide/Fedora-23-20150707.n.4
Traceback (most recent call last):
  File "/bin/pungi-koji", line 326, in <module>
    main()
  File "/bin/pungi-koji", line 180, in main
    run_compose(compose)
  File "/bin/pungi-koji", line 254, in run_compose
    buildinstall_phase.stop()
  File "/usr/lib/python2.7/site-packages/pungi/phases/base.py", line 68, in 
stop
    self.pool.stop()
  File "/usr/lib/python2.7/site-packages/kobo/threads.py", line 65, in run
    self.process(item, num)
  File "/usr/lib/python2.7/site-packages/pungi/phases/buildinstall.py", line 
340, in process
    koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
  File "/usr/lib/python2.7/site-packages/kobo/conf.py", line 90, in 
__getitem__
    return dict.__getitem__(self, name)
KeyError: 'koji_profile'

is this patch needed to make things work?

Dennis

[Pavol Babincak]

On 06/26/2015 10:00 AM, Daniel Mach wrote:
> cli/koji         | 137 ++++++++++++-------------------------------------------
>   docs/profiles    |  57 +++++++++++++++++++++++
>   koji/__init__.py | 130 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>   3 files changed, 215 insertions(+), 109 deletions(-)
>   create mode 100644 docs/profiles
>
> diff --git a/cli/koji b/cli/koji
> index f521d6d..29f26bb 100755
> --- a/cli/koji
> +++ b/cli/koji
> @@ -186,88 +186,9 @@ def get_options():
>           list_commands()
>           parser.error('Unknown command: %s' % args[0])
>           assert False
> # load local config
> defaults = {
>    'server' : 'http://localhost/kojihub',
>    'weburl' : 'http://localhost/koji',
>    'topurl' : None,
>    'pkgurl' : None,
>    'topdir' : '/mnt/koji',
>    'max_retries' : None,
>    'retry_interval': None,
>    'anon_retry' : None,
>    'offline_retry' : None,
>    'offline_retry_interval' : None,
>    'keepalive' : True,
>    'timeout' : None,
>    'use_fast_upload': False,
>    'poll_interval': 5,
>    'krbservice': 'host',
>    'cert': '~/.koji/client.crt',
>    'ca': '~/.koji/clientca.crt',
>    'serverca': '~/.koji/serverca.crt',
>    'authtype': None
>    }

This part of code was just moved between python modules (cli/koji -> 
koji/__init__.py). I believe it deserves its own commit where this will 
be obvious.

It seems you forget to rewrite koji.PathInfo in 
anon_handle_download_build().
> diff --git a/docs/profiles b/docs/profiles
> new file mode 100644
> index 0000000..2f21723
> --- /dev/null
> +++ b/docs/profiles

I believe this file should have .rst or .txt extension.
> diff --git a/koji/__init__.py b/koji/__init__.py
> index c0d5a30..dae52fc 100644
> --- a/koji/__init__.py
> +++ b/koji/__init__.py
> +def config_directory_contents(dir_name):
> configs = []
> try:
>    conf_dir_contents = os.listdir(dir_name)
> except OSError, exception:
>    if exception.errno != errno.ENOENT:
>        raise
> else:
>    for name in sorted(conf_dir_contents):
>        if not name.endswith('.conf'):
>            continue
>        config_full_name = os.path.join(dir_name, name)
>        configs.append(config_full_name)
> return configs

This function already exists in the same module.

I believe more obvious would have s/koji./koji_profile_module./ in the 
separate commit.

Because cli/koji isn't extensible python way we import the binary and 
add/modify some of the attributes (see [1]). It isn't obvious that I 
need to define global koji_profile_module otherwise some functions will 
fail. See for example linked_upload() or _printTaskInfo() functions.

[1] 
https://github.com/release-engineering/koji-containerbuild/blob/master/cli/k...