[PATCH] Better support for koji profiles.

[Daniel Mach]

Create python modules for each koji instance,
so it's easier to work with them in python code.
Also move profile related code from CLI to koji library.
See docs/profiles for more details.
---
 cli/koji         |  82 ++-------------------------------------
 docs/profiles    |  50 ++++++++++++++++++++++++
 koji/__init__.py | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 166 insertions(+), 80 deletions(-)
 create mode 100644 docs/profiles

diff --git a/cli/koji b/cli/koji
index 6eba62b..28533e1 100755
--- a/cli/koji
+++ b/cli/koji
@@ -202,84 +202,10 @@ def get_options():
         list_commands()
         parser.error('Unknown command: %s' % args[0])
         assert False
-    # load local config
-    defaults = {
-        'server' : 'http://localhost/kojihub',
-        'weburl' : 'http://localhost/koji',
-        'topurl' : None,
-        'pkgurl' : None,
-        'topdir' : '/mnt/koji',
-        'max_retries' : None,
-        'retry_interval': None,
-        'anon_retry' : None,
-        'offline_retry' : None,
-        'offline_retry_interval' : None,
-        'keepalive' : True,
-        'timeout' : None,
-        'use_fast_upload': False,
-        'poll_interval': 5,
-        'krbservice': 'host',
-        'cert': '~/.koji/client.crt',
-        'ca': '~/.koji/clientca.crt',
-        'serverca': '~/.koji/serverca.crt', 
-        'authtype': None
-        }
-    #note: later config files override earlier ones
-    configs = config_directory_contents('/etc/koji.conf.d')
-    if os.access('/etc/koji.conf', os.F_OK):
-        configs.append('/etc/koji.conf')
-    if options.configFile:
-        fn = os.path.expanduser(options.configFile)
-        if os.path.isdir(fn):
-            contents = config_directory_contents(fn)
-            if not contents:
-                parser.error("No config files found in directory: %s" % fn)
-            configs.extend(contents)
-        else:
-            if not os.access(fn, os.F_OK):
-                parser.error("No such file: %s" % fn)
-            configs.append(fn)
-    else:
-        user_config_dir = os.path.expanduser("~/.koji/config.d")
-        configs.extend(config_directory_contents(user_config_dir))
-        fn = os.path.expanduser("~/.koji/config")
-        if os.access(fn, os.F_OK):
-            configs.append(fn)
-    got_conf = False
-    for configFile in configs:
-        f = open(configFile)
-        config = ConfigParser.ConfigParser()
-        config.readfp(f)
-        f.close()
-        if config.has_section(options.profile):
-            got_conf = True
-            for name, value in config.items(options.profile):
-                #note the defaults dictionary also serves to indicate which
-                #options *can* be set via the config file. Such options should
-                #not have a default value set in the option parser.
-                if defaults.has_key(name):
-                    if name in ('anon_retry', 'offline_retry',
'keepalive', 'use_fast_upload'):
-                        defaults[name] = config.getboolean(options.profile, name)
-                    elif name in ('max_retries', 'retry_interval',
-                                  'offline_retry_interval',
'poll_interval', 'timeout'):
-                        try:
-                            defaults[name] = int(value)
-                        except ValueError:
-                            parser.error("value for %s config option must be a valid
integer" % name)
-                            assert False
-                    else:
-                        defaults[name] = value
-    if configs and not got_conf:
-        warn("Warning: no configuration for profile name: %s" %
options.profile)
-    for name, value in defaults.iteritems():
-        if getattr(options, name, None) is None:
-            setattr(options, name, value)
-    dir_opts = ('topdir', 'cert', 'ca', 'serverca')
-    for name in dir_opts:
-        # expand paths here, so we don't have to worry about it later
-        value = os.path.expanduser(getattr(options, name))
-        setattr(options, name, value)
-    
+
+    defaults = koji.read_config(options.profile, user_config=options.configFile)
+    options._update_loose(defaults.__dict__)
+
     #honor topdir
     if options.topdir:
         koji.BASEDIR = options.topdir
diff --git a/docs/profiles b/docs/profiles
new file mode 100644
index 0000000..82843fa
--- /dev/null
+++ b/docs/profiles
@@ -0,0 +1,50 @@
+Command Line Interface
+----------------------
+Koji client allows connecting to multiple koji instances from CLI
+by using profiles. The default profile is given by executable file name,
+which is 'koji'.
+
+To change koji profile, you can:
+* run koji with --profile=$profile_name argument
+* change executable file name by symlinking $profile_name -> koji
+
+
+Python Modules
+--------------
+In order to refer to python koji modules by name,
+you need to create new python module with following content:
+
+from koji import *
+
+BASEDIR = "/mnt/$profile_name-koji"
+
+
+class PathInfo(PathInfo):
+    @property
+    def basedir(self):
+        return BASEDIR
+
+
+pathinfo = PathInfo()
+config = read_config("$profile_name")
+
+
+Then you can do following:
+get koji profile_name from your config or code
+>>> koji_module = __import__(profile_name)
+>>> client = koji_module.ClientSession(koji_module.config.server, ...)
+
+
+Configuration Files
+-------------------
+Configuration files are located in following locations:
+* /etc/koji.conf
+* /etc/koji.conf.d/*.conf
+* ~/.koji/config.d/*.conf
+* user-specified config
+Koji reads them all, looking for [$profile_name] sections.
+
+
+TODO
+----
+* consider using pyxdg for user config locations
diff --git a/koji/__init__.py b/koji/__init__.py
index a1fae97..2dd5e24 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -1,7 +1,7 @@
 # Python module
 # Common functions
 
-# Copyright (c) 2005-2014 Red Hat, Inc.
+# Copyright (c) 2005-2015 Red Hat, Inc.
 #
 #    Koji is free software; you can redistribute it and/or
 #    modify it under the terms of the GNU Lesser General Public
@@ -28,6 +28,8 @@ except ImportError:
     sys.stderr.write("Warning: Could not install krbV module. Kerberos support will
be disabled.\n")
     sys.stderr.flush()
 import base64
+import ConfigParser
+import optparse
 import datetime
 import errno
 from fnmatch import fnmatch
@@ -1436,6 +1438,110 @@ def openRemoteFile(relpath, topurl=None, topdir=None):
     return fo
 
 
+CONFIG_DEFAULTS = {
+    'server': 'http://localhost/kojihub',
+    'weburl': 'http://localhost/koji',
+    'topurl': None,
+    'pkgurl': None,
+    'topdir': '/mnt/koji',
+    'max_retries': None,
+    'retry_interval': None,
+    'anon_retry': None,
+    'offline_retry': None,
+    'offline_retry_interval': None,
+    'keepalive': True,
+    'timeout': None,
+    'use_fast_upload': False,
+    'poll_interval': 5,
+    'krbservice': 'host',
+    'cert': '~/.koji/client.crt',
+    'ca': '~/.koji/clientca.crt',
+    'serverca': '~/.koji/serverca.crt',
+    'authtype': None
+}
+
+INT_OPTIONS = ['max_retries', 'retry_interval',
'offline_retry_interval', 'poll_interval', 'timeout']
+BOOL_OPTIONS = ['anon_retry', 'offline_retry', 'keepalive',
'use_fast_upload']
+PATH_OPTIONS = ['topdir', 'cert', 'ca', 'serverca']
+
+def config_directory_contents(dir_name):
+    configs = []
+    try:
+        conf_dir_contents = os.listdir(dir_name)
+    except OSError, exception:
+        if exception.errno != errno.ENOENT:
+            raise
+    else:
+        for name in sorted(conf_dir_contents):
+            if not name.endswith('.conf'):
+                continue
+            config_full_name = os.path.join(dir_name, name)
+            configs.append(config_full_name)
+    return configs
+
+
+def read_config(instance_name, user_config=None):
+    result = CONFIG_DEFAULTS.copy()
+    for option in CONFIG_DEFAULTS:
+        if option in PATH_OPTIONS:
+            result[option] = os.path.expanduser(result[option])
+
+    configs = []
+
+    # main config
+    configs.append("/etc/koji.conf")
+
+    # conf.d
+    configs.extend(config_directory_contents("/etc/koji.conf.d"))
+
+    # user config
+    configs.append(os.path.expanduser("~/.koji/config"))
+
+    # user conf.d
+   
configs.extend(config_directory_contents(os.path.expanduser("~/.koji/conf.d")))
+
+    # TODO: read configs via xdg.BaseDirectory.load_config_path("koji")
+
+    # user config specified in runtime
+    if user_config is not None:
+        configs.append(user_config)
+
+    # read configs in particular order, use the last value found
+    for config_path in configs:
+        if not os.access(config_path, os.F_OK):
+            continue
+        config = ConfigParser.SafeConfigParser()
+        config.readfp(open(config_path, "r"))
+
+        if instance_name not in config.sections():
+            continue
+
+        # check for invalid options
+        invalid_options = []
+        for option in config.options(instance_name):
+            if option not in result:
+                invalid_options.append(option)
+
+        if invalid_options:
+            raise ValueError("Invalid options: %s" % ",
".join(invalid_options))
+
+        for option in config.options(instance_name):
+            if option in BOOL_OPTIONS:
+                result[option] = config.getboolean(instance_name, option)
+            elif option in INT_OPTIONS:
+                result[option] = config.getint(instance_name, option)
+            else:
+                result[option] = config.get(instance_name, option)
+                if option in PATH_OPTIONS:
+                    result[option] = os.path.expanduser(result[option])
+
+    # convert dict to optparse Values
+    options = optparse.Values(result)
+    return options
+
+config = read_config("koji")
+
+
 class PathInfo(object):
     # ASCII numbers and upper- and lower-case letter for use in tmpdir()
     ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) + range(97, 123)]
@@ -1443,9 +1549,13 @@ class PathInfo(object):
     def __init__(self, topdir=None):
         self._topdir = topdir
 
+    @property
+    def basedir(self):
+        return str(BASEDIR)
+
     def topdir(self):
         if self._topdir is None:
-            self._topdir = str(BASEDIR)
+            self._topdir = self.basedir
         return self._topdir
 
     def _set_topdir(self, topdir):
-- 
2.1.0

[Mike McLean]

On 02/03/2015 09:32 AM, Daniel Mach wrote:
>  Create python modules for each koji instance,
>  so it's easier to work with them in python code.
>  Also move profile related code from CLI to koji library.
>  See docs/profiles for more details. 
At a glance this looks reasonable, but I need to take a little more time
to review since it conflicts with another patch.

>  ---
>   cli/koji         |  82 ++-------------------------------------
>   docs/profiles    |  50 ++++++++++++++++++++++++
>   koji/__init__.py | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
>   3 files changed, 166 insertions(+), 80 deletions(-)
>   create mode 100644 docs/profiles
>  
>  diff --git a/cli/koji b/cli/koji
>  index 6eba62b..28533e1 100755
>  --- a/cli/koji
>  +++ b/cli/koji
>  @@ -202,84 +202,10 @@ def get_options():
>           list_commands()
>           parser.error('Unknown command: %s' % args[0])
>           assert False
>  -    # load local config
>  -    defaults = {
>  -        'server' : 'http://localhost/kojihub',
>  -        'weburl' : 'http://localhost/koji',
>  -        'topurl' : None,
>  -        'pkgurl' : None,
>  -        'topdir' : '/mnt/koji',
>  -        'max_retries' : None,
>  -        'retry_interval': None,
>  -        'anon_retry' : None,
>  -        'offline_retry' : None,
>  -        'offline_retry_interval' : None,
>  -        'keepalive' : True,
>  -        'timeout' : None,
>  -        'use_fast_upload': False,
>  -        'poll_interval': 5,
>  -        'krbservice': 'host',
>  -        'cert': '~/.koji/client.crt',
>  -        'ca': '~/.koji/clientca.crt',
>  -        'serverca': '~/.koji/serverca.crt', 
>  -        'authtype': None
>  -        }
>  -    #note: later config files override earlier ones
>  -    configs = config_directory_contents('/etc/koji.conf.d')
>  -    if os.access('/etc/koji.conf', os.F_OK):
>  -        configs.append('/etc/koji.conf')
>  -    if options.configFile:
>  -        fn = os.path.expanduser(options.configFile)
>  -        if os.path.isdir(fn):
>  -            contents = config_directory_contents(fn)
>  -            if not contents:
>  -                parser.error("No config files found in directory: %s" % fn)
>  -            configs.extend(contents)
>  -        else:
>  -            if not os.access(fn, os.F_OK):
>  -                parser.error("No such file: %s" % fn)
>  -            configs.append(fn)
>  -    else:
>  -        user_config_dir = os.path.expanduser("~/.koji/config.d")
>  -        configs.extend(config_directory_contents(user_config_dir))
>  -        fn = os.path.expanduser("~/.koji/config")
>  -        if os.access(fn, os.F_OK):
>  -            configs.append(fn)
>  -    got_conf = False
>  -    for configFile in configs:
>  -        f = open(configFile)
>  -        config = ConfigParser.ConfigParser()
>  -        config.readfp(f)
>  -        f.close()
>  -        if config.has_section(options.profile):
>  -            got_conf = True
>  -            for name, value in config.items(options.profile):
>  -                #note the defaults dictionary also serves to indicate which
>  -                #options *can* be set via the config file. Such options should
>  -                #not have a default value set in the option parser.
>  -                if defaults.has_key(name):
>  -                    if name in ('anon_retry', 'offline_retry',
> 'keepalive', 'use_fast_upload'):
>  -                        defaults[name] = config.getboolean(options.profile, name)
>  -                    elif name in ('max_retries', 'retry_interval',
>  -                                  'offline_retry_interval',
> 'poll_interval', 'timeout'):
>  -                        try:
>  -                            defaults[name] = int(value)
>  -                        except ValueError:
>  -                            parser.error("value for %s config option must be a
> valid integer" % name)
>  -                            assert False
>  -                    else:
>  -                        defaults[name] = value
>  -    if configs and not got_conf:
>  -        warn("Warning: no configuration for profile name: %s" %
> options.profile)
>  -    for name, value in defaults.iteritems():
>  -        if getattr(options, name, None) is None:
>  -            setattr(options, name, value)
>  -    dir_opts = ('topdir', 'cert', 'ca', 'serverca')
>  -    for name in dir_opts:
>  -        # expand paths here, so we don't have to worry about it later
>  -        value = os.path.expanduser(getattr(options, name))
>  -        setattr(options, name, value)
>  -    
>  +
>  +    defaults = koji.read_config(options.profile, user_config=options.configFile)
>  +    options._update_loose(defaults.__dict__)
>  +
>       #honor topdir
>       if options.topdir:
>           koji.BASEDIR = options.topdir
>  diff --git a/docs/profiles b/docs/profiles
>  new file mode 100644
>  index 0000000..82843fa
>  --- /dev/null
>  +++ b/docs/profiles
>  @@ -0,0 +1,50 @@
>  +Command Line Interface
>  +----------------------
>  +Koji client allows connecting to multiple koji instances from CLI
>  +by using profiles. The default profile is given by executable file name,
>  +which is 'koji'.
>  +
>  +To change koji profile, you can:
>  +* run koji with --profile=$profile_name argument
>  +* change executable file name by symlinking $profile_name -> koji
>  +
>  +
>  +Python Modules
>  +--------------
>  +In order to refer to python koji modules by name,
>  +you need to create new python module with following content:
>  +
>  +from koji import *
>  +
>  +BASEDIR = "/mnt/$profile_name-koji"
>  +
>  +
>  +class PathInfo(PathInfo):
>  +    @property
>  +    def basedir(self):
>  +        return BASEDIR
>  +
>  +
>  +pathinfo = PathInfo()
>  +config = read_config("$profile_name")
>  +
>  +
>  +Then you can do following:
>  +get koji profile_name from your config or code
>  +>>> koji_module = __import__(profile_name)
>  +>>> client = koji_module.ClientSession(koji_module.config.server, ...)
>  +
>  +
>  +Configuration Files
>  +-------------------
>  +Configuration files are located in following locations:
>  +* /etc/koji.conf
>  +* /etc/koji.conf.d/*.conf
>  +* ~/.koji/config.d/*.conf
>  +* user-specified config
>  +Koji reads them all, looking for [$profile_name] sections.
>  +
>  +
>  +TODO
>  +----
>  +* consider using pyxdg for user config locations
>  diff --git a/koji/__init__.py b/koji/__init__.py
>  index a1fae97..2dd5e24 100644
>  --- a/koji/__init__.py
>  +++ b/koji/__init__.py
>  @@ -1,7 +1,7 @@
>   # Python module
>   # Common functions
>   
>  -# Copyright (c) 2005-2014 Red Hat, Inc.
>  +# Copyright (c) 2005-2015 Red Hat, Inc.
>   #
>   #    Koji is free software; you can redistribute it and/or
>   #    modify it under the terms of the GNU Lesser General Public
>  @@ -28,6 +28,8 @@ except ImportError:
>       sys.stderr.write("Warning: Could not install krbV module. Kerberos support will
> be disabled.\n")
>       sys.stderr.flush()
>   import base64
>  +import ConfigParser
>  +import optparse
>   import datetime
>   import errno
>   from fnmatch import fnmatch
>  @@ -1436,6 +1438,110 @@ def openRemoteFile(relpath, topurl=None, topdir=None):
>       return fo
>   
>   
>  +CONFIG_DEFAULTS = {
>  +    'server': 'http://localhost/kojihub',
>  +    'weburl': 'http://localhost/koji',
>  +    'topurl': None,
>  +    'pkgurl': None,
>  +    'topdir': '/mnt/koji',
>  +    'max_retries': None,
>  +    'retry_interval': None,
>  +    'anon_retry': None,
>  +    'offline_retry': None,
>  +    'offline_retry_interval': None,
>  +    'keepalive': True,
>  +    'timeout': None,
>  +    'use_fast_upload': False,
>  +    'poll_interval': 5,
>  +    'krbservice': 'host',
>  +    'cert': '~/.koji/client.crt',
>  +    'ca': '~/.koji/clientca.crt',
>  +    'serverca': '~/.koji/serverca.crt',
>  +    'authtype': None
>  +}
>  +
>  +INT_OPTIONS = ['max_retries', 'retry_interval',
> 'offline_retry_interval', 'poll_interval', 'timeout']
>  +BOOL_OPTIONS = ['anon_retry', 'offline_retry', 'keepalive',
> 'use_fast_upload']
>  +PATH_OPTIONS = ['topdir', 'cert', 'ca', 'serverca']
>  +
>  +def config_directory_contents(dir_name):
>  +    configs = []
>  +    try:
>  +        conf_dir_contents = os.listdir(dir_name)
>  +    except OSError, exception:
>  +        if exception.errno != errno.ENOENT:
>  +            raise
>  +    else:
>  +        for name in sorted(conf_dir_contents):
>  +            if not name.endswith('.conf'):
>  +                continue
>  +            config_full_name = os.path.join(dir_name, name)
>  +            configs.append(config_full_name)
>  +    return configs
>  +
>  +
>  +def read_config(instance_name, user_config=None):
>  +    result = CONFIG_DEFAULTS.copy()
>  +    for option in CONFIG_DEFAULTS:
>  +        if option in PATH_OPTIONS:
>  +            result[option] = os.path.expanduser(result[option])
>  +
>  +    configs = []
>  +
>  +    # main config
>  +    configs.append("/etc/koji.conf")
>  +
>  +    # conf.d
>  +    configs.extend(config_directory_contents("/etc/koji.conf.d"))
>  +
>  +    # user config
>  +    configs.append(os.path.expanduser("~/.koji/config"))
>  +
>  +    # user conf.d
>  +   
> configs.extend(config_directory_contents(os.path.expanduser("~/.koji/conf.d")))
>  +
>  +    # TODO: read configs via xdg.BaseDirectory.load_config_path("koji")
>  +
>  +    # user config specified in runtime
>  +    if user_config is not None:
>  +        configs.append(user_config)
>  +
>  +    # read configs in particular order, use the last value found
>  +    for config_path in configs:
>  +        if not os.access(config_path, os.F_OK):
>  +            continue
>  +        config = ConfigParser.SafeConfigParser()
>  +        config.readfp(open(config_path, "r"))
>  +
>  +        if instance_name not in config.sections():
>  +            continue
>  +
>  +        # check for invalid options
>  +        invalid_options = []
>  +        for option in config.options(instance_name):
>  +            if option not in result:
>  +                invalid_options.append(option)
>  +
>  +        if invalid_options:
>  +            raise ValueError("Invalid options: %s" % ",
> ".join(invalid_options))
>  +
>  +        for option in config.options(instance_name):
>  +            if option in BOOL_OPTIONS:
>  +                result[option] = config.getboolean(instance_name, option)
>  +            elif option in INT_OPTIONS:
>  +                result[option] = config.getint(instance_name, option)
>  +            else:
>  +                result[option] = config.get(instance_name, option)
>  +                if option in PATH_OPTIONS:
>  +                    result[option] = os.path.expanduser(result[option])
>  +
>  +    # convert dict to optparse Values
>  +    options = optparse.Values(result)
>  +    return options
>  +
>  +config = read_config("koji")
>  +
>  +
>   class PathInfo(object):
>       # ASCII numbers and upper- and lower-case letter for use in tmpdir()
>       ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) + range(97, 123)]
>  @@ -1443,9 +1549,13 @@ class PathInfo(object):
>       def __init__(self, topdir=None):
>           self._topdir = topdir
>   
>  +    @property
>  +    def basedir(self):
>  +        return str(BASEDIR)
>  +
>       def topdir(self):
>           if self._topdir is None:
>  -            self._topdir = str(BASEDIR)
>  +            self._topdir = self.basedir
>           return self._topdir
>   
>       def _set_topdir(self, topdir):
>

[Colin Walters]

On Tue, Feb 3, 2015, at 09:32 AM, Daniel Mach wrote:

>  +Then you can do following:
>  +get koji profile_name from your config or code
>  +>>> koji_module = __import__(profile_name) 
I'd say we should recommend against adding these to the global Python path; e.g. it
would be weird if "import centos" happened to be a koji configuration.  Options
here are to enforce a naming scheme like "kojiconfig_$profile", or use a
non-global import path like /usr/share/koji/profiles.