[PATCH] use createrepo_c instead of createrepo to make repos if it is available
by Dennis Gilmore
Signed-off-by: Dennis Gilmore <dennis(a)ausil.us>
---
builder/kojid | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/builder/kojid b/builder/kojid
index 3a92ec3..4e53fb5 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -4252,7 +4252,11 @@ class CreaterepoTask(BaseTaskHandler):
def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo):
koji.ensuredir(self.outdir)
- cmd = ['/usr/bin/createrepo', '-vd', '-o', self.outdir]
+ if os.path.isfile('/usr/bin/createrepo_c'):
+ cmd = ['/usr/bin/createrepo_c']
+ else:
+ cmd = ['/usr/bin/createrepo']
+ cmd.extend(['-vd', '-o', self.outdir])
if pkglist is not None:
cmd.extend(['-i', pkglist])
if os.path.isfile(groupdata):
--
2.4.0
7 years, 11 months
[PATCH 1/2] Move config processing from CLI to koji.read_config().
by Daniel Mach
---
cli/koji | 85 ++------------------------------------------
koji/__init__.py | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 109 insertions(+), 82 deletions(-)
diff --git a/cli/koji b/cli/koji
index 47d9691..66f4d99 100755
--- a/cli/koji
+++ b/cli/koji
@@ -186,88 +186,9 @@ def get_options():
list_commands()
parser.error('Unknown command: %s' % args[0])
assert False
- # load local config
- defaults = {
- 'server' : 'http://localhost/kojihub',
- 'weburl' : 'http://localhost/koji',
- 'topurl' : None,
- 'pkgurl' : None,
- 'topdir' : '/mnt/koji',
- 'max_retries' : None,
- 'retry_interval': None,
- 'anon_retry' : None,
- 'offline_retry' : None,
- 'offline_retry_interval' : None,
- 'keepalive' : True,
- 'timeout' : None,
- 'use_fast_upload': False,
- 'poll_interval': 5,
- 'krbservice': 'host',
- 'cert': '~/.koji/client.crt',
- 'ca': '~/.koji/clientca.crt',
- 'serverca': '~/.koji/serverca.crt',
- 'authtype': None
- }
- #note: later config files override earlier ones
- configs = koji.config_directory_contents('/etc/koji.conf.d')
- if os.access('/etc/koji.conf', os.F_OK):
- configs.append('/etc/koji.conf')
- if options.configFile:
- fn = os.path.expanduser(options.configFile)
- if os.path.isdir(fn):
- contents = koji.config_directory_contents(fn)
- if not contents:
- parser.error("No config files found in directory: %s" % fn)
- configs.extend(contents)
- else:
- if not os.access(fn, os.F_OK):
- parser.error("No such file: %s" % fn)
- configs.append(fn)
- else:
- user_config_dir = os.path.expanduser("~/.koji/config.d")
- configs.extend(koji.config_directory_contents(user_config_dir))
- fn = os.path.expanduser("~/.koji/config")
- if os.access(fn, os.F_OK):
- configs.append(fn)
- got_conf = False
- for configFile in configs:
- f = open(configFile)
- config = ConfigParser.ConfigParser()
- config.readfp(f)
- f.close()
- if config.has_section(options.profile):
- got_conf = True
- for name, value in config.items(options.profile):
- #note the defaults dictionary also serves to indicate which
- #options *can* be set via the config file. Such options should
- #not have a default value set in the option parser.
- if defaults.has_key(name):
- if name in ('anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload'):
- defaults[name] = config.getboolean(options.profile, name)
- elif name in ('max_retries', 'retry_interval',
- 'offline_retry_interval', 'poll_interval', 'timeout'):
- try:
- defaults[name] = int(value)
- except ValueError:
- parser.error("value for %s config option must be a valid integer" % name)
- assert False
- else:
- defaults[name] = value
- if configs and not got_conf:
- warn("Warning: no configuration for profile name: %s" % options.profile)
- for name, value in defaults.iteritems():
- if getattr(options, name, None) is None:
- setattr(options, name, value)
- dir_opts = ('topdir', 'cert', 'ca', 'serverca')
- for name in dir_opts:
- # expand paths here, so we don't have to worry about it later
- value = os.path.expanduser(getattr(options, name))
- setattr(options, name, value)
-
- #honor topdir
- if options.topdir:
- koji.BASEDIR = options.topdir
- koji.pathinfo.topdir = options.topdir
+
+ defaults = koji.read_config(options.profile, user_config=options.configFile)
+ options._update_loose(defaults.__dict__)
#pkgurl is obsolete
if options.pkgurl:
diff --git a/koji/__init__.py b/koji/__init__.py
index f45ff70..9951a28 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -28,6 +28,7 @@ except ImportError:
sys.stderr.write("Warning: Could not install krbV module. Kerberos support will be disabled.\n")
sys.stderr.flush()
import base64
+import ConfigParser
import datetime
import errno
from fnmatch import fnmatch
@@ -35,6 +36,7 @@ import httplib
import logging
import logging.handlers
from koji.util import md5_constructor
+import optparse
import os
import os.path
import pwd
@@ -1457,6 +1459,110 @@ def config_directory_contents(dir_name):
return configs
+def _config_directory_contents(dir_name):
+ configs = []
+ try:
+ conf_dir_contents = os.listdir(dir_name)
+ except OSError, exception:
+ if exception.errno != errno.ENOENT:
+ raise
+ else:
+ for name in sorted(conf_dir_contents):
+ if not name.endswith('.conf'):
+ continue
+ config_full_name = os.path.join(dir_name, name)
+ configs.append(config_full_name)
+ return configs
+
+
+def read_config(profile_name, user_config=None):
+ config_defaults = {
+ 'server': 'http://localhost/kojihub',
+ 'weburl': 'http://localhost/koji',
+ 'topurl': None,
+ 'pkgurl': None,
+ 'topdir': '/mnt/koji',
+ 'max_retries': None,
+ 'retry_interval': None,
+ 'anon_retry': None,
+ 'offline_retry': None,
+ 'offline_retry_interval': None,
+ 'keepalive': True,
+ 'timeout': None,
+ 'use_fast_upload': False,
+ 'poll_interval': 5,
+ 'krbservice': 'host',
+ 'cert': '~/.koji/client.crt',
+ 'ca': '~/.koji/clientca.crt',
+ 'serverca': '~/.koji/serverca.crt',
+ 'authtype': None
+ }
+
+ int_options = ['max_retries', 'retry_interval', 'offline_retry_interval', 'poll_interval', 'timeout']
+ bool_options = ['anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload']
+ path_options = ['topdir', 'cert', 'ca', 'serverca']
+
+ result = config_defaults.copy()
+ for option in config_defaults:
+ if option in path_options:
+ result[option] = os.path.expanduser(result[option])
+
+ configs = []
+
+ # main config
+ configs.append("/etc/koji.conf")
+
+ # conf.d
+ configs.extend(_config_directory_contents("/etc/koji.conf.d"))
+
+ # user config
+ configs.append(os.path.expanduser("~/.koji/config"))
+
+ # user conf.d
+ configs.extend(_config_directory_contents(os.path.expanduser("~/.koji/conf.d")))
+
+ # TODO: read configs via xdg.BaseDirectory.load_config_path("koji")
+
+ # user config specified in runtime
+ if user_config is not None:
+ configs.append(user_config)
+
+ # read configs in particular order, use the last value found
+ for config_path in configs:
+ if not os.access(config_path, os.F_OK):
+ continue
+ config = ConfigParser.SafeConfigParser()
+ config.readfp(open(config_path, "r"))
+
+ if profile_name not in config.sections():
+ continue
+
+ # check for invalid options
+ invalid_options = []
+ for option in config.options(profile_name):
+ if option not in result:
+ invalid_options.append(option)
+
+ if invalid_options:
+ raise ValueError("Invalid options: %s" % ", ".join(invalid_options))
+
+ for option in config.options(profile_name):
+ if option in bool_options:
+ result[option] = config.getboolean(profile_name, option)
+ elif option in int_options:
+ result[option] = config.getint(profile_name, option)
+ else:
+ result[option] = config.get(profile_name, option)
+ if option in path_options:
+ result[option] = os.path.expanduser(result[option])
+
+ result["profile"] = profile_name
+
+ # convert dict to optparse Values
+ options = optparse.Values(result)
+ return options
+
+
class PathInfo(object):
# ASCII numbers and upper- and lower-case letter for use in tmpdir()
ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) + range(97, 123)]
--
2.5.0
7 years, 11 months
[PATCH] PAM support for hub and BasicAuth for web
by Christos Triantafyllidis
Hello,
The following patch is adding support for PAM authentication for the
koji-hub and BasicAuth for the koji-web.
This is useful for our internal use case as it allows us to login without
the overhead of setting up either a CA or a kerberos realm for our users.
The configuration is backwards compatible and hopefully similar to the
other authntication methods.
To active PAM support on hub you define the option:
PAMService = koji
in hub.conf. The value will be the name of the PAM service. Note the call
to the PAM module is done via unpriviledged call thus the use of pam_unix
won't be possible.
Note that activating this option will have as result that username/password
combinations from the DB will no longer be checked (similarly to when
activating kerberos or SSL client auth).
The BasicAuth for koji-web requires 2 changes:
a) To enable WSGIPassAuthorization for /koji/login in httpd configuration.
That passes the authorization variable from the apache to the application.
b) Set the "BasicAuthRealm" option to the Basic Authentication Realm that
will be presented to the user to login.
Finally python-pam package has been added to the hub's dependencies.
Cheers,
Christos
Christos Triantafyllidis (1):
- Added PAM support for hub - Added BasicAuth support for web
hub/hub.conf | 4 +++-
hub/kojixmlrpc.py | 2 ++
koji.spec | 1 +
koji/auth.py | 33 +++++++++++++++++++++++++--------
koji/server.py | 2 ++
www/conf/kojiweb.conf | 5 +++++
www/conf/web.conf | 3 +++
www/kojiweb/index.py | 18 +++++++++++++++++-
www/kojiweb/wsgi_publisher.py | 9 +++++++--
9 files changed, 65 insertions(+), 12 deletions(-)
--
2.4.3
8 years
koji and mergerepo_c
by Thomas
Hi Folks,
Quick question, would you consider to use mergerepo_c with option
--all [1] in koji ?
I am happy to work on a patch if it would be accepted. Maybe, we can
keep default behavior and enable with an option on the tag.
My use case is to be able to ship different release built against fix
external repo package that may not be latest version
(Buildrequires:pkg = 1.0.0 while external repo has already 1.1.0)
[1]:
--all
Include all packages with the same name and arch if version or
release is different. If used --method argument is ignored!
--
Thomas
8 years
Koji and DNF status?
by Miroslav Suchý
I would like to ask what is current status of migrating Koji to use DNF for building?
I am mainly interested whether Koji will use DNF for building F23 packages, or you postpone it for F24?
Because in such case I would revert fedora-23-*.cfg default configs in Mock to use Yum so Mock is on pair with Koji.
--
Miroslav Suchy, RHCA
Red Hat, Senior Software Engineer, #brno, #devexp, #fedora-buildsys
8 years
[PATCH] Drop tables and functions only if exist
by Pavol Babincak
At the time of first initialization of Koji database tables and
functions don't exist. This isn't really a problem as SQL commands later
in the script create them.
For non-existent tables or functions psql prints NOTICE instead of an
ERROR. Therefore one can check if database was initialized correctly
with ON_ERROR_STOP set (see psql(1)):
psql returns 3 if an error occurred in a script and the variable
ON_ERROR_STOP was set.
---
docs/schema.sql | 70 ++++++++++++++++++++++++++++-----------------------------
1 file changed, 35 insertions(+), 35 deletions(-)
diff --git a/docs/schema.sql b/docs/schema.sql
index 56418c9..7a46f7f 100644
--- a/docs/schema.sql
+++ b/docs/schema.sql
@@ -1,56 +1,56 @@
-- vim:noet:sw=8
-- still needs work
-DROP TABLE build_notifications;
+DROP TABLE IF EXISTS build_notifications;
-DROP TABLE log_messages;
+DROP TABLE IF EXISTS log_messages;
-DROP TABLE buildroot_listing;
-DROP TABLE image_listing;
+DROP TABLE IF EXISTS buildroot_listing;
+DROP TABLE IF EXISTS image_listing;
-DROP TABLE rpminfo;
-DROP TABLE image_builds;
-DROP TABLE image_archives;
+DROP TABLE IF EXISTS rpminfo;
+DROP TABLE IF EXISTS image_builds;
+DROP TABLE IF EXISTS image_archives;
-DROP TABLE group_package_listing;
-DROP TABLE group_req_listing;
-DROP TABLE group_config;
-DROP TABLE groups;
+DROP TABLE IF EXISTS group_package_listing;
+DROP TABLE IF EXISTS group_req_listing;
+DROP TABLE IF EXISTS group_config;
+DROP TABLE IF EXISTS groups;
-DROP TABLE tag_listing;
-DROP TABLE tag_packages;
+DROP TABLE IF EXISTS tag_listing;
+DROP TABLE IF EXISTS tag_packages;
-DROP TABLE buildroot;
-DROP TABLE repo;
+DROP TABLE IF EXISTS buildroot;
+DROP TABLE IF EXISTS repo;
-DROP TABLE build_target_config;
-DROP TABLE build_target;
+DROP TABLE IF EXISTS build_target_config;
+DROP TABLE IF EXISTS build_target;
-DROP TABLE tag_config;
-DROP TABLE tag_inheritance;
-DROP TABLE tag;
+DROP TABLE IF EXISTS tag_config;
+DROP TABLE IF EXISTS tag_inheritance;
+DROP TABLE IF EXISTS tag;
-DROP TABLE build;
+DROP TABLE IF EXISTS build;
-DROP TABLE task;
+DROP TABLE IF EXISTS task;
-DROP TABLE host_channels;
-DROP TABLE host;
+DROP TABLE IF EXISTS host_channels;
+DROP TABLE IF EXISTS host;
-DROP TABLE channels;
-DROP TABLE package;
+DROP TABLE IF EXISTS channels;
+DROP TABLE IF EXISTS package;
-DROP TABLE user_groups;
-DROP TABLE user_perms;
-DROP TABLE permissions;
+DROP TABLE IF EXISTS user_groups;
+DROP TABLE IF EXISTS user_perms;
+DROP TABLE IF EXISTS permissions;
-DROP TABLE sessions;
-DROP TABLE users;
+DROP TABLE IF EXISTS sessions;
+DROP TABLE IF EXISTS users;
-DROP TABLE event_labels;
-DROP TABLE events;
-DROP FUNCTION get_event();
-DROP FUNCTION get_event_time(INTEGER);
+DROP TABLE IF EXISTS event_labels;
+DROP TABLE IF EXISTS events;
+DROP FUNCTION IF EXISTS get_event();
+DROP FUNCTION IF EXISTS get_event_time(INTEGER);
BEGIN WORK;
--
1.9.3
8 years, 1 month
[PATCH] pungi: Verify downloaded packages
by Marek Marczykowski-Górecki
Handle "repo --gpgkey" in kickstart to verify downloaded packages
Especially important for lorax and livecd-tools - those packages will
not verified in any way without setting yum options here.
---
pungi/gather.py | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
This depends on "repo --gpgkey" pykickstart patch here:
https://github.com/rhinstaller/pykickstart/pull/32
If possible, I'd like to have it also in 3.x branch. Cherry-pick works
just fine (there was a file name change, only that).
diff --git a/pungi/gather.py b/pungi/gather.py
index bcc2861..a5c9df9 100644
--- a/pungi/gather.py
+++ b/pungi/gather.py
@@ -281,7 +281,7 @@ class Pungi(PungiBase):
def _add_yum_repo(self, name, url, mirrorlist=False, groups=True,
cost=1000, includepkgs=None, excludepkgs=None,
- proxy=None):
+ proxy=None, gpgkey=None):
"""This function adds a repo to the yum object.
name: Name of the repo
url: Full url to the repo
@@ -318,6 +318,10 @@ class Pungi(PungiBase):
thisrepo.exclude = excludepkgs
thisrepo.includepkgs = includepkgs
thisrepo.cost = cost
+ if gpgkey:
+ thisrepo.gpgcheck = True
+ thisrepo.gpgkey = yum.parser.varReplace(gpgkey,
+ self.ayum.conf.yumvar)
# Yum doesn't like proxy being None
if proxy:
thisrepo.proxy = proxy
@@ -349,6 +353,7 @@ class Pungi(PungiBase):
yumconf.installroot = os.path.join(self.workdir, 'yumroot')
yumconf.uid = os.geteuid()
yumconf.cache = 0
+ yumconf.assumeyes = True
yumconf.failovermethod = 'priority'
yumconf.deltarpm = 0
yumvars = yum.config._getEnvVar()
@@ -379,7 +384,8 @@ class Pungi(PungiBase):
cost=repo.cost,
includepkgs=repo.includepkgs,
excludepkgs=repo.excludepkgs,
- proxy=repo.proxy)
+ proxy=repo.proxy,
+ gpgkey=repo.gpgkey)
else:
self._add_yum_repo(repo.name, repo.baseurl,
mirrorlist=False,
@@ -387,7 +393,8 @@ class Pungi(PungiBase):
cost=repo.cost,
includepkgs=repo.includepkgs,
excludepkgs=repo.excludepkgs,
- proxy=repo.proxy)
+ proxy=repo.proxy,
+ gpgkey=repo.gpgkey)
self.logger.info('Getting sacks for arches %s' % self.valid_arches)
self.ayum._getSacks(archlist=self.valid_arches)
--
2.1.0
--
Best Regards,
Marek Marczykowski-Górecki
Invisible Things Lab
A: Because it messes up the order in which people normally read text.
Q: Why is top-posting such a bad thing?
8 years, 1 month
RFC: dynamic generation of mock config from template?
by Clark Williams
The other day someone suggested that mock should dynamically create
configs from a set of templates, so that we wouldn't have to keep
delivering and deleting a series of config files each time a new Fedora
release goes out. Not sure who it was since I can't find the discussion
in my IRC logs, but might have been dgilmore. Nirik? Don't know...
Anyway, I started thinking about it and it seems doable. Right now
configs are named with three fields:
<distro>-<release>-<arch>.cfg
We could come up with templates for distro-arch that would be used to
generate a config. The idea is someone invokes mock with this command
line:
$ mock -r fedora-73-x86_64 --init
We go look in /etc/mock and find no fedora-73-x86_64.cfg, so we go
grab /etc/mock/template/fedora-x86_64 and substitute in '73' for the
release number, then write /etc/mock/fedora-73-x86_64.cfg. Then we
continue on our way, at least until the build fails due to non-existant
repositories (presuming that we're not talking about the year 2035
here).
Obviously you could fat-finger the release number and generate a bogus
config file. Worse you might want to be using F22 but mis-type '21' and
use a wrong, but existent config. Also, I don't really like the idea of
dynamically creating files in /etc, so we might need to move the
created configs off to a /var/mock/configs directory or something like
that.
The upside though is that we could stop having to add and delete
configs when new Fedora releases come out.
What do you guys think?
Clark
8 years, 1 month
