[koji PATCH] allow setting build priority
by Dan Horák
When building for multiple tags it could be useful to set different priorities
for the tasks, eg. rawhide builds will be lower prio than branched builds.
---
util/koji-shadow | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/util/koji-shadow b/util/koji-shadow
index 25492d5..8790cf8 100755
--- a/util/koji-shadow
+++ b/util/koji-shadow
@@ -140,6 +140,8 @@ def get_options():
help=_("tag sucessful builds into the tag we are building, default is to not tag"))
parser.add_option("--arches",
help=_("arches to use when creating tags"))
+ parser.add_option("--priority", type="int", default=5,
+ help=_("priority to set for submitted builds"))
#parse once to get the config file
(options, args) = parser.parse_args()
@@ -1080,7 +1082,7 @@ class BuildTracker(object):
#TODO ...and verify repo
repo_id, event_id = session.getTaskResult(task_id)
#kick off build
- task_id = session.build(src, None, opts={'repo_id': repo_id}, priority=5 )
+ task_id = session.build(src, None, opts={'repo_id': repo_id}, priority=options.priority )
return task_id
def report(self):
--
1.8.1.4
10 years, 3 months
[koji PATCH] fix format when printing rpms without buildroot
by Dan Horák
---
util/koji-shadow | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/util/koji-shadow b/util/koji-shadow
index 25492d5..da7b247 100755
--- a/util/koji-shadow
+++ b/util/koji-shadow
@@ -473,7 +473,7 @@ class TrackedBuild(object):
if brs and bad:
print "Warning: some rpms for %s lacked buildroots:" % self.nvr
for rinfo in bad:
- print " %(name)-%(version)-%(release).%(arch)" % rinfo
+ print " %(name)s-%(version)s-%(release)s.%(arch)s" % rinfo
return brs.keys()
def getDeps(self):
--
1.8.1.4
10 years, 3 months
[koji PATCH] allow setting the timeout for builds from config file
by Dan Horák
---
builder/kojid | 5 +++--
builder/kojid.conf | 3 +++
koji/__init__.py | 2 +-
3 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/builder/kojid b/builder/kojid
index f906d0b..18c1fa4 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -204,7 +204,7 @@ class BuildRoot(object):
for k in ('repoid', 'tag_name'):
if hasattr(self, k):
opts[k] = getattr(self, k)
- for k in ('mockdir', 'topdir', 'topurl', 'topurls', 'packager', 'vendor', 'distribution', 'mockhost', 'yum_proxy'):
+ for k in ('mockdir', 'topdir', 'topurl', 'topurls', 'packager', 'vendor', 'distribution', 'mockhost', 'yum_proxy', 'rpmbuild_timeout'):
if hasattr(self.options, k):
opts[k] = getattr(self.options, k)
opts['buildroot_id'] = self.id
@@ -3060,6 +3060,7 @@ def get_options():
'maven_repo_ignore': '*.md5 *.sha1 maven-metadata*.xml _maven.repositories '
'resolver-status.properties *.lastUpdated',
'failed_buildroot_lifetime' : 3600 * 4,
+ 'rpmbuild_timeout' : 3600 * 24,
'cert': '/etc/kojid/client.crt',
'ca': '/etc/kojid/clientca.crt',
'serverca': '/etc/kojid/serverca.crt'}
@@ -3067,7 +3068,7 @@ def get_options():
for name, value in config.items('kojid'):
if name in ['sleeptime', 'maxjobs', 'minspace', 'retry_interval',
'max_retries', 'offline_retry_interval', 'failed_buildroot_lifetime',
- 'timeout',]:
+ 'timeout', 'rpmbuild_timeout',]:
try:
defaults[name] = int(value)
except ValueError:
diff --git a/builder/kojid.conf b/builder/kojid.conf
index 1fa9ad4..05fe3b0 100644
--- a/builder/kojid.conf
+++ b/builder/kojid.conf
@@ -32,6 +32,9 @@
; The _host string to use in mock
; mockhost=koji-linux-gnu
+; Timeout for build duration (24 hours)
+; rpmbuild_timeout=86400
+
; The URL for the xmlrpc server
server=http://hub.example.com/kojihub
diff --git a/koji/__init__.py b/koji/__init__.py
index f42226b..8ab81a8 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -1266,7 +1266,7 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts)
# don't encourage network access from the chroot
'use_host_resolv': opts.get('use_host_resolv', False),
# Don't let a build last more than 24 hours
- 'rpmbuild_timeout': 86400
+ 'rpmbuild_timeout': opts.get('rpmbuild_timeout', 86400)
}
# bind_opts are used to mount parts (or all of) /dev if needed.
--
1.7.7.6
10 years, 3 months
[PATCH] builder: Upload recursively all files in all subdirs
by Mathieu Bridon
---
This will become necessary if we want to upload the results of a mash
task, as I am currently working on.
Should I have created a new uploadDirRecursively() method, instead of
changing uploadDir()?
builder/kojid | 33 ++++++++++++++++++++++++++-------
1 file changed, 26 insertions(+), 7 deletions(-)
diff --git a/builder/kojid b/builder/kojid
index ca73cad..fbd4e73 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -371,15 +371,34 @@ class BuildRoot(object):
append '.' + suffix to the filenames, so that successive uploads
of the same directory won't overwrite each other, if the files have
the same name but different contents."""
+ def walker(arg, dirname, filenames):
+ # `arg` is completely ignored here
+ if dirname == dirpath:
+ uploadpath = rootuploadpath
+
+ else:
+ dirrelpath = os.path.relpath(dirname, start=dirpath)
+ uploadpath = os.path.join(rootuploadpath, dirrelpath)
+
+ for filename in filenames:
+ filepath = os.path.join(dirname, filename)
+
+ if os.path.isdir(filepath):
+ # TODO: Should we do something about this?
+ continue
+
+ if os.stat(filepath).st_size > 0:
+ if suffix:
+ filename = '%s.%s' % (filename, suffix)
+
+ self.session.uploadWrapper(filepath, uploadpath, filename)
+
if not os.path.isdir(dirpath):
return
- uploadpath = self.getUploadPath()
- for filename in os.listdir(dirpath):
- filepath = os.path.join(dirpath, filename)
- if os.stat(filepath).st_size > 0:
- if suffix:
- filename = '%s.%s' % (filename, suffix)
- self.session.uploadWrapper(filepath, uploadpath, filename)
+
+ rootuploadpath = self.getUploadPath()
+
+ os.path.walk(dirpath, walker, None)
def init(self):
rv = self.mock(['--init'])
--
1.8.1.4
10 years, 4 months
[PATCH] cli: Set the mtime from urlgrabber/pycurl, make users happy. BZ 872631.
by James Antill
---
cli/koji | 9 +++++++++
1 files changed, 9 insertions(+), 0 deletions(-)
diff --git a/cli/koji b/cli/koji
index a870342..04d00bc 100755
--- a/cli/koji
+++ b/cli/koji
@@ -5726,6 +5726,15 @@ def anon_handle_download_build(options, session, args):
os.write(out, buf)
finally:
os.close(out)
+
+ # Set the mtime to the remote mtime, people like this. BZ 87263.
+ try:
+ import pycurl
+ mod_time = file.curl_obj.getinfo(pycurl.INFO_FILETIME)
+ os.utime(relpath, (mod_time, mod_time))
+ except:
+ pass # Ignore errors here.
+
file.close()
def anon_handle_wait_repo(options, session, args):
--
1.7.7.6
10 years, 4 months
strange build errors
by Peter Bojtos
Dear List,
I have a working koji setup with several clients. When I submit a few jobs, it works great. However when I submit lots of package (>100), some of the builds fail with the following error when mock is populating the buildroot (in root.log):
DEBUG util.py:264: Installed:
DEBUG util.py:264: bash.x86_64 0:4.1.2-15.el6_4
DEBUG util.py:264: bzip2.x86_64 0:1.0.5-7.el6_0
DEBUG util.py:264: coreutils.x86_64 0:8.4-19.el6_4.2
....
DEBUG util.py:264: xz.x86_64 0:4.999.9-0.5.beta.20091007git.slxrv3
DEBUG util.py:264: Dependency Installed:
DEBUG util.py:264: audit-libs.x86_64 0:2.2-2.el6
DEBUG util.py:264: basesystem.noarch 0:10.0-4.el6
....
DEBUG util.py:264: xz-lzma-compat.x86_64 0:4.999.9-0.5.beta.20091007git.slxrv3
DEBUG util.py:264: zlib.x86_64 0:1.2.3-29.el6
DEBUG util.py:354: Child return code was: 0
DEBUG util.py:314: Executing command: /usr/bin/repoquery -c /var/lib/mock/dist-slxrv3-build-5544-1386/root//etc/yum.conf -a --qf '%{nevra} %{buildtime} %{size} %{pkgid} %{repoid}' > /var/lib/mock/dist-slxrv3-build-5544-1386/result/available_pkgs with env {'LANG': 'en_US.UTF-8', 'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOSTNAME': 'mock', 'HOME': '/builddir', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin'}
DEBUG util.py:264: Traceback (most recent call last):
DEBUG util.py:264: File "/usr/bin/repoquery", line 1510, in <module>
DEBUG util.py:264: main(sys.argv)
DEBUG util.py:264: File "/usr/bin/repoquery", line 1504, in main
DEBUG util.py:264: repoq.runQuery(regexs)
DEBUG util.py:264: File "/usr/bin/repoquery", line 982, in runQuery
DEBUG util.py:264: pkgs = self.matchPkgs(items, plain_pkgs=plain_pkgs)
DEBUG util.py:264: File "/usr/bin/repoquery", line 903, in matchPkgs
DEBUG util.py:264: pkgs = self.returnPkgList(patterns=items)
DEBUG util.py:264: File "/usr/bin/repoquery", line 856, in returnPkgList
DEBUG util.py:264: pkgs = self.pkgSack.returnNewestByNameArch(**kwargs)
DEBUG util.py:264: File "/usr/lib/python2.7/site-packages/yum/__init__.py", line 1013, in <lambda>
DEBUG util.py:264: pkgSack = property(fget=lambda self: self._getSacks(),
DEBUG util.py:264: File "/usr/lib/python2.7/site-packages/yum/__init__.py", line 779, in _getSacks
DEBUG util.py:264: self.repos.populateSack(which=repos)
DEBUG util.py:264: File "/usr/lib/python2.7/site-packages/yum/repos.py", line 309, in populateSack
DEBUG util.py:264: self.doSetup()
DEBUG util.py:264: File "/usr/lib/python2.7/site-packages/yum/repos.py", line 134, in doSetup
DEBUG util.py:264: self.retrieveAllMD()
DEBUG util.py:264: File "/usr/lib/python2.7/site-packages/yum/repos.py", line 92, in retrieveAllMD
DEBUG util.py:264: downloading = repo._commonRetrieveDataMD_list(mdtypes)
DEBUG util.py:264: File "/usr/lib/python2.7/site-packages/yum/yumRepo.py", line 1540, in _commonRetrieveDataMD_list
DEBUG util.py:264: os.rename(local, local + '.old.tmp')
DEBUG util.py:264: OSError: [Errno 2] No such file or directory
DEBUG util.py:354: Child return code was: 1
DEBUG util.py:314: Executing command: ['/bin/umount', '-n', '-l', '/var/lib/mock/dist-slxrv3-build-5544-1386/root/dev/pts'] with env {'LANG': 'en_US.UTF-8', 'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOSTNAME': 'mock', 'HOME': '/builddir', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin'}
Is it because newRepo task is not finished but somehow creates the softlink to latest (and unfinished) koji repo? Have anyone seen the same error before?
Thanks for your reply,
Peter Bojtos
10 years, 4 months
[PATCH 1/2] cli: Check pkg. to move-pkgs is in src tag before spawning task. BZ 527296.
by James Antill
---
cli/koji | 12 ++++++++++++
1 files changed, 12 insertions(+), 0 deletions(-)
diff --git a/cli/koji b/cli/koji
index 04d00bc..b435f1a 100755
--- a/cli/koji
+++ b/cli/koji
@@ -5512,6 +5512,18 @@ def handle_move_build(opts, session, args):
builds.append(build)
for build in builds:
+ data = session.listTags(build=build['id'])
+ if not data:
+ print _("Package %s does not have any tags.") % arg
+ continue
+ data = [x['name'] for x in data]
+ if args[0] not in data:
+ msg = _("Package %s is not in %s, maybe you meant: %s")
+ arg = "%(package_name)s-%(version)s-%(release)s" % build
+ data = ",".join(data)
+ print msg % (arg, args[0], data)
+ continue
+
task_id = session.moveBuild(args[0], args[1], build['id'], options.force)
tasks.append(task_id)
print "Created task %s, moving %s" % (task_id, koji.buildLabel(build))
--
1.7.7.6
10 years, 4 months
[PATCH] cli: Simple text fix to make it more obvious what rpmdiff is doing. BZ 704129.
by James Antill
---
hub/kojihub.py | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/hub/kojihub.py b/hub/kojihub.py
index 6e256c4..9a4dafc 100644
--- a/hub/kojihub.py
+++ b/hub/kojihub.py
@@ -7162,8 +7162,8 @@ def rpmdiff(basepath, rpmlist):
status = proc.wait()
if os.WIFSIGNALED(status) or \
(os.WEXITSTATUS(status) != 0):
- raise koji.BuildError, 'mismatch when analyzing %s, rpmdiff output was:\n%s' % \
- (os.path.basename(first_rpm), output)
+ raise koji.BuildError, 'mismatch when analyzing %s (from multiple arches), rpmdiff output was:\n 1. %s\n 2. %s\n%s' % \
+ (os.path.basename(first_rpm), first_rpm, other_rpm, output)
def importImageInternal(task_id, build_id, imgdata):
"""
--
1.7.7.6
10 years, 4 months
Need Koji Server Setup Help
by John.Florian@dart.biz
Hi all,
I'm a complete Koji newb and am attempting to get a Koji server up and
running for internal use, but have run into a number of issues along the
way, despite following https://fedoraproject.org/wiki/Koji/ServerHowTo and
other similar pages found around the net. My setup is built atop Fedora
19, so I had to make several adjustments, such as:
* access directives for Apache httpd 2.4 (bug filed:
https://bugzilla.redhat.com/show_bug.cgi?id=1019323 and wiki likewise
edited)
* changed the ssl.cnf for the Koji CA to so that default_md=sha256
(instead of md5) under the [default_ca] section so that firefox wouldn't
complain about the trustworthiness of the certificate
I am able to use 'koji moshimoshi' both as kojiadmin and as myself from
the Koji server as well as my workstation. I am also able to use the
Koji-Web using http or https for some basic poking around. However, if I
try the login link on the web page, firefox comes up with a message
stating:
The page isn't redirecting properly
Firefox has detected that the server is redirecting the request for this
address in a way that will never complete.
This problem can sometimes be caused by disabling or refusing to accept
cookies.
>From what I can gather via /var/log/httpd/ssl_access_log and examination
of /usr/share/koji-web/scripts/index.py's login function the page is
trying to redirect the browser to force SSL (even if already using https
when clicking on login) and somehow this results in what would be an
infinite loop except that firefox halts the loop after a dozen or so
cycles. I'm rather stumped as to where my investigation should be focused
at this point however. Any advice?
--
John Florian
10 years, 4 months
Bootstrapping New Koji Server
by John.Florian@dart.biz
First off, I apologize for the lengthy message, but I hope to eventually
contribute to this project via code and/or docs. (I've already updated
the Server Setup wiki page where I felt knowledgeable enough.)
I'm loosely following the wiki page[1] for this process but may have
myself confused about what some steps do and what overall is required. I
want a Koji instance for building internally developed packages. I would
rarely, if ever, want to rebuild a package already provided in Fedora.
I've got lots of experience using rpmbuild and recently a little with
mock, but am trying for a full fledged setup with Koji now to improve my
workflow. For now, I am focusing on builds for F19 only, but will
eventually add others to basically follow Fedora's EoL schedule.
Thus far, I have:
A basic setup working and can now login thanks to Mike's excellent help.
A "f19" tag and a child "f19-build" tag for arches i386 and x86_64.
Successfully imported the SRPMs for my locally developed packages only. I
didn't import any binary RPMs as I wanted to see that Koji could do that
as a test. I also didn't import anything of the Fedora-provided packages
since I'm not interested in building those.
A "f19" build target.
A build group named "build" associated with my "f19-build" tag.
I've tried doing a build of one of my SRPMs that I had imported and keep
meeting failure. Examination of the root.log revealed only a few things,
but none that sounded to worrisome:
DEBUG util.py:266: There is no installed groups file.
DEBUG util.py:266: Warning: Group build does not have any packages to
install.
When I got to the mock_output.log however, I saw this which I'm guessing
is the reason for failure:
ERROR: Could not find useradd in chroot, maybe the install failed?
Seeing this I figured it was due to skipping the step for populating the
"build" group with packages to be installed in the minimal buildroot. I
skipped this because the instructions fail for me since I my FAS account
isn't set up sufficiently to let me to auth against the real Fedora Koji
instance, i.e., I don't have a signed user cert. Knowing that my current
mock setup can manage this, I dug into its config and stumbled upon the
"buildsys-build" package name and thus came up with:
$ yum group info buildsys-build
Loaded plugins: langpacks
There is no installed groups file.
Maybe run: yum groups mark convert
Group: Buildsystem building group
Group-Id: buildsys-build
Mandatory Packages:
bash
bzip2
coreutils
cpio
diffutils
fedora-release
findutils
gawk
gcc
gcc-c++
grep
gzip
info
make
patch
redhat-rpm-config
rpm-build
sed
shadow-utils
tar
unzip
util-linux
which
xz
This looks like a minimal buildroot to me, so I commenced with that list:
$ koji add-group-pkg f19-build build bash bzip2 coreutils cpio diffutils
fedora-release findutils gawk gcc gcc-c++ grep gzip info make patch
redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz
I believe these are now "in place" correctly since:
$ koji list-groups f19-build
build [f19-build]
bash: None, default [f19-build]
bzip2: None, default [f19-build]
coreutils: None, default [f19-build]
cpio: None, default [f19-build]
diffutils: None, default [f19-build]
fedora-release: None, default [f19-build]
findutils: None, default [f19-build]
gawk: None, default [f19-build]
gcc: None, default [f19-build]
gcc-c++: None, default [f19-build]
grep: None, default [f19-build]
gzip: None, default [f19-build]
info: None, default [f19-build]
make: None, default [f19-build]
patch: None, default [f19-build]
redhat-rpm-config: None, default [f19-build]
rpm-build: None, default [f19-build]
sed: None, default [f19-build]
shadow-utils: None, default [f19-build]
tar: None, default [f19-build]
unzip: None, default [f19-build]
util-linux: None, default [f19-build]
which: None, default [f19-build]
xz: None, default [f19-build]
However, when I resubmitted my build request, it failed and I see the same
warnings/errors in the logs. I also see this in /var/log/kojid.log:
2013-10-22 16:48:57,582 [DEBUG] koji.TaskManager: bins: {'1:noarch': 1,
'2:noarch': 1, '1:x86_64': 1, '1:i386': 1, '2:x86_64': 1, '2:i386': 1}
2013-10-22 16:49:12,598 [DEBUG] koji: Traceback (most recent call last):
File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1895, in
_callMethod
return self._sendCall(handler, headers, request)
File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1806, in
_sendCall
return self._sendOneCall(handler, headers, request)
File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1825, in
_sendOneCall
cnx.send(request)
File "/usr/lib64/python2.7/httplib.py", line 805, in send
self.sock.sendall(data)
File "/usr/lib/python2.7/site-packages/koji/ssl/SSLConnection.py", line
111, in sendall
self.close()
File "/usr/lib/python2.7/site-packages/koji/ssl/SSLConnection.py", line
82, in close
self.shutdown()
File "/usr/lib/python2.7/site-packages/koji/ssl/SSLConnection.py", line
53, in shutdown
self.__dict__["conn"].shutdown()
Error: []
2013-10-22 16:49:12,599 [INFO] koji: Try #1 for call 1328 (listBuildroots)
failed: []
I suspect I'm really close now, but have done something silly that's not
obvious to me. Any pointers will be greatly appreciated.
[1] https://fedoraproject.org/wiki/Koji/ServerBootstrap
--
John Florian
10 years, 5 months