Fails for build ISO by pungi+lorax on Fedora16
by Eric Zhong
OS: Fedora16
Tool: pungi + lorax (yum install pungi)
Error:
Traceback (most recent call last):
File "/usr/lib/python2.7/site-packages/yum/rpmtrans.py", line 446, in
callback
File "/usr/lib/python2.7/site-packages/yum/rpmtrans.py", line 524, in
_instProgress
File "/usr/lib/python2.7/site-packages/pylorax/yumhelper.py", line 102,
in event
UnicodeEncodeError: 'ascii' codec can't encode characters in position 0-3:
ordinal not in range(128)
error: python callback <bound method RPMTransaction.callback of
<yum.rpmtrans.RPMTransaction instance at 0xd0b27cc>> failed, aborting!
If there is a bug , how Fedora16 DVD was born ???
12 years, 3 months
Setting Environment Variables Into Mock Chroot
by Gordan Bobic
Is there a way to make mock explicitly set environment variables in the
build chroot?
Specifically, I want to set LD_PRELOAD explicitly during the build, but
I cannot find any way of telling mock to do that. Is there a way to do this?
Gordan
12 years, 3 months
Possible Koji-Hub Patch Code For Ticket # 119
by Jon Chiappetta
Ticket URL: https://fedorahosted.org/koji/ticket/119
$ diff -u ./koji-1.6.0/hub/kojihub.py /usr/share/koji-hub/kojihub.py
--- ./koji-1.6.0/hub/kojihub.py 2010-12-16 16:13:17.000000000 -0500
+++ /usr/share/koji-hub/kojihub.py 2011-12-04 20:17:40.460047608 -0500
@@ -51,6 +51,86 @@
import zipfile
from koji.context import context
+import hashlib
+import struct
+
+def sigsha2(fn, cm=None):
+ # http://www.iagora.com/~espel/rpm2cpio
+
+ nel = 0
+ f = open(fn, "r")
+
+ rpm = f.read(96)
+ if (len(rpm) != 96):
+ #print("error reading lead 96.0")
+ return cm
+ nel += len(rpm)
+
+ # http://perldoc.perl.org/functions/pack.html
+ # http://docs.python.org/library/struct.html
+
+ (magic, major, minor, rest) = struct.unpack(">LBB90s", rpm)
+
+ if (magic != 0xedabeedb):
+ #print("incorrect lead magic")
+ return cm
+
+ if ((major != 3) and (major != 4)):
+ #print("incorrect lead major")
+ return cm
+
+ # http://docs.python.org/library/stdtypes.html
+
+ while (1):
+ pos = nel
+ rpm = f.read(16)
+ if (len(rpm) != 16):
+ #print("error reading header 16.0")
+ return cm
+ nel += len(rpm)
+ (smagic, rest) = struct.unpack(">H14s", rpm)
+ if ((smagic == 0x1f8b) or (smagic == 0x425a) or (smagic == 0xfd37)):
+ break
+ if (pos & 0x7):
+ pos += 7
+ pos &= (~0x7)
+ f.seek(pos, 0)
+ nel = pos
+ rpm = f.read(16)
+ if (len(rpm) != 16):
+ #print("error reading header 16.1")
+ return cm
+ nel += len(rpm)
+ left = (len(rpm) - 16)
+ (magic, data, sections, bytes, rest) = struct.unpack(">4L" + str(left) + "s", rpm)
+ if (magic != 0x8eade801):
+ #print("incorrect header magic")
+ return cm
+ # beg custom
+ f.seek(pos + 16, 0)
+ tmp = f.read((16 * sections) + bytes)
+ head = (rpm + tmp)
+ # end custom
+ pos += 16
+ pos += (16 * sections)
+ pos += bytes
+ f.seek(pos, 0)
+ nel = pos
+
+ while (1):
+ tmp = f.read(16384)
+ if (not tmp):
+ break
+ rpm += tmp
+
+ f.close()
+ md5hex = hashlib.md5(head + rpm).hexdigest()
+
+ if ((cm != None) and (cm != md5hex)):
+ #print("sigmd5 mis-match")
+ return cm
+
+ return hashlib.sha256(head + rpm).hexdigest()
logger = logging.getLogger('koji.hub')
@@ -4092,12 +4172,13 @@
raise koji.GenericError, "srpm mismatch for %s: %s (expected %s)" \
% (fn,basename,srpmname)
+ hdr_md5 = hdr[rpm.RPMTAG_SIGMD5]
#add rpminfo entry
rpminfo['id'] = _singleValue("""SELECT nextval('rpminfo_id_seq')""")
rpminfo['build'] = buildinfo
rpminfo['build_id'] = buildinfo['id']
rpminfo['size'] = os.path.getsize(fn)
- rpminfo['payloadhash'] = koji.hex_string(hdr[rpm.RPMTAG_SIGMD5])
+ rpminfo['payloadhash'] = sigsha2(fn, koji.hex_string(hdr_md5))
rpminfo['brootid'] = brootid
koji.plugin.run_callbacks('preImport', type='rpm', rpm=rpminfo, build=buildinfo,
@@ -4539,7 +4620,9 @@
if not os.path.isdir(builddir):
raise koji.GenericError, "No such directory: %s" % builddir
rawhdr = koji.RawHeader(sighdr)
- sigmd5 = koji.hex_string(rawhdr.get(koji.RPM_SIGTAG_MD5))
+ rpm_path = "%s/%s" % (builddir, koji.pathinfo.rpm(rinfo))
+ hdr_md5 = rawhdr.get(koji.RPM_SIGTAG_MD5)
+ sigmd5 = sigsha2(rpm_path, koji.hex_string(hdr_md5))
if sigmd5 == rinfo['payloadhash']:
# note: payloadhash is a misnomer, that field is populated with sigmd5.
sigkey = rawhdr.get(koji.RPM_SIGTAG_GPG)
@@ -4554,7 +4637,7 @@
# we need to pull that differently as well
rpm_path = "%s/%s" % (builddir, koji.pathinfo.rpm(rinfo))
sigmd5, sigkey = _scan_sighdr(sighdr, rpm_path)
- sigmd5 = koji.hex_string(sigmd5)
+ sigmd5 = sigsha2(rpm_path, koji.hex_string(sigmd5))
if sigmd5 != rinfo['payloadhash']:
nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % rinfo
raise koji.GenericError, "wrong md5 for %s: %s" % (nvra, sigmd5)
$ koji list-tagged main-builds nledd
Build Tag Built by
---------------------------------------- -------------------- ----------------
nledd-2.52-7.fc15 main-builds admin
$ su - koji -c "psql -c \"select name,payloadhash from rpminfo where name LIKE 'nledd%';\""
name | payloadhash
-----------------+------------------------------------------------------------------
nledd | a6f6f123dde44e0967e30b016c8a60ad6c32575a4dd4c83ef779b3bdcebf0279
nledd | 988e2594303acc3c7a1e15bbf1ca621cd626f21faff24c24365c68551f48af3a
nledd-debuginfo | d9b6bf6a7d76f4f6cb966f3353f5d48075bd89184fe74e8a32c155dfd9abd402
(3 rows)
# Main Source Code Addition
import hashlib
import struct
def sigsha2(fn, cm=None):
# http://www.iagora.com/~espel/rpm2cpio
nel = 0
f = open(fn, "r")
rpm = f.read(96)
if (len(rpm) != 96):
#print("error reading lead 96.0")
return cm
nel += len(rpm)
# http://perldoc.perl.org/functions/pack.html
# http://docs.python.org/library/struct.html
(magic, major, minor, rest) = struct.unpack(">LBB90s", rpm)
if (magic != 0xedabeedb):
#print("incorrect lead magic")
return cm
if ((major != 3) and (major != 4)):
#print("incorrect lead major")
return cm
# http://docs.python.org/library/stdtypes.html
while (1):
pos = nel
rpm = f.read(16)
if (len(rpm) != 16):
#print("error reading header 16.0")
return cm
nel += len(rpm)
(smagic, rest) = struct.unpack(">H14s", rpm)
if ((smagic == 0x1f8b) or (smagic == 0x425a) or (smagic == 0xfd37)):
break
if (pos & 0x7):
pos += 7
pos &= (~0x7)
f.seek(pos, 0)
nel = pos
rpm = f.read(16)
if (len(rpm) != 16):
#print("error reading header 16.1")
return cm
nel += len(rpm)
left = (len(rpm) - 16)
(magic, data, sections, bytes, rest) = struct.unpack(">4L" + str(left) + "s", rpm)
if (magic != 0x8eade801):
#print("incorrect header magic")
return cm
# beg custom
f.seek(pos + 16, 0)
tmp = f.read((16 * sections) + bytes)
head = (rpm + tmp)
# end custom
pos += 16
pos += (16 * sections)
pos += bytes
f.seek(pos, 0)
nel = pos
while (1):
tmp = f.read(16384)
if (not tmp):
break
rpm += tmp
f.close()
md5hex = hashlib.md5(head + rpm).hexdigest()
if ((cm != None) and (cm != md5hex)):
#print("sigmd5 mis-match")
return cm
return hashlib.sha256(head + rpm).hexdigest()
12 years, 3 months
Possible Code Changes For Tickets #41 #76
by Jon Chiappetta
In terms of adding "tail-like" functionality to Koji "watch-logs" sub-command:
https://fedorahosted.org/koji/ticket/41
https://fedorahosted.org/koji/ticket/76
$ ./koji-1.6.0/cli/koji watch-logs 3559606 --log=build.log --tail=10
Watching logs (this may be safely interrupted)...
Wrote: /builddir/build/RPMS/calibre-0.8.29-1.fc15.i686.rpm
Wrote: /builddir/build/RPMS/calibre-debuginfo-0.8.29-1.fc15.i686.rpm
Executing(%clean): /bin/sh -e /var/tmp/rpm-tmp.RypJxm
+ umask 022
+ cd /builddir/build/BUILD
+ cd calibre
+ /bin/rm -rf /builddir/build/BUILDROOT/calibre-0.8.29-1.fc15.i386
+ exit 0
Child returncode was: 0
LEAVE do -->
$ diff -u /usr/bin/koji ./koji-1.6.0/cli/koji
--- /usr/bin/koji 2010-12-16 16:13:17.000000000 -0500
+++ ./koji-1.6.0/cli/koji 2011-12-03 17:09:42.225650495 -0500
@@ -433,6 +433,7 @@
offsets[task_id] = {}
lastlog = None
+ logline = ""
while True:
for task_id in tasklist[:]:
if _isDone(session, task_id):
@@ -458,15 +459,28 @@
currlog = "%d:%s:" % (task_id, log)
if currlog != lastlog:
if lastlog:
- sys.stdout.write("\n")
- sys.stdout.write("==> %s <==\n" % currlog)
+ logline += ("\n")
+ logline += ("==> %s <==\n" % currlog)
lastlog = currlog
- sys.stdout.write(contents)
+ logline += (contents)
if not tasklist:
break
time.sleep(options.poll_interval)
+
+ loglist = logline.strip("\n").split("\n")
+ endline = len(loglist)
+ try:
+ subvalue = int(opts.tail)
+ except:
+ subvalue = endline
+ if (subvalue > endline):
+ subvalue = endline
+ begline = (endline - subvalue)
+ for x in range(begline, endline):
+ print(loglist[x])
+
except (KeyboardInterrupt):
pass
@@ -5083,6 +5097,7 @@
usage += _("\n(Specify the --help global option for a list of other help options)")
parser = OptionParser(usage=usage)
parser.add_option("--log", help=_("Watch only a specific log"))
+ parser.add_option("--tail", help=_("Watch only the last N lines"))
(options, args) = parser.parse_args(args)
activate_session(session)
12 years, 3 months
Possible Starter Code For Ticket #119
by Jon Chiappetta
I saw a request for Koji to use a stronger cryptographic hash function such as SHA256
( https://fedorahosted.org/koji/ticket/119 ) on RPM packages. I took some time in reading how the
RPMTAG_SIGMD5 is calculated and implemented some Python code to hash the same RPM data
using hashlib's SHA256 module. The outputted SHA256 digest can be stored in Koji's DB and
also retrieved/calculated/checked with any new packages that are being submitted.
$ rpm -q --queryformat '%{RPMTAG_SIGMD5}' -p "gcc-4.6.2-1.fc17.1.src.rpm"
c88e96685e7eb3a124a5707b1bc41333
$ python sha2pay.py gcc-4.6.2-1.fc17.1.src.rpm
['c88e96685e7eb3a124a5707b1bc41333', 'c729d818d5468f8b1147cb70b266992f92f58c9dace218417c7582427d1e5561']
Source code:
import hashlib
import os
import struct
import subprocess
import sys
def sha2pay(fn):
# http://www.iagora.com/~espel/rpm2cpio
nel = 0
f = open(fn, "r")
rpm = f.read(96)
if (len(rpm) != 96):
#print("error reading lead 96.0")
return None
nel += len(rpm)
# http://perldoc.perl.org/functions/pack.html
# http://docs.python.org/library/struct.html
(magic, major, minor, rest) = struct.unpack(">LBB90s", rpm)
if (magic != 0xedabeedb):
#print("incorrect lead magic")
return None
if ((major != 3) and (major != 4)):
#print("incorrect lead major")
return None
# http://docs.python.org/library/stdtypes.html
while (1):
pos = nel
rpm = f.read(16)
if (len(rpm) != 16):
#print("error reading header 16.0")
return None
nel += len(rpm)
(smagic, rest) = struct.unpack(">H14s", rpm)
if ((smagic == 0x1f8b) or (smagic == 0x425a)):
break
if (pos & 0x7):
pos += 7
pos &= (~0x7)
f.seek(pos, 0)
nel = pos
rpm = f.read(16)
if (len(rpm) != 16):
#print("error reading header 16.1")
return None
nel += len(rpm)
left = (len(rpm) - 16)
(magic, data, sections, bytes, rest) = struct.unpack(">4L" + str(left) + "s", rpm)
if (magic != 0x8eade801):
#print("incorrect header magic")
return None
# beg custom
f.seek(pos + 16, 0)
tmp = f.read((16 * sections) + bytes)
head = (rpm + tmp)
# end custom
pos += 16
pos += (16 * sections)
pos += bytes
f.seek(pos, 0)
nel = pos
if ((smagic != 0x1f8b) and (smagic != 0x425a)):
#print("unknown compression format")
return None
while (1):
tmp = f.read(16384)
if (not tmp):
break
rpm += tmp
f.close()
return [hashlib.md5(head + rpm).hexdigest(), hashlib.sha256(head + rpm).hexdigest()]
12 years, 3 months
Some Possibly Useful Koji Changes
by Jon Chiappetta
Patch details:
- Mark any host that hasn't checked in the last 5 minutes as red
- Mark all other hosts as green
- Place all hosts on one page so one does not have to scroll thru and find any possibly downed hosts
--- ./koji-1.6.0/www/kojiweb/index.py 2010-12-16 16:13:17.000000000 -0500
+++ ./index.py.0 2011-12-01 13:53:54.169121521 -0500
@@ -1419,8 +1419,27 @@
host['last_update'] = lastUpdate
# Paginate after retrieving last update info so we can sort on it
- kojiweb.util.paginateList(values, hosts, start, 'hosts', 'host', order)
-
+ #kojiweb.util.paginateList(values, hosts, start, 'hosts', 'host', order)
+
+ if (order != None):
+ hosts.sort(kojiweb.util.sortByKeyFunc(order, False))
+ for host in hosts:
+ flag = 0
+ try:
+ utct = time.localtime()
+ utcs = int(time.strftime("%s", utct))
+ chkt = time.strptime(host['last_update'][0:19], "%Y-%m-%d %H:%M:%S")
+ chks = int(time.strftime("%s", chkt))
+ if ((utcs - (5 * 60)) >= chks):
+ flag = 1
+ except:
+ flag = 1
+ if (flag == 1):
+ host['name'] = ("<font color='#C00'>" + host['name'] + "</font>")
+ else:
+ host['name'] = ("<font color='#0C0'>" + host['name'] + "</font>")
+ kojiweb.util._populateValues(values, 'hosts', 'host', hosts, len(hosts), 0, len(hosts), len(hosts) + 5, order)
+
return _genHTML(req, 'hosts.chtml')
def hostinfo(req, hostID=None, userID=None):
Patch details:
- Produce a list of the latest buildArch attempts made by a host on the "hostinfo" page (up to max 50)
--- ./koji-1.6.0/www/kojiweb/index.py 2010-12-16 16:13:17.000000000 -0500
+++ ./index.py.1 2011-12-01 13:54:22.748122221 -0500
@@ -1423,6 +1423,18 @@
return _genHTML(req, 'hosts.chtml')
+def saferemo(inptstri, begistri, endistri, remoflag=True):
+ outpstri = inptstri
+ begilist = inptstri.split(begistri, 1)
+ if (len(begilist) > 1):
+ endilist = begilist[1].split(endistri, 1)
+ if (len(endilist) > 1):
+ if (remoflag):
+ outpstri = (begilist[0] + endilist[1])
+ else:
+ outpstri = (begistri + endilist[0] + endistri)
+ return outpstri
+
def hostinfo(req, hostID=None, userID=None):
values = _initValues(req, 'Host Info', 'hosts')
server = _getServer(req)
@@ -1461,7 +1473,16 @@
else:
values['perms'] = []
- return _genHTML(req, 'hostinfo.chtml')
+ hout = _genHTML(req, 'hostinfo.chtml')
+ hidn = hostID
+ tout = tasks(req, state='all', method='buildArch', view='flat', hostID=hidn)
+ tout = tout.replace("\t", "").replace("\r", "").replace("\n", "")
+ tout = saferemo(tout, "<form action=\"tasks\">", "</form>")
+ tout = saferemo(tout, "<table class=\"data-list\">", "</table>", remoflag=False)
+
+ hout = hout.replace("<p id=\"footer\">", tout + "<p id=\"footer\">")
+
+ return hout
def hostedit(req, hostID):
server = _getServer(req)
Patch details:
- Grab the last 5 buildArch statuses from Koji for each builder using ajax
- It will place that short build history beside each builder (sort of like a little "life-line")
--- ./koji-1.6.0/www/kojiweb/includes/header.chtml 2010-12-16 16:13:17.000000000 -0500
+++ ./header.chtml 2011-12-01 14:03:44.414136005 -0500
@@ -37,8 +37,95 @@
<link rel="stylesheet" type="text/css" media="screen" title="Koji Style" href="/koji-static/koji.css"/>
<link rel="alternate stylesheet" type="text/css" media="screen" title="Debug" href="/koji-static/debug.css"/>
<link rel="alternate" type="application/rss+xml" title="Koji: recent builds" href="/koji/recentbuilds"/>
+ <script>
+ var hostxmld = null, hostlist = [];
+ function hostresp()
+ {
+ if (hostxmld.readyState == 4)
+ {
+ if (hostxmld.status == 200)
+ {
+ var x, machobjc, infostri = "", infolist = [];
+ var regxfail = /^(<a href="taskinfo\?taskID=[0-9]*" class="taskfailed" title=")failed(">)(.*)(<\/a>)$/;
+ var regxpass = /^(<a href="taskinfo\?taskID=[0-9]*" class="taskclosed" title=")closed(">)(.*)(<\/a>)$/;
+ var taskdata = hostxmld.responseText.replace(/<a /g, "\n<a ").replace(/<\/a>/g, "</a>\n");
+ var tasklist = taskdata.split("\n");
+ for (x = 0; x < tasklist.length; ++x)
+ {
+ machobjc = regxfail.exec(tasklist[x]);
+ if (machobjc != null)
+ {
+ infolist.push([machobjc[1], machobjc[3], machobjc[2], "F", machobjc[4]]);
+ }
+ machobjc = regxpass.exec(tasklist[x]);
+ if (machobjc != null)
+ {
+ infolist.push([machobjc[1], machobjc[3], machobjc[2], "C", machobjc[4]]);
+ }
+ }
+ for (x = 0; (x < infolist.length) && (x < 5); ++x)
+ {
+ if (x != 0)
+ {
+ infostri += ".";
+ }
+ infostri += (infolist[x][0] + infolist[x][1] + infolist[x][2] + infolist[x][3] + infolist[x][4]);
+ }
+ if (infostri != "")
+ {
+ infostri = ("[ " + infostri + " ]");
+ }
+ hostlist[0][1].innerHTML = (hostlist[0][1].innerHTML + " " + infostri);
+ }
+ hostlist.shift();
+ hostxmld = null;
+ }
+ }
+ function hostrequ(hostnumb)
+ {
+ if (hostxmld != null)
+ {
+ return 0;
+ }
+ hostxmld = new XMLHttpRequest();
+ hostxmld.onreadystatechange = hostresp;
+ hostxmld.open("GET", "tasks?method=buildArch&state=all&view=flat&order=-id&hostID=" + hostnumb, true);
+ hostxmld.send();
+ return 1;
+ }
+ function hostloop()
+ {
+ if (hostlist.length > 0)
+ {
+ var hostiden = hostlist[0][0].innerHTML.replace(/[^0-9]/g, "");
+ hostrequ(hostiden);
+ setTimeout("hostloop();", 100);
+ }
+ }
+ function hostinfo()
+ {
+ if (location.href.match(/^.*\/hosts$/) || location.href.match(/^.*\/hosts\?[^\/]*$/))
+ {
+ var x, y;
+ var tabllist = document.getElementsByClassName("data-list");
+ for (x = 0; x < tabllist.length; ++x)
+ {
+ var rowslist = tabllist[x].getElementsByTagName("tr");
+ for (y = 0; y < rowslist.length; ++y)
+ {
+ if (rowslist[y].className.match(/^.*row-odd.*$/) || rowslist[y].className.match(/^.*row-even.*$/))
+ {
+ var colslist = rowslist[y].getElementsByTagName("td");
+ hostlist.push(colslist);
+ }
+ }
+ }
+ }
+ hostloop();
+ }
+ </script>
</head>
- <body id="$pageID">
+ <body id="$pageID" onload="hostinfo();">
<div id="wrap">
<div id="innerwrap">
12 years, 3 months