[FAF PATCH] save uReports that fail parsing for later debugging

Tuesday, 9 April 2013

Signed-off-by: Michal Toman <mtoman(a)redhat.com&gt;
---
 pyfaf/hub/menu.py                             |  3 +++
 pyfaf/hub/reports/forms.py                    | 30 ++++++++++++++++++++++++++-
 pyfaf/hub/reports/urls.py                     |  2 ++
 pyfaf/hub/reports/views.py                    | 30 +++++++++++++++++++++++++++
 pyfaf/hub/templates/reports/Makefile.am       |  2 ++
 pyfaf/hub/templates/reports/invalid.html      | 26 +++++++++++++++++++++++
 pyfaf/hub/templates/reports/invalid_item.html | 24 +++++++++++++++++++++
 pyfaf/storage/Makefile.am                     |  1 +
 pyfaf/storage/__init__.py                     |  1 +
 pyfaf/storage/debug.py                        | 14 +++++++++++++
 10 files changed, 132 insertions(+), 1 deletion(-)
 create mode 100644 pyfaf/hub/templates/reports/invalid.html
 create mode 100644 pyfaf/hub/templates/reports/invalid_item.html
 create mode 100644 pyfaf/storage/debug.py

diff --git a/pyfaf/hub/menu.py b/pyfaf/hub/menu.py
index 76263a2..05a5f0c 100644
--- a/pyfaf/hub/menu.py
+++ b/pyfaf/hub/menu.py
@@ -230,6 +230,9 @@ menu = (
                 placeholder=True, url_args_pattern=[42]),
             MenuItem("Diff", "pyfaf.hub.reports.views.diff",
                 placeholder=True, url_args_pattern=[42, 42]),
+            StaffMenuItem("Invalid uReports",
"pyfaf.hub.reports.views.invalid"),
+            StaffMenuItem("Invalid uReport",
+                "pyfaf.hub.reports.views.invalid_item", placeholder=True),
             StaffMenuItem("New", "pyfaf.hub.reports.views.new",
on_right=True),
             )),
         MenuItem("Statistics", "today_stats", menu=(
diff --git a/pyfaf/hub/reports/forms.py b/pyfaf/hub/reports/forms.py
index f77ba49..79984a0 100644
--- a/pyfaf/hub/reports/forms.py
+++ b/pyfaf/hub/reports/forms.py
@@ -1,10 +1,13 @@
+import datetime
 import json
+import logging
 
 from django import forms
 
 from pyfaf import ureport
 from pyfaf.hub.common.forms import (OsComponentFilterForm,
                                     FafMultipleChoiceField)
+from pyfaf.storage import getDatabase, InvalidUReport
 
 class ReportFilterForm(OsComponentFilterForm):
     status_values = ['new', 'processed']
@@ -29,18 +32,43 @@ class ReportFilterForm(OsComponentFilterForm):
 class NewReportForm(forms.Form):
     file = forms.FileField(label='uReport file')
 
+    def _save_invalid_ureport(self, ureport, errormsg, reporter=None):
+        try:
+            db = getDatabase()
+
+            new = InvalidUReport()
+            new.errormsg = errormsg
+            new.date = datetime.datetime.utcnow()
+            new.reporter = reporter
+            db.session.add(new)
+            db.session.flush()
+
+            new.save_lob("ureport", ureport)
+        except Exception as ex:
+            logging.error(str(ex))
+
     def clean_file(self):
         raw_data = self.cleaned_data['file'].read()
 
         try:
             data = json.loads(raw_data)
-        except:
+        except Exception as ex:
+            self._save_invalid_ureport(raw_data, str(ex))
             raise forms.ValidationError('Invalid JSON file')
 
         converted = ureport.convert_to_str(data)
         try:
             ureport.validate(converted)
         except Exception as exp:
+            reporter = None
+            if ("reporter" in converted and
+                "name" in converted["reporter"] and
+                "version" in converted["reporter"]):
+                reporter = "{0}
{1}".format(converted["reporter"]["name"],
+                                           
converted["reporter"]["version"])
+
+            self._save_invalid_ureport(json.dumps(data, indent=2),
+                                       str(exp), reporter=reporter)
             raise forms.ValidationError('Validation failed: %s' % exp)
 
         return dict(converted=converted, json=raw_data)
diff --git a/pyfaf/hub/reports/urls.py b/pyfaf/hub/reports/urls.py
index 34f9d3a..af6f052 100644
--- a/pyfaf/hub/reports/urls.py
+++ b/pyfaf/hub/reports/urls.py
@@ -9,6 +9,8 @@ urlpatterns = patterns('pyfaf.hub.reports.views',
     url(r'^new/$', 'new'),
     url(r'^attach/$', 'attach'),
     url(r'^bthash/(?P<bthash>[a-fA-F0-9]+)/$', 'bthash_forward'),
+    url(r'^invalid/$', 'invalid'),
+    url(r'^invalid/(?P<report_id>[0-9]+)/$', 'invalid_item'),
 
     url(r'^diff/(?P<lhs_id>\d+)/(?P<rhs_id>\d+)/$', 'diff'),
 
diff --git a/pyfaf/hub/reports/views.py b/pyfaf/hub/reports/views.py
index d933c48..1afd2a6 100644
--- a/pyfaf/hub/reports/views.py
+++ b/pyfaf/hub/reports/views.py
@@ -30,6 +30,7 @@ from pyfaf.storage.report import (Report,
                                   ReportPackage,
                                   ReportRhbz,
                                   ReportUnknownPackage)
+from pyfaf.storage.debug import InvalidUReport
 
 from pyfaf.hub.common.utils import paginate
 from pyfaf.hub.common.forms import OsComponentFilterForm
@@ -407,3 +408,32 @@ def bthash_forward(request, bthash):
                                    args=[reportbt.backtrace.report.id])
 
     return response
+
+def invalid(request):
+    if not request.user.is_staff:
+        raise Http404
+
+    db = pyfaf.storage.getDatabase()
+    reports = (db.session.query(InvalidUReport)
+                         .order_by(desc(InvalidUReport.date))
+                         .all())
+
+    return render_to_response("reports/invalid.html",
+        {"reports": paginate(reports, request)},
+        context_instance=RequestContext(request))
+
+def invalid_item(request, report_id):
+    if not request.user.is_staff:
+        raise Http404
+
+    db = pyfaf.storage.getDatabase()
+    report = (db.session.query(InvalidUReport)
+                        .filter(InvalidUReport.id == report_id)
+                        .first())
+
+    if report is None:
+        raise Http404
+
+    return render_to_response("reports/invalid_item.html",
+        {"report": report, "report_data":
report.get_lob("ureport")},
+        context_instance=RequestContext(request))
diff --git a/pyfaf/hub/templates/reports/Makefile.am
b/pyfaf/hub/templates/reports/Makefile.am
index 8d578c5..e23d33a 100644
--- a/pyfaf/hub/templates/reports/Makefile.am
+++ b/pyfaf/hub/templates/reports/Makefile.am
@@ -1,6 +1,8 @@
 reports_DATA = \
 	index.html \
 	item.html \
+	invalid.html \
+	invalid_item.html \
 	list.html \
 	new.html \
 	success.html \
diff --git a/pyfaf/hub/templates/reports/invalid.html
b/pyfaf/hub/templates/reports/invalid.html
new file mode 100644
index 0000000..4eae369
--- /dev/null
+++ b/pyfaf/hub/templates/reports/invalid.html
@@ -0,0 +1,26 @@
+{% extends "base.html" %}
+
+{% block title %}Invalid uReports list{% endblock %}
+
+{% block content %}
+    <table class="table table-bordered">
+        <tr>
+            <th>Date</th>
+            <th>Error</th>
+            <th>Reporter</th>
+            <th>uReport</th>
+        </tr>
+    {% for report in reports.object_list %}
+        <tr>
+            <td>{{ report.date|date:"Y-m-d" }}</td>
+            <td>{{ report.errormsg }}</td>
+            <td>{% if report.reporter %}{{ report.reporter }}{% endif
%}</td>
+            <td><a href="{% url pyfaf.hub.reports.views.invalid_item
report.id %}">Show</a></td>
+        </tr>
+    {% endfor %}
+    </table>
+
+    {% with reports as paginated %}
+      {% include "paginator.html" %}
+    {% endwith %}
+{% endblock %}
diff --git a/pyfaf/hub/templates/reports/invalid_item.html
b/pyfaf/hub/templates/reports/invalid_item.html
new file mode 100644
index 0000000..9114b53
--- /dev/null
+++ b/pyfaf/hub/templates/reports/invalid_item.html
@@ -0,0 +1,24 @@
+{% extends "base.html" %}
+
+{% block title %}Invalid uReport #{{ report.id }}{% endblock %}
+
+{% block submenu_append %}
+  <li class="active">
+    <a href="">Invalid uReport #{{ report.id }}</a>
+  </li>
+{% endblock %}
+
+{% block content %}
+  <dl class="dl-horizontal">
+    <dt>Occurred:</td>
+    <dd>{{ report.date|date:"Y-m-d H:i:s" }}</dd>
+
+    <dt>Reporter:</td>
+    <dd>{% if report.reporter %}{{ report.reporter }}{% else %}unknown{% endif
%}</dd>
+
+    <dt>Error message:</dt>
+    <dd>{{ report.errormsg }}</dd>
+  </dl>
+
+  <pre>{{ report_data }}</pre>
+{% endblock %}
diff --git a/pyfaf/storage/Makefile.am b/pyfaf/storage/Makefile.am
index 43a3357..2fab86d 100644
--- a/pyfaf/storage/Makefile.am
+++ b/pyfaf/storage/Makefile.am
@@ -3,6 +3,7 @@ SUBDIRS = fixtures
 pyfaf_storage_PYTHON = \
 	__init__.py \
 	common.py \
+	debug.py \
 	hub.py \
 	kb.py \
 	llvm.py \
diff --git a/pyfaf/storage/__init__.py b/pyfaf/storage/__init__.py
index 5492798..5a5cf40 100644
--- a/pyfaf/storage/__init__.py
+++ b/pyfaf/storage/__init__.py
@@ -163,6 +163,7 @@ from report import *
 from llvm import *
 from hub import *
 from kb import *
+from debug import *
 
 def getDatabase():
     db = Database.__instance__
diff --git a/pyfaf/storage/debug.py b/pyfaf/storage/debug.py
new file mode 100644
index 0000000..4b76f27
--- /dev/null
+++ b/pyfaf/storage/debug.py
@@ -0,0 +1,14 @@
+from . import Column
+from . import DateTime
+from . import GenericTable
+from . import Integer
+from . import String
+
+class InvalidUReport(GenericTable):
+    __tablename__ = "invalidureports"
+    __lobs__ = { "ureport": 1 << 22 }
+
+    id = Column(Integer, primary_key=True)
+    errormsg = Column(String(512), nullable=False)
+    reporter = Column(String(64), nullable=True)
+    date = Column(DateTime, nullable=False, index=True)
-- 
1.8.1.4


    

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009