[GNOME-ABRT PATCH] Show HTML titles of URLs from reported_to element

[Jakub Filak]

Signed-off-by: Jakub Filak <jfilak(a)redhat.com&gt;
---
 configure.ac                       |   1 +
 gnome-abrt.spec.in                 |   1 +
 pylintrc                           |   2 +-
 src/gnome-abrt                     |   7 ++
 src/gnome_abrt/Makefile.am         |   2 +-
 src/gnome_abrt/problems.py         |  41 +++++++--
 src/gnome_abrt/url/Makefile.am     |   3 +
 src/gnome_abrt/url/__init__.py     |  29 ++++++
 src/gnome_abrt/url/gliburltitle.py | 178 +++++++++++++++++++++++++++++++++++++
 src/gnome_abrt/url/urltitle.py     | 139 +++++++++++++++++++++++++++++
 src/gnome_abrt/views.py            |   8 +-
 11 files changed, 398 insertions(+), 13 deletions(-)
 create mode 100644 src/gnome_abrt/url/Makefile.am
 create mode 100644 src/gnome_abrt/url/__init__.py
 create mode 100644 src/gnome_abrt/url/gliburltitle.py
 create mode 100644 src/gnome_abrt/url/urltitle.py

diff --git a/configure.ac b/configure.ac
index 4a924ce..8f1a567 100644
--- a/configure.ac
+++ b/configure.ac
@@ -100,6 +100,7 @@ AC_CONFIG_FILES([
     src/Makefile
     src/gnome_abrt/Makefile
     src/gnome_abrt/wrappers/Makefile
+    src/gnome_abrt/url/Makefile
     po/Makefile.in
 ])
 
diff --git a/gnome-abrt.spec.in b/gnome-abrt.spec.in
index 5ed9a17..df30ac5 100644
--- a/gnome-abrt.spec.in
+++ b/gnome-abrt.spec.in
@@ -89,6 +89,7 @@ gtk-update-icon-cache %{_datadir}/icons/hicolor &>/dev/null || :
 %files -f %{name}.lang
 %doc COPYING
 %{python_sitearch}/gnome_abrt
+%{python_sitearch}/gnome_abrt/url
 %{_datadir}/%{name}
 %{_bindir}/%{name}
 %{_datadir}/applications/*
diff --git a/pylintrc b/pylintrc
index b5d55c5..fdb32d3 100644
--- a/pylintrc
+++ b/pylintrc
@@ -33,7 +33,7 @@ load-plugins=
 # can either give multiple identifier separated by comma (,) or put this option
 # multiple time (only on the command line, not in the configuration file where
 # it should appear only once).
-disable=C0111,W0212,R0201,R0903,I0011,W0603,W0622,I0001
+disable=C0111,W0212,R0201,R0903,R0904,I0011,W0603,W0622,I0001
 
 
 [REPORTS]
diff --git a/src/gnome-abrt b/src/gnome-abrt
index 633eb3a..d3417d2 100755
--- a/src/gnome-abrt
+++ b/src/gnome-abrt
@@ -34,6 +34,7 @@ from gi.repository import Gio
 
 # gnome-abrt
 import gnome_abrt
+import gnome_abrt.url
 from gnome_abrt.views import OopsWindow
 from gnome_abrt.controller import Controller
 from gnome_abrt.signals import glib_sigchld_signal_handler
@@ -45,6 +46,8 @@ from gnome_abrt.l10n import _
 from gnome_abrt.wrappers import show_events_list_dialog
 from gnome_abrt.config import get_configuration
 from gnome_abrt.dialogs import show_report_problem_with_abrt
+from gnome_abrt.url.gliburltitle import (GetURLTitleSourcePool,
+                                         GetURLTitleSourceCache)
 
 # dbus
 import dbus
@@ -54,6 +57,7 @@ from dbus.mainloop.glib import DBusGMainLoop
 GNOME_ABRT_APPLICATION_ID = 'org.freedesktop.GnomeAbrt'
 GNOME_ABRT_INTERFACE = 'org.freedesktop.GnomeAbrt'
 GNOME_ABRT_OBJECT_PATH = '/org/freedesktop/GnomeAbrt'
+GNOME_ABRT_URL_POOL_CAPACITY = 10
 
 
 # because of https://bugzilla.gnome.org/show_bug.cgi?id=682331
@@ -148,6 +152,9 @@ class OopsApplication(Gtk.Application):
         self.set_flags(Gio.ApplicationFlags.HANDLES_COMMAND_LINE)
         self.gcontext = None
         self.source = None
+        self._url_pool = GetURLTitleSourcePool(GNOME_ABRT_URL_POOL_CAPACITY)
+        self._url_cache = GetURLTitleSourceCache(self._url_pool)
+        gnome_abrt.url.set_async_worker(self._url_cache.get_url_title_async)
 
     #pylint: disable=W0613
     def _parse_command_line(self, sender, gcmdargs):
diff --git a/src/gnome_abrt/Makefile.am b/src/gnome_abrt/Makefile.am
index 60496f9..0f0e557 100644
--- a/src/gnome_abrt/Makefile.am
+++ b/src/gnome_abrt/Makefile.am
@@ -1,4 +1,4 @@
-SUBDIRS = wrappers
+SUBDIRS = wrappers url
 
 gnome_abrtexecdir = $(pyexecdir)/gnome_abrt
 
diff --git a/src/gnome_abrt/problems.py b/src/gnome_abrt/problems.py
index 4bf7695..1d3974e 100644
--- a/src/gnome_abrt/problems.py
+++ b/src/gnome_abrt/problems.py
@@ -19,6 +19,7 @@ import datetime
 import logging
 
 # gnome-abrt
+import gnome_abrt.url
 from gnome_abrt.application import find_application
 from gnome_abrt.errors import (InvalidProblem,
                                UnavailableSource,
@@ -70,10 +71,40 @@ class Problem:
         MSG = "MSG"
         BTHASH = "BTHASH"
 
-        def __init__(self, title, rtype, data):
-            self.title = title
-            self.rtype = rtype
-            self.data = data
+        def __init__(self, problem, name, rtype, data):
+            self._problem = problem
+            self._name = name
+            self._title = name
+            self._rtype = rtype
+            self._data = data
+            self._url_done = False
+
+        def _update_title_async_cb(self, result, unused_userdata):
+            if result[1]:
+                self._title = result[1]
+                self._problem.source.notify(ProblemSource.CHANGED_PROBLEM,
+                                            self._problem)
+
+        @property
+        def name(self):
+            return self._name
+
+        @property
+        def rtype(self):
+            return self._rtype
+
+        @property
+        def data(self):
+            return self._data
+
+        @property
+        def title(self):
+            if self._rtype == Problem.Submission.URL and not self._url_done:
+                self._url_done = True
+                gnome_abrt.url.get_url_title_async(self._data,
+                        self._update_title_async_cb, None)
+
+            return self._title
 
 
     def __init__(self, problem_id, source):
@@ -226,7 +257,7 @@ class Problem:
                     i += 1
 
                     self.submission.append(
-                        Problem.Submission(pfx, typ, line[i:]))
+                        Problem.Submission(self, pfx, typ, line[i:]))
 
         return self.submission
 
diff --git a/src/gnome_abrt/url/Makefile.am b/src/gnome_abrt/url/Makefile.am
new file mode 100644
index 0000000..195ebec
--- /dev/null
+++ b/src/gnome_abrt/url/Makefile.am
@@ -0,0 +1,3 @@
+gnome_abrt_url_PYTHON = __init__.py urltitle.py gliburltitle.py
+
+gnome_abrt_urldir = $(pyexecdir)/gnome_abrt/url
diff --git a/src/gnome_abrt/url/__init__.py b/src/gnome_abrt/url/__init__.py
new file mode 100644
index 0000000..5b77759
--- /dev/null
+++ b/src/gnome_abrt/url/__init__.py
@@ -0,0 +1,29 @@
+# coding=UTF-8
+
+## Copyright (C) 2013 ABRT team <crash-catcher(a)lists.fedorahosted.org&gt;
+## Copyright (C) 2013 Red Hat, Inc.
+
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2 of the License, or
+## (at your option) any later version.
+
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software
+## Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA  02110-1335  USA
+
+from gnome_abrt.url import gliburltitle
+
+__ASYNC_WORKER = gliburltitle.get_url_title_async
+
+def set_async_worker(worker):
+    global __ASYNC_WORKER
+    __ASYNC_WORKER = worker
+
+def get_url_title_async(url, callback, user_data):
+    __ASYNC_WORKER(url, callback, user_data)
diff --git a/src/gnome_abrt/url/gliburltitle.py b/src/gnome_abrt/url/gliburltitle.py
new file mode 100644
index 0000000..9a6f6e6
--- /dev/null
+++ b/src/gnome_abrt/url/gliburltitle.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+# coding=UTF-8
+
+## Copyright (C) 2013 ABRT team <crash-catcher(a)lists.fedorahosted.org&gt;
+## Copyright (C) 2013 Red Hat, Inc.
+
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2 of the License, or
+## (at your option) any later version.
+
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software
+## Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA  02110-1335  USA
+
+import sys
+from Queue import Queue
+
+# pygobject
+#pylint: disable=E0611
+from gi.repository import GLib
+
+from gnome_abrt.url.urltitle import GetURLTitleThread
+
+
+def get_url_title_async(url, readycallback, userdata, context=None):
+    """Gets url's title assynchronously
+    """
+
+    if context is None:
+        context = GLib.main_context_default()
+
+    source = GetURLTitleThreadSource(url, readycallback, userdata)
+    #pylint: disable=E1101
+    source.attach(context)
+
+
+class GetURLTitleThreadSource(GLib.Source):
+    """Runs thread and waits until the thread does not return the
url's title
+    """
+
+    #pylint: disable=W0613
+    def __new__(cls, *args):
+        return GLib.Source.__new__(cls)
+
+    def __init__(self, url, readycallback, userdata):
+        GLib.Source.__init__(self)
+
+        self._queue = Queue()
+        self._readycallback = readycallback
+        self._userdata = userdata
+        self._resolver = GetURLTitleThread(url, self._queue)
+        self._resolver.start()
+
+    def _is_ready(self):
+        return not self._queue.empty()
+
+    def prepare(self, *args):
+        return (self._is_ready(), 10)
+
+    def check(self, *args):
+        return self._is_ready()
+
+    def dispatch(self, *args):
+        if not self._is_ready():
+            raise RuntimeError("GetURLHTMLTitleThreadSource.dispatch()"
+            " called but result is not ready yet, or already consumed.")
+
+        # An attempty to destroy the resolver's thread ASAP
+        self._resolver = None
+
+        self._readycallback(self._queue.get(), self._userdata)
+
+        # Destroy this source
+        return False
+
+    def finalize(self, *args):
+        pass
+
+
+class GetURLTitleSourcePool(object):
+    """Pool for reducing number of running threads at time
+    """
+
+    def __init__(self, capacity):
+        self._capacity = capacity
+        self._running = 0
+        self._defered = list()
+
+    def get_url_title_async(self, url, readycallback, userdata):
+        if self._running == self._capacity:
+            self._defer_request(url, readycallback, userdata)
+        else:
+            self._start_resolving(url, readycallback, userdata)
+
+    def _defer_request(self, url, readycallback, userdata):
+        self._defered.append((url, readycallback, userdata))
+
+    def _start_resolving(self, url, readycallback, userdata):
+        self._running += 1
+        get_url_title_async(url, self._resolving_done,
+                (readycallback, userdata))
+
+    def _resolving_done(self, result, userdata):
+        try:
+            userdata[0](result, userdata[1])
+        finally:
+            self._running -= 1
+            if len(self._defered):
+                url, readycallback, userdata = self._defered.pop()
+                self._start_resolving(url, readycallback, userdata)
+
+
+class GetURLTitleSourceCache(object):
+    """Asynchronous cache for URL titles
+    """
+
+    def __init__(self, worker, max_size=1024):
+        self._worker = worker
+        self._max_size = max_size
+        self._cache = dict()
+        # The oldest is on the top
+        self._stamp = []
+
+    def get_url_title_async(self, url, readycallback, userdata):
+        if url in self._cache:
+            # Move url at the end
+            self._stamp.remove(url)
+            self._stamp.append(url)
+            # Callers expect that result will be delivered asynchronously
+            GLib.timeout_add(1,
+                lambda unused: readycallback((url, self._cache[url]), userdata),
+                None)
+            return
+
+        self._worker.get_url_title_async(url, self._resolved,
+                                         (readycallback, userdata))
+
+    def _resolved(self, result, userdata):
+        url, title = result
+        if title:
+            if len(self._cache.keys()) >= self._max_size:
+                # Remove the url on the top (the oldest)
+                del self._cache[self._stamp.pop()]
+
+            self._cache[url] = title
+            self._stamp.append(url)
+
+        readycallback, data = userdata
+        readycallback(result, data)
+
+
+if __name__ == "__main__":
+    def ready(result, userdata):
+        print result[1] or result[0]
+
+        userdata["count"] = userdata["count"] - 1
+        if userdata["count"] == 0:
+            userdata["gloop"].quit()
+
+    URLS = ["http://docs.python.org/2/library/htmlparser.html"]
+
+    if len(sys.argv) > 1:
+        URLS = sys.argv[1:]
+
+    MAINLOOP = GLib.MainLoop()
+    WORKER = GetURLTitleSourcePool(2)
+    CONSUMER = {"count":len(URLS), "gloop":MAINLOOP}
+
+    for u in URLS:
+        WORKER.get_url_title_async(u, ready, CONSUMER)
+
+    MAINLOOP.run()
diff --git a/src/gnome_abrt/url/urltitle.py b/src/gnome_abrt/url/urltitle.py
new file mode 100644
index 0000000..369b1af
--- /dev/null
+++ b/src/gnome_abrt/url/urltitle.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# coding=UTF-8
+
+## Copyright (C) 2013 ABRT team <crash-catcher(a)lists.fedorahosted.org&gt;
+## Copyright (C) 2013 Red Hat, Inc.
+
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2 of the License, or
+## (at your option) any later version.
+
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software
+## Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA  02110-1335  USA
+
+import sys
+import threading
+import logging
+from urllib import urlopen
+from HTMLParser import HTMLParser, HTMLParseError
+
+
+def get_url_title(url):
+    """Returns a title of the url
+
+    Function expects that the url points to a HTML page with the title element
+    in the head element. Tries to download the page and return a value of the
+    title.
+    """
+
+    try:
+        urlio = urlopen(url)
+    except IOError as ex:
+        logging.debug("{1} ('{0}')".format(url, str(ex)))
+        return None
+
+    encoding = urlio.info().getparam('charset')
+    # Can't find any documentation but try except needed.
+    data = urlio.read()
+    if encoding:
+        try:
+            data = data.decode(encoding)
+        except UnicodeError as ex:
+            logging.debug("{1} ('{0}')".format(url, str(ex)))
+            return None
+
+    return HTMLTitleGetter.parse(data)
+
+
+class HTMLTitleGetter(HTMLParser):
+    def __init__(self):
+        HTMLParser.__init__(self)
+
+        self._tag_head = False
+        self._tag_title = False
+        self._title = None
+
+    def handle_starttag(self, tag, attrs):
+        if not self._tag_head:
+            self._tag_head = tag.lower() == "head"
+
+        if self._tag_head and not self._tag_title:
+            self._tag_title = tag.lower() == "title"
+
+    def handle_endtag(self, tag):
+        if self._tag_head:
+            self._tag_head = tag.lower() != "head"
+
+        if self._tag_head and self._tag_title:
+            self._tag_title = tag.lower() != "title"
+
+    def handle_data(self, data):
+        if self._tag_title:
+            if not self._title:
+                self._title = data
+            else:
+                self._title += data
+
+    @staticmethod
+    def parse(data):
+        tgt = HTMLTitleGetter()
+        try:
+            tgt.feed(data)
+        except HTMLParseError as ex:
+            logging.debug("{1} ('{0}')".format(data, str(ex)))
+            # Hopefully title is parsed correctly
+
+        return tgt._title
+
+
+class GetURLTitleThread(threading.Thread):
+    """Simple thread which puts url's title into a Queue
+    """
+
+    def __init__(self, url, que):
+        super(GetURLTitleThread, self).__init__()
+
+        self._url = url
+        self._que = que
+
+    def run(self):
+        title = get_url_title(self._url)
+        self._que.put((self._url, title))
+
+
+if __name__ == "__main__":
+    URLS = ["http://docs.python.org/2/library/htmlparser.html"]
+
+    THREADS = False
+    if len(sys.argv) > 1:
+        i = 1
+        if "--threads" == sys.argv[i]:
+            THREADS = True
+            i += 1
+        URLS = sys.argv[i:]
+
+    if THREADS:
+        from Queue import Queue
+        QUEUE = Queue()
+        THREADS = []
+        for u in URLS:
+            t = GetURLTitleThread(u, QUEUE)
+            t.start()
+            THREADS.append(t)
+
+        for t in THREADS:
+            t.join()
+
+        while not QUEUE.empty():
+            ORIG_URL, GOT_TITLE = QUEUE.get()
+            print GOT_TITLE or ORIG_URL
+    else:
+        for u in URLS:
+            print get_url_title(u) or u
diff --git a/src/gnome_abrt/views.py b/src/gnome_abrt/views.py
index 5c1f6e2..6c2e5a9 100644
--- a/src/gnome_abrt/views.py
+++ b/src/gnome_abrt/views.py
@@ -256,8 +256,7 @@ class OopsWindow(Gtk.ApplicationWindow):
             except errors.InvalidProblem as ex:
                 self._remove_problem_from_storage(ex.problem_id)
                 logging.debug(ex.message)
-
-            return
+                return
 
         try:
             if problem in self._get_selected(self._builder.tvs_problems):
@@ -266,8 +265,6 @@ class OopsWindow(Gtk.ApplicationWindow):
             self._remove_problem_from_storage(ex.problem_id)
             logging.debug(ex.message)
 
-        return
-
     def _reload_problems(self, source):
         self._reloading = True
         old_selection = self._get_selected(self._builder.tvs_problems)
@@ -318,7 +315,6 @@ class OopsWindow(Gtk.ApplicationWindow):
         self._builder.tvs_problems.select_iter(pit)
         self._builder.tv_problems.scroll_to_cell(path)
 
-
     def _show_problem_links(self, submissions):
         need_align = False
         for sbm in submissions:
@@ -388,7 +384,7 @@ class OopsWindow(Gtk.ApplicationWindow):
                 if problem['not-reportable']:
                     self._show_problem_message(problem['not-reportable'])
                 elif (not problem['is_reported']
-                        or not any((s.title == "Bugzilla"
+                        or not any((s.name == "Bugzilla"
                                 for s in problem['submission']))):
                     self._show_problem_message(
     _("This problem hasn't been reported to <i>Bugzilla</i> yet.
"
-- 
1.8.1.4

[Martin Milata]

Works as expected --> pushed.