May 2011 - bodhi - Fedora Mailing-Lists

by Luke Macken

tools/pickledb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) New commits: commit c79000fc2c0d8d055d1d9f75c531cb964c251972 Author: Luke Macken <lmacken(a)redhat.com> Date: Fri May 20 13:27:17 2011 -0400 whitespace diff --git a/tools/pickledb.py b/tools/pickledb.py index ac4cc21..6241064 100755 --- a/tools/pickledb.py +++ b/tools/pickledb.py @@ -380,7 +380,7 @@ def load_sqlalchemy_db(): DBSession.commit() - print("Database migration complete!") + print("\n\nDatabase migration complete!") print(" * %d updates" % Update.query.count()) print(" * %d builds" % Build.query.count()) print(" * %d comments" % Comment.query.count())

13 years

1
0
0 / 0

Branch 'pyramid' - 3 commits - pyramid/bodhi

by Luke Macken

pyramid/bodhi/bodhi/models/models.py | 23 ++++++++++++++++------ pyramid/bodhi/bodhi/resources.py | 15 +++++++++++++- pyramid/bodhi/bodhi/tests/functional/test_wsgi.py | 7 ++++++ pyramid/bodhi/bodhi/tests/models/test_models.py | 5 +--- 4 files changed, 40 insertions(+), 10 deletions(-) New commits: commit d2f7f4c71512205025671b8e122b6d51057f6cc9 Author: Luke Macken <lmacken(a)redhat.com> Date: Tue May 17 16:41:20 2011 -0400 Add the ability to specify resource column filters diff --git a/pyramid/bodhi/bodhi/resources.py b/pyramid/bodhi/bodhi/resources.py index a14173a..6d15cad 100644 --- a/pyramid/bodhi/bodhi/resources.py +++ b/pyramid/bodhi/bodhi/resources.py @@ -25,9 +25,15 @@ class BodhiResource(object): __parent__ = None __model__ = None __column__ = None + __filter__ = None def __getitem__(self, key): session = DBSession() + if self.__filter__: + try: + key = self.__filter__(key) + except: + raise KeyError(key) try: return session.query(self.__model__).filter_by( **{self.__column__: key}).one() commit 4b3e9a2ff5dddb711487cacfe9c25384b27e7e5e Author: Luke Macken <lmacken(a)redhat.com> Date: Tue May 17 16:41:13 2011 -0400 Add our BugResource, and tests diff --git a/pyramid/bodhi/bodhi/models/models.py b/pyramid/bodhi/bodhi/models/models.py index a54a557..57f594e 100644 --- a/pyramid/bodhi/bodhi/models/models.py +++ b/pyramid/bodhi/bodhi/models/models.py @@ -744,6 +744,7 @@ class Update(Base): mail.send(self.get_maintainers(), 'unstable', self) comment = Comment(text=text, karma=karma, anonymous=anonymous) + if anonymous: author = u'anonymous' try: @@ -1017,6 +1018,9 @@ def populate(): session.add(build) update = Update(builds=[build], user=user, notes=u'Useful details!') update.type = UpdateType.bugfix + bug = Bug(bug_id=12345) + session.add(bug) + update.bugs.append(bug) session.add(update) session.commit() #session.flush() diff --git a/pyramid/bodhi/bodhi/resources.py b/pyramid/bodhi/bodhi/resources.py index a918be3..a14173a 100644 --- a/pyramid/bodhi/bodhi/resources.py +++ b/pyramid/bodhi/bodhi/resources.py @@ -3,7 +3,7 @@ import logging from sqlalchemy.orm.exc import NoResultFound from bodhi.models import initialize_sql, DBSession -from bodhi.models import Update, Package, Build, Release, User +from bodhi.models import Update, Package, Build, Release, User, Bug log = logging.getLogger(__name__) @@ -55,6 +55,12 @@ class UserResource(BodhiResource): __model__ = User __column__ = u'name' +class BugResource(BodhiResource): + __model__ = Bug + __column__ = u'bug_id' + __filter__ = int + + root = BodhiRoot() def default_get_root(request): @@ -65,6 +71,7 @@ def default_get_root(request): u'packages': PackageResource, u'releases': ReleaseResource, u'users': UserResource, + u'bugs': BugResource, }) return root diff --git a/pyramid/bodhi/bodhi/tests/functional/test_wsgi.py b/pyramid/bodhi/bodhi/tests/functional/test_wsgi.py index 7a21563..63e7d59 100644 --- a/pyramid/bodhi/bodhi/tests/functional/test_wsgi.py +++ b/pyramid/bodhi/bodhi/tests/functional/test_wsgi.py @@ -27,3 +27,10 @@ class FunctionalTests(unittest.TestCase): data = json.loads(res.body) eq_(data[u'entries'][0][u'name'], 'F15') + def test_releases_view_invalid_bug(self): + res = app.get('/bugs/abc', status=404) + + def test_releases_view_bug(self): + res = app.get('/bugs/12345', status=200) + data = json.loads(res.body) + eq_(data[u'context'][u'bug_id'], 12345) commit 66c00168512ee047e0b5cd41649a08974fa96888 Author: Luke Macken <lmacken(a)redhat.com> Date: Tue May 17 15:50:35 2011 -0400 Update our comment api to work with our new user model diff --git a/pyramid/bodhi/bodhi/models/models.py b/pyramid/bodhi/bodhi/models/models.py index fdbbe97..a54a557 100644 --- a/pyramid/bodhi/bodhi/models/models.py +++ b/pyramid/bodhi/bodhi/models/models.py @@ -719,9 +719,9 @@ class Update(Base): if not author: author = identity.current.user_name if not anonymous and karma != 0 and \ - not filter(lambda c: c.author == author and c.karma == karma, + not filter(lambda c: c.user.name == author and c.karma == karma, self.comments): - mycomments = [c.karma for c in self.comments if c.author == author] + mycomments = [c.karma for c in self.comments if c.user.name == author] if karma == 1 and -1 in mycomments: self.karma += 2 elif karma == -1 and 1 in mycomments: @@ -743,8 +743,15 @@ class Update(Base): self.obsolete() mail.send(self.get_maintainers(), 'unstable', self) - comment = Comment(text=text, karma=karma,author=author, - anonymous=anonymous) + comment = Comment(text=text, karma=karma, anonymous=anonymous) + if anonymous: + author = u'anonymous' + try: + user = User.query.filter_by(name=author).one() + except NoResultFound: + user = User(name=author) + DBSession.add(user) + user.comments.append(comment) DBSession.add(comment) self.comments.append(comment) @@ -755,9 +762,9 @@ class Update(Base): for person in self.get_maintainers(): people.add(person) for comment in self.comments: - if comment.anonymous or comment.author == 'bodhi': + if comment.anonymous or comment.user.name == u'bodhi': continue - people.add(comment.author) + people.add(comment.user.name) mail.send(people, 'comment', self) def unpush(self): diff --git a/pyramid/bodhi/bodhi/tests/models/test_models.py b/pyramid/bodhi/bodhi/tests/models/test_models.py index 03d91a1..ddd5a01 100644 --- a/pyramid/bodhi/bodhi/tests/models/test_models.py +++ b/pyramid/bodhi/bodhi/tests/models/test_models.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Test suite for the Bodhi models""" -import os import time from nose.tools import eq_, raises @@ -312,12 +311,12 @@ class TestUpdate(ModelTest): self.obj.status = UpdateStatus.testing self.obj.status_comment() eq_(len(self.obj.comments), 1) - eq_(self.obj.comments[0].author, u'bodhi') + eq_(self.obj.comments[0].user.name, u'bodhi') eq_(self.obj.comments[0].text, u'This update has been pushed to testing') self.obj.status = UpdateStatus.stable self.obj.status_comment() eq_(len(self.obj.comments), 2) - eq_(self.obj.comments[1].author, u'bodhi') + eq_(self.obj.comments[1].user.name, u'bodhi') eq_(self.obj.comments[1].text, u'This update has been pushed to stable') def test_get_url(self):

13 years

1
0
0 / 0

bodhi/controllers.py

by Luke Macken

bodhi/controllers.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) New commits: commit bf32225bf7addb7555986b5aa3208271ccb26001 Author: Luke Macken <lmacken(a)redhat.com> Date: Tue May 17 16:06:18 2011 -0400 Optimize querying for updates by bug numbers (#610) diff --git a/bodhi/controllers.py b/bodhi/controllers.py index 7ba07f6..7a234ff 100644 --- a/bodhi/controllers.py +++ b/bodhi/controllers.py @@ -262,6 +262,19 @@ class Root(controllers.RootController): title="%d %s found" % (num_items, num_items == 1 and 'update' or 'updates')) + # If we're looking for bugs specifically (#610) + if bugs: + updates = [] + bugs = bugs.replace('#', '').split(',') + for bug in bugs: + try: + bug = Bugzilla.byBz_id(int(bug)) + for update in bug.updates: + updates.append(update.__json__()) + except (SQLObjectNotFound, ValueError): + pass + return dict(updates=updates, num_items=len(updates)) + try: if release: # TODO: if a specific release is requested along with get_auth, @@ -359,12 +372,12 @@ class Root(controllers.RootController): # simply make them queries # Filter results by Bugs and/or CVEs - if bugs: - results = [] - for bug in map(Bugzilla.byBz_id, map(int, bugs.split(','))): - map(results.append, - filter(lambda x: bug in x.bugs, updates)) - updates = results + #if bugs: + # results = [] + # for bug in map(Bugzilla.byBz_id, map(int, bugs.split(','))): + # map(results.append, + # filter(lambda x: bug in x.bugs, updates)) + # updates = results if cves: results = [] for cve in map(CVE.byCve_id, cves.split(',')):

13 years

1
0
0 / 0

Branch 'pyramid' - pyramid/bodhi

by Luke Macken

pyramid/bodhi/bodhi/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) New commits: commit ffe713f2e6326ec9c4f67eafec06e26cdff0a1ae Author: Luke Macken <lmacken(a)redhat.com> Date: Tue May 17 15:37:02 2011 -0400 Fix our UserResource diff --git a/pyramid/bodhi/bodhi/resources.py b/pyramid/bodhi/bodhi/resources.py index cdb8881..a918be3 100644 --- a/pyramid/bodhi/bodhi/resources.py +++ b/pyramid/bodhi/bodhi/resources.py @@ -53,7 +53,7 @@ class ReleaseResource(BodhiResource): class UserResource(BodhiResource): __model__ = User - __keys__ = u'name' + __column__ = u'name' root = BodhiRoot()

13 years

1
0
0 / 0

Branch 'pyramid' - 3 commits - pyramid/bodhi

by Luke Macken

pyramid/bodhi/bodhi/models/models.py | 3 pyramid/bodhi/tools/pickledb.py | 192 +++++++++++++++++++++++++---------- 2 files changed, 141 insertions(+), 54 deletions(-) New commits: commit 22c4bbba9d7f47b2a02a886b574bc22c0f238d98 Author: Luke Macken <lmacken(a)redhat.com> Date: Tue May 17 14:41:34 2011 -0400 Initial port of pickledb to handle SQLAlchemy migration diff --git a/pyramid/bodhi/tools/pickledb.py b/pyramid/bodhi/tools/pickledb.py index c35ef29..2be0382 100755 --- a/pyramid/bodhi/tools/pickledb.py +++ b/pyramid/bodhi/tools/pickledb.py @@ -26,16 +26,17 @@ import time import cPickle as pickle from os.path import isfile -from sqlobject import SQLObjectNotFound -from turbogears.database import PackageHub +#from sqlobject import SQLObjectNotFound +#from turbogears.database import PackageHub -from bodhi.util import ProgressBar, load_config +#from bodhi.util import ProgressBar, load_config +from bodhi.util import ProgressBar from bodhi.exceptions import (DuplicateEntryError, SQLiteIntegrityError, PostgresIntegrityError) -from bodhi.model import (PackageUpdate, Release, Comment, Bugzilla, CVE, - Package, PackageBuild) - -hub = __connection__ = PackageHub("bodhi") +#from bodhi.model import (PackageUpdate, Release, Comment, Bugzilla, CVE, +# Package, PackageBuild) +# +#hub = __connection__ = PackageHub("bodhi") def save_db(): ## Save each release and it's metrics @@ -186,99 +187,183 @@ def load_sqlalchemy_db(): db = file(sys.argv[2], 'r') data = pickle.load(db) + from bodhi.models import initialize_sql, DBSession + from bodhi.models import Release, Update, Build, Comment, User, Bug, CVE + from bodhi.models import Package + from bodhi.models import UpdateType, UpdateStatus, UpdateRequest + from sqlalchemy import create_engine + from sqlalchemy.orm.exc import NoResultFound + + engine = create_engine('sqlite:///bodhi.db') + initialize_sql(engine) + # Legacy format was just a list of update dictionaries # Now we'll pull things out into an organized dictionary: # {'updates': [], 'releases': []} if isinstance(data, dict): for release in data['releases']: try: - Release.byName(release['name']) - except SQLObjectNotFound: - Release(**release) + Release.query.filter_by(name=release['name']).one() + except NoResultFound: + r = Release(**release) + DBSession.add(r) data = data['updates'] progress = ProgressBar(maxValue=len(data)) for u in data: try: - release = Release.byName(u['release'][0]) - except SQLObjectNotFound: + release = Release.query.filter_by(name=u['release'][0]).one() + except NoResultFound: release = Release(name=u['release'][0], long_name=u['release'][1], - id_prefix=u['release'][2], dist_tag=u['release'][3]) + id_prefix=u['release'][2], + dist_tag=u['release'][3]) + DBSession.add(release) ## Backwards compatbility request = u['request'] if u['request'] == 'move': - request = 'stable' + u['request'] = 'stable' elif u['request'] == 'push': - request = 'testing' + u['request'] = 'testing' elif u['request'] == 'unpush': - request = 'obsolete' - if u['approved'] in (True, False): + u['request'] = 'obsolete' + if u['approved'] not in (True, False): u['approved'] = None if u.has_key('update_id'): u['updateid'] = u['update_id'] if not u.has_key('date_modified'): u['date_modified'] = None + # Port to new enum types + if u['request']: + if u['request'] == 'stable': + u['request'] = UpdateRequest.stable + elif u['request'] == 'testing': + u['request'] = UpdateRequest.testing + else: + raise Exception("Unknown request: %s" % u['request']) + + if u['type'] == 'bugfix': + u['type'] = UpdateType.bugfix + elif u['type'] == 'newpackage': + u['type'] = UpdateType.newpackage + elif u['type'] == 'enhancement': + u['type'] = UpdateType.enhancement + elif u['type'] == 'security': + u['type'] = UpdateType.security + else: + raise Exception("Unknown type: %r" % u['type']) + + if u['status'] == 'pending': + u['status'] = UpdateStatus.pending + elif u['status'] == 'testing': + u['status'] = UpdateStatus.testing + elif u['status'] == 'obsolete': + u['status'] = UpdateStatus.obsolete + elif u['status'] == 'stable': + u['status'] = UpdateStatus.stable + elif u['status'] == 'unpushed': + u['status'] = UpdateStatus.unpushed + else: + raise Exception("Unknown status: %r" % u['status']) + try: - update = PackageUpdate.byTitle(u['title']) - except SQLObjectNotFound: - update = PackageUpdate(title=u['title'], - date_submitted=u['date_submitted'], - date_pushed=u['date_pushed'], - date_modified=u['date_modified'], - release=release, - submitter=u['submitter'], - updateid=u['updateid'], - type=u['type'], - status=u['status'], - pushed=u['pushed'], - notes=u['notes'], - karma=u['karma'], - request=request, - approved=u['approved']) + update = Update.query.filter_by(title=u['title']).one() + except NoResultFound: + update = Update(_title=u['title'], + date_submitted=u['date_submitted'], + date_pushed=u['date_pushed'], + date_modified=u['date_modified'], + #release=release, + alias=u['updateid'], + pushed=u['pushed'], + notes=u['notes'], + karma=u['karma'], + type=u['type'], + status=u['status'], + request=u['request'], + ) + #approved=u['approved']) + DBSession.add(update) - ## Create Package and PackageBuild objects + try: + user = User.query.filter_by(name=u['submitter']).one() + except NoResultFound: + user = User(name=u['submitter']) + DBSession.add(user) + user.updates.append(update) + + ## Create Package and Build objects for pkg, nvr in u['builds']: try: - package = Package.byName(pkg) - except SQLObjectNotFound: + package = Package.query.filter_by(name=pkg).one() + except NoResultFound: package = Package(name=pkg) + DBSession.add(package) try: - build = PackageBuild.byNvr(nvr) - except SQLObjectNotFound: - build = PackageBuild(nvr=nvr, package=package) - update.addPackageBuild(build) + build = Build.query.filter_by(nvr=nvr).one() + except NoResultFound: + build = Build(nvr=nvr, package=package) + DBSession.add(build) + update.builds.append(build) ## Create all Bugzilla objects for this update for bug_num, bug_title, security, parent in u['bugs']: try: - bug = Bugzilla.byBz_id(bug_num) - except SQLObjectNotFound: - bug = Bugzilla(bz_id=bug_num, security=security, parent=parent) - bug.title = bug_title - update.addBugzilla(bug) + bug = Bug.query.filter_by(bug_id=bug_num).one() + except NoResultFound: + bug = Bug(bug_id=bug_num, security=security, parent=parent, + title=bug_title) + DBSession.add(bug) + update.bugs.append(bug) ## Create all CVE objects for this update for cve_id in u['cves']: try: - cve = CVE.byCve_id(cve_id) - except SQLObjectNotFound: + cve = CVE.query.filter_by(cve_id=cve_id).one() + except NoResultFound: cve = CVE(cve_id=cve_id) - update.addCVE(cve) + DBSession.add(cve) + update.cves.append(cve) + + ## Create all Comments for this update for timestamp, author, text, karma, anonymous in u['comments']: - comment = Comment(timestamp=timestamp, author=author, text=text, - karma=karma, update=update, anonymous=anonymous) + comment = Comment(timestamp=timestamp, text=text, + karma=karma, anonymous=anonymous) + DBSession.add(comment) + update.comments.append(comment) + if anonymous: + name = 'anonymous' + else: + name = author + try: + user = User.query.filter_by(name=name).one() + except NoResultFound: + user = User(name=name) + DBSession.add(user) + user.comments.append(comment) + + DBSession.flush() progress() + DBSession.commit() + + print("Database migration complete!") + print(" * %d updates" % Update.query.count()) + print(" * %d builds" % Build.query.count()) + print(" * %d comments" % Comment.query.count()) + print(" * %d users" % User.query.count()) + print(" * %d bugs" % Bug.query.count()) + print(" * %d CVEs" % CVE.query.count()) + def usage(): print "Usage: ./pickledb.py [ save | load <file> ]" sys.exit(-1) def main(): - load_config() + #load_config() if len(sys.argv) < 2: usage() elif sys.argv[1] == 'save': @@ -292,10 +377,11 @@ def main(): hub.commit() elif sys.argv[1] == 'migrate' and len(sys.argv) == 3: try: - hub.begin() + #hub.begin() load_sqlalchemy_db() finally: - hub.commit() + pass + #hub.commit() else: usage() commit 9288b78d83c32c5a3cfdef4cf200047fd03cf173 Author: Luke Macken <lmacken(a)redhat.com> Date: Tue May 17 14:41:15 2011 -0400 Remove unnecessary author column from Comment, since we now have a User model diff --git a/pyramid/bodhi/bodhi/models/models.py b/pyramid/bodhi/bodhi/models/models.py index 78955d1..fdbbe97 100644 --- a/pyramid/bodhi/bodhi/models/models.py +++ b/pyramid/bodhi/bodhi/models/models.py @@ -831,7 +831,6 @@ class Comment(Base): __tablename__ = 'comments' id = Column(Integer, primary_key=True) - author = Column(Unicode(50), nullable=False) karma = Column(Integer, default=0) text = Column(UnicodeText) anonymous = Column(Boolean, default=False) commit 06d5c96db7e76eef6af3f74f20755f4c9115653f Author: Luke Macken <lmacken(a)redhat.com> Date: Tue May 17 14:40:54 2011 -0400 Add a 'query' property to our Base model diff --git a/pyramid/bodhi/bodhi/models/models.py b/pyramid/bodhi/bodhi/models/models.py index 38a0207..78955d1 100644 --- a/pyramid/bodhi/bodhi/models/models.py +++ b/pyramid/bodhi/bodhi/models/models.py @@ -70,6 +70,8 @@ class BodhiBase(object): Base = declarative_base(cls=BodhiBase) metadata = Base.metadata DBSession = scoped_session(sessionmaker()) +Base.query = DBSession.query_property() + ## ## Enumerated type declarations

13 years

1
0
0 / 0

Branch 'pyramid' - pyramid/bodhi

by Luke Macken

pyramid/bodhi/tools/pickledb.py | 303 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 303 insertions(+) New commits: commit ca96967cc07bc100105d4197896e50d583795a9c Author: Luke Macken <lmacken(a)redhat.com> Date: Tue May 17 13:57:38 2011 -0400 Add our original pickledb tool, which we need to modify to support migrating to our SQLAlchemy model. diff --git a/pyramid/bodhi/tools/pickledb.py b/pyramid/bodhi/tools/pickledb.py new file mode 100755 index 0000000..c35ef29 --- /dev/null +++ b/pyramid/bodhi/tools/pickledb.py @@ -0,0 +1,303 @@ +#!/usr/bin/python -tt +# $Id: $ +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +""" +This script pickles all updates/bugs/cves/comments and writes it out to disk +in the format of bodhi-pickledb-YYYYMMDD.HHMM +""" + +__requires__ = 'bodhi' + +import sys +import time +import cPickle as pickle + +from os.path import isfile +from sqlobject import SQLObjectNotFound +from turbogears.database import PackageHub + +from bodhi.util import ProgressBar, load_config +from bodhi.exceptions import (DuplicateEntryError, SQLiteIntegrityError, + PostgresIntegrityError) +from bodhi.model import (PackageUpdate, Release, Comment, Bugzilla, CVE, + Package, PackageBuild) + +hub = __connection__ = PackageHub("bodhi") + +def save_db(): + ## Save each release and it's metrics + releases = [] + for release in Release.select(): + rel = {} + for attr in ('name', 'long_name', 'id_prefix', 'dist_tag', + 'locked', 'metrics'): + rel[attr] = getattr(release, attr) + releases.append(rel) + + updates = [] + all_updates = PackageUpdate.select() + progress = ProgressBar(maxValue=all_updates.count()) + + for update in all_updates: + data = {} + data['title'] = update.title + data['builds'] = [(build.package.name, build.nvr) for build in update.builds] + data['date_submitted'] = update.date_submitted + data['date_pushed'] = update.date_pushed + data['date_modified'] = update.date_modified + data['release'] = [update.release.name, update.release.long_name, + update.release.id_prefix, update.release.dist_tag] + data['submitter'] = update.submitter + data['update_id'] = hasattr(update, 'update_id') and update.update_id or update.updateid + data['type'] = update.type + data['karma'] = update.karma + data['cves'] = [cve.cve_id for cve in update.cves] + data['bugs'] = [] + for bug in update.bugs: + data['bugs'].append([bug.bz_id, bug.title, bug.security]) + if hasattr(bug, 'parent'): + data['bugs'][-1].append(bug.parent) + else: + data['bugs'][-1].append(False) + data['status'] = update.status + data['pushed'] = update.pushed + data['notes'] = update.notes + data['request'] = update.request + data['comments'] = [(c.timestamp, c.author, c.text, c.karma, c.anonymous) for c in update.comments] + if hasattr(update, 'approved'): + data['approved'] = update.approved + else: + data['approved'] = None + + updates.append(data) + progress() + + dump = file('bodhi-pickledb-%s' % time.strftime("%y%m%d.%H%M"), 'w') + pickle.dump({'updates': updates, 'releases': releases}, dump) + dump.close() + +def load_db(): + print "\nLoading pickled database %s" % sys.argv[2] + db = file(sys.argv[2], 'r') + data = pickle.load(db) + + # Legacy format was just a list of update dictionaries + # Now we'll pull things out into an organized dictionary: + # {'updates': [], 'releases': []} + if isinstance(data, dict): + for release in data['releases']: + try: + Release.byName(release['name']) + except SQLObjectNotFound: + Release(**release) + data = data['updates'] + + progress = ProgressBar(maxValue=len(data)) + + for u in data: + try: + release = Release.byName(u['release'][0]) + except SQLObjectNotFound: + release = Release(name=u['release'][0], long_name=u['release'][1], + id_prefix=u['release'][2], dist_tag=u['release'][3]) + + ## Backwards compatbility + request = u['request'] + if u['request'] == 'move': + request = 'stable' + elif u['request'] == 'push': + request = 'testing' + elif u['request'] == 'unpush': + request = 'obsolete' + if u['approved'] in (True, False): + u['approved'] = None + if u.has_key('update_id'): + u['updateid'] = u['update_id'] + if not u.has_key('date_modified'): + u['date_modified'] = None + + try: + update = PackageUpdate.byTitle(u['title']) + except SQLObjectNotFound: + update = PackageUpdate(title=u['title'], + date_submitted=u['date_submitted'], + date_pushed=u['date_pushed'], + date_modified=u['date_modified'], + release=release, + submitter=u['submitter'], + updateid=u['updateid'], + type=u['type'], + status=u['status'], + pushed=u['pushed'], + notes=u['notes'], + karma=u['karma'], + request=request, + approved=u['approved']) + + ## Create Package and PackageBuild objects + for pkg, nvr in u['builds']: + try: + package = Package.byName(pkg) + except SQLObjectNotFound: + package = Package(name=pkg) + try: + build = PackageBuild.byNvr(nvr) + except SQLObjectNotFound: + build = PackageBuild(nvr=nvr, package=package) + update.addPackageBuild(build) + + ## Create all Bugzilla objects for this update + for bug_num, bug_title, security, parent in u['bugs']: + try: + bug = Bugzilla.byBz_id(bug_num) + except SQLObjectNotFound: + bug = Bugzilla(bz_id=bug_num, security=security, parent=parent) + bug.title = bug_title + update.addBugzilla(bug) + + ## Create all CVE objects for this update + for cve_id in u['cves']: + try: + cve = CVE.byCve_id(cve_id) + except SQLObjectNotFound: + cve = CVE(cve_id=cve_id) + update.addCVE(cve) + for timestamp, author, text, karma, anonymous in u['comments']: + comment = Comment(timestamp=timestamp, author=author, text=text, + karma=karma, update=update, anonymous=anonymous) + + progress() + +def load_sqlalchemy_db(): + print "\nLoading pickled database %s" % sys.argv[2] + db = file(sys.argv[2], 'r') + data = pickle.load(db) + + # Legacy format was just a list of update dictionaries + # Now we'll pull things out into an organized dictionary: + # {'updates': [], 'releases': []} + if isinstance(data, dict): + for release in data['releases']: + try: + Release.byName(release['name']) + except SQLObjectNotFound: + Release(**release) + data = data['updates'] + + progress = ProgressBar(maxValue=len(data)) + + for u in data: + try: + release = Release.byName(u['release'][0]) + except SQLObjectNotFound: + release = Release(name=u['release'][0], long_name=u['release'][1], + id_prefix=u['release'][2], dist_tag=u['release'][3]) + + ## Backwards compatbility + request = u['request'] + if u['request'] == 'move': + request = 'stable' + elif u['request'] == 'push': + request = 'testing' + elif u['request'] == 'unpush': + request = 'obsolete' + if u['approved'] in (True, False): + u['approved'] = None + if u.has_key('update_id'): + u['updateid'] = u['update_id'] + if not u.has_key('date_modified'): + u['date_modified'] = None + + try: + update = PackageUpdate.byTitle(u['title']) + except SQLObjectNotFound: + update = PackageUpdate(title=u['title'], + date_submitted=u['date_submitted'], + date_pushed=u['date_pushed'], + date_modified=u['date_modified'], + release=release, + submitter=u['submitter'], + updateid=u['updateid'], + type=u['type'], + status=u['status'], + pushed=u['pushed'], + notes=u['notes'], + karma=u['karma'], + request=request, + approved=u['approved']) + + ## Create Package and PackageBuild objects + for pkg, nvr in u['builds']: + try: + package = Package.byName(pkg) + except SQLObjectNotFound: + package = Package(name=pkg) + try: + build = PackageBuild.byNvr(nvr) + except SQLObjectNotFound: + build = PackageBuild(nvr=nvr, package=package) + update.addPackageBuild(build) + + ## Create all Bugzilla objects for this update + for bug_num, bug_title, security, parent in u['bugs']: + try: + bug = Bugzilla.byBz_id(bug_num) + except SQLObjectNotFound: + bug = Bugzilla(bz_id=bug_num, security=security, parent=parent) + bug.title = bug_title + update.addBugzilla(bug) + + ## Create all CVE objects for this update + for cve_id in u['cves']: + try: + cve = CVE.byCve_id(cve_id) + except SQLObjectNotFound: + cve = CVE(cve_id=cve_id) + update.addCVE(cve) + for timestamp, author, text, karma, anonymous in u['comments']: + comment = Comment(timestamp=timestamp, author=author, text=text, + karma=karma, update=update, anonymous=anonymous) + + progress() + +def usage(): + print "Usage: ./pickledb.py [ save | load <file> ]" + sys.exit(-1) + +def main(): + load_config() + if len(sys.argv) < 2: + usage() + elif sys.argv[1] == 'save': + print "Pickling database..." + save_db() + elif sys.argv[1] == 'load' and len(sys.argv) == 3: + try: + hub.begin() + load_db() + finally: + hub.commit() + elif sys.argv[1] == 'migrate' and len(sys.argv) == 3: + try: + hub.begin() + load_sqlalchemy_db() + finally: + hub.commit() + else: + usage() + +if __name__ == '__main__': + main()

13 years

1
0
0 / 0

Branch 'pyramid' - 2 commits - pyramid/bodhi

by Luke Macken

pyramid/bodhi/bodhi/models/models.py | 8 ++++---- pyramid/bodhi/bodhi/tests/views/test_views.py | 17 ++++++++++++++++- pyramid/bodhi/bodhi/views.py | 12 +++++++----- 3 files changed, 27 insertions(+), 10 deletions(-) New commits: commit 356bdce860f47aa68fd07a247a5cdce5d126bb69 Author: Luke Macken <lmacken(a)redhat.com> Date: Tue May 17 13:53:55 2011 -0400 Get our views.py back up to 100% coverage diff --git a/pyramid/bodhi/bodhi/tests/views/test_views.py b/pyramid/bodhi/bodhi/tests/views/test_views.py index ed58a63..f9d0484 100644 --- a/pyramid/bodhi/bodhi/tests/views/test_views.py +++ b/pyramid/bodhi/bodhi/tests/views/test_views.py @@ -18,9 +18,24 @@ class ViewTests(unittest.TestCase): info = view_model_instance_json(context, request) self.assertEqual(info['context'], context.__json__()) - def test_view_model(self): + def test_view_model_json(self): from bodhi.resources import ReleaseResource from bodhi.views import view_model_json request = testing.DummyRequest() info = view_model_json(ReleaseResource, request) self.assertEqual(len(info['entries']), 2) + + def test_view_model_instance(self): + from bodhi.views import view_model_instance + request = testing.DummyRequest() + context = DBSession.query(Release).first() + info = view_model_instance(context, request) + self.assertEqual(type(info['context']), context.__class__) + + def test_view_model(self): + from bodhi.resources import ReleaseResource + from bodhi.views import view_model + request = testing.DummyRequest() + info = view_model(ReleaseResource, request) + self.assertEqual(len(info['page']), 2) + self.assertEqual(info['caption'], 'Releases') commit 75246c4615eeda9d1de97fb28e09782a62c7cf8f Author: Luke Macken <lmacken(a)redhat.com> Date: Tue May 17 13:48:48 2011 -0400 Paginate JSON and optimize our Grid by removing an extra SELECT diff --git a/pyramid/bodhi/bodhi/models/models.py b/pyramid/bodhi/bodhi/models/models.py index 83385fc..38a0207 100644 --- a/pyramid/bodhi/bodhi/models/models.py +++ b/pyramid/bodhi/bodhi/models/models.py @@ -56,11 +56,11 @@ class BodhiBase(object): items.append((col.name, prop)) return dict(items) - @property - def grid_columns(self): + @classmethod + def grid_columns(cls): columns = [] - exclude = getattr(self, '__exclude_columns__', []) - for col in self.__table__.columns: + exclude = getattr(cls, '__exclude_columns__', []) + for col in cls.__table__.columns: if col.name in exclude: continue columns.append(col.name) diff --git a/pyramid/bodhi/bodhi/views.py b/pyramid/bodhi/bodhi/views.py index afc8f50..0ec2e33 100644 --- a/pyramid/bodhi/bodhi/views.py +++ b/pyramid/bodhi/bodhi/views.py @@ -10,9 +10,11 @@ def view_model_instance_json(context, request): def view_model_json(context, request): session = DBSession() - # TODO: pagination - entries = session.query(context.__model__).all() - return {'entries': [entry.__json__() for entry in entries]} + entries = session.query(context.__model__) + current_page = int(request.params.get('page', 1)) + items_per_page = int(request.params.get('items_per_page', 20)) + page = Page(entries, page=current_page, items_per_page=items_per_page) + return {'entries': [entry.__json__() for entry in page]} ## Mako templated views @@ -27,7 +29,7 @@ def view_model(context, request): page_url = PageURL_WebOb(request) page = Page(entries, page=current_page, url=page_url, items_per_page=items_per_page) - # FIXME: entries[0] triggers another query - grid = Grid([entry.__json__() for entry in page], entries[0].grid_columns) + grid = Grid([entry.__json__() for entry in page], + context.__model__.grid_columns()) return {'caption': context.__model__.__name__ + 's', 'grid': grid, 'page': page}

13 years

1
0
0 / 0

Branch 'pyramid' - 5 commits - pyramid/bodhi

by Luke Macken

pyramid/bodhi/bodhi/__init__.py | 19 +++++++++++-- pyramid/bodhi/bodhi/models/models.py | 15 +++++++++-- pyramid/bodhi/bodhi/templates/model.mak | 16 +++++++++++ pyramid/bodhi/bodhi/tests/functional/test_wsgi.py | 29 +++++++++++++++++++++ pyramid/bodhi/bodhi/tests/views/test_views.py | 10 +++---- pyramid/bodhi/bodhi/views.py | 30 ++++++++++++++++++---- pyramid/bodhi/development.ini | 1 7 files changed, 105 insertions(+), 15 deletions(-) New commits: commit eb0b2761d321100689d292bbcc38bca3b80ddd63 Author: Luke Macken <lmacken(a)redhat.com> Date: Mon May 16 12:11:31 2011 -0400 Add functional WSGI stack tests diff --git a/pyramid/bodhi/bodhi/tests/functional/__init__.py b/pyramid/bodhi/bodhi/tests/functional/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyramid/bodhi/bodhi/tests/functional/test_wsgi.py b/pyramid/bodhi/bodhi/tests/functional/test_wsgi.py new file mode 100644 index 0000000..7a21563 --- /dev/null +++ b/pyramid/bodhi/bodhi/tests/functional/test_wsgi.py @@ -0,0 +1,29 @@ +import unittest +import json + +from nose.tools import eq_ + +app = None + +def setup(): + from bodhi import main + from webtest import TestApp + global app + app = main({}, **{'sqlalchemy.url': 'sqlite://'}) + app = TestApp(app) + +class FunctionalTests(unittest.TestCase): + + def test_release_view_json(self): + res = app.get('/releases/F15', status=200) + data = json.loads(res.body) + eq_(data['context']['name'], 'F15') + + def test_invalid_release(self): + app.get('/releases/F16', status=404) + + def test_releases_view_json(self): + res = app.get('/releases', status=200) + data = json.loads(res.body) + eq_(data[u'entries'][0][u'name'], 'F15') + commit 7d7d99cff82ed0fc3c12d94141ea85d54d7e1e82 Author: Luke Macken <lmacken(a)redhat.com> Date: Mon May 16 12:11:19 2011 -0400 Add our generic model grid mako template diff --git a/pyramid/bodhi/bodhi/templates/model.mak b/pyramid/bodhi/bodhi/templates/model.mak new file mode 100644 index 0000000..6854880 --- /dev/null +++ b/pyramid/bodhi/bodhi/templates/model.mak @@ -0,0 +1,16 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml"> +<head> + <meta http-equiv="content-type" content="text/html; charset=utf-8" /> + <title></title> +</head> + <body> + <table class="stylized"> + ${page.pager('$link_previous ~3~ $link_next (Page $page of $page_count)')} + <caption>${caption}</caption> + <col class="c1" /> + ${grid} + </table> + </body> +</html> commit 012dba7807a3319484540058296cdb7670703664 Author: Luke Macken <lmacken(a)redhat.com> Date: Mon May 16 12:10:47 2011 -0400 Add a BodhiBase.grid_columns property for use by our Grid view diff --git a/pyramid/bodhi/bodhi/models/models.py b/pyramid/bodhi/bodhi/models/models.py index 92c0bd4..83385fc 100644 --- a/pyramid/bodhi/bodhi/models/models.py +++ b/pyramid/bodhi/bodhi/models/models.py @@ -56,6 +56,17 @@ class BodhiBase(object): items.append((col.name, prop)) return dict(items) + @property + def grid_columns(self): + columns = [] + exclude = getattr(self, '__exclude_columns__', []) + for col in self.__table__.columns: + if col.name in exclude: + continue + columns.append(col.name) + return columns + + Base = declarative_base(cls=BodhiBase) metadata = Base.metadata DBSession = scoped_session(sessionmaker()) commit 9cde4c9ae7c53f568f00a55f4a4f53b7bd7089d5 Author: Luke Macken <lmacken(a)redhat.com> Date: Mon May 16 12:10:44 2011 -0400 Exclude some columns from JSON output diff --git a/pyramid/bodhi/bodhi/models/models.py b/pyramid/bodhi/bodhi/models/models.py index b09148b..92c0bd4 100644 --- a/pyramid/bodhi/bodhi/models/models.py +++ b/pyramid/bodhi/bodhi/models/models.py @@ -32,7 +32,7 @@ identity = Bunch(current=Bunch(user_name=u'Bob')) class BodhiBase(object): """ Our custom model base class """ - __exclude_columns___ = None # List of columns to exclude from JSON + __exclude_columns___ = ('id',) # List of columns to exclude from JSON def __init__(self, **kw): """ Automatically mapping attributes """ @@ -115,7 +115,7 @@ bug_cve_table = Table('bug_cve_table', metadata, class Release(Base): __tablename__ = 'releases' - __exclude_columns__ = ('metrics') + __exclude_columns__ = ('id', 'metrics') id = Column(Integer, primary_key=True) name = Column(Unicode(10), unique=True, nullable=False) commit 56252f87d5f2c5286a896c9d441dc398e0983421 Author: Luke Macken <lmacken(a)redhat.com> Date: Mon May 16 12:10:22 2011 -0400 Add our JSON and Mako grid views diff --git a/pyramid/bodhi/bodhi/__init__.py b/pyramid/bodhi/bodhi/__init__.py index 923ad53..d3fc95c 100644 --- a/pyramid/bodhi/bodhi/__init__.py +++ b/pyramid/bodhi/bodhi/__init__.py @@ -28,11 +28,24 @@ def main(global_config, **settings): #config.set_request_factory(BodhiRequest) config.add_static_view('static', 'bodhi:static') - config.add_view('bodhi.views.view_model_instance', + # JSON views + config.add_view('bodhi.views.view_model_instance_json', context='bodhi.models.Base', + accept='application/json', renderer='json') - config.add_view('bodhi.views.view_model', + config.add_view('bodhi.views.view_model_json', context='bodhi.resources.BodhiResource', + accept='application/json', renderer='json') - #config.scan() + + # Mako template views + config.add_view('bodhi.views.view_model_instance', + context='bodhi.models.Base', + accept='text/html', + renderer='bodhi:templates/instance.mak') + config.add_view('bodhi.views.view_model', + accept='text/html', + context='bodhi.resources.BodhiResource', + renderer='bodhi:templates/model.mak') + return config.make_wsgi_app() diff --git a/pyramid/bodhi/bodhi/tests/views/test_views.py b/pyramid/bodhi/bodhi/tests/views/test_views.py index f7477c7..ed58a63 100644 --- a/pyramid/bodhi/bodhi/tests/views/test_views.py +++ b/pyramid/bodhi/bodhi/tests/views/test_views.py @@ -11,16 +11,16 @@ class ViewTests(unittest.TestCase): def tearDown(self): testing.tearDown() - def test_view_model_instance(self): - from bodhi.views import view_model_instance + def test_view_model_instance_json(self): + from bodhi.views import view_model_instance_json request = testing.DummyRequest() context = DBSession.query(Release).first() - info = view_model_instance(context, request) + info = view_model_instance_json(context, request) self.assertEqual(info['context'], context.__json__()) def test_view_model(self): from bodhi.resources import ReleaseResource - from bodhi.views import view_model + from bodhi.views import view_model_json request = testing.DummyRequest() - info = view_model(ReleaseResource, request) + info = view_model_json(ReleaseResource, request) self.assertEqual(len(info['entries']), 2) diff --git a/pyramid/bodhi/bodhi/views.py b/pyramid/bodhi/bodhi/views.py index b0b70dc..afc8f50 100644 --- a/pyramid/bodhi/bodhi/views.py +++ b/pyramid/bodhi/bodhi/views.py @@ -1,13 +1,33 @@ -#from pyramid.view import view_config -from pyramid.response import Response +from webhelpers.html.grid import Grid +from webhelpers.paginate import Page, PageURL_WebOb -from bodhi.models import Update, Package, Build, Release, DBSession +from bodhi.models import DBSession -def view_model_instance(context, request): +## JSON views + +def view_model_instance_json(context, request): return {'context': context.__json__()} -def view_model(context, request): +def view_model_json(context, request): session = DBSession() # TODO: pagination entries = session.query(context.__model__).all() return {'entries': [entry.__json__() for entry in entries]} + +## Mako templated views + +def view_model_instance(context, request): + return {'context': context} + +def view_model(context, request): + session = DBSession() + entries = session.query(context.__model__) + current_page = int(request.params.get('page', 1)) + items_per_page = int(request.params.get('items_per_page', 20)) + page_url = PageURL_WebOb(request) + page = Page(entries, page=current_page, url=page_url, + items_per_page=items_per_page) + # FIXME: entries[0] triggers another query + grid = Grid([entry.__json__() for entry in page], entries[0].grid_columns) + return {'caption': context.__model__.__name__ + 's', + 'grid': grid, 'page': page} diff --git a/pyramid/bodhi/development.ini b/pyramid/bodhi/development.ini index bef0deb..adb4df9 100644 --- a/pyramid/bodhi/development.ini +++ b/pyramid/bodhi/development.ini @@ -7,6 +7,7 @@ debug_routematch = false debug_templates = true default_locale_name = en sqlalchemy.url = sqlite:///%(here)s/bodhi.db +mako.directories = bodhi:templates [pipeline:main] pipeline =

13 years

1
0
0 / 0

Branch 'pyramid' - pyramid/bodhi

by Luke Macken

pyramid/bodhi/bodhi/resources.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) New commits: commit e3fd34aa6bd623dd6bbb8a641cf1304fee65b89e Author: Luke Macken <lmacken(a)redhat.com> Date: Sat May 14 14:10:01 2011 -0400 Minor cleanups diff --git a/pyramid/bodhi/bodhi/resources.py b/pyramid/bodhi/bodhi/resources.py index ac73d62..cdb8881 100644 --- a/pyramid/bodhi/bodhi/resources.py +++ b/pyramid/bodhi/bodhi/resources.py @@ -29,22 +29,11 @@ class BodhiResource(object): def __getitem__(self, key): session = DBSession() try: - return session.query(self.__model__).filter_by(**{self.__column__: key}).one() + return session.query(self.__model__).filter_by( + **{self.__column__: key}).one() except NoResultFound: raise KeyError(key) - #def get(self, key, default=None): - # try: - # item = self.__getitem__(key) - # except KeyError: - # item = default - # return item - - #def __iter__(self): - # session = DBSession() - # query = session.query(self.__model__) - # return iter(query) - class PackageResource(BodhiResource): __model__ = Package

13 years

1
0
0 / 0

Branch 'pyramid' - 2 commits - pyramid/bodhi

by Luke Macken

pyramid/bodhi/bodhi/__init__.py | 22 +++++++++++----------- pyramid/bodhi/bodhi/models/models.py | 1 + pyramid/bodhi/bodhi/resources.py | 24 +++++++++++------------- 3 files changed, 23 insertions(+), 24 deletions(-) New commits: commit 7b6c8c7c5fb0693fed3e384d90730077f3da6709 Author: Luke Macken <lmacken(a)redhat.com> Date: Sat May 14 14:02:15 2011 -0400 Exclude the Release.metrics column from JSON diff --git a/pyramid/bodhi/bodhi/models/models.py b/pyramid/bodhi/bodhi/models/models.py index 0cefdf0..b09148b 100644 --- a/pyramid/bodhi/bodhi/models/models.py +++ b/pyramid/bodhi/bodhi/models/models.py @@ -115,6 +115,7 @@ bug_cve_table = Table('bug_cve_table', metadata, class Release(Base): __tablename__ = 'releases' + __exclude_columns__ = ('metrics') id = Column(Integer, primary_key=True) name = Column(Unicode(10), unique=True, nullable=False) commit f0065b5584620d9a3af36cdd78f6236e08030c7a Author: Luke Macken <lmacken(a)redhat.com> Date: Sat May 14 14:02:00 2011 -0400 Minor cleanups diff --git a/pyramid/bodhi/bodhi/__init__.py b/pyramid/bodhi/bodhi/__init__.py index 2f0099f..923ad53 100644 --- a/pyramid/bodhi/bodhi/__init__.py +++ b/pyramid/bodhi/bodhi/__init__.py @@ -7,17 +7,17 @@ from pyramid.config import Configurator from bodhi.resources import appmaker -class BodhiRequest(Request): - @reify - def user(self): - # <your database connection, however you get it, the below line - # is just an example> - dbconn = self.registry.settings['dbconn'] - userid = unauthenticated_userid(self) - if userid is not None: - # this should return None if the user doesn't exist - # in the database - return dbconn['users'].query({'id':userid}) +#class BodhiRequest(Request): +# @reify +# def user(self): +# # <your database connection, however you get it, the below line +# # is just an example> +# dbconn = self.registry.settings['dbconn'] +# userid = unauthenticated_userid(self) +# if userid is not None: +# # this should return None if the user doesn't exist +# # in the database +# return dbconn['users'].query({'id':userid}) def main(global_config, **settings): diff --git a/pyramid/bodhi/bodhi/resources.py b/pyramid/bodhi/bodhi/resources.py index 7839bf6..ac73d62 100644 --- a/pyramid/bodhi/bodhi/resources.py +++ b/pyramid/bodhi/bodhi/resources.py @@ -28,24 +28,22 @@ class BodhiResource(object): def __getitem__(self, key): session = DBSession() - log.debug('%s(%r)'% (self.__class__.__name__, key)) try: return session.query(self.__model__).filter_by(**{self.__column__: key}).one() except NoResultFound: - log.debug('NoResultsFound') raise KeyError(key) - def get(self, key, default=None): - try: - item = self.__getitem__(key) - except KeyError: - item = default - return item - - def __iter__(self): - session = DBSession() - query = session.query(self.__model__) - return iter(query) + #def get(self, key, default=None): + # try: + # item = self.__getitem__(key) + # except KeyError: + # item = default + # return item + + #def __iter__(self): + # session = DBSession() + # query = session.query(self.__model__) + # return iter(query) class PackageResource(BodhiResource):

13 years

1
0
0 / 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

bodhi May 2011