[copr] bkabrda-workspace: Enable just creating the data dir in manage.py for easier use with alembic (91c40fa)
by bkabrda＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : bkabrda-workspace
>---------------------------------------------------------------
commit 91c40fa603935b1932ba53e71de71f892c1c9dcb
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Mon Nov 19 08:34:35 2012 +0100
Enable just creating the data dir in manage.py for easier use with alembic
>---------------------------------------------------------------
wsgi/manage.py | 14 ++++++++++++--
1 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/wsgi/manage.py b/wsgi/manage.py
index 1d4a72a..d032f26 100755
--- a/wsgi/manage.py
+++ b/wsgi/manage.py
@@ -9,18 +9,26 @@ class DBManager(object):
def __init__(self, db):
self.db = db
- def create_db(self):
+ def create_sqlite_file(self):
if app.config['SQLALCHEMY_DATABASE_URI'].startswith('sqlite'):
# strip sqlite:///
datadir_name = os.path.dirname(app.config['SQLALCHEMY_DATABASE_URI'][10:])
if not os.path.exists(datadir_name):
os.makedirs(datadir_name)
+
+ def create_db(self):
+ self.create_sqlite_file()
self.db.create_all()
def delete_db(self):
self.db.drop_all()
parser = argparse.ArgumentParser(description = 'Manage the app')
+parser.add_argument('-s', '--create-sqlite-file',
+ required = False,
+ help = 'Create the sqlite DB file (not the tables). User for alembic, the -c does this automatically.',
+ action = 'store_true')
+
parser.add_argument('-c', '--create-db',
required = False,
help = 'Create the DB scheme',
@@ -34,7 +42,9 @@ parser.add_argument('-d', '--delete-db',
args = parser.parse_args()
manager = DBManager(db)
-if args.create_db:
+if args.create_sqlite_file:
+ manager.create_sqlite_file()
+elif args.create_db:
manager.create_db()
elif args.delete_db:
manager.delete_db()
8 years, 2 months
[copr] bkabrda-workspace: Pull the app logic out from views into separate module (9a4f904)
by bkabrda＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : bkabrda-workspace
>---------------------------------------------------------------
commit 9a4f90459e08c0a3466f93f1fae9375d16fd646b
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Mon Nov 19 07:59:27 2012 +0100
Pull the app logic out from views into separate module
>---------------------------------------------------------------
wsgi/coprs/exceptions.py | 2 +
wsgi/coprs/forms.py | 8 +--
wsgi/coprs/logic/.coprs_logic.py.swo | Bin 0 -> 12288 bytes
wsgi/coprs/{views => logic}/__init__.py | 0
wsgi/coprs/logic/coprs_logic.py | 75 +++++++++++++++++++++
wsgi/coprs/views/coprs_ns/builds.py | 5 +-
wsgi/coprs/views/coprs_ns/general.py | 111 +++++++-----------------------
7 files changed, 110 insertions(+), 91 deletions(-)
diff --git a/wsgi/coprs/exceptions.py b/wsgi/coprs/exceptions.py
new file mode 100644
index 0000000..5995fec
--- /dev/null
+++ b/wsgi/coprs/exceptions.py
@@ -0,0 +1,2 @@
+class DuplicateCoprNameException(BaseException):
+ pass
diff --git a/wsgi/coprs/forms.py b/wsgi/coprs/forms.py
index df2a10d..3d71c72 100644
--- a/wsgi/coprs/forms.py
+++ b/wsgi/coprs/forms.py
@@ -8,6 +8,7 @@ from flask.ext import wtf
from coprs import constants
from coprs import helpers
from coprs import models
+from coprs.logic import coprs_logic
class UrlListValidator(object):
def __init__(self, message = None):
@@ -51,12 +52,9 @@ class CoprUniqueNameValidator(object):
self.message = message
def __call__(self, form, field):
- existing = models.Copr.query.filter(models.Copr.name == field.data).\
- filter(models.Copr.owner_id == flask.g.user.id)
- if form.id.data:
- existing = existing.filter(models.Copr.id != form.id.data)
+ existing = coprs_logic.CoprsLogic.exists_for_current_user(flask.g.user, field.data).first()
- if existing.first():
+ if existing and str(existing.id) != form.id.data:
raise wtf.ValidationError(self.message.format(field.data))
diff --git a/wsgi/coprs/logic/.coprs_logic.py.swo b/wsgi/coprs/logic/.coprs_logic.py.swo
new file mode 100644
index 0000000..b244106
Binary files /dev/null and b/wsgi/coprs/logic/.coprs_logic.py.swo differ
diff --git a/wsgi/coprs/views/__init__.py b/wsgi/coprs/logic/__init__.py
similarity index 100%
copy from wsgi/coprs/views/__init__.py
copy to wsgi/coprs/logic/__init__.py
diff --git a/wsgi/coprs/logic/coprs_logic.py b/wsgi/coprs/logic/coprs_logic.py
new file mode 100644
index 0000000..6b639ba
--- /dev/null
+++ b/wsgi/coprs/logic/coprs_logic.py
@@ -0,0 +1,75 @@
+import flask
+
+from coprs import db
+from coprs import exceptions
+from coprs import models
+
+class CoprsLogic(object):
+ """Used for manipulating Coprs. All methods accept user object as a first argument, as this may be needed in future."""
+ @classmethod
+ def get(cls, user, username, coprname, **kwargs):
+ with_builds = kwargs.get('with_builds', False)
+ with_permissions = kwargs.get('with_permissions', False)
+
+ query = db.session.query(models.Copr).\
+ join(models.Copr.owner).\
+ options(db.contains_eager(models.Copr.owner)).\
+ filter(models.Copr.name == coprname).\
+ filter(models.User.openid_name == models.User.openidize_name(username))
+
+ if with_builds:
+ query = query.outerjoin(models.Copr.builds).\
+ options(db.contains_eager(models.Copr.builds)).\
+ order_by(models.Build.submitted_on.desc())
+
+ if with_permissions:
+ query = query.outerjoin(models.Copr.copr_permissions).\
+ options(db.contains_eager(models.Copr.copr_permissions))
+
+ return query
+
+ @classmethod
+ def get_multiple(cls, user, **kwargs):
+ user_relation = kwargs.get('user_relation', None)
+ username = kwargs.get('username', None)
+
+ query = db.session.query(models.Copr).\
+ join(models.Copr.owner).\
+ options(db.contains_eager(models.Copr.owner))
+ if user_relation == 'owned':
+ query = query.filter(models.User.openid_name == models.User.openidize_name(username))
+ elif user_relation == 'allowed':
+ aliased_user = db.aliased(models.User)
+ query = query.join(models.CoprPermission, models.Copr.copr_permissions).\
+ filter(models.CoprPermission.approved == True).\
+ join(aliased_user, models.CoprPermission.user).\
+ filter(aliased_user.openid_name == models.User.openidize_name(username))
+ return query
+
+ @classmethod
+ def new(cls, user, copr, check_for_duplicates = True):
+ if check_for_duplicates and cls.exists_for_current_user(user, copr.name):
+ raise exceptions.DuplicateCoprNameException
+ db.session.add(copr)
+ db.session.commit()
+
+ @classmethod
+ def update(cls, user, copr, check_for_duplicates = True):
+ if check_for_duplicates and cls.exists_for_current_user(user, copr.name):
+ raise exceptions.DuplicateCoprNameException
+ db.session.add(copr)
+ db.session.commit()
+
+ @classmethod
+ def exists_for_current_user(cls, user, coprname):
+ existing = models.Copr.query.filter(models.Copr.name == coprname).\
+ filter(models.Copr.owner_id == user.id)
+
+ return existing
+
+ @classmethod
+ def increment_build_count(cls, user, owner, coprname): #TODO API of this method is different, maybe change?
+ models.Copr.query.filter(models.Copr.name == coprname).\
+ filter(models.Copr.owner_id == owner.id).\
+ update({models.Copr.build_count: models.Copr.build_count + 1})
+ db.session.commit()
diff --git a/wsgi/coprs/views/coprs_ns/builds.py b/wsgi/coprs/views/coprs_ns/builds.py
index db94b60..6464cfa 100644
--- a/wsgi/coprs/views/coprs_ns/builds.py
+++ b/wsgi/coprs/views/coprs_ns/builds.py
@@ -7,8 +7,9 @@ from coprs import forms
from coprs import helpers
from coprs import models
-from coprs.views.misc import login_required
+from coprs.logic import coprs_logic
+from coprs.views.misc import login_required
from coprs.views.coprs_ns.general import coprs_ns
@coprs_ns.route('/detail/<username>/<coprname>/builds/', defaults = {'page': 1})
@@ -55,6 +56,8 @@ def copr_add_build(username, coprname):
build.memory_reqs = form.memory_reqs.data
build.timeout = form.timeout.data
+ # increment copr build count (will be part of method logic/build_logic, so that it doesn't have to be done manually)
+ coprs_logic.CoprsLogic.increment_build_count(flask.g.user, copr.owner, coprname)
db.session.add(build)
db.session.commit()
diff --git a/wsgi/coprs/views/coprs_ns/general.py b/wsgi/coprs/views/coprs_ns/general.py
index 327a7eb..4ed669e 100644
--- a/wsgi/coprs/views/coprs_ns/general.py
+++ b/wsgi/coprs/views/coprs_ns/general.py
@@ -7,6 +7,8 @@ from coprs import models
from coprs.views.misc import login_required
+from coprs.logic import coprs_logic
+
coprs_ns = flask.Blueprint('coprs_ns',
__name__,
url_prefix = '/coprs')
@@ -14,55 +16,31 @@ coprs_ns = flask.Blueprint('coprs_ns',
@coprs_ns.route('/', defaults = {'page': 1})
@coprs_ns.route('/<int:page>/')
def coprs_show(page = 1):
- query = db.session.query(models.Copr, db.func.count(models.Build.id)).\
- join(models.Copr.owner).\
- outerjoin(models.Copr.builds).\
- options(db.contains_eager(models.Copr.owner)).\
- group_by(models.Copr.id)
+ query = coprs_logic.CoprsLogic.get_multiple(flask.g.user)
paginator = helpers.Paginator(query, query.count(), page)
- if paginator.sliced_query:
- coprs, build_counts = zip(*paginator.sliced_query)
- else:
- coprs, build_counts = [], []
- return flask.render_template('coprs/show.html', coprs = coprs, build_counts = build_counts, paginator = paginator)
+ coprs = paginator.sliced_query
+ return flask.render_template('coprs/show.html', coprs = coprs, paginator = paginator)
@coprs_ns.route('/owned/<username>/', defaults = {'page': 1})
@coprs_ns.route('/owned/<username>/<int:page>/')
def coprs_by_owner(username = None, page = 1):
- query = db.session.query(models.Copr, db.func.count(models.Build.id)).\
- join(models.Copr.owner).\
- outerjoin(models.Copr.builds).\
- options(db.contains_eager(models.Copr.owner)).\
- filter(models.User.openid_name == models.User.openidize_name(username)).\
- group_by(models.Copr.id)
+ query = coprs_logic.CoprsLogic.get_multiple(flask.g.user, user_relation = 'owned', username = username)
paginator = helpers.Paginator(query, query.count(), page)
- if paginator.sliced_query:
- coprs, build_counts = zip(*paginator.sliced_query)
- else:
- coprs, build_counts = [], []
- return flask.render_template('coprs/show.html', coprs = coprs, build_counts = build_counts, paginator = paginator)
+ coprs = paginator.sliced_query
+ return flask.render_template('coprs/show.html', coprs = coprs, paginator = paginator)
@coprs_ns.route('/allowed/<username>/', defaults = {'page': 1})
@coprs_ns.route('/allowed/<username>/<int:page>/')
def coprs_by_allowed(username = None, page = 1):
- query = db.session.query(models.Copr, models.CoprPermission, models.User, db.func.count(models.Build.id)).\
- outerjoin(models.Copr.builds).\
- join(models.Copr.copr_permissions).\
- join(models.CoprPermission.user).\
- filter(models.User.openid_name == models.User.openidize_name(username)).\
- filter(models.CoprPermission.approved == True).\
- group_by(models.Copr.id)
+ query = coprs_logic.CoprsLogic.get_multiple(flask.g.user, user_relation = 'allowed', username = username)
paginator = helpers.Paginator(query, query.count(), page)
- if paginator.sliced_query:
- coprs, build_counts = zip(*map(lambda x: (x[0], x[3]), paginator.sliced_query))
- else:
- coprs, build_counts = [], []
- return flask.render_template('coprs/show.html', coprs = coprs, build_counts = build_counts, paginator = paginator)
+ coprs = paginator.sliced_query
+ return flask.render_template('coprs/show.html', coprs = coprs, paginator = paginator)
@coprs_ns.route('/add/')
@@ -83,8 +61,7 @@ def copr_new():
chroots = ' '.join(form.chroots),
repos = form.repos.data.replace('\n', ' '),
owner = flask.g.user)
- db.session.add(copr)
- db.session.commit()
+ coprs_logic.CoprsLogic.new(flask.g.user, copr, check_for_duplicates = False) # form validation checks for duplicates
flask.flash('New entry was successfully posted')
return flask.redirect(flask.url_for('coprs_ns.coprs_show'))
@@ -95,40 +72,25 @@ def copr_new():
@coprs_ns.route('/detail/<username>/<coprname>/')
def copr_detail(username, coprname):
form = forms.BuildForm()
- try: # query will raise an index error, if Copr doesn't exist
- copr = models.Copr.query.outerjoin(models.Copr.builds).\
- join(models.Copr.owner).\
- options(db.contains_eager(models.Copr.builds), db.contains_eager(models.Copr.owner)).\
- filter(models.Copr.name == coprname).\
- filter(models.User.openid_name == models.User.openidize_name(username)).\
- order_by(models.Build.submitted_on.desc())[0:10][0] # we retrieved all builds, but we got one copr in a list...
+ try: # query[0:10][0] will raise an index error, if Copr doesn't exist
+ query = coprs_logic.CoprsLogic.get(flask.g.user, username, coprname, with_builds = True, with_permissions = True)
+ copr = query[0:10][0]# we retrieved all builds, but we got one copr in a list...
except IndexError:
return page_not_found('Copr with name {0} does not exist.'.format(coprname))
- permissions = models.CoprPermission.query.join(models.CoprPermission.user).\
- options(db.contains_eager(models.CoprPermission.user)).\
- filter(models.CoprPermission.copr_id == copr.id).\
- all()
- return flask.render_template('coprs/detail.html', copr = copr, form = form, permissions = permissions)
+ return flask.render_template('coprs/detail.html', copr = copr, form = form, permissions = copr.copr_permissions)
@coprs_ns.route('/detail/<username>/<coprname>/edit/')
@login_required
def copr_edit(username, coprname):
- query = db.session.query(models.Copr, models.CoprPermission).\
- outerjoin(models.CoprPermission).\
- outerjoin(models.Copr.owner).\
- options(db.contains_eager(models.Copr.owner)).\
- filter(models.Copr.name == coprname).\
- filter(models.User.openid_name == models.User.openidize_name(username)).\
- all()
- copr = query[0][0]
+ query = coprs_logic.CoprsLogic.get(flask.g.user, username, coprname, with_permissions = True)
+ copr = query.first()
if not copr:
return page_not_found('Copr with name {0} does not exist.'.format(coprname))
form = forms.CoprForm(obj = copr)
- permissions = map(lambda x: x[1], query)
- permissions = filter(lambda x: x, permissions) # throw away None away
+ permissions = copr.copr_permissions
permissions_form = forms.DynamicPermissionsFormFactory.create_form_cls(permissions)()
return flask.render_template('coprs/edit.html',
@@ -142,12 +104,7 @@ def copr_edit(username, coprname):
@login_required
def copr_update(username, coprname):
form = forms.CoprForm()
- copr = models.Copr.query.\
- join(models.Copr.owner).\
- options(db.contains_eager(models.Copr.owner)).\
- filter(models.Copr.name == coprname).\
- filter(models.User.openid_name == models.User.openidize_name(username)).\
- first()
+ copr = coprs_logic.CoprsLogic.get(flask.g.user, username, coprname).first()
# only owner can update a copr
if flask.g.user != copr.owner:
flask.flash('Only owners may update their Coprs.')
@@ -155,16 +112,11 @@ def copr_update(username, coprname):
if form.validate_on_submit():
# we don't change owner (yet)
- copr = models.Copr.query.join(models.Copr.owner).\
- filter(models.Copr.name == coprname).\
- filter(models.User.openid_name == models.User.openidize_name(username)).\
- first()
copr.name = form.name.data
copr.chroots = ' '.join(form.chroots)
copr.repos = form.repos.data.replace('\n', ' ')
- db.session.add(copr)
- db.session.commit()
+ coprs_logic.CoprsLogic.update(flask.g.user, copr, check_for_duplicates = False) # form validation checks for duplicates
flask.flash('Copr was updated successfully.')
return flask.redirect(flask.url_for('coprs_ns.copr_detail', username = username, coprname = form.name.data))
else:
@@ -201,15 +153,8 @@ def copr_apply_for_building(username, coprname):
@coprs_ns.route('/detail/<username>/<coprname>/give_up_building/', methods = ['POST'])
@login_required
def copr_give_up_building(username, coprname):
- query = db.session.query(models.Copr, models.CoprPermission).\
- join(models.Copr.owner).\
- outerjoin(models.CoprPermission).\
- options(db.contains_eager(models.Copr.owner)).\
- filter(models.Copr.name == coprname).\
- filter(models.User.openid_name == models.User.openidize_name(username)).\
- filter(models.CoprPermission.user == flask.g.user).\
- first()
- copr = query[0]
+ query = coprs_logic.CoprsLogic.get(flask.g.user, username, coprname, with_permissions = True)
+ copr = query.first()
if not copr:
return page_not_found('Copr with name {0} does not exist.'.format(name))
@@ -226,13 +171,9 @@ def copr_give_up_building(username, coprname):
@coprs_ns.route('/detail/<username>/<coprname>/update_permissions/', methods = ['POST'])
@login_required
def copr_update_permissions(username, coprname):
- copr = models.Copr.query.\
- join(models.Copr.owner).\
- options(db.contains_eager(models.Copr.owner)).\
- filter(models.User.name == username).\
- filter(models.Copr.name == coprname).\
- first()
- permissions = models.CoprPermission.query.filter(models.CoprPermission.copr_id == copr.id).all()
+ query = coprs_logic.CoprsLogic.get(flask.g.user, username, coprname)
+ copr = query.first()
+ permissions = copr.copr_permissions
permissions_form = forms.DynamicPermissionsFormFactory.create_form_cls(permissions)()
# only owner can update copr permissions
8 years, 2 months
[copr] bkabrda-workspace: Add build counts to copr model to make things faster and easier to query (6d34286)
by bkabrda＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : bkabrda-workspace
>---------------------------------------------------------------
commit 6d34286f730c3281631c86afc89fa19fa0c66452
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Mon Nov 19 07:57:31 2012 +0100
Add build counts to copr model to make things faster and easier to query
>---------------------------------------------------------------
...b_setup.py => 17aef4202992_initial_db_setup.py} | 25 ++++++++++---------
wsgi/coprs/models.py | 2 +
wsgi/coprs/templates/coprs/show.html | 2 +-
3 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/wsgi/alembic/versions/2cb3b8a9de3a_initial_db_setup.py b/wsgi/alembic/versions/17aef4202992_initial_db_setup.py
similarity index 94%
rename from wsgi/alembic/versions/2cb3b8a9de3a_initial_db_setup.py
rename to wsgi/alembic/versions/17aef4202992_initial_db_setup.py
index 3435690..3d8ce25 100644
--- a/wsgi/alembic/versions/2cb3b8a9de3a_initial_db_setup.py
+++ b/wsgi/alembic/versions/17aef4202992_initial_db_setup.py
@@ -1,13 +1,13 @@
"""Initial db setup
-Revision ID: 2cb3b8a9de3a
+Revision ID: 17aef4202992
Revises: None
-Create Date: 2012-11-16 11:38:30.605131
+Create Date: 2012-11-18 16:05:02.407902
"""
# revision identifiers, used by Alembic.
-revision = '2cb3b8a9de3a'
+revision = '17aef4202992'
down_revision = None
from alembic import op
@@ -27,10 +27,19 @@ def upgrade():
sa.Column('name', sa.String(length=100), nullable=False),
sa.Column('chroots', sa.Text(), nullable=False),
sa.Column('repos', sa.Text(), nullable=True),
+ sa.Column('build_count', sa.Integer(), nullable=True),
sa.Column('owner_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['owner_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
+ op.create_table('copr_permission',
+ sa.Column('approved', sa.Boolean(), nullable=True),
+ sa.Column('user_id', sa.Integer(), nullable=False),
+ sa.Column('copr_id', sa.Integer(), nullable=False),
+ sa.ForeignKeyConstraint(['copr_id'], ['copr.id'], ),
+ sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
+ sa.PrimaryKeyConstraint('user_id', 'copr_id')
+ )
op.create_table('build',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('pkgs', sa.Text(), nullable=True),
@@ -49,21 +58,13 @@ def upgrade():
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
- op.create_table('copr_permission',
- sa.Column('approved', sa.Boolean(), nullable=True),
- sa.Column('user_id', sa.Integer(), nullable=False),
- sa.Column('copr_id', sa.Integer(), nullable=False),
- sa.ForeignKeyConstraint(['copr_id'], ['copr.id'], ),
- sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
- sa.PrimaryKeyConstraint('user_id', 'copr_id')
- )
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
- op.drop_table('copr_permission')
op.drop_table('build')
+ op.drop_table('copr_permission')
op.drop_table('copr')
op.drop_table('user')
### end Alembic commands ###
diff --git a/wsgi/coprs/models.py b/wsgi/coprs/models.py
index cce24da..1061678 100644
--- a/wsgi/coprs/models.py
+++ b/wsgi/coprs/models.py
@@ -86,6 +86,8 @@ class Copr(db.Model, Serializer):
name = db.Column(db.String(100), nullable = False)
chroots = db.Column(db.Text, nullable = False)
repos = db.Column(db.Text)
+ # duplicate information, but speeds up a lot and makes queries simpler
+ build_count = db.Column(db.Integer, default = 0)
# relations
owner_id = db.Column(db.Integer, db.ForeignKey('user.id'))
diff --git a/wsgi/coprs/templates/coprs/show.html b/wsgi/coprs/templates/coprs/show.html
index 24e146d..1bbec24 100644
--- a/wsgi/coprs/templates/coprs/show.html
+++ b/wsgi/coprs/templates/coprs/show.html
@@ -8,7 +8,7 @@
<div class=copr>
<h2><a href="{{ url_for('coprs_ns.copr_detail', username = copr.owner.name, coprname = copr.name) }}">{{ copr.owner.name }}/{{ copr.name }}</a></h2>
chroots: {{ copr.chroots_list|join(', ') }}<br>
- builds: {{ build_counts[loop.index0] }}<br>
+ builds: {{ copr.build_count }}<br>
{% if copr.repos %}
repos: {{ copr.repos_list|length }}
{% endif %}
8 years, 2 months
[copr] skvidal-backend: - move the json job file out to the results dir - more logging output (ffcbad7)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit ffcbad79c8d087a35e5db77a148f0e7998c648e7
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Sun Nov 18 22:55:23 2012 -0500
- move the json job file out to the results dir
- more logging output
>---------------------------------------------------------------
backend/dispatcher.py | 3 ++-
copr-be.py | 1 +
2 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index f1c7f90..ef621be 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -142,7 +142,8 @@ class Worker(multiprocessing.Process):
def return_results(self, job):
self.callback.log('%s status %s. Took %s seconds' % (job.build_id, job.status, job.ended_on - job.started_on))
- os.unlink(job.jobfile)
+ jobfilename = os.path.dirname(job.jobfile)
+ os.rename(job.jobfile, job.destdir + '/' + jobfilename)
#FIXME - this should either return job status/results
# into a queue or it should submit results directly to the frontend
diff --git a/copr-be.py b/copr-be.py
index 7ebc4c4..d638ddd 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -108,6 +108,7 @@ class CoprBackend(object):
w = Worker(self.opts, self.jobs, worker_num)
self.workers.append(w)
w.start()
+ self.log("Finished starting worker processes")
# FIXME - prune out workers
#if len(self.workers) > self.opts.num_workers:
# killnum = len(self.workers) - self.opts.num_workers
8 years, 2 months
[copr] skvidal-backend: should run pyflakes more often (2e29020)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 2e29020a22e23edb9ca1b62df97e36ae8878d973
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 15:57:31 2012 -0500
should run pyflakes more often
>---------------------------------------------------------------
backend/dispatcher.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 8b2733f..f1c7f90 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -96,7 +96,7 @@ class Worker(multiprocessing.Process):
self.callback = callback
if not self.callback:
self.logfile = self.opts.worker_logdir + '/worker-%s.log' % self.worker_num
- self.callback = WorkerCallback(logfile = logfile)
+ self.callback = WorkerCallback(logfile = self.logfile)
self.callback.log('creating worker: %s' % ip)
8 years, 2 months
[copr] skvidal-backend: started_on not startedon (a3f4780)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit a3f478092da7f88eec50dc57a34420437a3be7bc
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 15:41:46 2012 -0500
started_on not startedon
>---------------------------------------------------------------
backend/dispatcher.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index e216c4a..8b2733f 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -141,7 +141,7 @@ class Worker(multiprocessing.Process):
return jobdata
def return_results(self, job):
- self.callback.log('%s status %s. Took %s seconds' % (job.build_id, job.status, job.ended_on - job.startedon))
+ self.callback.log('%s status %s. Took %s seconds' % (job.build_id, job.status, job.ended_on - job.started_on))
os.unlink(job.jobfile)
#FIXME - this should either return job status/results
# into a queue or it should submit results directly to the frontend
8 years, 2 months
[copr] skvidal-backend: catch the common errors and make sure we're logging as much as we can from mockremote(and the rest is silent) (55d6d74)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 55d6d74dec3ecc8964075296fea7a12a807994fd
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 15:40:57 2012 -0500
catch the common errors and make sure we're logging as much as we can
from mockremote(and the rest is silent)
>---------------------------------------------------------------
backend/dispatcher.py | 20 ++++++++++++++++----
1 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 639eb10..e216c4a 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -95,8 +95,8 @@ class Worker(multiprocessing.Process):
self.kill_received = False
self.callback = callback
if not self.callback:
- lf = self.opts.worker_logdir + '/worker-%s.log' % self.worker_num
- self.callback = WorkerCallback(logfile = lf)
+ self.logfile = self.opts.worker_logdir + '/worker-%s.log' % self.worker_num
+ self.callback = WorkerCallback(logfile = logfile)
self.callback.log('creating worker: %s' % ip)
@@ -177,11 +177,23 @@ class Worker(multiprocessing.Process):
status = 1
job.started_on = time.time()
for chroot in job.chroots:
+
+ # setup our target dir locally
+ if not os.path.exists(job.destdir):
+ try:
+ os.makedirs(job.destdir, mode=0755)
+ except (OSError, IOError), e:
+ msg = "Could not make results dir for job: %s - %s" % (job.destdir, str(e))
+ self.callback.log(msg)
+ status = 0
+ continue
+
self.callback.log('mockremote %s %s %s %s %s' % (ip, job.timeout, job.destdir, chroot, str(job.repos)))
try:
mr = mockremote.MockRemote(builder=ip, timeout=job.timeout,
destdir=job.destdir, chroot=chroot, cont=True, recurse=True,
- repos=job.repos, callback=None)
+ repos=job.repos,
+ callback=mockremote.DefaultCallBack(quiet=True,logfn=self.logfile))
mr.build_pkgs(job.pkgs)
except mockremote.MockRemoteError, e:
# record and break
@@ -192,7 +204,7 @@ class Worker(multiprocessing.Process):
job.status = status
self.return_results(job)
self.callback.log('worker finished build: %s' % ip)
- # call terminate on the instance
+ # FIXME call terminate on the instance
8 years, 2 months
[copr] skvidal-backend: best if you don't cause tracebacks in your logging calls, dumbass (2e6ae6d)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 2e6ae6d2a259a96d3f2bb9451808f1ce69cd808e
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 15:26:31 2012 -0500
best if you don't cause tracebacks in your logging calls, dumbass
>---------------------------------------------------------------
backend/dispatcher.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 8320b5b..639eb10 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -177,7 +177,7 @@ class Worker(multiprocessing.Process):
status = 1
job.started_on = time.time()
for chroot in job.chroots:
- self.callback.log('mockremote %s %s %s %s %s' % (ip, job.timeout, job.destdir, chroot, str(repos)))
+ self.callback.log('mockremote %s %s %s %s %s' % (ip, job.timeout, job.destdir, chroot, str(job.repos)))
try:
mr = mockremote.MockRemote(builder=ip, timeout=job.timeout,
destdir=job.destdir, chroot=chroot, cont=True, recurse=True,
8 years, 2 months
[copr] skvidal-backend: more logging (78b5e37)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 78b5e3703481d5e1840794fec5f489a6013e5fa9
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 15:24:11 2012 -0500
more logging
>---------------------------------------------------------------
backend/dispatcher.py | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 5478719..8320b5b 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -101,6 +101,7 @@ class Worker(multiprocessing.Process):
self.callback.log('creating worker: %s' % ip)
def spawn_instance(self):
+ self.callback.log('spawning instance begin')
stats = callbacks.AggregateStats()
playbook_cb = SilentPlaybookCallbacks(verbose=False)
@@ -112,6 +113,7 @@ class Worker(multiprocessing.Process):
remote_user='root')
play.run()
+ self.callback.log('spawning instance end')
if self.ip:
return self.ip
8 years, 2 months
[copr] skvidal-backend: running from el6 and using transport=ssh ends in tears :) (6619282)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 6619282faaec188a845422ecea1ff3a55c395967
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 15:20:35 2012 -0500
running from el6 and using transport=ssh ends in tears :)
>---------------------------------------------------------------
backend/dispatcher.py | 2 +-
backend/mockremote.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index cd387da..5478719 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -139,7 +139,7 @@ class Worker(multiprocessing.Process):
return jobdata
def return_results(self, job):
- self.callback.log('%s status %s. Took %s seconds' % (job.id, job.status, job.ended_on - job.startedon))
+ self.callback.log('%s status %s. Took %s seconds' % (job.build_id, job.status, job.ended_on - job.startedon))
os.unlink(job.jobfile)
#FIXME - this should either return job status/results
# into a queue or it should submit results directly to the frontend
diff --git a/backend/mockremote.py b/backend/mockremote.py
index 44e9fd9..7a48857 100755
--- a/backend/mockremote.py
+++ b/backend/mockremote.py
@@ -97,7 +97,7 @@ def get_ans_results(results, hostname):
def _create_ans_conn(hostname, username, timeout):
ans_conn = ansible.runner.Runner(remote_user=username,
host_list=[hostname], pattern=hostname, forks=1,
- timeout=timeout, transport='ssh')
+ timeout=timeout)
return ans_conn
def check_for_ans_error(results, hostname, err_codes=[], success_codes=[0],
@@ -362,7 +362,7 @@ class Builder(object):
# connect as user
ans = ansible.runner.Runner(host_list=[self.hostname], pattern='*',
- remote_user=self.username, forks=1, timeout=20, transport='ssh')
+ remote_user=self.username, forks=1, timeout=20)
ans.module_name = "shell"
ans.module_args = str("/bin/rpm -q mock rsync")
res = ans.run()
8 years, 2 months