[copr] bkabrda-workspace: Move TODOs to one file, so that they're not present in source code changes (f742e83)
by bkabrda＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : bkabrda-workspace
>---------------------------------------------------------------
commit f742e83759eccad608a57a459a4ee7e2d9dd7d3b
Author: Slavek Kabrda <bkabrda(a)redhat.com>
Date: Fri Nov 16 13:21:26 2012 +0100
Move TODOs to one file, so that they're not present in source code changes
>---------------------------------------------------------------
wsgi/TODO | 8 ++++++++
wsgi/coprs/config.py | 2 +-
wsgi/coprs/forms.py | 1 -
wsgi/coprs/models.py | 2 --
wsgi/coprs/templates/coprs/_coprs_forms.html | 1 -
wsgi/coprs/views/coprs_ns/general.py | 4 +---
6 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/wsgi/TODO b/wsgi/TODO
new file mode 100644
index 0000000..1cf014d
--- /dev/null
+++ b/wsgi/TODO
@@ -0,0 +1,8 @@
+- Unit tests
+- Query optimization
+- Build details
+- Figure out some sane production settings
+- More form validations?
+- It would be nice to be able to change allowed release arches dynamically by JS while creating/editing coprs
+- Convert all remaining forms into wtforms (login, etc)
+- Backend API :)
diff --git a/wsgi/coprs/config.py b/wsgi/coprs/config.py
index d5d720c..fcf29d3 100644
--- a/wsgi/coprs/config.py
+++ b/wsgi/coprs/config.py
@@ -17,7 +17,7 @@ class Config(object):
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.abspath(DATABASE)
class ProductionConfig(Config):
- pass # TODO: some sane production settings
+ pass
class DevelopmentConfig(Config):
DEBUG = True
diff --git a/wsgi/coprs/forms.py b/wsgi/coprs/forms.py
index 8387ba5..df2a10d 100644
--- a/wsgi/coprs/forms.py
+++ b/wsgi/coprs/forms.py
@@ -72,7 +72,6 @@ class StringListFilter(object):
return regex.sub(lambda x: '\n', result)
class CoprForm(wtf.Form):
- # TODO: validations
# also use id here, to be able to find out whether user is updating a copr
# if so, we don't want to shout that name already exists
id = wtf.HiddenField()
diff --git a/wsgi/coprs/models.py b/wsgi/coprs/models.py
index fa51cc0..cce24da 100644
--- a/wsgi/coprs/models.py
+++ b/wsgi/coprs/models.py
@@ -18,8 +18,6 @@ class Serializer(object):
Options can also specify whether to include foo_id when displaying related foo object
(__included_ids__, defaults to True). This doesn't apply when __columns_only__ is specified.
-
- TODO: write some unit tests for this :)
"""
result = {}
columns = self.serializable_attributes
diff --git a/wsgi/coprs/templates/coprs/_coprs_forms.html b/wsgi/coprs/templates/coprs/_coprs_forms.html
index feece30..f655ffa 100644
--- a/wsgi/coprs/templates/coprs/_coprs_forms.html
+++ b/wsgi/coprs/templates/coprs/_coprs_forms.html
@@ -7,7 +7,6 @@
{{ form.csrf_token }}
{{ render_field(form.id, hidden = True) }}
{{ render_field(form.name) }}
- {# TODO: validate/change arches dynamically by JS #}
{{ render_field(form.release) }}
{{ render_field(form.arches, size = 2) }}
{{ render_field(form.repos, rows = 10, cols = 50) }}
diff --git a/wsgi/coprs/views/coprs_ns/general.py b/wsgi/coprs/views/coprs_ns/general.py
index 7e4ec87..327a7eb 100644
--- a/wsgi/coprs/views/coprs_ns/general.py
+++ b/wsgi/coprs/views/coprs_ns/general.py
@@ -83,7 +83,6 @@ def copr_new():
chroots = ' '.join(form.chroots),
repos = form.repos.data.replace('\n', ' '),
owner = flask.g.user)
- # TODO: verify that user doesn't already have a copr with this name
db.session.add(copr)
db.session.commit()
@@ -156,7 +155,6 @@ def copr_update(username, coprname):
if form.validate_on_submit():
# we don't change owner (yet)
- # TODO: verify that this user doesn't already have copr named like this
copr = models.Copr.query.join(models.Copr.owner).\
filter(models.Copr.name == coprname).\
filter(models.User.openid_name == models.User.openidize_name(username)).\
@@ -227,7 +225,7 @@ def copr_give_up_building(username, coprname):
@coprs_ns.route('/detail/<username>/<coprname>/update_permissions/', methods = ['POST'])
@login_required
-def copr_update_permissions(username, coprname): #TODO: optimize!
+def copr_update_permissions(username, coprname):
copr = models.Copr.query.\
join(models.Copr.owner).\
options(db.contains_eager(models.Copr.owner)).\
11 years, 5 months
[copr] bkabrda-workspace: Filter user by username, not coprname :) (78eb27f)
by bkabrda＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : bkabrda-workspace
>---------------------------------------------------------------
commit 78eb27f544af55ece06a7760ffd14c5b2d74de1d
Author: Slavek Kabrda <bkabrda(a)redhat.com>
Date: Fri Nov 16 13:12:51 2012 +0100
Filter user by username, not coprname :)
>---------------------------------------------------------------
wsgi/coprs/views/coprs_ns/general.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/wsgi/coprs/views/coprs_ns/general.py b/wsgi/coprs/views/coprs_ns/general.py
index a9a2860..7e4ec87 100644
--- a/wsgi/coprs/views/coprs_ns/general.py
+++ b/wsgi/coprs/views/coprs_ns/general.py
@@ -181,7 +181,7 @@ def copr_apply_for_building(username, coprname):
outerjoin(models.CoprPermission).\
options(db.contains_eager(models.Copr.owner)).\
filter(models.Copr.name == coprname).\
- filter(models.User.openid_name == models.User.openidize_name(coprname)).\
+ filter(models.User.openid_name == models.User.openidize_name(username)).\
filter(db.or_(models.CoprPermission.user == flask.g.user, models.CoprPermission.user == None)).\
first()
copr = query[0]
11 years, 5 months
[copr] bkabrda-workspace: Fix applying for building to use username/coprname scheme (0a6a967)
by bkabrda＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : bkabrda-workspace
>---------------------------------------------------------------
commit 0a6a967d7c8d620b2f907895aeb4e8a845a86f97
Author: Slavek Kabrda <bkabrda(a)redhat.com>
Date: Fri Nov 16 13:09:34 2012 +0100
Fix applying for building to use username/coprname scheme
>---------------------------------------------------------------
wsgi/coprs/templates/coprs/_coprs_forms.html | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/wsgi/coprs/templates/coprs/_coprs_forms.html b/wsgi/coprs/templates/coprs/_coprs_forms.html
index d5016ae..feece30 100644
--- a/wsgi/coprs/templates/coprs/_coprs_forms.html
+++ b/wsgi/coprs/templates/coprs/_coprs_forms.html
@@ -57,13 +57,13 @@
{% endmacro %}
{% macro copr_apply_for_building(copr) %}
- <form action="{{ url_for('coprs_ns.copr_apply_for_building', name = copr.name) }}" method=post>
+ <form action="{{ url_for('coprs_ns.copr_apply_for_building', username = copr.owner.name, coprname = copr.name) }}" method=post>
<input type=submit value="Apply for building">
</form>
{% endmacro %}
{% macro copr_give_up_building(copr) %}
- <form action="{{ url_for('coprs_ns.copr_give_up_building', name = copr.name) }}" method=post>
+ <form action="{{ url_for('coprs_ns.copr_give_up_building', username = copr.owner.name, coprname = copr.name) }}" method=post>
<input type=submit value="Give up building">
</form>
{% endmacro %}
11 years, 5 months
[copr] bkabrda-workspace: Make sure that user can't create two coprs with the same name (3e1d319)
by bkabrda＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : bkabrda-workspace
>---------------------------------------------------------------
commit 3e1d3192be12dea4850597f5a5ae25216313e8ec
Author: Slavek Kabrda <bkabrda(a)redhat.com>
Date: Fri Nov 16 13:06:07 2012 +0100
Make sure that user can't create two coprs with the same name
>---------------------------------------------------------------
wsgi/coprs/forms.py | 25 +++++++++++++++++++++++--
wsgi/coprs/templates/coprs/_coprs_forms.html | 1 +
2 files changed, 24 insertions(+), 2 deletions(-)
diff --git a/wsgi/coprs/forms.py b/wsgi/coprs/forms.py
index 351ecbe..8387ba5 100644
--- a/wsgi/coprs/forms.py
+++ b/wsgi/coprs/forms.py
@@ -1,10 +1,13 @@
import re
import urlparse
+import flask
+
from flask.ext import wtf
from coprs import constants
from coprs import helpers
+from coprs import models
class UrlListValidator(object):
def __init__(self, message = None):
@@ -41,6 +44,21 @@ class AllowedArchesValidator(object):
if a not in constants.CHROOTS[form.release.data]:
raise wtf.ValidationError(self.message.format(a))
+class CoprUniqueNameValidator(object):
+ def __init__(self, message = None):
+ if not message:
+ message = 'You already have copr named "{0}".'
+ self.message = message
+
+ def __call__(self, form, field):
+ existing = models.Copr.query.filter(models.Copr.name == field.data).\
+ filter(models.Copr.owner_id == flask.g.user.id)
+ if form.id.data:
+ existing = existing.filter(models.Copr.id != form.id.data)
+
+ if existing.first():
+ raise wtf.ValidationError(self.message.format(field.data))
+
class StringListFilter(object):
def __call__(self, value):
@@ -55,9 +73,13 @@ class StringListFilter(object):
class CoprForm(wtf.Form):
# TODO: validations
+ # also use id here, to be able to find out whether user is updating a copr
+ # if so, we don't want to shout that name already exists
+ id = wtf.HiddenField()
name = wtf.TextField('Name',
validators = [wtf.Required(),
- wtf.Regexp(re.compile(r'^[\w-]+$'), message = 'Name must contain only letters, digits, underscores and dashes.')])
+ wtf.Regexp(re.compile(r'^[\w-]+$'), message = 'Name must contain only letters, digits, underscores and dashes.'),
+ CoprUniqueNameValidator()])
# choices must be list of tuples
# => make list like [(fedora-18, fedora-18), ...]
release = wtf.SelectField('Release', choices = [(x, x) for x in constants.CHROOTS])
@@ -72,7 +94,6 @@ class CoprForm(wtf.Form):
def chroots(self):
return ['{0}-{1}'.format(self.release.data, arch) for arch in self.arches.data ]
-
class BuildForm(wtf.Form):
pkgs = wtf.TextAreaField('Pkgs',
validators = [wtf.Required(), UrlListValidator()],
diff --git a/wsgi/coprs/templates/coprs/_coprs_forms.html b/wsgi/coprs/templates/coprs/_coprs_forms.html
index 1786477..d5016ae 100644
--- a/wsgi/coprs/templates/coprs/_coprs_forms.html
+++ b/wsgi/coprs/templates/coprs/_coprs_forms.html
@@ -5,6 +5,7 @@
<form action="{% if copr %}{{ url_for(view, username = copr.owner.name, coprname = copr.name) }}{% else %}{{ url_for(view) }}{% endif %}" method=post class=add-entry>
<dl>
{{ form.csrf_token }}
+ {{ render_field(form.id, hidden = True) }}
{{ render_field(form.name) }}
{# TODO: validate/change arches dynamically by JS #}
{{ render_field(form.release) }}
11 years, 5 months
[copr] bkabrda-workspace: A little fix to the autogenerated migration (02c5c76)
by bkabrda＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : bkabrda-workspace
>---------------------------------------------------------------
commit 02c5c7689191ac1a1360c9aa02740cea55767d8c
Author: Slavek Kabrda <bkabrda(a)redhat.com>
Date: Fri Nov 16 11:44:16 2012 +0100
A little fix to the autogenerated migration
>---------------------------------------------------------------
.../versions/2cb3b8a9de3a_initial_db_setup.py | 3 ---
1 files changed, 0 insertions(+), 3 deletions(-)
diff --git a/wsgi/alembic/versions/2cb3b8a9de3a_initial_db_setup.py b/wsgi/alembic/versions/2cb3b8a9de3a_initial_db_setup.py
index 85887db..3435690 100644
--- a/wsgi/alembic/versions/2cb3b8a9de3a_initial_db_setup.py
+++ b/wsgi/alembic/versions/2cb3b8a9de3a_initial_db_setup.py
@@ -20,7 +20,6 @@ def upgrade():
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('openid_name', sa.String(length=100), nullable=False),
sa.Column('proven', sa.Boolean(), nullable=True),
- sa.CheckConstraint('TODO'),
sa.PrimaryKeyConstraint('id')
)
op.create_table('copr',
@@ -46,7 +45,6 @@ def upgrade():
sa.Column('timeout', sa.Integer(), nullable=True),
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('copr_id', sa.Integer(), nullable=True),
- sa.CheckConstraint('TODO'),
sa.ForeignKeyConstraint(['copr_id'], ['copr.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
@@ -55,7 +53,6 @@ def upgrade():
sa.Column('approved', sa.Boolean(), nullable=True),
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('copr_id', sa.Integer(), nullable=False),
- sa.CheckConstraint('TODO'),
sa.ForeignKeyConstraint(['copr_id'], ['copr.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('user_id', 'copr_id')
11 years, 5 months
[copr] bkabrda-workspace: Drop copr.name uniqueness. Don't create more alembic revisions until we actually deploy it somewhere, this makes it easier (17cc75b)
by bkabrda＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : bkabrda-workspace
>---------------------------------------------------------------
commit 17cc75b29cf309ce8435ae3de9326c3837e4604f
Author: Slavek Kabrda <bkabrda(a)redhat.com>
Date: Fri Nov 16 11:40:56 2012 +0100
Drop copr.name uniqueness. Don't create more alembic revisions until we actually deploy it somewhere, this makes it easier
>---------------------------------------------------------------
...b_setup.py => 2cb3b8a9de3a_initial_db_setup.py} | 32 ++++++++++---------
wsgi/coprs/models.py | 2 +-
2 files changed, 18 insertions(+), 16 deletions(-)
diff --git a/wsgi/alembic/versions/5e4f5077380_initial_db_setup.py b/wsgi/alembic/versions/2cb3b8a9de3a_initial_db_setup.py
similarity index 90%
rename from wsgi/alembic/versions/5e4f5077380_initial_db_setup.py
rename to wsgi/alembic/versions/2cb3b8a9de3a_initial_db_setup.py
index 4b70894..85887db 100644
--- a/wsgi/alembic/versions/5e4f5077380_initial_db_setup.py
+++ b/wsgi/alembic/versions/2cb3b8a9de3a_initial_db_setup.py
@@ -1,13 +1,13 @@
-"""Initial DB setup
+"""Initial db setup
-Revision ID: 5e4f5077380
+Revision ID: 2cb3b8a9de3a
Revises: None
-Create Date: 2012-11-09 08:56:39.836459
+Create Date: 2012-11-16 11:38:30.605131
"""
# revision identifiers, used by Alembic.
-revision = '5e4f5077380'
+revision = '2cb3b8a9de3a'
down_revision = None
from alembic import op
@@ -20,6 +20,7 @@ def upgrade():
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('openid_name', sa.String(length=100), nullable=False),
sa.Column('proven', sa.Boolean(), nullable=True),
+ sa.CheckConstraint('TODO'),
sa.PrimaryKeyConstraint('id')
)
op.create_table('copr',
@@ -29,16 +30,7 @@ def upgrade():
sa.Column('repos', sa.Text(), nullable=True),
sa.Column('owner_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['owner_id'], ['user.id'], ),
- sa.PrimaryKeyConstraint('id'),
- sa.UniqueConstraint('name')
- )
- op.create_table('copr_permission',
- sa.Column('approved', sa.Boolean(), nullable=True),
- sa.Column('user_id', sa.Integer(), nullable=False),
- sa.Column('copr_id', sa.Integer(), nullable=False),
- sa.ForeignKeyConstraint(['copr_id'], ['copr.id'], ),
- sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
- sa.PrimaryKeyConstraint('user_id', 'copr_id')
+ sa.PrimaryKeyConstraint('id')
)
op.create_table('build',
sa.Column('id', sa.Integer(), nullable=False),
@@ -54,17 +46,27 @@ def upgrade():
sa.Column('timeout', sa.Integer(), nullable=True),
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('copr_id', sa.Integer(), nullable=True),
+ sa.CheckConstraint('TODO'),
sa.ForeignKeyConstraint(['copr_id'], ['copr.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('id')
)
+ op.create_table('copr_permission',
+ sa.Column('approved', sa.Boolean(), nullable=True),
+ sa.Column('user_id', sa.Integer(), nullable=False),
+ sa.Column('copr_id', sa.Integer(), nullable=False),
+ sa.CheckConstraint('TODO'),
+ sa.ForeignKeyConstraint(['copr_id'], ['copr.id'], ),
+ sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
+ sa.PrimaryKeyConstraint('user_id', 'copr_id')
+ )
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
- op.drop_table('build')
op.drop_table('copr_permission')
+ op.drop_table('build')
op.drop_table('copr')
op.drop_table('user')
### end Alembic commands ###
diff --git a/wsgi/coprs/models.py b/wsgi/coprs/models.py
index c2a27d8..fa51cc0 100644
--- a/wsgi/coprs/models.py
+++ b/wsgi/coprs/models.py
@@ -85,7 +85,7 @@ class User(db.Model, Serializer):
class Copr(db.Model, Serializer):
id = db.Column(db.Integer, primary_key = True)
- name = db.Column(db.String(100), nullable = False, unique = True)
+ name = db.Column(db.String(100), nullable = False)
chroots = db.Column(db.Text, nullable = False)
repos = db.Column(db.Text)
11 years, 5 months
[copr] bkabrda-workspace: Bulk of changes to have unique coprs only for each user, not across the whole system (will need some more fixing, probably doesn't work well, yet). (575d718)
by bkabrda＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : bkabrda-workspace
>---------------------------------------------------------------
commit 575d7183cc7f3d5d17ec976a217f9da1bb57c786
Author: Slavek Kabrda <bkabrda(a)redhat.com>
Date: Thu Nov 15 11:53:38 2012 +0100
Bulk of changes to have unique coprs only for each user, not across the whole system (will need some more fixing, probably doesn't work well, yet).
>---------------------------------------------------------------
wsgi/coprs/templates/coprs/_coprs_forms.html | 8 +-
wsgi/coprs/templates/coprs/detail.html | 10 +--
wsgi/coprs/templates/coprs/edit.html | 2 +-
wsgi/coprs/templates/coprs/show.html | 2 +-
wsgi/coprs/templates/coprs/show_builds.html | 2 +-
wsgi/coprs/templates/layout.html | 4 +-
wsgi/coprs/views/coprs_ns/builds.py | 33 +++++---
wsgi/coprs/views/coprs_ns/general.py | 107 ++++++++++++++++---------
8 files changed, 101 insertions(+), 67 deletions(-)
diff --git a/wsgi/coprs/templates/coprs/_coprs_forms.html b/wsgi/coprs/templates/coprs/_coprs_forms.html
index 4979836..1786477 100644
--- a/wsgi/coprs/templates/coprs/_coprs_forms.html
+++ b/wsgi/coprs/templates/coprs/_coprs_forms.html
@@ -1,8 +1,8 @@
{% from "_helpers.html" import render_field %}
-{% macro copr_form(form, view, name = None) %}
+{% macro copr_form(form, view, copr = None) %}
{# if using for updating, we need to pass name to url_for, but otherwise we need to pass nothing #}
- <form action="{% if name %}{{ url_for(view, name = name) }}{% else %}{{ url_for(view) }}{% endif %}" method=post class=add-entry>
+ <form action="{% if copr %}{{ url_for(view, username = copr.owner.name, coprname = copr.name) }}{% else %}{{ url_for(view) }}{% endif %}" method=post class=add-entry>
<dl>
{{ form.csrf_token }}
{{ render_field(form.name) }}
@@ -31,7 +31,7 @@
{% endmacro %}
{% macro copr_build_form(form, view, copr) %}
- <form action="{{ url_for(view, name = copr.name) }}" method=post class=add-entry>
+ <form action="{{ url_for(view, username = copr.owner.name, coprname = copr.name) }}" method=post class=add-entry>
<dl>
{{ form.csrf_token }}
{{ render_field(form.pkgs, rows = 10, cols = 50) }}
@@ -49,7 +49,7 @@
{% macro copr_build_cancel_form(build) %}
{% if build.cancelable %}
- <form action="{{ url_for('coprs_ns.copr_cancel_build', name = build.copr.name, build_id = build.id)}}">
+ <form action="{{ url_for('coprs_ns.copr_cancel_build', username = build.copr.owner.name, coprname = build.copr.name, build_id = build.id)}}">
<input type=submit value=Cancel>
</form>
{% endif %}
diff --git a/wsgi/coprs/templates/coprs/detail.html b/wsgi/coprs/templates/coprs/detail.html
index d2db453..271c8f6 100644
--- a/wsgi/coprs/templates/coprs/detail.html
+++ b/wsgi/coprs/templates/coprs/detail.html
@@ -1,6 +1,6 @@
{% extends "layout.html" %}
-{% block title %}{{ copr.name }} Copr{% endblock %}
-{% block header %}Information for Copr {{ copr.name }}{% endblock %}
+{% block title %}{{ copr.owner.name }}/{{ copr.name }} Copr{% endblock %}
+{% block header %}Information for Copr {{ copr.owner.name }}/{{ copr.name }}{% endblock %}
{% from "coprs/_builds_table.html" import builds_table %}
{% from "coprs/_permissions_table.html" import permissions_table %}
{% from "coprs/_coprs_forms.html" import copr_build_form,
@@ -9,8 +9,6 @@
copr_give_up_building with context %}
{% block body %}
- <h3>owner</h3>
- {{ copr.owner.name }}
<h3>chroots</h3>
<ul class=chroots-list>
{% for chroot in copr.chroots_list %}
@@ -41,12 +39,12 @@
{% endif %}
{% endif %}
{% if g.user and g.user.id == copr.owner_id %}
- <a href="{{ url_for('coprs_ns.copr_edit', name = copr.name) }}">Edit this Copr</a>
+ <a href="{{ url_for('coprs_ns.copr_edit', username = copr.owner.name, coprname = copr.name) }}">Edit this Copr</a>
{% endif %}
{% if copr.builds %}
<h3>Latest 10 in this repo</h3>
{{ builds_table(copr.builds) }}
- <a href="{{ url_for('coprs_ns.copr_show_builds', name = copr.name) }}">Show all builds</a>
+ <a href="{{ url_for('coprs_ns.copr_show_builds', username = copr.owner.name, coprname = copr.name) }}">Show all builds</a>
{% else %}
<h3>No builds so far</h3>
{% endif %}
diff --git a/wsgi/coprs/templates/coprs/edit.html b/wsgi/coprs/templates/coprs/edit.html
index c3c504f..62cc1bd 100644
--- a/wsgi/coprs/templates/coprs/edit.html
+++ b/wsgi/coprs/templates/coprs/edit.html
@@ -9,6 +9,6 @@
<h3>Permissions</h3>
{{ copr_permissions_form(permissions_form, copr, permissions) }}
<h3>Other attributes</h3>
- {{ copr_form(form, view = 'coprs_ns.copr_update', name = copr.name) }}
+ {{ copr_form(form, view = 'coprs_ns.copr_update', copr = copr) }}
{% endblock %}
diff --git a/wsgi/coprs/templates/coprs/show.html b/wsgi/coprs/templates/coprs/show.html
index 5f5e34f..24e146d 100644
--- a/wsgi/coprs/templates/coprs/show.html
+++ b/wsgi/coprs/templates/coprs/show.html
@@ -6,7 +6,7 @@
<a href={{ url_for('coprs_ns.copr_add') }}>Add a new Copr</a>
{% for copr in coprs %}
<div class=copr>
- <h2><a href="{{ url_for('coprs_ns.copr_detail', name = copr.name) }}">{{ copr.name }}</a></h2>
+ <h2><a href="{{ url_for('coprs_ns.copr_detail', username = copr.owner.name, coprname = copr.name) }}">{{ copr.owner.name }}/{{ copr.name }}</a></h2>
chroots: {{ copr.chroots_list|join(', ') }}<br>
builds: {{ build_counts[loop.index0] }}<br>
{% if copr.repos %}
diff --git a/wsgi/coprs/templates/coprs/show_builds.html b/wsgi/coprs/templates/coprs/show_builds.html
index e81b682..2432088 100644
--- a/wsgi/coprs/templates/coprs/show_builds.html
+++ b/wsgi/coprs/templates/coprs/show_builds.html
@@ -9,5 +9,5 @@
<div class=pagination>
{{ render_pagination(request, paginator) }}
</div>
- <a href="{{ url_for('coprs_ns.copr_detail', name = request.view_args['name']) }}">Back to Copr</a>
+ <a href="{{ url_for('coprs_ns.copr_detail', username = request.view_args['username'], coprname = request.view_args['coprname']) }}">Back to Copr</a>
{% endblock %}
diff --git a/wsgi/coprs/templates/layout.html b/wsgi/coprs/templates/layout.html
index c05bd8e..a9c6c6b 100644
--- a/wsgi/coprs/templates/layout.html
+++ b/wsgi/coprs/templates/layout.html
@@ -10,8 +10,8 @@
{% block menu %}
<a href="{{ url_for('coprs_ns.coprs_show') }}">All Coprs</a>
{% if g.user %}
- <a href="{{ url_for('coprs_ns.coprs_by_owner', name = g.user.name) }}">Owned Coprs</a>
- <a href="{{ url_for('coprs_ns.coprs_by_allowed', name = g.user.name) }}">Coprs I Can Build In</a>
+ <a href="{{ url_for('coprs_ns.coprs_by_owner', username = g.user.name) }}">Owned Coprs</a>
+ <a href="{{ url_for('coprs_ns.coprs_by_allowed', username = g.user.name) }}">Coprs I Can Build In</a>
<a class=log-url href="{{ url_for('misc.logout') }}">Log out</a>
{% else %}
<a class=log-url href="{{ url_for('misc.login') }}">Log in</a>
diff --git a/wsgi/coprs/views/coprs_ns/builds.py b/wsgi/coprs/views/coprs_ns/builds.py
index 61f2555..db94b60 100644
--- a/wsgi/coprs/views/coprs_ns/builds.py
+++ b/wsgi/coprs/views/coprs_ns/builds.py
@@ -11,31 +11,38 @@ from coprs.views.misc import login_required
from coprs.views.coprs_ns.general import coprs_ns
-(a)coprs_ns.route('/detail/<name>/builds/', defaults = {'page': 1})
-(a)coprs_ns.route('/detail/<name>/builds/<int:page>/')
-def copr_show_builds(name, page = 1):
+(a)coprs_ns.route('/detail/<username>/<coprname>/builds/', defaults = {'page': 1})
+(a)coprs_ns.route('/detail/<username>/<coprname>/builds/<int:page>/')
+def copr_show_builds(username, coprname, page = 1):
query = models.Build.query.join(models.Copr.builds).\
+ join(models.Copr.owner).\
options(db.contains_eager(models.Build.copr)).\
- filter(models.Copr.name == name).\
+ filter(models.Copr.name == coprname).\
+ filter(models.User.openid_name == models.User.openidize_name(username)).\
order_by(models.Build.submitted_on.desc())
build_count = query.count()
if build_count == 0: # no builds => we still need Copr
copr = models.Copr.query.filter(models.Copr.name == name).first()
if not copr: # hey, this Copr doesn't exist
- return page_not_found('Copr with name {0} does not exist.'.format(name))
+ return page_not_found('Copr with name {0} does not exist.'.format(coprname))
paginator = helpers.Paginator(query, build_count, page, per_page_override = 20)
return flask.render_template('coprs/show_builds.html', builds = paginator.sliced_query, paginator = paginator)
-(a)coprs_ns.route('/detail/<name>/add_build/', methods = ["POST"])
+(a)coprs_ns.route('/detail/<username>/<coprname>/add_build/', methods = ["POST"])
@login_required
-def copr_add_build(name = None):
+def copr_add_build(username, coprname):
form = forms.BuildForm()
- copr = models.Copr.query.filter(models.Copr.name == name).first()
+ copr = models.Copr.query.\
+ join(models.Copr.owner).\
+ options(db.contains_eager(models.Copr.owner)).\
+ filter(models.Copr.name == coprname).\
+ filter(models.User.openid_name == models.User.openidize_name(username)).\
+ first()
if not copr: # hey, this Copr doesn't exist
- return page_not_found('Copr with name {0} does not exist.'.format(name))
+ return page_not_found('Copr with name {0} does not exist.'.format(coprname))
if form.validate_on_submit() and flask.g.user.can_build_in(copr):
build = models.Build(pkgs = form.pkgs.data.replace('\n', ' '),
@@ -52,14 +59,14 @@ def copr_add_build(name = None):
db.session.commit()
flask.flash("Build was added")
- return flask.redirect(flask.url_for('coprs_ns.copr_detail', name = copr.name))
+ return flask.redirect(flask.url_for('coprs_ns.copr_detail', username = username, coprname = copr.name))
else:
return flask.render_template('coprs/detail.html', copr = copr, form = form)
-(a)coprs_ns.route('/detail/<name>/cancel_build/<int:build_id>/')
+(a)coprs_ns.route('/detail/<username>/<coprname>/cancel_build/<int:build_id>/')
@login_required
-def copr_cancel_build(name, build_id):
+def copr_cancel_build(username, coprname, build_id):
# only the user who ran the build can cancel it
build = models.Build.query.filter(models.Build.id == build_id).first()
if not build: # hey, this Copr doesn't exist
@@ -71,4 +78,4 @@ def copr_cancel_build(name, build_id):
db.session.commit()
flask.flash('Build was canceled')
- return flask.redirect(flask.url_for('coprs_ns.copr_detail', name = name))
+ return flask.redirect(flask.url_for('coprs_ns.copr_detail', username = username, coprname = coprname))
diff --git a/wsgi/coprs/views/coprs_ns/general.py b/wsgi/coprs/views/coprs_ns/general.py
index ef90be3..a9a2860 100644
--- a/wsgi/coprs/views/coprs_ns/general.py
+++ b/wsgi/coprs/views/coprs_ns/general.py
@@ -15,7 +15,9 @@ coprs_ns = flask.Blueprint('coprs_ns',
@coprs_ns.route('/<int:page>/')
def coprs_show(page = 1):
query = db.session.query(models.Copr, db.func.count(models.Build.id)).\
+ join(models.Copr.owner).\
outerjoin(models.Copr.builds).\
+ options(db.contains_eager(models.Copr.owner)).\
group_by(models.Copr.id)
paginator = helpers.Paginator(query, query.count(), page)
@@ -26,13 +28,14 @@ def coprs_show(page = 1):
return flask.render_template('coprs/show.html', coprs = coprs, build_counts = build_counts, paginator = paginator)
-(a)coprs_ns.route('/owned/<name>/', defaults = {'page': 1})
-(a)coprs_ns.route('/owned/<name>/<int:page>/')
-def coprs_by_owner(name = None, page = 1):
+(a)coprs_ns.route('/owned/<username>/', defaults = {'page': 1})
+(a)coprs_ns.route('/owned/<username>/<int:page>/')
+def coprs_by_owner(username = None, page = 1):
query = db.session.query(models.Copr, db.func.count(models.Build.id)).\
join(models.Copr.owner).\
outerjoin(models.Copr.builds).\
- filter(models.User.openid_name == models.User.openidize_name(name)).\
+ options(db.contains_eager(models.Copr.owner)).\
+ filter(models.User.openid_name == models.User.openidize_name(username)).\
group_by(models.Copr.id)
paginator = helpers.Paginator(query, query.count(), page)
@@ -43,14 +46,14 @@ def coprs_by_owner(name = None, page = 1):
return flask.render_template('coprs/show.html', coprs = coprs, build_counts = build_counts, paginator = paginator)
-(a)coprs_ns.route('/allowed/<name>/', defaults = {'page': 1})
-(a)coprs_ns.route('/allowed/<name>/<int:page>/')
-def coprs_by_allowed(name = None, page = 1):
+(a)coprs_ns.route('/allowed/<username>/', defaults = {'page': 1})
+(a)coprs_ns.route('/allowed/<username>/<int:page>/')
+def coprs_by_allowed(username = None, page = 1):
query = db.session.query(models.Copr, models.CoprPermission, models.User, db.func.count(models.Build.id)).\
outerjoin(models.Copr.builds).\
join(models.Copr.copr_permissions).\
join(models.CoprPermission.user).\
- filter(models.User.openid_name == models.User.openidize_name(name)).\
+ filter(models.User.openid_name == models.User.openidize_name(username)).\
filter(models.CoprPermission.approved == True).\
group_by(models.Copr.id)
paginator = helpers.Paginator(query, query.count(), page)
@@ -80,7 +83,7 @@ def copr_new():
chroots = ' '.join(form.chroots),
repos = form.repos.data.replace('\n', ' '),
owner = flask.g.user)
-
+ # TODO: verify that user doesn't already have a copr with this name
db.session.add(copr)
db.session.commit()
@@ -90,17 +93,18 @@ def copr_new():
return flask.render_template('coprs/add.html', form = form)
-(a)coprs_ns.route('/detail/<name>/')
-def copr_detail(name = None):
+(a)coprs_ns.route('/detail/<username>/<coprname>/')
+def copr_detail(username, coprname):
form = forms.BuildForm()
try: # query will raise an index error, if Copr doesn't exist
copr = models.Copr.query.outerjoin(models.Copr.builds).\
join(models.Copr.owner).\
options(db.contains_eager(models.Copr.builds), db.contains_eager(models.Copr.owner)).\
- filter(models.Copr.name == name).\
+ filter(models.Copr.name == coprname).\
+ filter(models.User.openid_name == models.User.openidize_name(username)).\
order_by(models.Build.submitted_on.desc())[0:10][0] # we retrieved all builds, but we got one copr in a list...
except IndexError:
- return page_not_found('Copr with name {0} does not exist.'.format(name))
+ return page_not_found('Copr with name {0} does not exist.'.format(coprname))
permissions = models.CoprPermission.query.join(models.CoprPermission.user).\
options(db.contains_eager(models.CoprPermission.user)).\
filter(models.CoprPermission.copr_id == copr.id).\
@@ -109,16 +113,19 @@ def copr_detail(name = None):
return flask.render_template('coprs/detail.html', copr = copr, form = form, permissions = permissions)
-(a)coprs_ns.route('/edit/<name>/')
+(a)coprs_ns.route('/detail/<username>/<coprname>/edit/')
@login_required
-def copr_edit(name = None):
+def copr_edit(username, coprname):
query = db.session.query(models.Copr, models.CoprPermission).\
outerjoin(models.CoprPermission).\
- filter(models.Copr.name == name).\
+ outerjoin(models.Copr.owner).\
+ options(db.contains_eager(models.Copr.owner)).\
+ filter(models.Copr.name == coprname).\
+ filter(models.User.openid_name == models.User.openidize_name(username)).\
all()
copr = query[0][0]
if not copr:
- return page_not_found('Copr with name {0} does not exist.'.format(name))
+ return page_not_found('Copr with name {0} does not exist.'.format(coprname))
form = forms.CoprForm(obj = copr)
permissions = map(lambda x: x[1], query)
@@ -132,35 +139,49 @@ def copr_edit(name = None):
permissions_form = permissions_form)
-(a)coprs_ns.route('/update/<name>/', methods = ['POST'])
+(a)coprs_ns.route('/detail/<username>/<coprname>/update/', methods = ['POST'])
@login_required
-def copr_update(name = None):
+def copr_update(username, coprname):
form = forms.CoprForm()
- copr = models.Copr.query.filter(models.Copr.name == name).first()
+ copr = models.Copr.query.\
+ join(models.Copr.owner).\
+ options(db.contains_eager(models.Copr.owner)).\
+ filter(models.Copr.name == coprname).\
+ filter(models.User.openid_name == models.User.openidize_name(username)).\
+ first()
# only owner can update a copr
if flask.g.user != copr.owner:
flask.flash('Only owners may update their Coprs.')
- return flask.redirect(flask.url_for('coprs_ns.copr_detail', name = form.name.data))
+ return flask.redirect(flask.url_for('coprs_ns.copr_detail', username = copr.owner.name, coprname = form.name.data))
if form.validate_on_submit():
# we don't change owner (yet)
- updated = models.Copr.query.filter(models.Copr.name == name).\
- update({'name': form.name.data,
- 'chroots': ' '.join(form.chroots),
- 'repos': form.repos.data.replace('\n', ' ')})
+ # TODO: verify that this user doesn't already have copr named like this
+ copr = models.Copr.query.join(models.Copr.owner).\
+ filter(models.Copr.name == coprname).\
+ filter(models.User.openid_name == models.User.openidize_name(username)).\
+ first()
+ copr.name = form.name.data
+ copr.chroots = ' '.join(form.chroots)
+ copr.repos = form.repos.data.replace('\n', ' ')
+
+ db.session.add(copr)
db.session.commit()
flask.flash('Copr was updated successfully.')
- return flask.redirect(flask.url_for('coprs_ns.copr_detail', name = form.name.data))
+ return flask.redirect(flask.url_for('coprs_ns.copr_detail', username = username, coprname = form.name.data))
else:
return flask.render_template('coprs/edit.html', copr = copr, form = form)
-(a)coprs_ns.route('/apply_for_building/<name>/', methods = ['POST'])
+(a)coprs_ns.route('/detail/<username>/<coprname>/apply_for_building', methods = ['POST'])
@login_required
-def copr_apply_for_building(name = None):
+def copr_apply_for_building(username, coprname):
query = db.session.query(models.Copr, models.CoprPermission).\
+ join(models.Copr.owner).\
outerjoin(models.CoprPermission).\
- filter(models.Copr.name == name).\
+ options(db.contains_eager(models.Copr.owner)).\
+ filter(models.Copr.name == coprname).\
+ filter(models.User.openid_name == models.User.openidize_name(coprname)).\
filter(db.or_(models.CoprPermission.user == flask.g.user, models.CoprPermission.user == None)).\
first()
copr = query[0]
@@ -176,15 +197,18 @@ def copr_apply_for_building(name = None):
db.session.commit()
flask.flash('You have successfuly applied for building in Copr "{0}".'.format(copr.name))
- return flask.redirect(flask.url_for('coprs_ns.copr_detail', name = copr.name))
+ return flask.redirect(flask.url_for('coprs_ns.copr_detail', username = copr.owner.name, coprname = copr.name))
-(a)coprs_ns.route('/give_up_building/<name>/', methods = ['POST'])
+(a)coprs_ns.route('/detail/<username>/<coprname>/give_up_building/', methods = ['POST'])
@login_required
-def copr_give_up_building(name = None):
+def copr_give_up_building(username, coprname):
query = db.session.query(models.Copr, models.CoprPermission).\
+ join(models.Copr.owner).\
outerjoin(models.CoprPermission).\
- filter(models.Copr.name == name).\
+ options(db.contains_eager(models.Copr.owner)).\
+ filter(models.Copr.name == coprname).\
+ filter(models.User.openid_name == models.User.openidize_name(username)).\
filter(models.CoprPermission.user == flask.g.user).\
first()
copr = query[0]
@@ -198,20 +222,25 @@ def copr_give_up_building(name = None):
db.session.commit()
flask.flash('You have successfuly given up building in Copr "{0}".'.format(copr.name))
- return flask.redirect(flask.url_for('coprs_ns.copr_detail', name = copr.name))
+ return flask.redirect(flask.url_for('coprs_ns.copr_detail', username = copr.owner.name, coprname = copr.name))
-(a)coprs_ns.route('/update_permissions/<name>/', methods = ['POST'])
+(a)coprs_ns.route('/detail/<username>/<coprname>/update_permissions/', methods = ['POST'])
@login_required
-def copr_update_permissions(name = None): #TODO: optimize!
- copr = models.Copr.query.filter(models.Copr.name == name).first()
+def copr_update_permissions(username, coprname): #TODO: optimize!
+ copr = models.Copr.query.\
+ join(models.Copr.owner).\
+ options(db.contains_eager(models.Copr.owner)).\
+ filter(models.User.name == username).\
+ filter(models.Copr.name == coprname).\
+ first()
permissions = models.CoprPermission.query.filter(models.CoprPermission.copr_id == copr.id).all()
permissions_form = forms.DynamicPermissionsFormFactory.create_form_cls(permissions)()
# only owner can update copr permissions
if flask.g.user != copr.owner:
flask.flash('Only owners may update their Coprs permissions.')
- return flask.redirect(flask.url_for('coprs_ns.copr_detail', name = copr.name))
+ return flask.redirect(flask.url_for('coprs_ns.copr_detail', username = copr.owner.name, coprname = copr.name))
if permissions_form.validate_on_submit():
# we don't change owner (yet)
@@ -221,6 +250,6 @@ def copr_update_permissions(name = None): #TODO: optimize!
update({'approved': permissions_form['user_{0}'.format(perm.user_id)].data})
db.session.commit()
flask.flash('Copr permissions were updated successfully.')
- return flask.redirect(flask.url_for('coprs_ns.copr_detail', name = copr.name))
+ return flask.redirect(flask.url_for('coprs_ns.copr_detail', username = copr.owner.name, coprname = copr.name))
else:
return flask.render_template('coprs/edit.html', copr = copr, form = form)
11 years, 5 months
[copr] skvidal-backend: make it create the dirs we need (9938a2f)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 9938a2fce9d5add9186b8dc14c7a8e8d205e7410
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Nov 15 16:21:05 2012 -0500
make it create the dirs we need
>---------------------------------------------------------------
copr-be.py | 7 +++++++
1 files changed, 7 insertions(+), 0 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 39ebb48..0e3e01a 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -29,6 +29,13 @@ class CoprBackend(object):
self.config_file = config_file
self.opts = self.read_config()
+ logdir = os.path.dirname(self.opts.logfile)
+ if not os.path.exists(logdir):
+ os.makedirs(logdir, mode=0750)
+
+ if not os.path.exists(self.opts.worker_logdir):
+ os.makedirs(self.opts.worker_logdir, module=0750)
+
self.jobs = multiprocessing.Queue()
self.workers = []
self.added_jobs = []
11 years, 5 months
[copr] master: rename file to README and explain where to look (5fdd930)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 5fdd9307d4abde89f760d65cecc0f5efef2606db
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Nov 15 12:20:36 2012 -0500
rename file to README and explain where to look
>---------------------------------------------------------------
README | 7 +++++++
text | 1 -
2 files changed, 7 insertions(+), 1 deletions(-)
diff --git a/README b/README
new file mode 100644
index 0000000..92034f0
--- /dev/null
+++ b/README
@@ -0,0 +1,7 @@
+This is the master copr branch.
+Right now all the work is going on in 2 branches:
+ - skvidal-backend - this is the backend/builder code
+ - bkabrda-workspace - this is the frontend/webinterface code
+
+
+
diff --git a/text b/text
deleted file mode 100644
index 5b9e8fe..0000000
--- a/text
+++ /dev/null
@@ -1 +0,0 @@
-open it up
11 years, 5 months
[copr] skvidal-backend: add errors file for exceptions separately continue to flesh out functionality (75b9e88)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 75b9e88a80b3c3d379c2a4ad3866e405e4e91de4
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Nov 15 02:24:56 2012 -0500
add errors file for exceptions separately
continue to flesh out functionality
>---------------------------------------------------------------
backend/dispatcher.py | 148 ++++++++++++++++++++++++++++++++-----------------
backend/errors.py | 12 ++++
copr-be.py | 99 +++++++++++++++-----------------
3 files changed, 154 insertions(+), 105 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 748e519..4948512 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -1,18 +1,18 @@
#!/usr/bin/python -tt
-import sys
import os
-import glob
-import subprocess
+import sys
import multiprocessing
import time
import Queue
import json
import mockremote
from bunch import Bunch
+import errors
import ansible
import ansible.playbook
+import ansible.errors
from ansible import callbacks
@@ -62,25 +62,21 @@ class SilentPlaybookCallbacks(callbacks.object):
def on_stats(self, stats):
callbacks.call_callback_module('playbook_on_stats', stats)
-def spawn_instance(opts, ip):
- # FIXME - setup silent callbacks
- # - check errors in setup
- # - playbook variablized
- stats = callbacks.AggregateStats()
- playbook_cb = SilentPlaybookCallbacks(verbose=False)
- runner_cb = callbacks.DefaultRunnerCallbacks()
- play = ansible.playbook.PlayBook(stats=stats, playbook='/srv/copr-work/provision/builderpb.yml',
- callbacks=playbook_cb, runner_callbacks=runner_cb, remote_user='root')
+class WorkerCallback(object):
+ def __init__(self, logfile=None):
+ self.logfile = logfile
+
+ def log(self, msg):
+ if not self.logfile:
+ return
- play.run()
- if ip:
- return ip
+ now = time.time()
+ try:
+ open(self.logfile, 'a').write(str(now) + ':' + msg + '\n')
+ except (IOError, OSError), e:
+ print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfile, str(e))
- for i in play.SETUP_CACHE:
- if i =='localhost':
- continue
- return i
class Worker(multiprocessing.Process):
def __init__(self, opts, jobs, ip=None, create=True, callback=None):
@@ -88,17 +84,63 @@ class Worker(multiprocessing.Process):
# base class initialization
multiprocessing.Process.__init__(self, name="worker-builder")
+
# job management stuff
self.jobs = jobs
self.ip = ip
self.opts = opts
self.kill_received = False
- print 'creating worker: %s' % ip
-
-
- def parse_job(job):
+ self.callback = callback
+ if not self.callback:
+ lf = self.opts.worker_logdir + '/worker-%s.log' % self.pid
+ self.callback = WorkerCallback(logfile = lf)
+
+ self.callback.log('creating worker: %s' % ip)
+
+ def spawn_instance(self):
+
+ stats = callbacks.AggregateStats()
+ playbook_cb = SilentPlaybookCallbacks(verbose=False)
+ runner_cb = callbacks.DefaultRunnerCallbacks()
+ # fixme - extra_vars to include ip as a var if we need to specify ips
+ # also to include info for instance type to handle the memory requirements of builds
+ play = ansible.playbook.PlayBook(stats=stats, playbook=self.opts.playbook,
+ callbacks=playbook_cb, runner_callbacks=runner_cb,
+ remote_user='root')
+
+ play.run()
+ if self.ip:
+ return self.ip
+
+ for i in play.SETUP_CACHE:
+ if i =='localhost':
+ continue
+ return i
+ return None
+
+ def parse_job(self, jobfile):
# read the json of the job in
- # break out what we need return a structured
+ # break out what we need return a bunch of the info we need
+ d = json.load(open(jobfile))
+ build = d['builds'][0]
+ jobdata = Bunch()
+ jobdata.pkgs = build['pkgs'].split(' ')
+ jobdata.repos = build['repos'].split(' ')
+ jobdata.chroots = build['chroots'].split(' ')
+ jobdata.memory_reqs = build['memory_reqs']
+ jobdata.timeout = build['timeout']
+ jobdata.destdir = self.opts.destdir + '/' + build['copr']['owner']['name'] + '/' + build['copr']['name'] + '/'
+ jobdata.build_id = build['id']
+ jobdata.copr_id = build['copr']['id']
+ jobdata.user_id = build['user_id']
+ return jobdata
+
+ def return_results(self, job):
+ self.log('%s status %s. Took %s seconds' % (job.id, job.status, build.ended_on - build.startedon)
+ os.unlink(job.jobfile)
+ #FIXME - this should either return job status/results
+ # into a queue or it should submit results directly to the frontend
+
def run(self):
# worker should startup and check if it can function
# for each job it takes from the jobs queue
@@ -108,41 +150,43 @@ class Worker(multiprocessing.Process):
while not self.kill_received:
try:
- job = self.jobs.get()
+ jobfile = self.jobs.get()
except Queue.Empty:
break
- self.cur_job = job
- f = open(self.opts.get('destdir', '/') + '/' + job, 'w')
- f.write('')
- f.close()
-
# parse the job json into our info
- # pkgs
- # repos
- # chroot(s)
- # memory needed
- # timeout
- # make up a destdir
-
- #print 'start up instance %s using %s' % (self.ip, self.opts.get('playbook', None))
- ip = spawn_instance(self.opts, ip=ip)
+ job = self.parse_job(jobfile)
- destdir = construct_something_here
+ job.jobfile = jobfile
+ # spin up our build instance
try:
- mr = mockremote.MockRemote(builder=ip, timeout=timeout,
- destdir=destdir, chroot=chroot, cont=True, recurse=True,
- repos=repos, callback=None)
- mr.build_pkgs(pkgs)
- except mockremote.MockRemoteError, e:
- # record and break
- print '%s - %s' % (ip, e)
- break
+ ip = self.spawn_instance()
+ if not ip:
+ raise errors.CoprWorkerError, "No IP found from creating instance"
+
+ except ansible.errors.AnsibleError, e:
+ self.callback.log('failure to setup instance: %s' % e)
+ raise
+
+ status = 1
+ job.started_on = time.time()
+ for chroot in job.chroots:
+ self.callback.log('mockremote on %s - %s' % (ip, jobfile))
+ try:
+ mr = mockremote.MockRemote(builder=ip, timeout=job.timeout,
+ destdir=job.destdir, chroot=chroot, cont=True, recurse=True,
+ repos=job.repos, callback=None)
+ mr.build_pkgs(job.pkgs)
+ except mockremote.MockRemoteError, e:
+ # record and break
+ self.callback.log('%s - %s' % (ip, e))
+ status = 0 # failure
+
+ job.ended_on = time.time()
+ job.status = status
+ self.return_results(job)
+
- # run mockremote to that ip with the args from above
- print 'mockremote on %s - %s' % (ip, job)
- time.sleep(30)
- #print 'terminate-instance %s' % (self. ip)
diff --git a/backend/errors.py b/backend/errors.py
new file mode 100644
index 0000000..ae8ac34
--- /dev/null
+++ b/backend/errors.py
@@ -0,0 +1,12 @@
+# copr error/exceptions
+class CoprBackendError(Exception):
+
+ def __init__(self, msg):
+ self.msg = msg
+
+ def __str__(self):
+ return self.msg
+
+class CoprWorkerError(CoprBackendError):
+ pass
+
diff --git a/copr-be.py b/copr-be.py
index 4e8945b..39ebb48 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -5,9 +5,9 @@ import sys
import os
import glob
import time
-import json
import multiprocessing
from backend.dispatcher import Worker
+from backend import errors
from bunch import Bunch
from ConfigParser import ConfigParser
@@ -17,13 +17,6 @@ def _get_conf(cp, section, option, default):
return cp.get(section, option)
return default
-class CoprBackendError(Exception):
-
- def __init__(self, msg):
- self.msg = msg
-
- def __str__(self):
- return self.msg
class CoprBackend(object):
def __init__(self, config_file=None):
@@ -31,7 +24,7 @@ class CoprBackend(object):
# put all the config items into a single self.opts bunch
if not config_file:
- raise CoprBackendError, "Must specify config_file"
+ raise errors.CoprBackendError, "Must specify config_file"
self.config_file = config_file
self.opts = self.read_config()
@@ -58,73 +51,73 @@ class CoprBackend(object):
opts.sleeptime = int(_get_conf(cp, 'backend', 'sleeptime', 10))
opts.num_workers = int(_get_conf(cp, 'backend', 'num_workers', 8))
opts.timeout = int(_get_conf(cp, 'builder', 'timeout', 1800))
- opts.logfile = _get_conf(cp, 'backend', 'logfile', '/var/log/copr-be.log')
+ opts.logfile = _get_conf(cp, 'backend', 'logfile', '/var/log/copr/backend.log')
+ opts.worker_logdir = _get_conf(cp, 'backend', 'worker_logdir', '/var/log/copr/worker/')
# thoughts for later
# ssh key for connecting to builders?
# cloud key stuff?
#
except ConfigParser.Error, e:
- raise CoprBackendError, 'Error parsing config file: %s: %s' % (self.config_file, e)
+ raise errors.CoprBackendError, 'Error parsing config file: %s: %s' % (self.config_file, e)
if not opts.jobsdir or not opts.destdir:
- raise CoprBackendError, "Incomplete Config - must specify jobsdir and destdir in configuration"
+ raise errors.CoprBackendError, "Incomplete Config - must specify jobsdir and destdir in configuration"
return opts
def log(self, msg):
- if self.logfile:
- now = time.time()
- try:
- open(self.logfn, 'a').write(str(now) + ':' + msg + '\n')
- except (IOError, OSError), e:
- print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.lf, str(e))
- if not self.quiet:
- print msg
+ now = time.time()
+ try:
+ open(self.logfile, 'a').write(str(now) + ':' + msg + '\n')
+ except (IOError, OSError), e:
+ print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfile, str(e))
def run(self):
- # start processing builds, etc
- # setup and run our workers
- for i in range(opts.num_workers):
- w = backend.dispatcher.Worker(opts, jobs)
- workers.append(w)
- w.start()
abort = False
while not abort:
print 'adding jobs'
- for f in sorted(glob.glob(jobsdir + '/*.json')):
+ for f in sorted(glob.glob(self.opts.jobsdir + '/*.json')):
n = os.path.basename(f).replace('.json', '')
- if not is_completed(n) and n not in added:
- #jobdata = open(f).read()
- jobs.put(n)
- added.append(n)
- print 'adding %s' % n
-
-
-
- print "# jobs in queue: %s" % jobs.qsize()
-
-
- # FIXME:
- # look up number of workers in config
- # see if it changed and update accordingly?
- # poison pill? if opts.num_workers < len(workers)?
- time.sleep(opts.sleeptime)
+ if n not in self.added_jobs:
+ self.jobs.put(f)
+ self.added_jobs.append(n)
+ self.log('adding %s' % n)
+
+ # this handles starting/growing the number of workers
+ if len(self.workers) < self.opts.num_workers:
+ for i in range(self.opts.num_workers - len(self.workers)):
+ w = dispatcher.Worker(self.opts, self.jobs)
+ self.workers.append(w)
+ w.start()
+ # FIXME - prune out workers
+ #if len(self.workers) > self.opts.num_workers:
+ # killnum = len(self.workers) - self.opts.num_workers
+ # for w in self.workers[:killnum]:
+ # #insert a poison pill? Kill after something? I dunno.
+ # FIXME - if a worker bombs out - we need to check them
+ # and startup a new one if it happens
+
+ self.log("# jobs in queue: %s" % jobs.qsize())
+
+ time.sleep(self.opts.sleeptime)
-def is_completed(jobid):
-
- if glob.glob(destdir + '/' + jobid + '*'):
- return True
- return False
-
def main(args):
-
-
-
+ if len(args) < 1 or not os.path.exists(args[0]):
+ print 'Must pass in config file'
+ sys.exit(1)
+ try:
+ cbe = CoprBackend(args[0])
+ cbe.run()
+ except Exception, e:
+ print 'Killing/Dying'
+ for w in cbe.workers:
+ w.terminate()
+ raise
if __name__ == '__main__':
@@ -134,6 +127,6 @@ if __name__ == '__main__':
print "ERROR: %s - %s" % (str(type(e)), str(e))
sys.exit(1)
except KeyboardInterrupt, e:
- print "\nUser cancelled, need cleanup\n"
+ print "\nUser cancelled, may need cleanup\n"
sys.exit(0)
11 years, 5 months