[copr] master: initial and incomplete doc for setting up copr instance (03fe7f8)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 03fe7f84e032b54bfd3f3a220f4bd6497e6fa161
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Tue Jan 15 00:52:25 2013 -0500
initial and incomplete doc for setting up copr instance
>---------------------------------------------------------------
copr-setup.txt | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 92 insertions(+), 0 deletions(-)
diff --git a/copr-setup.txt b/copr-setup.txt
new file mode 100644
index 0000000..72305d4
--- /dev/null
+++ b/copr-setup.txt
@@ -0,0 +1,92 @@
+WORK IN PROGRESS
+
+- How coprs works
+ - SEE README FOR OVERVIEW
+
+
+
+- need at least 1 server
+- some instances/systems for builders
+
+Backend:
+ansible playbook:
+ http://infrastructure.fedoraproject.org/infra/ansible/playbooks/hosts/cop...
+
+Required pkgs:
+ - ansible - 0.9 or higher
+ - [lig]httpd
+ - euca2ools (or other cloud tools)
+ - rsync
+ - openssh-clients
+ - mock - recent - from epel - including mockchain
+ - yum-utils
+ - createrepo
+ - python-bunch
+ - python-requests
+
+
+- add copr user
+- give write access to a dir somewhere not in your homedir
+- git clone git://git.fedorahosted.org/copr.git into that dir
+- setup copr-be.conf
+- dirs needed to be setup?
+- setup webserver - point to results path from copr-be.conf
+- dirindexes enabled
+
+- builder provisioning
+3. spawn instance playbook -
+ http://infrastructure.fedoraproject.org/infra/ansible/files/copr/provisio...
+ - just an ansible playbook which sets up and provisions an instance
+ in our case it is using eucalyptus but it doesn't have to be
+ any cloud provider - it just needs to be able to get an ip an
+ ssh into it - cloud providers or private clouds just make more
+ sense
+4. provisioning dir
+ http://infrastructure.fedoraproject.org/infra/ansible/files/copr/provisio...
+ - what gets pushed to the other side - ssh keys, users, mock
+ configs, etc
+5. terminate instance playbook -
+ http://infrastructure.fedoraproject.org/infra/ansible/files/copr/provisio...
+
+
+
+Frontend:
+rhel6ish - add epel
+
+Required pkgs:
+ - httpd
+ - mod_wsgi
+ - postgresql-server
+ - postgresql-devel
+ - python-virtualenv
+ - gcc
+
+- apache setup
+ coprs.conf:
+
+<VirtualHost *:80>
+ ServerName yourservername.here
+
+ WSGIDaemonProcess yourservername.hereuser=copr-fe group=copr-fe threads=5
+ WSGIScriptAlias / /path/to/coprs_git_tree/coprs_frontend/application
+ WSGIProcessGroup yourservername.here
+ WSGIPassAuthorization On
+
+ ErrorLog logs/error_coprs
+ CustomLog logs/access_coprs common
+
+ <Directory /var/www/coprs_frontend>
+ WSGIApplicationGroup %{GLOBAL}
+ Order deny,allow
+ Allow from all
+ </Directory>
+</VirtualHost>
+
+- virtenv of flask and friends
+ - virtualenv --distribute virtenv
+ - source virtenv/bin/activate
+ - pip install flask flask-sqlalchemy flask-openid flask-wtf alembic
+ - pip install -I SQLAlchemy MarkupSafe
+ - pip install pytest
+
+
11 years, 3 months
[copr] master: - add the job base repo to the list of repos to include in mock - for later builds - record the build id when finished so we can match them up (89f33ea)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 89f33ea9d6e768c7e40394394ae2507274131e30
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Mon Jan 14 12:55:57 2013 -0500
- add the job base repo to the list of repos to include in mock - for later builds
- record the build id when finished so we can match them up
>---------------------------------------------------------------
backend/dispatcher.py | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index eb0ab06..92395ce 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -163,6 +163,8 @@ class Worker(multiprocessing.Process):
jobdata.destdir = self.opts.destdir + '/' + build['copr']['owner']['name'] + '/' + build['copr']['name'] + '/'
jobdata.build_id = build['id']
jobdata.results = self.opts.results_baseurl + '/' + build['copr']['owner']['name'] + '/' + build['copr']['name'] + '/'
+ # add the results dir to the jobdata.repos
+ jobdata.repos.append(jobdata.results)
jobdata.copr_id = build['copr']['id']
jobdata.user_id = build['user_id']
return jobdata
@@ -293,7 +295,7 @@ class Worker(multiprocessing.Process):
# check if any pkgs didn't build
if mr.failed:
status = 0
- self.callback.log('Finished build: builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos)))
+ self.callback.log('Finished build: id=%r builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (job.build_id, ip, job.timeout, job.destdir, chroot, str(job.repos)))
job.ended_on = time.time()
11 years, 3 months
[copr] master: change the log file name from meaningless mockremote to build-## (1b4ce65)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 1b4ce65e6b3d1c69a63e8159be20c33cb12d387a
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Mon Jan 14 10:47:33 2013 -0500
change the log file name from meaningless mockremote to build-##
>---------------------------------------------------------------
backend/dispatcher.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index d5bc7be..eb0ab06 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -278,7 +278,7 @@ class Worker(multiprocessing.Process):
self.callback.log('Starting build: id=%r builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (job.build_id,ip, job.timeout, job.destdir, chroot, str(job.repos)))
self.callback.log('building pkgs: %s' % ' '.join(job.pkgs))
try:
- chrootlogfile = chroot_destdir + '/mockremote-%s.log' % job.build_id
+ chrootlogfile = chroot_destdir + '/build-%s.log' % job.build_id
mr = mockremote.MockRemote(builder=ip, timeout=job.timeout,
destdir=job.destdir, chroot=chroot, cont=True, recurse=True,
repos=job.repos,
11 years, 3 months
[copr] master: Display build ids in builds table. (7969871)
by bkabrda＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 796987131c4377f9d691c93ba1fc9bd234d0cfc8
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Mon Jan 14 07:53:07 2013 +0100
Display build ids in builds table.
>---------------------------------------------------------------
.../coprs/templates/coprs/_builds_table.html | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/coprs_frontend/coprs/templates/coprs/_builds_table.html b/coprs_frontend/coprs/templates/coprs/_builds_table.html
index f6840d9..072a7e3 100644
--- a/coprs_frontend/coprs/templates/coprs/_builds_table.html
+++ b/coprs_frontend/coprs/templates/coprs/_builds_table.html
@@ -4,6 +4,7 @@
{% if builds %}
<table class=builds-table>
<tr>
+ <th>Id</th>
<th>Submitted on</th>
<th>Submitted by</th>
<th>Started on</th>
@@ -14,6 +15,7 @@
</tr>
{% for build in builds %}
<tr class=build-{{ build.state }}>
+ <td>{{ build.id }}</td>
<td>{{ build.submitted_on|date_from_secs }}</td>
<td>{{ build.user.name }}</td>
<td>{{ build.started_on|date_from_secs or 'Not yet' }}</td>
@@ -29,7 +31,7 @@
</td>
</tr>
<tr>
- <td colspan=7>
+ <td colspan=8>
<h4>Package URLs:</h4>
<ul class=packages-list>
{% if build.pkgs is not none %}
11 years, 3 months
[copr] master: - update readme with simple overview - remove done TODO list item (07c7d60)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 07c7d6058260c8a015dbe0728e045f841fa96ade
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Jan 11 17:23:24 2013 -0500
- update readme with simple overview
- remove done TODO list item
>---------------------------------------------------------------
README | 15 +++++++++++++++
TODO-backend | 2 --
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/README b/README
index 92034f0..99991b3 100644
--- a/README
+++ b/README
@@ -5,3 +5,18 @@ Right now all the work is going on in 2 branches:
+Overview of use:
+- frontend allows users to create a copr for themselves.
+ - then they are able to submit urls to pkgs they wish to build
+
+- the frontend makes the build requests available as JSON
+
+- the backend polls the frontend for new builds
+- when it finds a new build it spins off a builder using
+ the ansible playbook to create a new builder instance
+ (normally in a private or public cloud)
+- it notifies the frontend that the build is starting
+- it submits the pkg/repo/etc to be built on the instance with mockchain
+- it retrieves the results and saves them into the results dir
+- it returns the build status and where to find the results to the front end
+
diff --git a/TODO-backend b/TODO-backend
index 4cc5ca1..058330e 100644
--- a/TODO-backend
+++ b/TODO-backend
@@ -15,5 +15,3 @@
- work on a way to find and cancel a specific build that's happening other than just killing the instance
- determine if it is properly checking the timeout from a dead instance
- maybe dump out the PID of the worker that is running so we know which one to kill?
-- failure/success not being returned correctly. Should check for 'fail' in the directories and return based on
- that. also anything lacking success is a failure.
11 years, 4 months
[copr] master: make the mockchain log include a buildid - so if we have multiple running we don't have a race (c141ecd)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit c141ecd64d3b2b0f3f62acfa594279d5aabada27
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Jan 11 17:24:22 2013 -0500
make the mockchain log include a buildid - so if we have multiple running
we don't have a race
>---------------------------------------------------------------
backend/dispatcher.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index a0115fc..d5bc7be 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -278,7 +278,7 @@ class Worker(multiprocessing.Process):
self.callback.log('Starting build: id=%r builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (job.build_id,ip, job.timeout, job.destdir, chroot, str(job.repos)))
self.callback.log('building pkgs: %s' % ' '.join(job.pkgs))
try:
- chrootlogfile = chroot_destdir + '/mockremote.log'
+ chrootlogfile = chroot_destdir + '/mockremote-%s.log' % job.build_id
mr = mockremote.MockRemote(builder=ip, timeout=job.timeout,
destdir=job.destdir, chroot=chroot, cont=True, recurse=True,
repos=job.repos,
11 years, 4 months
yum plugin for copr
by Pierre-Yves Chibon
Hi,
Discussing with Seth today, he was telling me that the current idea
would be to kind of loop through a list of packages to find the one
requested by the user.
While I was actually more thinking of something like
--enablerepo=copr-<repo_name>
Even if we do a
--enablerepo=copr-all
which would eventually still do the loop up through all packages in all
repo.
I think the copr-<repo_name> has several advantages though:
- no extra infrastructure needed
- faster (only one call)
- allow multiple copr to have the same rpm (with eventually different
option in the build or tweaks or <insert here your idea if you follow
mine>)
Just my 2cts on the subject :)
Pierre
11 years, 4 months
[copr] master: Properly prefill the permissions apply form (ed727a5)
by bkabrda＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit ed727a50dfb80d5755a3d92d3ed33e6664e98ae1
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Fri Jan 11 15:00:58 2013 +0100
Properly prefill the permissions apply form
>---------------------------------------------------------------
.../coprs/views/coprs_ns/coprs_general.py | 3 ++-
1 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
index 2cd455c..73ac8e7 100644
--- a/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
+++ b/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
@@ -108,7 +108,8 @@ def copr_detail(username, coprname, build_form = None):
if flask.g.user.can_edit(copr):
permissions_form = forms.PermissionsFormFactory.create_form_cls(permissions)()
else:
- permissions_applier_form = forms.PermissionsApplierFormFactory.create_form_cls(user_perm)()
+ # https://github.com/ajford/flask-wtf/issues/58
+ permissions_applier_form = forms.PermissionsApplierFormFactory.create_form_cls(user_perm)(formdata=None)
return flask.render_template('coprs/detail.html',
copr = copr,
11 years, 4 months
[copr] master: Fix commit a2b66fdd in a proper way. (364d1dd)
by bkabrda＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 364d1dde9bd2908e58ffcd76a523a00c790e3e5f
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Fri Jan 11 14:58:27 2013 +0100
Fix commit a2b66fdd in a proper way.
>---------------------------------------------------------------
.../coprs/views/coprs_ns/coprs_builds.py | 3 +--
1 files changed, 1 insertions(+), 2 deletions(-)
diff --git a/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py b/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py
index 7addf6e..5f65321 100644
--- a/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py
+++ b/coprs_frontend/coprs/views/coprs_ns/coprs_builds.py
@@ -54,8 +54,7 @@ def copr_add_build(username, coprname):
flask.flash("Build was added")
return flask.redirect(flask.url_for('coprs_ns.copr_detail', username = username, coprname = copr.name))
else:
- # TODO: figure out a more elegant way to get the function
- return coprs_general.copr_detail(username = username, coprname = coprname, build_form = form)
+ return flask.current_app.view_functions['coprs_ns.copr_detail'](username = username, coprname = coprname, build_form = form)
@coprs_ns.route('/detail/<username>/<coprname>/cancel_build/<int:build_id>/', methods = ['POST'])
@login_required
11 years, 4 months
[copr] master: Allow dots in copr names (bccd9ad)
by bkabrda＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit bccd9ad6856e9a922f677adb653cca3d9503eb52
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Fri Jan 11 13:08:41 2013 +0100
Allow dots in copr names
>---------------------------------------------------------------
coprs_frontend/coprs/forms.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/coprs_frontend/coprs/forms.py b/coprs_frontend/coprs/forms.py
index bb8b7a8..461e7c0 100644
--- a/coprs_frontend/coprs/forms.py
+++ b/coprs_frontend/coprs/forms.py
@@ -83,7 +83,7 @@ class CoprForm(wtf.Form):
id = wtf.HiddenField()
name = wtf.TextField('Name',
validators = [wtf.Required(),
- wtf.Regexp(re.compile(r'^[\w-]+$'), message = 'Name must contain only letters, digits, underscores and dashes.'),
+ wtf.Regexp(re.compile(r'^[\w.-]+$'), message = 'Name must contain only letters, digits, underscores, dashes and dots.'),
CoprUniqueNameValidator()])
# choices must be list of tuples
# => make list like [(fedora-18, fedora-18), ...]
11 years, 4 months
