[copr] master: initialize multiprocessing base object (3e18ad1)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 3e18ad1e4697ae867e8fbec2f941d749d5c3327f
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Tue Jan 15 11:44:04 2013 -0500
initialize multiprocessing base object
>---------------------------------------------------------------
copr-be.py | 3 +++
1 files changed, 3 insertions(+), 0 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 9391968..881c30d 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -26,6 +26,9 @@ class CoprJobGrab(multiprocessing.Process):
"""Fetch jobs from the Frontend - submit them to the jobs queue for workers"""
def __init__(self, opts, events, jobs):
+ # base class initialization
+ multiprocessing.Process.__init__(self, name="jobgrab")
+
self.opts = opts
self.events = events
self.jobs = jobs
11 years, 4 months
[copr] master: first cut and refactoring for event queue and better logging (1cfded9)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 1cfded9339138e940b588ab2e346164f31677079
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Tue Jan 15 11:35:24 2013 -0500
first cut and refactoring for event queue and better logging
>---------------------------------------------------------------
backend/dispatcher.py | 33 +++++++---
copr-be.py | 161 +++++++++++++++++++++++++++++++++----------------
2 files changed, 131 insertions(+), 63 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 92395ce..1485285 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -68,18 +68,16 @@ class WorkerCallback(object):
self.logfile = logfile
def log(self, msg):
- if not self.logfile:
- return
-
- now = time.strftime('%F %T')
- try:
- open(self.logfile, 'a').write(str(now) + ': ' + msg + '\n')
- except (IOError, OSError), e:
- print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfile, str(e))
+ if self.logfile:
+ now = time.strftime('%F %T')
+ try:
+ open(self.logfile, 'a').write(str(now) + ': ' + msg + '\n')
+ except (IOError, OSError), e:
+ print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfile, str(e))
class Worker(multiprocessing.Process):
- def __init__(self, opts, jobs, worker_num, ip=None, create=True, callback=None):
+ def __init__(self, opts, jobs, events, worker_num, ip=None, create=True, callback=None):
# base class initialization
multiprocessing.Process.__init__(self, name="worker-builder")
@@ -87,6 +85,7 @@ class Worker(multiprocessing.Process):
# job management stuff
self.jobs = jobs
+ self.events = events # event queue for communicating back to dispatcher
self.worker_num = worker_num
self.ip = ip
self.opts = opts
@@ -99,11 +98,23 @@ class Worker(multiprocessing.Process):
if ip:
self.callback.log('creating worker: %s' % ip)
+ self.event('creating worker: %s' % ip)
else:
self.callback.log('creating worker: dynamic ip')
+ self.event('creating worker: dynamic ip')
+
+ def event(self, what):
+ if self.ip:
+ who = 'worker-%s-%s' % (self.worker_num, self.ip)
+ else:
+ who = 'worker-%s' % (self.worker_num)
+
+ self.events.put({'when':time.time(), 'who':who, 'what':what})
def spawn_instance(self):
"""call the spawn playbook to startup/provision a building instance"""
+
+
self.callback.log('spawning instance begin')
start = time.time()
@@ -195,6 +206,7 @@ class Worker(multiprocessing.Process):
# maybe we move this to the callback?
def mark_started(self, job):
+ self.event('job start: user:%s copr:%s build:%s ip:%s pid:%s' % (job.user_id, job.copr_id, job.build_id, self.ip, self.pid))
build = {'id':job.build_id,
'started_on': job.started_on,
'results': job.results,
@@ -206,8 +218,9 @@ class Worker(multiprocessing.Process):
# maybe we move this to the callback?
def return_results(self, job):
- self.callback.log('%s status %s. Took %s seconds' % (job.build_id, job.status, job.ended_on - job.started_on))
+ self.event('job end: user:%s copr:%s build:%s ip:%s pid:%s' % (job.user_id, job.copr_id, job.build_id, self.ip, self.pid))
+ self.callback.log('%s status %s. Took %s seconds' % (job.build_id, job.status, job.ended_on - job.started_on))
build = {'id':job.build_id,
'ended_on': job.ended_on,
'status': job.status,
diff --git a/copr-be.py b/copr-be.py
index 6631ac4..9391968 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -21,7 +21,98 @@ def _get_conf(cp, section, option, default):
return default
+
+class CoprJobGrab(multiprocessing.Process):
+ """Fetch jobs from the Frontend - submit them to the jobs queue for workers"""
+
+ def __init__(self, opts, events, jobs):
+ self.opts = opts
+ self.events = events
+ self.jobs = jobs
+ self.added_jobs = []
+
+ def event(self, what):
+ self.events.put({'when':time.time(), 'who':'job', 'what':what})
+
+ def fetch_jobs(self):
+ self.event('fetching jobs')
+ try:
+ r = requests.get('%s/waiting_builds/' % self.opts.frontend_url) # auth stuff here? maybe/maybenot
+ except requests.RequestException, e:
+ self.event('Error retrieving jobs from %s: %s' % (self.opts.frontend_url, e))
+ else:
+ try:
+ r_json = json.loads(r.content) # using old requests on el6 :(
+ except ValueError, e:
+ self.event('Error getting JSON build list from FE %s' % e)
+ return
+
+ if 'builds' in r_json and r_json['builds']:
+ self.event('%s jobs returned' % len(r_json['builds']))
+ count = 0
+ for b in r_json['builds']:
+ if 'id' in b:
+ jobfile = self.opts.jobsdir + '/%s.json' % b['id']
+ if not os.path.exists(jobfile) and b['id'] not in self.added_jobs:
+ count += 1
+ open(jobfile, 'w').write(json.dumps(b))
+ self.event('Wrote job: %s' % b['id'])
+ if count:
+ self.event('New jobs: %s' % count)
+
+ def run(self):
+ abort = False
+ while not abort:
+ self.fetch_jobs()
+ for f in sorted(glob.glob(self.opts.jobsdir + '/*.json')):
+ n = os.path.basename(f).replace('.json', '')
+ if n not in self.added_jobs:
+ self.jobs.put(f)
+ self.added_jobs.append(n)
+ self.event('adding to work queue id %s' % n)
+
+class CoprLog(multiprocessing.Process):
+ """log mechanism where items from the events queue get recorded"""
+ def __init__(self, opts, events):
+
+ # base class initialization
+ multiprocessing.Process.__init__(self, name="logger")
+
+ self.opts = opts
+ self.events = events
+
+ logdir = os.path.dirname(self.opts.logfile)
+ if not os.path.exists(logdir):
+ os.makedirs(logdir, mode=0750)
+
+ if not os.path.exists(self.opts.destdir):
+ os.makedirs(self.opts.destdir, mode=0755)
+
+ # setup a log file to write to
+ self.logfile = self.opts.logfile
+
+ def log(self, event):
+
+ when = time.strftime('%F %T', time.gmtime(event['when']))
+ msg = '%s : %s %s' % (when, event['who'], event['what'].strip())
+
+ try:
+ open(self.logfile, 'a').write(msg + '\n')
+ except (IOError, OSError), e:
+ print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfile, str(e))
+
+
+ # event format is a dict {when:time, who:[worker|logger|job|main], what:str}
+ def run(self):
+ abort = False
+ while not abort:
+ for e in self.events.get():
+ if 'when' in e and 'who' in e and 'what' in e:
+ self.log(e)
+
class CoprBackend(object):
+ """core process - starts/stops/initializes workers"""
+
def __init__(self, config_file=None, ext_opts=None):
# read in config file
# put all the config items into a single self.opts bunch
@@ -33,25 +124,28 @@ class CoprBackend(object):
self.ext_opts = ext_opts # to stow our cli options for read_conf()
self.opts = self.read_conf()
- logdir = os.path.dirname(self.opts.logfile)
- if not os.path.exists(logdir):
- os.makedirs(logdir, mode=0750)
+ self.jobs = multiprocessing.Queue() # job is a path to a jobfile on the localfs
+ self.events = multiprocessing.Queue()
+ # event format is a dict {when:time, who:[worker|logger|job|main], what:str}
- if not os.path.exists(self.opts.destdir):
- os.makedirs(self.opts.destdir, mode=0755)
- # setup a log file to write to
- self.logfile = self.opts.logfile
- self.log("Starting up new copr-be instance")
+ # create logger
+ self._logger = CoprLog(self.opts, self.events)
+ self._logger.start()
+ self.event('Starting up Job Grabber')
+ # create job grabber
+ self._jobgrab = CoprJobGrab(self.opts, self.events, self.jobs)
+ self._jobgrab.start()
if not os.path.exists(self.opts.worker_logdir):
os.makedirs(self.opts.worker_logdir, mode=0750)
- self.jobs = multiprocessing.Queue()
self.workers = []
self.added_jobs = []
+ def event(self, what):
+ self.events.put({'when':time.time(), 'who':'main', 'what':what})
def read_conf(self):
"read in config file - return Bunch of config data"
@@ -90,64 +184,25 @@ class CoprBackend(object):
return opts
- def log(self, msg):
- now = time.strftime('%F %T')
- output = str(now) + ': ' + msg
- if not self.opts.daemonize:
- print output
-
- try:
- open(self.logfile, 'a').write(output + '\n')
- except (IOError, OSError), e:
- print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfile, str(e))
-
-
- def fetch_jobs(self):
- self.log('fetching jobs')
- try:
- r = requests.get('%s/waiting_builds/' % self.opts.frontend_url) # auth stuff here? maybe/maybenot
- except requests.RequestException, e:
- self.log('Error retrieving jobs from %s: %s' % (self.opts.frontend_url, e))
- else:
- r_json = json.loads(r.content) # using old requests on el6 :(
- if 'builds' in r_json:
- self.log('%s jobs returned' % len(r_json['builds']))
- count = 0
- for b in r_json['builds']:
- if 'id' in b:
- jobfile = self.opts.jobsdir + '/%s.json' % b['id']
- if not os.path.exists(jobfile) and b['id'] not in self.added_jobs:
- count += 1
- open(jobfile, 'w').write(json.dumps(b))
- self.log('Wrote job: %s' % b['id'])
- self.log('New jobs: %s' % count)
def run(self):
abort = False
while not abort:
- self.fetch_jobs()
- for f in sorted(glob.glob(self.opts.jobsdir + '/*.json')):
- n = os.path.basename(f).replace('.json', '')
- if n not in self.added_jobs:
- self.jobs.put(f)
- self.added_jobs.append(n)
- self.log('adding to work queue id %s' % n)
-
# re-read config into opts
self.opts = self.read_conf()
if self.jobs.qsize():
- self.log("# jobs in queue: %s" % self.jobs.qsize())
+ self.event("# jobs in queue: %s" % self.jobs.qsize())
# this handles starting/growing the number of workers
if len(self.workers) < self.opts.num_workers:
- self.log("Spinning up more workers for jobs")
+ self.event("Spinning up more workers for jobs")
for i in range(self.opts.num_workers - len(self.workers)):
worker_num = len(self.workers) + 1
- w = Worker(self.opts, self.jobs, worker_num)
+ w = Worker(self.opts, self.jobs, self.events, worker_num)
self.workers.append(w)
w.start()
- self.log("Finished starting worker processes")
+ self.event("Finished starting worker processes")
# FIXME - prune out workers
#if len(self.workers) > self.opts.num_workers:
# killnum = len(self.workers) - self.opts.num_workers
@@ -158,7 +213,7 @@ class CoprBackend(object):
# check for dead workers and abort
for w in self.workers:
if not w.is_alive():
- self.log('Worker %d died unexpectedly' % w.worker_num)
+ self.event('Worker %d died unexpectedly' % w.worker_num)
if self.opts.exit_on_worker:
raise errors.CoprBackendError, "Worker died unexpectedly, exiting"
else:
11 years, 4 months
[copr] master: Merge branch 'master' of ssh://git.fedorahosted.org/git/copr (a4715bf)
by bkabrda＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit a4715bf1373275cb43b853c32a7f5231e9584ae1
Merge: cbaa66e 03fe7f8
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Tue Jan 15 12:05:05 2013 +0100
Merge branch 'master' of ssh://git.fedorahosted.org/git/copr
>---------------------------------------------------------------
backend/dispatcher.py | 6 ++-
copr-setup.txt | 92 +++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 96 insertions(+), 2 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index d5bc7be..92395ce 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -163,6 +163,8 @@ class Worker(multiprocessing.Process):
jobdata.destdir = self.opts.destdir + '/' + build['copr']['owner']['name'] + '/' + build['copr']['name'] + '/'
jobdata.build_id = build['id']
jobdata.results = self.opts.results_baseurl + '/' + build['copr']['owner']['name'] + '/' + build['copr']['name'] + '/'
+ # add the results dir to the jobdata.repos
+ jobdata.repos.append(jobdata.results)
jobdata.copr_id = build['copr']['id']
jobdata.user_id = build['user_id']
return jobdata
@@ -278,7 +280,7 @@ class Worker(multiprocessing.Process):
self.callback.log('Starting build: id=%r builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (job.build_id,ip, job.timeout, job.destdir, chroot, str(job.repos)))
self.callback.log('building pkgs: %s' % ' '.join(job.pkgs))
try:
- chrootlogfile = chroot_destdir + '/mockremote-%s.log' % job.build_id
+ chrootlogfile = chroot_destdir + '/build-%s.log' % job.build_id
mr = mockremote.MockRemote(builder=ip, timeout=job.timeout,
destdir=job.destdir, chroot=chroot, cont=True, recurse=True,
repos=job.repos,
@@ -293,7 +295,7 @@ class Worker(multiprocessing.Process):
# check if any pkgs didn't build
if mr.failed:
status = 0
- self.callback.log('Finished build: builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos)))
+ self.callback.log('Finished build: id=%r builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (job.build_id, ip, job.timeout, job.destdir, chroot, str(job.repos)))
job.ended_on = time.time()
diff --git a/copr-setup.txt b/copr-setup.txt
new file mode 100644
index 0000000..72305d4
--- /dev/null
+++ b/copr-setup.txt
@@ -0,0 +1,92 @@
+WORK IN PROGRESS
+
+- How coprs works
+ - SEE README FOR OVERVIEW
+
+
+
+- need at least 1 server
+- some instances/systems for builders
+
+Backend:
+ansible playbook:
+ http://infrastructure.fedoraproject.org/infra/ansible/playbooks/hosts/cop...
+
+Required pkgs:
+ - ansible - 0.9 or higher
+ - [lig]httpd
+ - euca2ools (or other cloud tools)
+ - rsync
+ - openssh-clients
+ - mock - recent - from epel - including mockchain
+ - yum-utils
+ - createrepo
+ - python-bunch
+ - python-requests
+
+
+- add copr user
+- give write access to a dir somewhere not in your homedir
+- git clone git://git.fedorahosted.org/copr.git into that dir
+- setup copr-be.conf
+- dirs needed to be setup?
+- setup webserver - point to results path from copr-be.conf
+- dirindexes enabled
+
+- builder provisioning
+3. spawn instance playbook -
+ http://infrastructure.fedoraproject.org/infra/ansible/files/copr/provisio...
+ - just an ansible playbook which sets up and provisions an instance
+ in our case it is using eucalyptus but it doesn't have to be
+ any cloud provider - it just needs to be able to get an ip an
+ ssh into it - cloud providers or private clouds just make more
+ sense
+4. provisioning dir
+ http://infrastructure.fedoraproject.org/infra/ansible/files/copr/provisio...
+ - what gets pushed to the other side - ssh keys, users, mock
+ configs, etc
+5. terminate instance playbook -
+ http://infrastructure.fedoraproject.org/infra/ansible/files/copr/provisio...
+
+
+
+Frontend:
+rhel6ish - add epel
+
+Required pkgs:
+ - httpd
+ - mod_wsgi
+ - postgresql-server
+ - postgresql-devel
+ - python-virtualenv
+ - gcc
+
+- apache setup
+ coprs.conf:
+
+<VirtualHost *:80>
+ ServerName yourservername.here
+
+ WSGIDaemonProcess yourservername.hereuser=copr-fe group=copr-fe threads=5
+ WSGIScriptAlias / /path/to/coprs_git_tree/coprs_frontend/application
+ WSGIProcessGroup yourservername.here
+ WSGIPassAuthorization On
+
+ ErrorLog logs/error_coprs
+ CustomLog logs/access_coprs common
+
+ <Directory /var/www/coprs_frontend>
+ WSGIApplicationGroup %{GLOBAL}
+ Order deny,allow
+ Allow from all
+ </Directory>
+</VirtualHost>
+
+- virtenv of flask and friends
+ - virtualenv --distribute virtenv
+ - source virtenv/bin/activate
+ - pip install flask flask-sqlalchemy flask-openid flask-wtf alembic
+ - pip install -I SQLAlchemy MarkupSafe
+ - pip install pytest
+
+
11 years, 4 months
[copr] master: Refactor the messages from manage.py (cbaa66e)
by bkabrda＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit cbaa66e69a5d69edcd66768d0b78ff5c2ba65e1c
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Tue Jan 15 12:02:57 2013 +0100
Refactor the messages from manage.py
>---------------------------------------------------------------
coprs_frontend/manage.py | 22 ++++++++++++++++------
1 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/coprs_frontend/manage.py b/coprs_frontend/manage.py
index be5a5b1..766ca46 100755
--- a/coprs_frontend/manage.py
+++ b/coprs_frontend/manage.py
@@ -43,6 +43,16 @@ class DropDBCommand(Command):
db.drop_all()
class ChrootCommand(Command):
+ def print_invalid_format(self, chroot_name):
+ print '{0} - invalid chroot format, must be "{release}-{version}-{arch}".'.format(chroot_name)
+
+ def print_already_exists(self, chroot_name):
+ print '{0} - already exists.'.format(chroot_name)
+
+ def print_doesnt_exist(self, chroot_name):
+ print '{0} - chroot doesn\'t exist.'.format(chroot_name)
+
+
option_list = (
Option('chroot_names',
help='Chroot name, e.g. fedora-18-x86_64.',
@@ -55,9 +65,9 @@ class CreateChrootCommand(ChrootCommand):
for chroot_name in chroot_names:
split_chroot = chroot_name.split('-')
if len(split_chroot) < 3:
- print '{0} - Invalid chroot format, must be "{release}-{version}-{arch}".'.format(chroot_name)
+ self.print_invalid_format(chroot_name)
elif models.MockChroot.get(*split_chroot):
- print '{0} - already exists.'.format(chroot_name)
+ self.print_already_exists(chroot_name)
else:
new_chroot = models.MockChroot(os_release=split_chroot[0],
os_version=split_chroot[1],
@@ -72,10 +82,10 @@ class AlterChrootCommand(ChrootCommand):
for chroot_name in chroot_names:
split_chroot = chroot_name.split('-')
if len(split_chroot) < 3:
- print '{0} - invalid chroot format, must be "{release}-{version}-{arch}".'.format(chroot_name)
+ self.print_invalid_format(chroot_name)
chroot = models.MockChroot.get(*split_chroot)
if not chroot:
- print '{0} - chroot doesn\'t exist.'.format(chroot_name)
+ self.print_doesnt_exist(chroot_name)
else:
if action == 'activate':
chroot.is_active = True
@@ -100,10 +110,10 @@ class DropChrootCommand(ChrootCommand):
for chroot_name in chroot_names:
split_chroot = chroot_name.split('-')
if len(split_chroot) < 3:
- print '{0} - invalid chroot format, must be "{release}-{version}-{arch}".'.format(chroot_name)
+ self.print_invalid_format(chroot_name)
chroot = models.MockChroot.get(*split_chroot)
if not chroot:
- print '{0} - chroot doesn\'t exist.'.format(chroot_name)
+ self.print_doesnt_exist(chroot_name)
else:
db.session.delete(chroot)
db.session.commit()
11 years, 4 months
[copr] master: Let chroot commands accept multiple chroots as arguments (78beac9)
by bkabrda＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 78beac97a43d1c45675e046e65f1d06ccf781d85
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Tue Jan 15 11:58:11 2013 +0100
Let chroot commands accept multiple chroots as arguments
>---------------------------------------------------------------
coprs_frontend/manage.py | 78 +++++++++++++++++++++++++--------------------
1 files changed, 43 insertions(+), 35 deletions(-)
diff --git a/coprs_frontend/manage.py b/coprs_frontend/manage.py
index e2461d8..be5a5b1 100755
--- a/coprs_frontend/manage.py
+++ b/coprs_frontend/manage.py
@@ -44,41 +44,46 @@ class DropDBCommand(Command):
class ChrootCommand(Command):
option_list = (
- Option('chroot_name',
- help='Chroot name, e.g. fedora-18-x86_64.'),
+ Option('chroot_names',
+ help='Chroot name, e.g. fedora-18-x86_64.',
+ nargs='+'),
)
class CreateChrootCommand(ChrootCommand):
'Creates a mock chroot in DB'
- def run(self, chroot_name):
- split_chroot = chroot_name.split('-')
- if len(split_chroot) < 3:
- print 'Invalid chroot format, must be "{release}-{version}-{arch}".'
- return
- if models.MockChroot.get(*split_chroot):
- print 'Already exists.'
- return
- new_chroot = models.MockChroot(os_release=split_chroot[0],
- os_version=split_chroot[1],
- arch=split_chroot[2],
- is_active=True)
- db.session.add(new_chroot)
- db.session.commit()
+ def run(self, chroot_names):
+ for chroot_name in chroot_names:
+ split_chroot = chroot_name.split('-')
+ if len(split_chroot) < 3:
+ print '{0} - Invalid chroot format, must be "{release}-{version}-{arch}".'.format(chroot_name)
+ elif models.MockChroot.get(*split_chroot):
+ print '{0} - already exists.'.format(chroot_name)
+ else:
+ new_chroot = models.MockChroot(os_release=split_chroot[0],
+ os_version=split_chroot[1],
+ arch=split_chroot[2],
+ is_active=True)
+ db.session.add(new_chroot)
+ db.session.commit()
class AlterChrootCommand(ChrootCommand):
'Activates or deactivates a chroot'
- def run(self, chroot_name, action):
- split_chroot = chroot_name.split('-')
- if len(split_chroot) < 3:
- print 'Invalid chroot format, must be "{release}-{version}-{arch}".'
- chroot = models.MockChroot.get(*split_chroot)
- if action == 'activate':
- chroot.is_active = True
- else:
- chroot.is_active = False
-
- db.session.add(chroot)
- db.session.commit()
+ def run(self, chroot_names, action):
+ for chroot_name in chroot_names:
+ split_chroot = chroot_name.split('-')
+ if len(split_chroot) < 3:
+ print '{0} - invalid chroot format, must be "{release}-{version}-{arch}".'.format(chroot_name)
+ chroot = models.MockChroot.get(*split_chroot)
+ if not chroot:
+ print '{0} - chroot doesn\'t exist.'.format(chroot_name)
+ else:
+ if action == 'activate':
+ chroot.is_active = True
+ else:
+ chroot.is_active = False
+
+ db.session.add(chroot)
+ db.session.commit()
option_list = ChrootCommand.option_list + (
Option('--action',
@@ -91,14 +96,17 @@ class AlterChrootCommand(ChrootCommand):
class DropChrootCommand(ChrootCommand):
'Activates or deactivates a chroot'
- def run(self, chroot_name):
- split_chroot = chroot_name.split('-')
- if len(split_chroot) < 3:
- print 'Invalid chroot format, must be "{release}-{version}-{arch}".'
- chroot = models.MockChroot.get(*split_chroot)
- if chroot:
- db.session.delete(chroot)
- db.session.commit()
+ def run(self, chroot_names):
+ for chroot_name in chroot_names:
+ split_chroot = chroot_name.split('-')
+ if len(split_chroot) < 3:
+ print '{0} - invalid chroot format, must be "{release}-{version}-{arch}".'.format(chroot_name)
+ chroot = models.MockChroot.get(*split_chroot)
+ if not chroot:
+ print '{0} - chroot doesn\'t exist.'.format(chroot_name)
+ else:
+ db.session.delete(chroot)
+ db.session.commit()
class DisplayChrootsCommand(Command):
'Displays current mock chroots'
11 years, 4 months
[copr] master: Fix the failing builds after mandating at least chroot for copr (d1e54b5)
by bkabrda＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit d1e54b5bc2f561be63b492c8efcf5912280717ce
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Tue Jan 15 11:45:24 2013 +0100
Fix the failing builds after mandating at least chroot for copr
>---------------------------------------------------------------
.../tests/test_views/test_coprs_ns/test_general.py | 22 ++++++++++---------
1 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/coprs_frontend/tests/test_views/test_coprs_ns/test_general.py b/coprs_frontend/tests/test_views/test_coprs_ns/test_general.py
index 931e70c..c5bf51d 100644
--- a/coprs_frontend/tests/test_views/test_coprs_ns/test_general.py
+++ b/coprs_frontend/tests/test_views/test_coprs_ns/test_general.py
@@ -60,19 +60,19 @@ class TestCoprsAllowed(CoprsTestCase):
class TestCoprNew(CoprsTestCase):
success_string = 'New copr was successfully created'
- def test_copr_new_normal(self, f_users):
+ def test_copr_new_normal(self, f_users, f_mock_chroots):
with self.tc as c:
with c.session_transaction() as s:
s['openid'] = self.u1.openid_name
- r = c.post('/coprs/new/', data = {'name': 'foo', 'release': 'fedora-rawhide', 'arches': ['i386']}, follow_redirects = True)
+ r = c.post('/coprs/new/', data = {'name': 'foo', 'fedora-rawhide-i386': 'y', 'arches': ['i386']}, follow_redirects = True)
assert self.models.Copr.query.filter(self.models.Copr.name == 'foo').first()
assert self.success_string in r.data
# make sure no initial build was submitted
assert self.models.Build.query.first() == None
- def test_copr_new_exists_for_another_user(self, f_users, f_coprs):
+ def test_copr_new_exists_for_another_user(self, f_users, f_coprs, f_mock_chroots):
with self.tc as c:
with c.session_transaction() as s:
s['openid'] = self.u3.openid_name
@@ -81,12 +81,13 @@ class TestCoprNew(CoprsTestCase):
foocoprs = len(self.models.Copr.query.filter(self.models.Copr.name == self.c1.name).all())
assert foocoprs > 0
- r = c.post('/coprs/new/', data = {'name': self.c1.name, 'release': 'fedora-rawhide', 'arches': ['i386']}, follow_redirects = True)
+ r = c.post('/coprs/new/', data = {'name': self.c1.name, 'fedora-rawhide-i386': 'y'}, follow_redirects = True)
+ print r.data
self.db.session.add(self.c1)
assert len(self.models.Copr.query.filter(self.models.Copr.name == self.c1.name).all()) == foocoprs + 1
assert self.success_string in r.data
- def test_copr_new_exists_for_this_user(self, f_users, f_coprs):
+ def test_copr_new_exists_for_this_user(self, f_users, f_coprs, f_mock_chroots):
with self.tc as c:
with c.session_transaction() as s:
s['openid'] = self.u1.openid_name
@@ -95,17 +96,17 @@ class TestCoprNew(CoprsTestCase):
foocoprs = len(self.models.Copr.query.filter(self.models.Copr.name == self.c1.name).all())
assert foocoprs > 0
- r = c.post('/coprs/new/', data = {'name': self.c1.name, 'release': 'fedora-rawhide', 'arches': ['i386']}, follow_redirects = True)
+ r = c.post('/coprs/new/', data = {'name': self.c1.name, 'fedora-rawhide-i386': 'y'}, follow_redirects = True)
self.db.session.add(self.c1)
assert len(self.models.Copr.query.filter(self.models.Copr.name == self.c1.name).all()) == foocoprs
assert "You already have copr named" in r.data
- def test_copr_new_with_initial_pkgs(self, f_users):
+ def test_copr_new_with_initial_pkgs(self, f_users, f_mock_chroots):
with self.tc as c:
with c.session_transaction() as s:
s['openid'] = self.u1.openid_name
- r = c.post('/coprs/new/', data = {'name': 'foo', 'release': 'fedora-rawhide', 'arches': ['i386'], 'initial_pkgs': ['http://f', 'http://b']}, follow_redirects = True)
+ r = c.post('/coprs/new/', data = {'name': 'foo', 'fedora-rawhide-i386': 'y', 'initial_pkgs': ['http://f', 'http://b']}, follow_redirects = True)
copr = self.models.Copr.query.filter(self.models.Copr.name == 'foo').first()
assert copr
assert self.success_string in r.data
@@ -210,7 +211,7 @@ class TestCoprEdit(CoprsTestCase):
class TestCoprUpdate(CoprsTestCase):
- def test_update_no_changes(self, f_users, f_coprs):
+ def test_update_no_changes(self, f_users, f_coprs, f_mock_chroots):
with self.tc as c:
with c.session_transaction() as s:
s['openid'] = self.u1.openid_name
@@ -221,7 +222,7 @@ class TestCoprUpdate(CoprsTestCase):
follow_redirects = True)
assert 'Copr was updated successfully' in r.data
- def test_copr_admin_can_update(self, f_users, f_coprs, f_copr_permissions):
+ def test_copr_admin_can_update(self, f_users, f_coprs, f_copr_permissions, f_mock_chroots):
with self.tc as c:
with c.session_transaction() as s:
s['openid'] = self.u1.openid_name
@@ -230,6 +231,7 @@ class TestCoprUpdate(CoprsTestCase):
r = c.post('/coprs/detail/{0}/{1}/update/'.format(self.u2.name, self.c3.name),
data = {'name': self.c3.name, 'fedora-rawhide-i386': 'y', 'id': self.c3.id},
follow_redirects = True)
+ print r.data
assert 'Copr was updated successfully' in r.data
def test_update_multiple_chroots(self, f_users, f_coprs, f_copr_permissions, f_mock_chroots):
11 years, 4 months
[copr] master: Only bind the testing mock_chroots to coprs if we use coprs fixture (8cb3525)
by bkabrda＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 8cb3525da922781b9d0a7720299cee6775d091c0
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Tue Jan 15 11:44:44 2013 +0100
Only bind the testing mock_chroots to coprs if we use coprs fixture
>---------------------------------------------------------------
coprs_frontend/tests/coprs_test_case.py | 30 ++++++++++++++++--------------
1 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/coprs_frontend/tests/coprs_test_case.py b/coprs_frontend/tests/coprs_test_case.py
index d711c7f..c0d2838 100644
--- a/coprs_frontend/tests/coprs_test_case.py
+++ b/coprs_frontend/tests/coprs_test_case.py
@@ -59,20 +59,22 @@ class CoprsTestCase(object):
self.mc3 = models.MockChroot(os_release='fedora', os_version='17', arch='i386', is_active=True)
self.mc4 = models.MockChroot(os_release='fedora', os_version='rawhide', arch='i386', is_active=True)
- cc1 = models.CoprChroot()
- cc1.mock_chroot = self.mc1
- self.c1.copr_chroots.append(cc1)
-
- cc2 = models.CoprChroot()
- cc2.mock_chroot = self.mc2
- cc3 = models.CoprChroot()
- cc3.mock_chroot = self.mc3
- self.c2.copr_chroots.append(cc2)
- self.c2.copr_chroots.append(cc3)
-
- cc4 = models.CoprChroot()
- cc4.mock_chroot = self.mc4
- self.c3.copr_chroots.append(cc4)
+ # only bind to coprs if the test has used the f_coprs fixture
+ if hasattr(self, 'c1'):
+ cc1 = models.CoprChroot()
+ cc1.mock_chroot = self.mc1
+ self.c1.copr_chroots.append(cc1)
+
+ cc2 = models.CoprChroot()
+ cc2.mock_chroot = self.mc2
+ cc3 = models.CoprChroot()
+ cc3.mock_chroot = self.mc3
+ self.c2.copr_chroots.append(cc2)
+ self.c2.copr_chroots.append(cc3)
+
+ cc4 = models.CoprChroot()
+ cc4.mock_chroot = self.mc4
+ self.c3.copr_chroots.append(cc4)
self.db.session.add_all([self.mc1, self.mc2, self.mc3, self.mc4])
self.db.session.commit()
11 years, 4 months
[copr] master: Mandate at least one mock chroot per copr (734f797)
by bkabrda＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 734f7978bd2a81eea1dc4a6d95a33cca564f4322
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Tue Jan 15 10:48:04 2013 +0100
Mandate at least one mock chroot per copr
>---------------------------------------------------------------
coprs_frontend/coprs/forms.py | 16 ++++++++++++++++
.../coprs/templates/coprs/_coprs_forms.html | 3 +++
2 files changed, 19 insertions(+), 0 deletions(-)
diff --git a/coprs_frontend/coprs/forms.py b/coprs_frontend/coprs/forms.py
index 3a8fc59..cfca500 100644
--- a/coprs_frontend/coprs/forms.py
+++ b/coprs_frontend/coprs/forms.py
@@ -90,6 +90,22 @@ class CoprFormFactory(object):
selected.append(ch)
return selected
+ def validate(self):
+ if not super(F, self).validate():
+ return False
+
+ if not self.validate_mock_chroots_not_empty():
+ self._mock_chroots_error = 'At least one chroot must be selected'
+ return False
+ return True
+
+ def validate_mock_chroots_not_empty(self):
+ have_any = False
+ for c in self.chroots_list:
+ if getattr(self, c).data:
+ have_any = True
+ return have_any
+
F.chroots_list = map(lambda x: x.chroot_name, models.MockChroot.query.filter(models.MockChroot.is_active==True).all())
F.chroots_list.sort()
F.chroots_sets = {} # sets of chroots according to how we should print them in columns
diff --git a/coprs_frontend/coprs/templates/coprs/_coprs_forms.html b/coprs_frontend/coprs/templates/coprs/_coprs_forms.html
index 7b90310..4256bdc 100644
--- a/coprs_frontend/coprs/templates/coprs/_coprs_forms.html
+++ b/coprs_frontend/coprs/templates/coprs/_coprs_forms.html
@@ -4,6 +4,9 @@
{# if using for updating, we need to pass name to url_for, but otherwise we need to pass nothing #}
<form action="{% if copr %}{{ url_for(view, username = copr.owner.name, coprname = copr.name) }}{% else %}{{ url_for(view) }}{% endif %}" method=post class=add-entry>
<dl>
+ {% if form._mock_chroots_error %}
+ <p style="color:red;">{{ form._mock_chroots_error }}</p>
+ {% endif %}
{{ form.csrf_token }}
{{ render_field(form.id, hidden = True) }}
{{ render_field(form.name) }}
11 years, 4 months
[copr] master: Improve the migration to actually migrate data, support migrating down (07fd65b)
by bkabrda＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 07fd65b8cca7fcf48367aa480b7483979c6325ec
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Tue Jan 15 10:22:13 2013 +0100
Improve the migration to actually migrate data, support migrating down
>---------------------------------------------------------------
.../versions/2fa80e062525_add_mock_chroots.py | 37 ++++++++++++++++++-
1 files changed, 35 insertions(+), 2 deletions(-)
diff --git a/coprs_frontend/alembic/versions/2fa80e062525_add_mock_chroots.py b/coprs_frontend/alembic/versions/2fa80e062525_add_mock_chroots.py
index 7d77a35..fdd88b3 100644
--- a/coprs_frontend/alembic/versions/2fa80e062525_add_mock_chroots.py
+++ b/coprs_frontend/alembic/versions/2fa80e062525_add_mock_chroots.py
@@ -13,7 +13,6 @@ down_revision = '2e30169e58ce'
from alembic import op
import sqlalchemy as sa
-
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.create_table('mock_chroot',
@@ -31,6 +30,40 @@ def upgrade():
sa.ForeignKeyConstraint(['mock_chroot_id'], ['mock_chroot.id'], ),
sa.PrimaryKeyConstraint('mock_chroot_id', 'copr_id')
)
+
+
+ # transfer the data - we can't assume how the code looks like when
+ # running the migration, so do everything from scratch
+ session = sa.orm.sessionmaker(bind=op.get_bind())()
+ metadata = sa.MetaData()
+ # just what we need of copr table
+ coprs_table = sa.Table('copr', metadata, sa.Column('chroots', sa.Text()), sa.Column('id', sa.Integer()))
+ # get chroots
+ chroots = set()
+ for cs in op.get_bind().execute(sa.select([coprs_table.c.chroots])):
+ chroots.update(set(cs[0].split(' ')))
+ chroots = list(chroots)
+
+ mc_table = sa.Table('mock_chroot', metadata,
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('os_release', sa.String(length=50), nullable=False),
+ sa.Column('os_version', sa.String(length=50), nullable=False),
+ sa.Column('arch', sa.String(length=50), nullable=False),
+ sa.Column('is_active', sa.Boolean(), nullable=False),
+ )
+ cc_table = sa.Table('copr_chroot', metadata,
+ sa.Column('mock_chroot_id', sa.Integer(), nullable=False),
+ sa.Column('copr_id', sa.Integer(), nullable=False),
+ )
+ for i, c in enumerate(chroots): # each mock_chroot now has id of value i + 1 (not to include 0)
+ sc = c.split('-')
+ op.bulk_insert(mc_table, [{'id': i + 1, 'os_release': sc[0], 'os_version': sc[1], 'arch': sc[2], 'is_active': True}])
+ # insert proper copr_chroots for every copr
+ for row in op.get_bind().execute(sa.select([coprs_table.c.id, coprs_table.c.chroots])):
+ for c in row[1].split(' '):
+ op.bulk_insert(cc_table, [{'mock_chroot_id': chroots.index(c) + 1, 'copr_id': row[0]}])
+
+
if op.get_bind().dialect.name == 'sqlite':
op.rename_table('copr', 'copr_1')
op.create_table('copr',
@@ -52,7 +85,7 @@ def upgrade():
def downgrade():
### commands auto generated by Alembic - please adjust! ###
- op.add_column('copr', sa.Column(u'chroots', sa.TEXT(), nullable=False))
+ op.add_column('copr', sa.Column(u'chroots', sa.TEXT(), nullable=False, server_default='fedora-rawhide-x86_64'))
op.drop_table('copr_chroot')
op.drop_table('mock_chroot')
### end Alembic commands ###
11 years, 4 months
[copr] master: Add a new testcase for chroots, couple of fixes for the fixtures (6020ac3)
by bkabrda＠fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 6020ac369859abb7477cfa69b80122cbba062d2b
Author: Bohuslav Kabrda <bkabrda(a)redhat.com>
Date: Tue Jan 15 08:35:16 2013 +0100
Add a new testcase for chroots, couple of fixes for the fixtures
>---------------------------------------------------------------
coprs_frontend/tests/coprs_test_case.py | 17 ++++++++++-------
.../tests/test_views/test_coprs_ns/test_general.py | 15 +++++++++++++++
2 files changed, 25 insertions(+), 7 deletions(-)
diff --git a/coprs_frontend/tests/coprs_test_case.py b/coprs_frontend/tests/coprs_test_case.py
index bae7b73..d711c7f 100644
--- a/coprs_frontend/tests/coprs_test_case.py
+++ b/coprs_frontend/tests/coprs_test_case.py
@@ -54,10 +54,10 @@ class CoprsTestCase(object):
@pytest.fixture
def f_mock_chroots(self):
- self.mc1 = models.MockChroot(os_release='fedora', os_version='18', arch='x86_64')
- self.mc2 = models.MockChroot(os_release='fedora', os_version='17', arch='x86_64')
- self.mc3 = models.MockChroot(os_release='fedora', os_version='17', arch='i386')
- self.mc4 = models.MockChroot(os_release='fedora', os_version='rawhide', arch='i386')
+ self.mc1 = models.MockChroot(os_release='fedora', os_version='18', arch='x86_64', is_active=True)
+ self.mc2 = models.MockChroot(os_release='fedora', os_version='17', arch='x86_64', is_active=True)
+ self.mc3 = models.MockChroot(os_release='fedora', os_version='17', arch='i386', is_active=True)
+ self.mc4 = models.MockChroot(os_release='fedora', os_version='rawhide', arch='i386', is_active=True)
cc1 = models.CoprChroot()
cc1.mock_chroot = self.mc1
@@ -66,13 +66,16 @@ class CoprsTestCase(object):
cc2 = models.CoprChroot()
cc2.mock_chroot = self.mc2
cc3 = models.CoprChroot()
- cc3.mock_chroot = self.m3
+ cc3.mock_chroot = self.mc3
self.c2.copr_chroots.append(cc2)
self.c2.copr_chroots.append(cc3)
- cc4 = models.CoprChroots()
+ cc4 = models.CoprChroot()
cc4.mock_chroot = self.mc4
- self.c4.copr_chroots.append(cc4)
+ self.c3.copr_chroots.append(cc4)
+
+ self.db.session.add_all([self.mc1, self.mc2, self.mc3, self.mc4])
+ self.db.session.commit()
@pytest.fixture
def f_builds(self):
diff --git a/coprs_frontend/tests/test_views/test_coprs_ns/test_general.py b/coprs_frontend/tests/test_views/test_coprs_ns/test_general.py
index ac87e44..931e70c 100644
--- a/coprs_frontend/tests/test_views/test_coprs_ns/test_general.py
+++ b/coprs_frontend/tests/test_views/test_coprs_ns/test_general.py
@@ -232,6 +232,21 @@ class TestCoprUpdate(CoprsTestCase):
follow_redirects = True)
assert 'Copr was updated successfully' in r.data
+ def test_update_multiple_chroots(self, f_users, f_coprs, f_copr_permissions, f_mock_chroots):
+ with self.tc as c:
+ with c.session_transaction() as s:
+ s['openid'] = self.u1.openid_name
+
+ self.db.session.add_all([self.u1, self.c1, self.mc2, self.mc3])
+ r = c.post('/coprs/detail/{0}/{1}/update/'.format(self.u1.name, self.c1.name),
+ data = {'name': self.c1.name, self.mc2.chroot_name: 'y', self.mc3.chroot_name: 'y', 'id': self.c1.id},
+ follow_redirects = True)
+ self.db.session.add_all([self.mc1, self.mc2, self.mc3])
+ assert 'Copr was updated successfully' in r.data
+ assert self.mc2.chroot_name in r.data
+ assert self.mc3.chroot_name in r.data
+ assert self.mc1.chroot_name not in r.data
+
class TestCoprApplyForPermissions(CoprsTestCase):
def test_apply(self, f_users, f_coprs):
11 years, 4 months
