[copr] skvidal-backend: add some more logging - trying to figure out why the mockremote call is failing (81c2267)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 81c2267ca82c9f94f36dc534c472e190a92b4168
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 15:03:25 2012 -0500
add some more logging - trying to figure out why the mockremote call is failing
>---------------------------------------------------------------
backend/dispatcher.py | 2 +-
copr-be.py | 4 +---
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 0fc1ada..cd387da 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -175,7 +175,7 @@ class Worker(multiprocessing.Process):
status = 1
job.started_on = time.time()
for chroot in job.chroots:
- self.callback.log('mockremote on %s - %s' % (ip, jobfile))
+ self.callback.log('mockremote %s %s %s %s %s' % (ip, job.timeout, job.destdir, chroot, str(repos)))
try:
mr = mockremote.MockRemote(builder=ip, timeout=job.timeout,
destdir=job.destdir, chroot=chroot, cont=True, recurse=True,
diff --git a/copr-be.py b/copr-be.py
index 2d2df43..7ebc4c4 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -97,7 +97,7 @@ class CoprBackend(object):
self.log('adding %s' % n)
if self.jobs.qsize():
-
+ self.log("# jobs in queue: %s" % self.jobs.qsize())
# re-read config into opts
self.opts = self.read_conf()
# this handles starting/growing the number of workers
@@ -116,8 +116,6 @@ class CoprBackend(object):
# FIXME - if a worker bombs out - we need to check them
# and startup a new one if it happens
- self.log("# jobs in queue: %s" % self.jobs.qsize())
-
time.sleep(self.opts.sleeptime)
11 years, 5 months
[copr] skvidal-backend: add a few log outputs so I know where it is (f43545a)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit f43545a565e1866133d0ffdec2cae6bc3965da0f
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 14:47:15 2012 -0500
add a few log outputs so I know where it is
>---------------------------------------------------------------
copr-be.py | 9 +++++++--
1 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 7e9e3c4..2d2df43 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -33,6 +33,11 @@ class CoprBackend(object):
if not os.path.exists(logdir):
os.makedirs(logdir, mode=0750)
+ # setup a log file to write to
+ self.logfile = self.opts.logfile
+ self.log("Starting up new copr-be instance")
+
+
if not os.path.exists(self.opts.worker_logdir):
os.makedirs(self.opts.worker_logdir, mode=0750)
@@ -40,8 +45,6 @@ class CoprBackend(object):
self.workers = []
self.added_jobs = []
- # setup a log file to write to
- self.logfile = self.opts.logfile
def read_conf(self):
"read in config file - return Bunch of config data"
@@ -94,10 +97,12 @@ class CoprBackend(object):
self.log('adding %s' % n)
if self.jobs.qsize():
+
# re-read config into opts
self.opts = self.read_conf()
# this handles starting/growing the number of workers
if len(self.workers) < self.opts.num_workers:
+ self.log("Spinning up more workers for jobs")
for i in range(self.opts.num_workers - len(self.workers)):
worker_num = len(self.workers) + 1
w = Worker(self.opts, self.jobs, worker_num)
11 years, 5 months
[copr] skvidal-backend: logs should go to a better place now (1412938)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 1412938b5e9411c72ab41f5563e0e493b6e6867c
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 14:41:26 2012 -0500
logs should go to a better place now
>---------------------------------------------------------------
backend/dispatcher.py | 7 +++++--
copr-be.py | 3 ++-
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 3c0e160..0fc1ada 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -81,7 +81,7 @@ class WorkerCallback(object):
class Worker(multiprocessing.Process):
- def __init__(self, opts, jobs, ip=None, create=True, callback=None):
+ def __init__(self, opts, jobs, worker_num, ip=None, create=True, callback=None):
# base class initialization
multiprocessing.Process.__init__(self, name="worker-builder")
@@ -89,12 +89,13 @@ class Worker(multiprocessing.Process):
# job management stuff
self.jobs = jobs
+ self.worker_num = worker_num
self.ip = ip
self.opts = opts
self.kill_received = False
self.callback = callback
if not self.callback:
- lf = self.opts.worker_logdir + '/worker-%s.log' % self.pid
+ lf = self.opts.worker_logdir + '/worker-%s.log' % self.worker_num
self.callback = WorkerCallback(logfile = lf)
self.callback.log('creating worker: %s' % ip)
@@ -188,6 +189,8 @@ class Worker(multiprocessing.Process):
job.ended_on = time.time()
job.status = status
self.return_results(job)
+ self.callback.log('worker finished build: %s' % ip)
+ # call terminate on the instance
diff --git a/copr-be.py b/copr-be.py
index da97128..7e9e3c4 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -99,7 +99,8 @@ class CoprBackend(object):
# this handles starting/growing the number of workers
if len(self.workers) < self.opts.num_workers:
for i in range(self.opts.num_workers - len(self.workers)):
- w = Worker(self.opts, self.jobs)
+ worker_num = len(self.workers) + 1
+ w = Worker(self.opts, self.jobs, worker_num)
self.workers.append(w)
w.start()
# FIXME - prune out workers
11 years, 5 months
[copr] skvidal-backend: log to the right place add a fixme for later (3ca4f10)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 3ca4f109ded164a0d70ce2181ed15bbe4befeee5
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 13:22:17 2012 -0500
log to the right place
add a fixme for later
>---------------------------------------------------------------
backend/dispatcher.py | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 7a8f126..3c0e160 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -19,6 +19,8 @@ from ansible import callbacks
+#FIXME - this should be emitting to the per-worker log file
+# so we can know what is going on and where
class SilentPlaybookCallbacks(callbacks.PlaybookCallbacks):
''' playbook callbacks - quietly! '''
@@ -136,7 +138,7 @@ class Worker(multiprocessing.Process):
return jobdata
def return_results(self, job):
- self.log('%s status %s. Took %s seconds' % (job.id, job.status, job.ended_on - job.startedon))
+ self.callback.log('%s status %s. Took %s seconds' % (job.id, job.status, job.ended_on - job.startedon))
os.unlink(job.jobfile)
#FIXME - this should either return job status/results
# into a queue or it should submit results directly to the frontend
11 years, 5 months
[copr] skvidal-backend: locals: (bf225bb)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit bf225bbaebd9d5003a8febe9a4052fa8f73291f8
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 09:52:07 2012 -0500
locals:
>---------------------------------------------------------------
copr-be.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index fcf3f44..da97128 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -124,7 +124,7 @@ def main(args):
cbe.run()
except Exception, e:
print 'Killing/Dying'
- if cbe in locals()
+ if 'cbe' in locals():
for w in cbe.workers:
w.terminate()
raise
11 years, 5 months
[copr] skvidal-backend: only spin up any workers if we have jobs (947827f)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 947827fa5d0862aa57a46c438538df02c159b77d
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 09:49:01 2012 -0500
only spin up any workers if we have jobs
>---------------------------------------------------------------
copr-be.py | 34 +++++++++++++++++-----------------
1 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 9db9f17..fcf3f44 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -86,7 +86,6 @@ class CoprBackend(object):
abort = False
while not abort:
- print 'adding jobs'
for f in sorted(glob.glob(self.opts.jobsdir + '/*.json')):
n = os.path.basename(f).replace('.json', '')
if n not in self.added_jobs:
@@ -94,25 +93,26 @@ class CoprBackend(object):
self.added_jobs.append(n)
self.log('adding %s' % n)
- # re-read config into opts
- self.opts= self.read_conf()
- # this handles starting/growing the number of workers
- if len(self.workers) < self.opts.num_workers:
- for i in range(self.opts.num_workers - len(self.workers)):
- w = Worker(self.opts, self.jobs)
- self.workers.append(w)
- w.start()
- # FIXME - prune out workers
- #if len(self.workers) > self.opts.num_workers:
- # killnum = len(self.workers) - self.opts.num_workers
- # for w in self.workers[:killnum]:
- # #insert a poison pill? Kill after something? I dunno.
- # FIXME - if a worker bombs out - we need to check them
- # and startup a new one if it happens
+ if self.jobs.qsize():
+ # re-read config into opts
+ self.opts = self.read_conf()
+ # this handles starting/growing the number of workers
+ if len(self.workers) < self.opts.num_workers:
+ for i in range(self.opts.num_workers - len(self.workers)):
+ w = Worker(self.opts, self.jobs)
+ self.workers.append(w)
+ w.start()
+ # FIXME - prune out workers
+ #if len(self.workers) > self.opts.num_workers:
+ # killnum = len(self.workers) - self.opts.num_workers
+ # for w in self.workers[:killnum]:
+ # #insert a poison pill? Kill after something? I dunno.
+ # FIXME - if a worker bombs out - we need to check them
+ # and startup a new one if it happens
self.log("# jobs in queue: %s" % self.jobs.qsize())
- time.sleep(self.opts.sleeptime)
+ time.sleep(self.opts.sleeptime)
def main(args):
11 years, 5 months
[copr] skvidal-backend: thinkos/cleanups and make sure we reread the config file before starting up builders (7aa741d)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 7aa741d82b5268f118f6d380b3ac9a61c0c7ed5f
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 09:45:57 2012 -0500
thinkos/cleanups and make sure we reread the config file before
starting up builders
>---------------------------------------------------------------
copr-be.py | 8 +++++---
1 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 313f075..9db9f17 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -9,7 +9,7 @@ import multiprocessing
from backend.dispatcher import Worker
from backend import errors
from bunch import Bunch
-from ConfigParser import ConfigParser
+import ConfigParser
def _get_conf(cp, section, option, default):
"""to make returning items from config parser less irritating"""
@@ -27,14 +27,14 @@ class CoprBackend(object):
raise errors.CoprBackendError, "Must specify config_file"
self.config_file = config_file
- self.opts = self.read_config()
+ self.opts = self.read_conf()
logdir = os.path.dirname(self.opts.logfile)
if not os.path.exists(logdir):
os.makedirs(logdir, mode=0750)
if not os.path.exists(self.opts.worker_logdir):
- os.makedirs(self.opts.worker_logdir, module=0750)
+ os.makedirs(self.opts.worker_logdir, mode=0750)
self.jobs = multiprocessing.Queue()
self.workers = []
@@ -94,6 +94,8 @@ class CoprBackend(object):
self.added_jobs.append(n)
self.log('adding %s' % n)
+ # re-read config into opts
+ self.opts= self.read_conf()
# this handles starting/growing the number of workers
if len(self.workers) < self.opts.num_workers:
for i in range(self.opts.num_workers - len(self.workers)):
11 years, 5 months
[copr] skvidal-backend: when we abort- show why (f8ae662)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit f8ae662188d968a793c5c0ccfd2232b3e3bd7a8b
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 09:39:38 2012 -0500
when we abort- show why
>---------------------------------------------------------------
copr-be.py | 5 +++--
1 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index fcf493a..313f075 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -122,8 +122,9 @@ def main(args):
cbe.run()
except Exception, e:
print 'Killing/Dying'
- for w in cbe.workers:
- w.terminate()
+ if cbe in locals()
+ for w in cbe.workers:
+ w.terminate()
raise
11 years, 5 months
[copr] skvidal-backend: callbacks fix (753cefd)
by skvidal＠fedorahosted.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 753cefd248ccf1e0648ca5e5b9a97a15f7bfccb2
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 09:36:52 2012 -0500
callbacks fix
>---------------------------------------------------------------
backend/dispatcher.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 3c03d87..7a8f126 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -19,7 +19,7 @@ from ansible import callbacks
-class SilentPlaybookCallbacks(callbacks.object):
+class SilentPlaybookCallbacks(callbacks.PlaybookCallbacks):
''' playbook callbacks - quietly! '''
def __init__(self, verbose=False):
11 years, 5 months
[copr] skvidal-backend: fix up a few tyops (ee51389)
by Seth Vidal
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit ee51389a7802fa93f7f6e491fbd3fa10c0048f98
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Nov 16 09:33:34 2012 -0500
fix up a few tyops
>---------------------------------------------------------------
backend/dispatcher.py | 2 +-
copr-be.py | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index 4948512..3c03d87 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -136,7 +136,7 @@ class Worker(multiprocessing.Process):
return jobdata
def return_results(self, job):
- self.log('%s status %s. Took %s seconds' % (job.id, job.status, build.ended_on - build.startedon)
+ self.log('%s status %s. Took %s seconds' % (job.id, job.status, job.ended_on - job.startedon))
os.unlink(job.jobfile)
#FIXME - this should either return job status/results
# into a queue or it should submit results directly to the frontend
diff --git a/copr-be.py b/copr-be.py
index 0e3e01a..fcf493a 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -50,7 +50,7 @@ class CoprBackend(object):
try:
cp.read(self.config_file)
opts.results_baseurl = _get_conf(cp,'backend', 'results_baseurl', 'http://copr')
- opts.frontend_url = _get_config(cp, 'backend', 'frontend_url', 'http://coprs/rest/api')
+ opts.frontend_url = _get_conf(cp, 'backend', 'frontend_url', 'http://coprs/rest/api')
opts.frontend_auth = _get_conf(cp,'backend', 'frontend_auth', 'PASSWORDHERE')
opts.playbook = _get_conf(cp,'backend','playbook', '/etc/copr/builder_playbook.yml')
opts.jobsdir = _get_conf(cp, 'backend', 'jobsdir', None)
@@ -97,7 +97,7 @@ class CoprBackend(object):
# this handles starting/growing the number of workers
if len(self.workers) < self.opts.num_workers:
for i in range(self.opts.num_workers - len(self.workers)):
- w = dispatcher.Worker(self.opts, self.jobs)
+ w = Worker(self.opts, self.jobs)
self.workers.append(w)
w.start()
# FIXME - prune out workers
@@ -108,7 +108,7 @@ class CoprBackend(object):
# FIXME - if a worker bombs out - we need to check them
# and startup a new one if it happens
- self.log("# jobs in queue: %s" % jobs.qsize())
+ self.log("# jobs in queue: %s" % self.jobs.qsize())
time.sleep(self.opts.sleeptime)
11 years, 5 months