3:53 a.m.
---
backend/backend/daemons/backend.py | 11 ++++++-----
backend/backend/daemons/job_grab.py | 7 ++++---
backend/backend/frontend.py | 9 ++++++---
backend/run/copr_run_job_grab.py | 4 +---
4 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py
index 8818f71..934a566 100644
--- a/backend/backend/daemons/backend.py
+++ b/backend/backend/daemons/backend.py
@@ -52,11 +52,11 @@ class CoprBackend(object):
self.task_queues = {}
- self.frontend_client = FrontendClient(self.opts)
- self.is_running = False
-
self.log = get_redis_logger(self.opts, "backend.main",
"backend")
+ self.frontend_client = FrontendClient(self.opts, self.log)
+ self.is_running = False
+
def clean_task_queues(self):
"""
Make sure there is nothing in our task queues
@@ -162,7 +162,7 @@ class CoprBackend(object):
try:
self.log.info("Rescheduling unfinished builds before stop")
- self.frontend_client.reschedule_all_running()
+ self.frontend_client.reschedule_all_running(5)
except RequestException as err:
self.log.exception(err)
return
@@ -178,7 +178,8 @@ class CoprBackend(object):
try:
self.log.info("Rescheduling old unfinished builds")
- self.frontend_client.reschedule_all_running()
+ # 120*5 = 10 minutes
+ self.frontend_client.reschedule_all_running(120)
except RequestException as err:
self.log.exception(err)
return
diff --git a/backend/backend/daemons/job_grab.py b/backend/backend/daemons/job_grab.py
index 558ae57..21871ae 100644
--- a/backend/backend/daemons/job_grab.py
+++ b/backend/backend/daemons/job_grab.py
@@ -13,6 +13,8 @@ from requests import get, RequestException
from retask.task import Task
from retask.queue import Queue
+from backend.frontend import FrontendClient
+
from ..actions import Action
from ..constants import JOB_GRAB_TASK_END_PUBSUB
from ..helpers import get_redis_connection, get_redis_logger
@@ -32,12 +34,11 @@ class CoprJobGrab(object):
:param Munch opts: backend config
- :type frontend_client: FrontendClient
:param lock: :py:class:`multiprocessing.Lock` global backend lock
"""
- def __init__(self, opts, frontend_client):
+ def __init__(self, opts):
""" base class initialization """
self.opts = opts
@@ -51,13 +52,13 @@ class CoprJobGrab(object):
self.added_jobs_dict = dict() # task_id -> task dict
- self.frontend_client = frontend_client
self.rc = None
self.channel = None
self.ps_thread = None
self.log = get_redis_logger(self.opts, "backend.job_grab",
"job_grab")
+ self.frontend_client = FrontendClient(self.opts, self.log)
def connect_queues(self):
"""
diff --git a/backend/backend/frontend.py b/backend/backend/frontend.py
index 52c91b6..6f442d3 100644
--- a/backend/backend/frontend.py
+++ b/backend/backend/frontend.py
@@ -8,12 +8,13 @@ class FrontendClient(object):
Object to send data back to fronted
"""
- def __init__(self, opts):
+ def __init__(self, opts, logger=None):
super(FrontendClient, self).__init__()
self.frontend_url = "{}/backend".format(opts.frontend_base_url)
self.frontend_auth = opts.frontend_auth
self.msg = None
+ self.log = logger
def _post_to_frontend(self, data, url_path):
"""
@@ -43,6 +44,8 @@ class FrontendClient(object):
"""
for i in range(max_repeats):
try:
+ if i and self.log:
+ self.log.warning("failed to post data to frontend, repeat
#{0}".format(i))
return self._post_to_frontend(data, url_path)
except RequestException:
time.sleep(5)
@@ -74,7 +77,7 @@ class FrontendClient(object):
data = {"build_id": build_id, "chroot": chroot_name}
self._post_to_frontend_repeatedly(data, "reschedule_build_chroot")
- def reschedule_all_running(self):
- response = self._post_to_frontend({}, "reschedule_all_running")
+ def reschedule_all_running(self, attempts):
+ response = self._post_to_frontend_repeatedly({},
"reschedule_all_running", attempts)
if response.status_code != 200:
raise RequestException("Failed to reschedule all running jobs")
diff --git a/backend/run/copr_run_job_grab.py b/backend/run/copr_run_job_grab.py
index b39fabe..d1ce730 100755
--- a/backend/run/copr_run_job_grab.py
+++ b/backend/run/copr_run_job_grab.py
@@ -11,13 +11,11 @@ sys.path.append("/usr/share/copr/")
from backend.helpers import get_backend_opts
from backend.daemons.job_grab import CoprJobGrab
-from backend.frontend import FrontendClient
def main():
opts = get_backend_opts()
- fc = FrontendClient(opts)
- grabber = CoprJobGrab(opts, frontend_client=fc)
+ grabber = CoprJobGrab(opts)
grabber.run()
--
2.5.0