cumin-developers December 2010

cumin-developers@lists.fedorahosted.org

4 participants
18 discussions

r4445 - branches/scale_testing/mint/python/mint
by tmckay＠fedoraproject.org 23 Dec '10

23 Dec '10

Author: tmckay Date: 2010-12-23 20:14:11 +0000 (Thu, 23 Dec 2010) New Revision: 4445 Modified: branches/scale_testing/mint/python/mint/update.py Log: Fix up null object stats cases. Apparently Python 2.4 deals with None + int differently than 2.6 (or running on grid0 directly causes a different execution path) Modified: branches/scale_testing/mint/python/mint/update.py =================================================================== --- branches/scale_testing/mint/python/mint/update.py 2010-12-22 13:32:50 UTC (rev 4444) +++ branches/scale_testing/mint/python/mint/update.py 2010-12-23 20:14:11 UTC (rev 4445) @@ -116,6 +116,13 @@ self.objects_deleted_by_class = defaultdict(int) self.objects_dropped_by_class = defaultdict(int) + for pkg in app.model._packages: + for cls in pkg._classes: + self.objects_created_by_class[cls] = 0 + self.objects_updated_by_class[cls] = 0 + self.objects_deleted_by_class[cls] = 0 + self.objects_dropped_by_class[cls] = 0 + self.sql_ops = 0 self.errors = 0 @@ -351,6 +358,8 @@ except: cls = None if cls != None: + if thread.stats.objects_dropped_by_class == None: + thread.stats.objects_dropped_by_class = 0 thread.stats.objects_dropped_by_class[cls] += 1 except: @@ -510,6 +519,8 @@ self.model.print_event(3, "Created %s", obj) stats.objects_created += 1 + if stats.objects_created_by_class[cls] == None: + stats.objects_created_by_class[cls] = 0 stats.objects_created_by_class[cls] += 1 return obj @@ -571,6 +582,8 @@ self.model.print_event(4, "Updated %s", obj) stats.objects_updated += 1 + if stats.objects_updated_by_class[cls] == None: + stats.objects_updated_by_class[cls] = 0 stats.objects_updated_by_class[cls] += 1 def delete_object(self, cursor, stats, obj): @@ -579,6 +592,8 @@ self.model.print_event(3, "Deleted %s", obj) stats.objects_deleted += 1 + if stats.objects_deleted_by_class[cls] == None: + stats.objects_deleted_by_class[cls] = 0 stats.objects_deleted_by_class[obj._class] += 1 def process_properties(self, obj, columns, cursor):

1 0

r4444 - branches/scale_testing/mint/python/mint
by tmckay＠fedoraproject.org 22 Dec '10

22 Dec '10

Author: tmckay Date: 2010-12-22 13:32:50 +0000 (Wed, 22 Dec 2010) New Revision: 4444 Modified: branches/scale_testing/mint/python/mint/update.py Log: Tweak stats collecting, get rid of exceptions and bad values on zero message intervals or dropped updates Modified: branches/scale_testing/mint/python/mint/update.py =================================================================== --- branches/scale_testing/mint/python/mint/update.py 2010-12-21 20:15:51 UTC (rev 4443) +++ branches/scale_testing/mint/python/mint/update.py 2010-12-22 13:32:50 UTC (rev 4444) @@ -36,6 +36,10 @@ def enqueue(self, update): update.enqueue_time = time.time() + # make sure these exist to avoid exceptions if processing + # is truncated for some reason + update.start_process = update.finish_process = \ + update.after_process = update.after_commit = update.enqueue_time self.updates.put(update) @@ -149,6 +153,7 @@ update.after_commit - update.after_process except: print "record update exception" + print_exc() def capture(self): now = copy.copy(self) @@ -261,7 +266,7 @@ def get_update_durations(self): total = self.now.record_samples - if total == 0: + if total == 0 or self.now.dequeued == self.then.dequeued: create_dur = proc_dur = que_dur = init_proc_time = commit_time = 0 else: create_dur = self.now.update_queue_duration / total @@ -322,6 +327,7 @@ def process(self, thread): self.start_process = time.time() + log.debug("Processing %s", self) try: @@ -330,10 +336,15 @@ thread.conn.commit() self.after_commit = time.time() except UpdateDropped: + # Don't want to throw off stats with negatives if + # these are never set. Assume the time to reach + # this point. + self.after_process = time.time() + self.after_commit = self.after_process + log.debug("Update dropped") thread.conn.rollback() - thread.stats.dropped += 1 try: cls = self.get_class() @@ -341,7 +352,6 @@ cls = None if cls != None: thread.stats.objects_dropped_by_class[cls] += 1 - self.after_commit = time.time() except: log.exception("Update failed")

1 0

r4443 - in branches/scale_testing: cumin/bin cumin/python/cumin mint/python/mint
by tmckay＠fedoraproject.org 21 Dec '10

21 Dec '10

Author: tmckay Date: 2010-12-21 20:15:51 +0000 (Tue, 21 Dec 2010) New Revision: 4443 Modified: branches/scale_testing/cumin/bin/cumin-data branches/scale_testing/cumin/python/cumin/config.py branches/scale_testing/mint/python/mint/main.py branches/scale_testing/mint/python/mint/session.py branches/scale_testing/mint/python/mint/update.py Log: Changes to collect stats in CSV files for scale testing. Modified: branches/scale_testing/cumin/bin/cumin-data =================================================================== --- branches/scale_testing/cumin/bin/cumin-data 2010-12-20 14:12:38 UTC (rev 4442) +++ branches/scale_testing/cumin/bin/cumin-data 2010-12-21 20:15:51 UTC (rev 4443) @@ -16,6 +16,8 @@ parser = CuminOptionParser(values) parser.add_option("--print-stats", action="store_true") + parser.add_option("--scale-stats", action="store_true") + parser.add_option("--object-stats", action="store_true") parser.add_option("--print-events", type="int", default=0, metavar="LEVEL") opts, args = parser.parse_args() @@ -26,7 +28,7 @@ broker_uris = [x.strip() for x in opts.brokers.split(",")] - mint = Mint(model_dir, broker_uris, opts.database) + mint = Mint(model_dir, broker_uris, opts.database, values.queue_size) mint.print_event_level = opts.print_events @@ -65,19 +67,40 @@ if opts.print_stats: print "[Starred columns are the number of events per second]" + # set up for scale stats here + if opts.scale_stats: + stats.init_stats_path(mint, config.home) + + if opts.print_stats or \ + opts.scale_stats or \ + opts.object_stats: while True: - if count % 20 == 0: - stats.print_headings() + stats.capture() + if opts.print_stats: + if count % 20 == 0: + stats.print_headings() + count += 1 + stats.print_values() + + if opts.object_stats: + stats.print_values_by_class(mint) - count += 1 + if opts.scale_stats: + stats.write_values_by_class(mint, time.localtime()) + stats.write_values(time.localtime()) - stats.print_values() - sleep(5) else: while True: sleep(86400) + except Exception as inst: + print inst + finally: + # close the scale stats CSV file here: + if opts.scale_stats: + pass + mint.stop() if __name__ == "__main__": Modified: branches/scale_testing/cumin/python/cumin/config.py =================================================================== --- branches/scale_testing/cumin/python/cumin/config.py 2010-12-20 14:12:38 UTC (rev 4442) +++ branches/scale_testing/cumin/python/cumin/config.py 2010-12-21 20:15:51 UTC (rev 4443) @@ -56,6 +56,9 @@ param = ConfigParameter(data, "vacuum-interval", int) param.default = 60 * 60 # 1 hour + param = ConfigParameter(data, "queue-size", int) + param.default = 1000 + def parse(self): paths = list() Modified: branches/scale_testing/mint/python/mint/main.py =================================================================== --- branches/scale_testing/mint/python/mint/main.py 2010-12-20 14:12:38 UTC (rev 4442) +++ branches/scale_testing/mint/python/mint/main.py 2010-12-21 20:15:51 UTC (rev 4443) @@ -10,14 +10,14 @@ log = logging.getLogger("mint.main") class Mint(object): - def __init__(self, model_dir, broker_uris, database_dsn): + def __init__(self, model_dir, broker_uris, database_dsn, queue_max=1000): self.model = MintModel(self, model_dir) self.model.sql_logging_enabled = False self.session = MintSession(self, broker_uris) self.database = MintDatabase(self, database_dsn) - self.update_thread = UpdateThread(self) + self.update_thread = UpdateThread(self,queue_max) self.expire_enabled = True self.expire_thread = ExpireThread(self) Modified: branches/scale_testing/mint/python/mint/session.py =================================================================== --- branches/scale_testing/mint/python/mint/session.py 2010-12-20 14:12:38 UTC (rev 4442) +++ branches/scale_testing/mint/python/mint/session.py 2010-12-21 20:15:51 UTC (rev 4443) @@ -64,14 +64,17 @@ def brokerConnected(self, qmf_broker): message = "Broker %s:%i is connected" self.model.print_event(1, message, qmf_broker.host, qmf_broker.port) + self.model.app.update_thread.ignore() def brokerInfo(self, qmf_broker): message = "Broker info from %s:%i" self.model.print_event(1, message, qmf_broker.host, qmf_broker.port) + self.model.app.update_thread.ignore() def brokerDisconnected(self, qmf_broker): message = "Broker %s:%i is disconnected" self.model.print_event(1, message, qmf_broker.host, qmf_broker.port) + self.model.app.update_thread.ignore() def newAgent(self, qmf_agent): self.model.print_event(3, "Creating %s", qmf_agent) @@ -94,9 +97,11 @@ def newPackage(self, name): self.model.print_event(2, "New package %s", name) + self.model.app.update_thread.ignore() def newClass(self, kind, classKey): self.model.print_event(2, "New class %s", classKey) + self.model.app.update_thread.ignore() def objectProps(self, broker, qmf_object): up = ObjectUpdate(self.model, qmf_object) @@ -108,6 +113,7 @@ def event(self, broker, event): self.model.print_event(4, "New event %s from %s", broker, event) + self.model.app.update_thread.ignore() def methodResponse(self, broker, seq, response): message = "Method response for request %i received from %s" @@ -121,3 +127,4 @@ callback(response.text, response.outArgs) finally: self.model.lock.release() + self.model.app.update_thread.method_response() Modified: branches/scale_testing/mint/python/mint/update.py =================================================================== --- branches/scale_testing/mint/python/mint/update.py 2010-12-20 14:12:38 UTC (rev 4442) +++ branches/scale_testing/mint/python/mint/update.py 2010-12-21 20:15:51 UTC (rev 4443) @@ -1,6 +1,8 @@ import copy import resource import pickle +import os.path +import csv from psycopg2 import IntegrityError, TimestampFromTicks from psycopg2.extensions import cursor as Cursor @@ -15,10 +17,10 @@ sample_window_max = 60 * 5 class UpdateThread(MintDaemonThread): - def __init__(self, app): + def __init__(self, app, queue_max=1000): super(UpdateThread, self).__init__(app) - self.updates = ConcurrentQueue(maxsize=1000) + self.updates = ConcurrentQueue(maxsize=queue_max) self.stats = UpdateStats(self.app) self.conn = None @@ -33,24 +35,37 @@ self.cursor.stats = self.stats def enqueue(self, update): + update.enqueue_time = time.time() + self.updates.put(update) - + self.stats.enqueued += 1 + def ignore(self): + self.stats.ignore += 1 + + def method_response(self): + self.stats.method_response +=1 + def run(self): - while True: - if self.stop_requested: - break + try: + while True: + if self.stop_requested: + break - try: - update = self.updates.get(True, 1) - except Empty: - continue + try: + update = self.updates.get(True) #, 1) + except Empty: + continue + update.dequeue_time = time.time() + self.stats.dequeued += 1 - self.stats.dequeued += 1 + update.process(self) + self.conn.commit() + + except: + print "uh oh" - update.process(self) - class UpdateStats(object): group_names = ("Updates", "Agents", "Objects") groups = "%43s | %32s | %32s |" % group_names @@ -60,6 +75,7 @@ "*Created", "*Updated", "*Deleted", "*Created", "*Updated", "*Deleted", "*Sql Ops", "Errors", "Cpu (%)", "Mem (M)") + headings_fmt = \ "%10s %10s %10s %10s | " + \ "%10s %10s %10s | " + \ @@ -67,7 +83,6 @@ "%10s %10s %10s %10s" headings = headings_fmt % heading_names - values_fmt = \ "%10i %10.1f %10.1f %10.1f | " + \ "%10.1f %10.1f %10.1f | " + \ @@ -81,6 +96,8 @@ self.enqueued = 0 self.dequeued = 0 self.dropped = 0 + self.ignore = 0 + self.method_response = 0 self.agents_created = 0 self.agents_updated = 0 @@ -93,6 +110,7 @@ self.objects_created_by_class = defaultdict(int) self.objects_updated_by_class = defaultdict(int) self.objects_deleted_by_class = defaultdict(int) + self.objects_dropped_by_class = defaultdict(int) self.sql_ops = 0 self.errors = 0 @@ -101,11 +119,41 @@ self.cpu = 0 self.memory = 0 + self.init_update_times() + + def init_update_times(self): + # Track average time of update from arrival to + # processing complete. Refreshed per interval. + self.update_queue_duration = 0 + self.update_process_duration = 0 + self.queue_time = 0 + self.initial_process_time = 0 + self.commit_time = 0 + self.record_samples = 0 + self.init_update_flag = False + + def record_update_times(self,update): + try: + if self.init_update_flag: + self.init_update_times() + self.record_samples += 1 + self.update_queue_duration += \ + update.start_process - update.create_time + self.update_process_duration += \ + update.finish_process - update.start_process + self.queue_time += \ + update.dequeue_time - update.enqueue_time + self.initial_process_time += \ + update.after_process - update.start_process + self.commit_time += \ + update.after_commit - update.after_process + except: + print "record update exception" + def capture(self): now = copy.copy(self) now.time = time.time() - rusage = resource.getrusage(resource.RUSAGE_SELF) now.cpu = rusage[0] + rusage[1] @@ -113,6 +161,7 @@ UpdateStats.then = UpdateStats.now UpdateStats.now = now + self.init_update_flag = True def get_resident_pages(self): try: @@ -126,12 +175,7 @@ print self.groups print self.headings - def print_values(self): - self.capture() - - if not self.then: - return - + def _gather_values(self): values = [self.now.enqueued - self.then.enqueued, self.now.dequeued - self.then.dequeued, self.now.dropped - self.then.dropped, @@ -153,23 +197,118 @@ values.append(self.errors) values.append(int((self.now.cpu - self.then.cpu) / secs * 100)) values.append(self.now.memory / 1000000.0) + return values - print self.values_fmt % tuple(values) + def print_values(self): +# moved capture call to cumin-data so that write_values can +# be run as well +# self.capture() + if not self.then: + return - def print_values_by_class(self): - names = ("Class", "Created", "Updated", "Deleted") - print "%20s %10s %10s %10s" % names + print self.values_fmt % tuple(self._gather_values()) + def print_values_by_class(self,mint): + names = ("Class", "Created", "Updated", "Deleted", "Dropped") + print "%20s %10s %10s %10s %10s" % names + for pkg in mint.model._packages: for cls in pkg._classes: - created = stats.created_by_class[cls] - updated = stats.updated_by_class[cls] - deleted = stats.deleted_by_class[cls] + created = self.objects_created_by_class[cls] + updated = self.objects_updated_by_class[cls] + deleted = self.objects_deleted_by_class[cls] + dropped = self.objects_dropped_by_class[cls] + if created or updated or deleted or dropped: + args = (cls._name, created, updated, deleted, dropped) + print "%-20s %10i %10i %10i %10i" % args + + def init_stats_path(self, mint, path): + p = os.path.join(path, \ + "stats_"+ time.strftime("%m%d%y_%H%M%S",time.localtime())) + + if not os.path.exists(p): + os.makedirs(p) + + # init the path for general stats + self.stats_path = os.path.join(p, "stats.csv") + + # init the paths for stats by class + self.class_stats_paths = {} + for pkg in mint.model._packages: + for cls in pkg._classes: + self.class_stats_paths[cls._name] = \ + os.path.join(p, cls._name+".csv") + + def write_values_by_class(self, mint, localtime_): + names = ("Time", "Created", "Updated", "Deleted", "Dropped") + + for pkg in mint.model._packages: + for cls in pkg._classes: + created = self.objects_created_by_class[cls] + updated = self.objects_updated_by_class[cls] + deleted = self.objects_deleted_by_class[cls] + dropped = self.objects_dropped_by_class[cls] + if created or updated or deleted: - args = (cls._name, created, updated, deleted) - print "%-20s %10i %10i %10i" % args - + fname = self.class_stats_paths[cls._name] + write_header = not os.path.isfile(fname) + db = csv.writer(open(fname, "a+")) + if write_header: + db.writerow(names) + db.writerow((time.strftime("%H:%M:%S",localtime_), + created, updated, deleted, dropped)) + + def get_update_durations(self): + total = self.now.record_samples + if total == 0: + create_dur = proc_dur = que_dur = init_proc_time = commit_time = 0 + else: + create_dur = self.now.update_queue_duration / total + proc_dur = self.now.update_process_duration / total + que_dur = self.now.queue_time / total + init_proc_time = self.now.initial_process_time / total + commit_time = self.now.commit_time / total + return (create_dur, proc_dur, que_dur, init_proc_time, commit_time) + + + def write_values(self, localtime_): + stats_headings = \ + ("Time", "Total Msgs", + "Depth", "*Avg Que", "*Avg Create", "*Avg Proc", + "*Init Proc", "*Commit", + "*Enqueued", "*Dequeued", "*Dropped", + "*Create Agent", "*Update Agent", "*Delete Agent", + "*Create", "*Update", "*Delete", + "*Sql Ops", "Errors", "Cpu (%)", "Mem (M)") + + if not self.then: + return + try: + values = self._gather_values() + values.insert(0, time.strftime("%H:%M:%S",localtime_)) + values.insert(1, self.now.enqueued) + + # include object update times + avg_create_dur,avg_process_dur,avg_que_dur, \ + avg_init_proc,avg_commit = self.get_update_durations() + + values.insert(3, avg_que_dur) + values.insert(4, avg_create_dur) + values.insert(5, avg_process_dur) + values.insert(6, avg_init_proc) + values.insert(7, avg_commit) + + write_header = not os.path.isfile(self.stats_path) + db = csv.writer(open(self.stats_path, "a+")) + if write_header: + db.writerow(stats_headings) + db.writerow(values) + + except Exception as inst: + print "exception in write values" + print inst + class UpdateCursor(Cursor): def execute(self, sql, args=None): super(UpdateCursor, self).execute(sql, args) @@ -179,35 +318,50 @@ class Update(object): def __init__(self, model): self.model = model + self.create_time = time.time() def process(self, thread): + self.start_process = time.time() log.debug("Processing %s", self) try: self.do_process(thread.cursor, thread.stats) - + self.after_process = time.time() thread.conn.commit() + self.after_commit = time.time() except UpdateDropped: log.debug("Update dropped") thread.conn.rollback() thread.stats.dropped += 1 + try: + cls = self.get_class() + except: + cls = None + if cls != None: + thread.stats.objects_dropped_by_class[cls] += 1 + self.after_commit = time.time() + except: log.exception("Update failed") thread.conn.rollback() - thread.stats.errors += 1 - #print_exc() + print_exc() if thread.halt_on_error: raise + self.finish_process = time.time() + thread.stats.record_update_times(self) def do_process(self, cursor, stats): raise Exception("Not implemented") + def get_class(self): + return None + def __repr__(self): return self.__class__.__name__ @@ -346,7 +500,7 @@ self.model.print_event(3, "Created %s", obj) stats.objects_created += 1 - #stats.objects_created_by_class[cls] += 1 + stats.objects_created_by_class[cls] += 1 return obj @@ -380,7 +534,9 @@ # force a write if it's been too long, even if the values match if object_columns \ - or obj._save_time < obj._qmf_update_time - timedelta(hours=1): + or (obj._save_time != None and \ + obj._qmf_update_time != None and \ + obj._save_time < obj._qmf_update_time - timedelta(hours=1)): object_columns.append(cls.sql_table._qmf_update_time) sql = cls.sql_update_object.emit(object_columns) @@ -405,7 +561,7 @@ self.model.print_event(4, "Updated %s", obj) stats.objects_updated += 1 - #stats.objects_updated_by_class[cls] += 1 + stats.objects_updated_by_class[cls] += 1 def delete_object(self, cursor, stats, obj): obj.delete(cursor) @@ -413,7 +569,7 @@ self.model.print_event(3, "Deleted %s", obj) stats.objects_deleted += 1 - #stats.objects_deleted_by_class[obj._class] += 1 + stats.objects_deleted_by_class[obj._class] += 1 def process_properties(self, obj, columns, cursor): cls = obj._class @@ -605,7 +761,7 @@ count = cls.delete_selection(cursor, _qmf_agent_id=agent.id) stats.objects_deleted += count - #stats.objects_deleted_by_class[cls] += count + stats.objects_deleted_by_class[cls] += count cursor.connection.commit()

1 0

r4442 - trunk/cumin/python/cumin
by eallen＠fedoraproject.org 20 Dec '10

20 Dec '10

Author: eallen Date: 2010-12-20 14:12:38 +0000 (Mon, 20 Dec 2010) New Revision: 4442 Modified: trunk/cumin/python/cumin/model.py Log: Removing obsolete CuminClass and RemoteClass code Modified: trunk/cumin/python/cumin/model.py =================================================================== --- trunk/cumin/python/cumin/model.py 2010-12-20 13:53:33 UTC (rev 4441) +++ trunk/cumin/python/cumin/model.py 2010-12-20 14:12:38 UTC (rev 4442) @@ -667,189 +667,6 @@ self.ad_properties.append(prop) self.ad_properties_by_name[prop.name] = prop -class CuminClass(object): - def __init__(self, model, name, mint_class): - self.model = model - self.cumin_name = name - self.mint_class = mint_class - - self.properties = list() - self.ad_properties = list() - self.ad_properties_by_name = dict() - self.stats = list() - self.actions = list() - - self.tasks = list() - self.tasks_by_class = dict() - - self.frame = None - - self.model.add_class(self) - - def add_property(self, prop): - self.properties.append(prop) - setattr(self, prop.name, prop) - - def add_ad_property(self, prop): - self.ad_properties.append(prop) - self.ad_properties_by_name[prop.name] = prop - - def add_stat(self, stat): - self.stats.append(stat) - setattr(self, stat.name, stat) - - def add_action(self, action): - self.actions.append(action) - setattr(self, action.name, action) - - def add_task(self, task): - self.tasks.append(task) - self.tasks_by_class[task.__class__] = task - - def init(self): - log.debug("Initializing class %s", self) - - for prop in self.properties: - prop.init() - - for ad_prop in self.ad_properties: - ad_prop.init() - - for stat in self.stats: - stat.init() - - for action in self.actions: - action.init() - - for task in self.tasks: - log.debug("Initializing task %s", task) - task.init() - - def get_title(self, session): - return "Object" - - def get_plural_title(self, session): - return self.get_title(session) + "s" - - def get_icon_href(self, session): - return "resource?name=action-36.png" - - def get_object_href(self, session, object): - branch = session.branch() - self.show_object(branch, object) - return branch.marshal() - - def get_object_href_by_id(self, session, id): - return self.get_object_href(session, Identifiable(id)) - - def show_object(self, session, object): - return self.frame.show_object(session, object) - - def get_object_title(self, session, object): - title = self.get_title(session) - name = self.get_object_name(object) - return "%s '%s'" % (title, name) - - def get_object_name(self, object): - return object.name - - def write_event_xml(self, writer, object): - writer.write("<events errors=\"%i\" warnings=\"%i\"/>" % (0, 0)) - - def write_stat_xml(self, session, writer, object): - for stat in self.stats: - stat.write_xml(writer, object) - - def write_xml(self, session, writer, object): - object.sync() - - writer.write("<%s id=\"%s\" name=\"%s\">" % \ - (self.cumin_name, str(object.id), - self.get_object_name(object))) - - self.write_event_xml(writer, object) - self.write_stat_xml(session, writer, object) - - writer.write("</%s>" % self.cumin_name) - - def __str__(self): - return "%s.%s" % (self.__module__, self.__class__.__name__) - -class RemoteClass(CuminClass): - def __init__(self, model, name, mint_class, mint_stats_class): - super(RemoteClass, self).__init__(model, name, mint_class) - - self.mint_stats_class = mint_stats_class - - prop = CuminProperty(self, "qmfObjectId") - prop.title = "Management ID" - - prop = CuminProperty(self, "qmfCreateTime") - prop.title = "Creation Time" - - prop = CuminProperty(self, "qmfDeleteTime") - prop.title = "Deletion Time" - - prop = CuminProperty(self, "qmfUpdateTime") - prop.title = "Last Updated" - prop.summary = True - -class CuminGrid(RemoteClass): - def __init__(self, model): - super(CuminGrid, self).__init__(model, "grid", Grid, GridStats) - - prop = CuminProperty(self, "Name") - prop.title = "Name" - - prop = CuminProperty(self, "Pool") - prop.title = "Pool" - - prop = CuminProperty(self, "JobLimit") - prop.title = "Job Limit" - - prop = CuminProperty(self, "SubmitLimit") - prop.title = "Submit Limit" - - prop = CuminProperty(self, "GridResourceUnavailableTime") - prop.title = "Grid Resource Unavailable Time" - - stat = CuminStat(self, "NumJobs") - stat.title = "Num Jobs" - stat.category = "general" - - stat = CuminStat(self, "SubmitsInProgress") - stat.title = "Submits In Progress" - stat.category = "general" - - stat = CuminStat(self, "SubmitsQueued") - stat.title = "Submits Queued" - stat.category = "general" - - stat = CuminStat(self, "SubmitsAllowed") - stat.title = "Submits Allowed" - stat.category = "general" - - stat = CuminStat(self, "SubmitsWanted") - stat.title = "Submits Wanted" - stat.category = "general" - - stat = CuminStat(self, "RunningJobs") - stat.title = "Running Jobs" - stat.category = "general" - - stat = CuminStat(self, "IdleJobs") - stat.title = "Idle Jobs" - stat.category = "general" - - def init(self): - super(CuminGrid, self).init() - - def get_title(self, session): - return "Grid" - - def get_icon_href(self, session): - return "resource?name=pool-36.png" - # XXX "do_" on this doesn't make sense def do_bind(session, queue_name, binding_info): for exchange in binding_info: @@ -863,573 +680,6 @@ binding_key=binding_key, arguments=binding_info[exchange]["arguments"]) -class CuminQueue(RemoteClass): - def __init__(self, model): - super(CuminQueue, self).__init__(model, "queue", Queue, QueueStats) - - prop = CuminProperty(self, "name") - prop.title = "Name" - - prop = CuminProperty(self, "durable") - prop.title = "Durable?" - prop.summary = True - - prop = CuminProperty(self, "autoDelete") - prop.title = "Auto Delete?" - prop.summary = True - - prop = CuminProperty(self, "exclusive") - prop.title = "Exclusive?" - prop.summary = True - - prop = CuminProperty(self, "arguments") - prop.title = "Arguments" - prop.prefix = "qpid." - prop.escape = False - - stat = CuminStat(self, "consumerCount") - stat.title = "Consumers" - stat.unit = "consumer" - stat.highlow = True - - stat = CuminStat(self, "bindingCount") - stat.title = "Bindings" - stat.unit = "binding" - stat.highlow = True - - stat = CuminStat(self, "msgDepth") - stat.title = "Message Depth" - stat.highlow = True - - stat = CuminStat(self, "msgTotalEnqueues") - stat.title = "Msgs. Enqueued" - stat.unit = "message" - stat.category = "io" - - stat = CuminStat(self, "msgTotalDequeues") - stat.title = "Msgs. Dequeued" - stat.unit = "message" - stat.category = "io" - - stat = CuminStat(self, "byteDepth") - stat.title = "Byte Depth" - stat.unit = "byte" - stat.highlow = True - - stat = CuminStat(self, "byteTotalEnqueues") - stat.title = "Bytes Enqueued" - stat.unit = "byte" - stat.category = "io" - - stat = CuminStat(self, "byteTotalDequeues") - stat.title = "Bytes Dequeued" - stat.unit = "byte" - stat.category = "io" - - stat = CuminStat(self, "unackedMessages") - stat.title = "Msgs. Unacked" - stat.unit = "message" - stat.category = "io" - stat.highlow = True - - stat = CuminStat(self, "messageLatencyMin") - stat.title = "Min. Msg. Latency" - stat.unit = "nanosecond" - stat.category = "io" - - stat = CuminStat(self, "messageLatencyMax") - stat.title = "Max. Msg. Latency" - stat.unit = "nanosecond" - stat.category = "io" - - stat = CuminStat(self, "messageLatencyAverage") - stat.title = "Avg. Msg. Latency" - stat.unit = "nanosecond" - stat.category = "io" - - # Transactional - - stat = CuminStat(self, "msgTxnEnqueues") - stat.title = "Msgs. Enqueued" - stat.unit = "message" - stat.category = "io.txn" - - stat = CuminStat(self, "msgTxnDequeues") - stat.title = "Msgs. Dequeued" - stat.unit = "message" - stat.category = "io.txn" - - stat = CuminStat(self, "byteTxnEnqueues") - stat.title = "Bytes Enqueued" - stat.unit = "byte" - stat.category = "io.txn" - - stat = CuminStat(self, "byteTxnDequeues") - stat.title = "Bytes Dequeued" - stat.unit = "byte" - stat.category = "io.txn" - - stat = CuminStat(self, "enqueueTxnStarts") - stat.title = "Enq. Trans. Started" - stat.unit = "transaction" - stat.category = "txn" - - stat = CuminStat(self, "enqueueTxnCommits") - stat.title = "Enq. Trans. Committed" - stat.unit = "transaction" - stat.category = "txn" - - stat = CuminStat(self, "enqueueTxnRejects") - stat.title = "Enq. Trans. Rejected" - stat.unit = "transaction" - stat.category = "txn" - - stat = CuminStat(self, "enqueueTxnCount") - stat.title = "Enq. Trans. Pending" - stat.unit = "transaction" - stat.category = "txn" - stat.highlow = True - - stat = CuminStat(self, "dequeueTxnStarts") - stat.title = "Deq. Trans. Started" - stat.unit = "transaction" - stat.category = "txn" - - stat = CuminStat(self, "dequeueTxnCommits") - stat.title = "Deq. Trans. Committed" - stat.unit = "transaction" - stat.category = "txn" - - stat = CuminStat(self, "dequeueTxnRejects") - stat.title = "Deq. Trans. Rejected" - stat.unit = "transaction" - stat.category = "txn" - - stat = CuminStat(self, "dequeueTxnCount") - stat.title = "Deq. Trans. Pending" - stat.unit = "transaction" - stat.category = "txn" - stat.highlow = True - - # Durable - - stat = CuminStat(self, "msgPersistEnqueues") - stat.title = "Msgs. Enqueued" - stat.unit = "message" - stat.category = "io.durable" - - stat = CuminStat(self, "msgPersistDequeues") - stat.title = "Msgs. Dequeued" - stat.unit = "message" - stat.category = "io.durable" - - stat = CuminStat(self, "bytePersistEnqueues") - stat.title = "Bytes Enqueued" - stat.unit = "byte" - stat.category = "io.durable" - - stat = CuminStat(self, "bytePersistDequeues") - stat.title = "Bytes Dequeued" - stat.unit = "byte" - stat.category = "io.durable" - - def init(self): - super(CuminQueue, self).init() - - self.frame = self.model.frame.messaging.broker.queue - - def get_title(self, session): - return "Queue" - - def get_icon_href(self, session): - return "resource?name=queue-36.png" - -class CuminExchange(RemoteClass): - def __init__(self, model): - super(CuminExchange, self).__init__(model, "exchange", - Exchange, ExchangeStats) - - prop = self.NameProperty(self, "name") - prop.title = "Name" - - prop = CuminProperty(self, "type") - prop.title = "Type" - prop.summary = True - - prop = CuminProperty(self, "durable") - prop.title = "Durable?" - prop.summary = True - - prop = CuminProperty(self, "arguments") - prop.title = "Arguments" - prop.prefix = "qpid." - prop.escape = False - - stat = CuminStat(self, "producerCount") - stat.title = "Producers" - stat.unit = "producer" - stat.highlow = True - - stat = CuminStat(self, "bindingCount") - stat.title = "Bindings" - stat.unit = "binding" - stat.highlow = True - - stat = CuminStat(self, "msgReceives") - stat.title = "Msgs. Received" - stat.unit = "message" - stat.category = "io" - - stat = CuminStat(self, "msgRoutes") - stat.title = "Msgs. Routed" - stat.unit = "message" - stat.category = "io" - - stat = CuminStat(self, "msgDrops") - stat.title = "Msgs. Dropped" - stat.unit = "message" - stat.category = "io" - - stat = CuminStat(self, "byteReceives") - stat.title = "Bytes Received" - stat.unit = "message" - stat.category = "io" - - stat = CuminStat(self, "byteRoutes") - stat.title = "Bytes Routed" - stat.unit = "message" - stat.category = "io" - - stat = CuminStat(self, "byteDrops") - stat.title = "Bytes Dropped" - stat.unit = "message" - stat.category = "io" - - def init(self): - super(CuminExchange, self).init() - - self.frame = self.model.frame.messaging.broker.exchange - - def get_title(self, session): - return "Exchange" - - def get_icon_href(self, session): - return "resource?name=exchange-36.png" - - def get_object_title(self, session, object): - name = super(CuminExchange, self).get_object_name(object) - - if name: - return super(CuminExchange, self).get_object_title(session, object) - else: - return self.get_object_name(object) - - def get_object_name(self, object): - return object.name and object.name or "Default Exchange" - - class NameProperty(CuminProperty): - def value(self, session, object): - value = getattr(object, self.name, None) - return value and value or "Default Exchange" - -class CuminBinding(RemoteClass): - def __init__(self, model): - super(CuminBinding, self).__init__(model, "binding", - Binding, BindingStats) - - prop = CuminProperty(self, "bindingKey") - prop.title = "Binding Key" - - stat = CuminStat(self, "msgMatched") - stat.title = "Msgs. Matched" - stat.unit = "message" - - def get_title(self, session): - return "Binding" - - def get_object_name(self, binding): - return "between %s and %s" % (binding.exchange.name, binding.queue.name) - -class CuminRoute(RemoteClass): - def __init__(self, model): - super(CuminRoute, self).__init__ \ - (model, "route", Bridge, BridgeStats) - - prop = CuminProperty(self, "key") - prop.title = "Route Key" - - prop = CuminProperty(self, "tag") - prop.title = "Tag" - - prop = CuminProperty(self, "excludes") - prop.title = "Excludes" - - prop = CuminProperty(self, "durable") - prop.title = "Durable" - - prop = CuminProperty(self, "dest") - prop.title = "Exchange" - - def get_title(self, session): - return "Route" - - def get_object_name(self, route): - return route.src - -class CuminConnection(RemoteClass): - def __init__(self, model): - super(CuminConnection, self).__init__(model, "connection", - ClientConnection, - ClientConnectionStats) - - prop = CuminProperty(self, "address") - prop.title = "Address" - - prop = CuminProperty(self, "SystemConnection") - prop.title = "System Connection?" - - prop = CuminProperty(self, "authIdentity") - prop.title = "Auth Identity?" - - prop = CuminProperty(self, "federationLink") - prop.title = "Federation Link" - - stat = CuminStat(self, "closing") - stat.title = "Closing Down" - stat.category = "general" - - stat = CuminStat(self, "bytesFromClient") - stat.title = "Bytes Sent" - stat.unit = "byte" - stat.category = "io" - - stat = CuminStat(self, "bytesToClient") - stat.title = "Bytes Received" - stat.unit = "byte" - stat.category = "io" - - stat = CuminStat(self, "framesFromClient") - stat.title = "Frames Sent" - stat.unit = "frame" - stat.category = "io" - - stat = CuminStat(self, "framesToClient") - stat.title = "Frames Received" - stat.unit = "frame" - stat.category = "io" - - def init(self): - super(CuminConnection, self).init() - - self.frame = self.model.frame.messaging.broker.connection - - def get_title(self, session): - return "Connection" - - def get_icon_href(self, session): - return "resource?name=client-36.png" - - def get_object_name(self, conn): - return conn.address - -class CuminLink(RemoteClass): - def __init__(self, model): - super(CuminLink, self).__init__(model, "link", Link, LinkStats) - - prop = CuminProperty(self, "useSsl") - prop.title = "Use SSL" - prop.summary = True - - prop = CuminProperty(self, "durable") - prop.title = "Durable" - prop.summary = True - - stat = CuminStat(self, "closing") - stat.title = "Closing Down" - - stat = CuminStat(self, "framesFromPeer") - stat.title = "Frames from Peer" - stat.unit = "frame" - stat.category = "io" - - stat = CuminStat(self, "framesToPeer") - stat.title = "Frames to Peer" - stat.unit = "frame" - stat.category = "io" - - stat = CuminStat(self, "bytesFromPeer") - stat.title = "Bytes from Peer" - stat.unit = "byte" - stat.category = "io" - - stat = CuminStat(self, "bytesToPeer") - stat.title = "Bytes to Peer" - stat.unit = "byte" - stat.category = "io" - - def get_title(self, session): - return "Broker Link" - - def get_object_name(self, link): - return "%s:%d" % (link.host, link.port) - -class CuminBrokerStoreModule(RemoteClass): - def __init__(self, model): - super(CuminBrokerStoreModule, self).__init__ \ - (model, "store", Store, StoreStats) - - prop = CuminProperty(self, "location") - prop.title = "Location" - -class CuminJournal(RemoteClass): - def __init__(self, model): - super(CuminJournal, self).__init__(model, "journal", - Journal, JournalStats) - - prop = CuminProperty(self, "name") - prop.title = "Name" - prop.summary = True - - prop = CuminProperty(self, "directory") - prop.title = "Directory" - - stat = CuminStat(self, "initialFileCount") - stat.title = "Initial File Count" - stat.unit = "file" - stat.category = "io.journal" - - stat = CuminStat(self, "dataFileSize") - stat.title = "Data File Size" - stat.unit = "byte" - stat.category = "io.journal" - - stat = CuminStat(self, "recordDepth") - stat.title = "Record Depth" - stat.unit = "record" - stat.category = "io.journal" - - stat = CuminStat(self, "recordEnqueues") - stat.title = "Record Enqueues" - stat.unit = "record" - stat.category = "io.journal" - - stat = CuminStat(self, "outstandingAIOs") - stat.title = "Outstanding AIOs" - stat.unit = "aio" - stat.category = "io.journal" - - stat = CuminStat(self, "freeFileCount") - stat.title = "Free Files" - stat.unit = "file" - stat.category = "io.journal" - - stat = CuminStat(self, "availableFileCount") - stat.title = "Avail. Files" - stat.unit = "file" - stat.category = "io.journal" - - stat = CuminStat(self, "writeWaitFailures") - stat.title = "Write Wait Failures" - stat.unit = "failure" - stat.category = "io.journal" - - stat = CuminStat(self, "writeBusyFailures") - stat.title = "Write Busy Failures" - stat.unit = "failure" - stat.category = "io.journal" - - stat = CuminStat(self, "readRecordCount") - stat.title = "Read Records" - stat.unit = "record" - stat.category = "io.journal" - - stat = CuminStat(self, "readBusyFailures") - stat.title = "Read Busy Failures" - stat.unit = "failure" - stat.category = "io.journal" - - stat = CuminStat(self, "writePageCacheDepth") - stat.title = "Write Page Cache Depth" - stat.unit = "page" - stat.category = "io.journal" - - stat = CuminStat(self, "readPageCacheDepth") - stat.title = "Read Page Cache Depth" - stat.unit = "page" - stat.category = "io.journal" - -class CuminBrokerAclModule(RemoteClass): - def __init__(self, model): - super(CuminBrokerAclModule, self).__init__ \ - (model, "store", Acl, AclStats) - - prop = CuminProperty(self, "policyFile") - prop.title = "Policy File" - - prop = CuminProperty(self, "enforcingAcl") - prop.title = "Enforcing?" - - prop = CuminProperty(self, "transferAcl") - prop.title = "Transfer?" - - prop = CuminProperty(self, "lastAclLoad") - prop.title = "Last Loaded" - - stat = CuminStat(self, "aclDenyCount") - stat.title = "Denials" - stat.unit = "denial" - -class CuminBrokerClusterModule(RemoteClass): - def __init__(self, model): - super(CuminBrokerClusterModule, self).__init__ \ - (model, "broker_cluster_module", Cluster, ClusterStats) - - prop = CuminProperty(self, "clusterName") - prop.title = "Name" - - prop = CuminProperty(self, "clusterID") - prop.title = "ID" - - prop = CuminProperty(self, "publishedURL") - prop.title = "URL" - - prop = CuminProperty(self, "clusterSize") - prop.title = "Size" - - prop = CuminProperty(self, "status") - prop.title = "Status" - - prop = CuminProperty(self, "members") - prop.title = "Members" - -class CuminLimit(CuminClass): - def __init__(self, model): - super(CuminLimit, self).__init__ \ - (model, "limit", None) - - prop = CuminProperty(self, "name") - prop.title = "Name" - - prop = CuminProperty(self, "curr") - prop.summary = True - prop.title = "Current" - - prop = CuminProperty(self, "max") - prop.summary = True - prop.title = "Maximum" - - def init(self): - super(CuminLimit, self).init() - - #self.frame = self.model.frame.grid.pool.limit - - def get_title(self, session): - return "Concurrency Limit" - - def get_object_title(self, session, limit): - title = self.get_title(session) - return "%s '%s'" % (title, limit) - class JobMetaData(MetaData): def __init__(self, name): super(JobMetaData, self).__init__(name) @@ -1759,181 +1009,6 @@ log.exception(e) return self.do_wait() -class CuminScheduler(RemoteClass): - def __init__(self, model): - super(CuminScheduler, self).__init__(model, "scheduler", - Scheduler, SchedulerStats) - - prop = CuminProperty(self, "Name") - prop.title = "Name" - - prop = CuminProperty(self, "JobQueueBirthdate") - prop.title = "Job Queue Birthdate" - - prop = CuminProperty(self, "MaxJobsRunning") - prop.title = "Max Jobs Running" - - prop = CuminProperty(self, "MyAddress") - prop.title = "Contact Address" - - prop = CuminProperty(self, "PublicNetworkIpAddr") - prop.title = "Public IP Address" - - prop = CuminProperty(self, "Machine") - prop.title = "Machine" - prop.summary = True - - prop = CuminProperty(self, "System") - prop.title = "System" - - stat = CuminStat(self, "NumUsers") - stat.title = "Users" - - stat = CuminStat(self, "TotalRunningJobs") - stat.title = "Running Jobs" - - stat = CuminStat(self, "TotalIdleJobs") - stat.title = "Idle Jobs" - - stat = CuminStat(self, "TotalHeldJobs") - stat.title = "Held Jobs" - - stat = CuminStat(self, "TotalJobAds") - stat.title = "Job Ads" - - stat = CuminStat(self, "TotalRemovedJobs") - stat.title = "Removed Jobs" - - action = GetStartedAction(self, "GetStarted") - action.navigable = False - - def init(self): - super(CuminScheduler, self).init() - - # self.frame = self.model.frame.grid.pool.scheduler - - def get_title(self, session): - return "Scheduler" - - def get_object_name(self, sched): - return sched.Name - -class CuminSubmission(RemoteClass): - def __init__(self, model): - super(CuminSubmission, self).__init__(model, "submission", - Submission, SubmissionStats) - - prop = CuminProperty(self, "Name") - prop.title = "Name" - - stat = CuminStat(self, "Idle") - stat.title = "Idle Jobs" - - stat = CuminStat(self, "Running") - stat.title = "Running Jobs" - - stat = CuminStat(self, "Removed") - stat.title = "Removed Jobs" - - stat = CuminStat(self, "Completed") - stat.title = "Completed Jobs" - - stat = CuminStat(self, "Held") - stat.title = "Held Jobs" - - def get_title(self, session): - return "Submission" - - def get_object_name(self, sub): - return sub.Name - -class CuminSubmitter(RemoteClass): - def __init__(self, model): - super(CuminSubmitter, self).__init__(model, "submitter", - Submitter, SubmitterStats) - - prop = CuminProperty(self, "Name") - prop.title = "Name" - - stat = CuminStat(self, "RunningJobs") - stat.title = "Running Jobs" - - stat = CuminStat(self, "IdleJobs") - stat.title = "Idle Jobs" - - stat = CuminStat(self, "HeldJobs") - stat.title = "Held Jobs" - - def init(self): - super(CuminSubmitter, self).init() - - #self.frame = self.model.frame.grid.pool.scheduler.submitter - - def get_title(self, session): - return "Submitter" - - def get_object_name(self, sub): - return sub.Name - - def show_object(self, session, sub): - pool = self.model.get_main_pool() - - assert pool - - self.model.frame.grid.pool.set_object(session, pool) - - return self.frame.show_object(session, sub) - -class CuminCollector(RemoteClass): - def __init__(self, model): - super(CuminCollector, self).__init__(model, "collector", - Collector, CollectorStats) - - prop = CuminProperty(self, "Name") - prop.title = "Name" - - prop = CuminProperty(self, "PublicNetworkIpAddr") - prop.title = "IP Address" - - action = GetStartedAction(self, "GetStarted") - action.navigable = False - - stat = CuminStat(self, "RunningJobs") - stat.title = "Running Jobs" - - stat = CuminStat(self, "IdleJobs") - stat.title = "Idle Jobs" - - stat = CuminStat(self, "HostsClaimed") - stat.title = "Claimed Slots" - - stat = CuminStat(self, "HostsUnclaimed") - stat.title = "Unclaimed Slots" - - stat = CuminStat(self, "HostsOwner") - stat.title = "Owner Slots" - - stat = CuminStat(self, "HostsTotal") - stat.title = "Total Slots" - - def init(self): - super(CuminCollector, self).init() - - #self.frame = self.model.frame.grid.pool.collector - - def get_title(self, session): - return "Collector" - - def get_object_name(self, coll): - return coll.Name - -class CuminSubject(CuminClass): - def __init__(self, model): - super(CuminSubject, self).__init__(model, "subject", Subject) - - def get_title(self, session): - return "Subject" - class Pool(object): def __init__(self, id): self.id = id

1 0

r4441 - trunk/wooly/python/wooly
by eallen＠fedoraproject.org 20 Dec '10

20 Dec '10

Author: eallen Date: 2010-12-20 13:53:33 +0000 (Mon, 20 Dec 2010) New Revision: 4441 Modified: trunk/wooly/python/wooly/datatable.py Log: Avoid potential exception if formatted values are None Modified: trunk/wooly/python/wooly/datatable.py =================================================================== --- trunk/wooly/python/wooly/datatable.py 2010-12-17 18:51:54 UTC (rev 4440) +++ trunk/wooly/python/wooly/datatable.py 2010-12-20 13:53:33 UTC (rev 4441) @@ -43,7 +43,7 @@ value = record[self.index] if self.format is not None: - value = self.format % value + value = value and self.format % value or None return value

1 0

r4440 - in trunk/cumin/python/cumin: . grid inventory messaging
by eallen＠fedoraproject.org 17 Dec '10

17 Dec '10

Author: eallen Date: 2010-12-17 18:51:54 +0000 (Fri, 17 Dec 2010) New Revision: 4440 Modified: trunk/cumin/python/cumin/grid/job.py trunk/cumin/python/cumin/grid/negotiator.py trunk/cumin/python/cumin/grid/pool.py trunk/cumin/python/cumin/grid/scheduler.py trunk/cumin/python/cumin/grid/slot.py trunk/cumin/python/cumin/grid/submission.py trunk/cumin/python/cumin/grid/submitter.py trunk/cumin/python/cumin/inventory/system.py trunk/cumin/python/cumin/messaging/binding.py trunk/cumin/python/cumin/messaging/broker.py trunk/cumin/python/cumin/messaging/brokerlink.py trunk/cumin/python/cumin/messaging/connection.py trunk/cumin/python/cumin/messaging/exchange.py trunk/cumin/python/cumin/messaging/queue.py trunk/cumin/python/cumin/messaging/subscription.py trunk/cumin/python/cumin/model.py trunk/cumin/python/cumin/objectselector.py trunk/cumin/python/cumin/objectselector.strings trunk/cumin/python/cumin/qmfadapter.py Log: -Added MonitorSelfStats to Negotiator and Submitter lists and overview pages. -Added search boxes to ObjectTable lists to perform SQL LIKE searches Modified: trunk/cumin/python/cumin/grid/job.py =================================================================== --- trunk/cumin/python/cumin/grid/job.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/grid/job.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -199,7 +199,7 @@ super(JobSelector, self).__init__(app, name, cls) - self.adapter = JobSummariesAdapter(app, cls) + self.table.adapter = JobSummariesAdapter(app, cls) self.submission = submission frame = "main.grid.pool.submission.job" @@ -220,6 +220,8 @@ self.qmf_response = Attribute(app, "qmf_response") self.add_attribute(self.qmf_response) + self.add_search_filter(cls.GlobalJobId) + JobSelectionHold(app, self, "held") JobSelectionRelease(app, self, "released") JobSelectionRemove(app, self, "removed") Modified: trunk/cumin/python/cumin/grid/negotiator.py =================================================================== --- trunk/cumin/python/cumin/grid/negotiator.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/grid/negotiator.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -45,6 +45,21 @@ self.add_attribute_column(cls.Machine) self.add_attribute_column(cls.System) + stat = MonitorSelfStatColumn(app, cls.MonitorSelfAge.name, cls.MonitorSelfAge) + self.add_column(stat) + stat = MonitorSelfStatColumn(app, cls.MonitorSelfCPUUsage.name, cls.MonitorSelfCPUUsage) + self.add_column(stat) + stat = MonitorSelfStatColumn(app, cls.MonitorSelfImageSize.name, cls.MonitorSelfImageSize) + self.add_column(stat) + #stat = MonitorSelfStatColumn(app, cls.MonitorSelfRegisteredSocketCount.name, cls.MonitorSelfRegisteredSocketCount) + #self.add_column(stat) + #stat = MonitorSelfStatColumn(app, cls.MonitorSelfResidentSetSize.name, cls.MonitorSelfResidentSetSize) + #self.add_column(stat) + stat = MonitorSelfStatColumn(app, cls.MonitorSelfTime.name, cls.MonitorSelfTime) + self.add_column(stat) + + self.add_search_filter(cls.Name) + #self.start = DaemonSelectionStart(app, self, "NEGOTIATOR") #self.stop = DaemonSelectionStop(app, self, "NEGOTIATOR") Modified: trunk/cumin/python/cumin/grid/pool.py =================================================================== --- trunk/cumin/python/cumin/grid/pool.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/grid/pool.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -53,7 +53,7 @@ when = "now() - interval '1 day'" filter = SqlComparisonFilter(table._qmf_update_time, when, ">") - self.adapter.query.add_filter(filter) + self.table.adapter.query.add_filter(filter) class PoolNameColumn(ObjectLinkColumn): def render_header_content(self, session): Modified: trunk/cumin/python/cumin/grid/scheduler.py =================================================================== --- trunk/cumin/python/cumin/grid/scheduler.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/grid/scheduler.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -86,6 +86,17 @@ self.add_attribute_column(cls.TotalRunningJobs) self.add_attribute_column(cls.TotalHeldJobs) + stat = MonitorSelfStatColumn(app, cls.MonitorSelfAge.name, cls.MonitorSelfAge) + self.add_column(stat) + stat = MonitorSelfStatColumn(app, cls.MonitorSelfCPUUsage.name, cls.MonitorSelfCPUUsage) + self.add_column(stat) + stat = MonitorSelfStatColumn(app, cls.MonitorSelfImageSize.name, cls.MonitorSelfImageSize) + self.add_column(stat) + stat = MonitorSelfStatColumn(app, cls.MonitorSelfTime.name, cls.MonitorSelfTime) + self.add_column(stat) + + self.add_search_filter(cls.Name) + #self.start = DaemonSelectionStart(app, self, "SCHEDD") #self.stop = DaemonSelectionStop(app, self, "SCHEDD") @@ -95,7 +106,10 @@ self.attrs = ("NumUsers", "TotalRunningJobs", "TotalIdleJobs", "TotalHeldJobs", - "TotalJobAds", "TotalRemovedJobs") + "TotalJobAds", "TotalRemovedJobs", + "MonitorSelfAge", "MonitorSelfCPUUsage", + "MonitorSelfImageSize","MonitorSelfRegisteredSocketCount", + "MonitorSelfResidentSetSize","MonitorSelfTime") class SchedulerOverview(Widget): def __init__(self, app, name, scheduler): Modified: trunk/cumin/python/cumin/grid/slot.py =================================================================== --- trunk/cumin/python/cumin/grid/slot.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/grid/slot.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -24,6 +24,7 @@ self.pool = pool self.add_filter(self.pool, cls.Pool, cls.Pool) + self.add_search_filter(cls.Name) frame = "main.grid.pool.slot" col = ObjectLinkColumn(app, "name", cls.Name, cls._id, frame) @@ -451,7 +452,7 @@ return item[0][2] and "class='%s'" % item[0][2] or "" class SlotStates(ItemSet): - states = ("Unclaimed", "Claimed", "Owner", "Matched", "Preempting") + states = ("Unclaimed", "Claimed", "Unavailable", "Matched", "Preempting") def do_get_items(self, session): return self.states Modified: trunk/cumin/python/cumin/grid/submission.py =================================================================== --- trunk/cumin/python/cumin/grid/submission.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/grid/submission.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -63,13 +63,15 @@ super(SubmissionSelector, self).__init__(app, name, cls) - self.adapter = SubmissionData(app) + self.table.adapter = SubmissionData(app) self.add_attribute_column(cls.Idle) self.add_attribute_column(cls.Running) self.add_attribute_column(cls.Completed) self.add_attribute_column(cls.Held) + self.add_search_filter(cls.Name) + class PoolSubmissionSelector(SubmissionSelector): def __init__(self, app, name, pool): super(PoolSubmissionSelector, self).__init__(app, name) @@ -108,7 +110,7 @@ super(PoolSubmissionSchedulerSelector, self).__init__(app, name, pool) scheduler = app.model.com_redhat_grid.Scheduler - self.SchedulerJoin(app, self.adapter.query, self.cls.sql_table, + self.SchedulerJoin(app, self.table.adapter.query, self.cls.sql_table, self.cls.jobserverRef.sql_column, "jid") frame = "main.grid.pool.scheduler" Modified: trunk/cumin/python/cumin/grid/submitter.py =================================================================== --- trunk/cumin/python/cumin/grid/submitter.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/grid/submitter.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -50,6 +50,8 @@ self.add_attribute_column(cls.RunningJobs) self.add_attribute_column(cls.HeldJobs) + self.add_search_filter(cls.Name) + class SubmitterGeneralStatSet(StatSet): def __init__(self, app, name, object): super(SubmitterGeneralStatSet, self).__init__(app, name, object) Modified: trunk/cumin/python/cumin/inventory/system.py =================================================================== --- trunk/cumin/python/cumin/inventory/system.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/inventory/system.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -30,6 +30,8 @@ self.add_attribute_column(cls.memFree) self.add_attribute_column(cls.loadAverage1Min) + self.add_search_filter(cls.nodeName) + class SystemFrame(ObjectFrame): def __init__(self, app, name): cls = app.model.com_redhat_sesame.Sysimage Modified: trunk/cumin/python/cumin/messaging/binding.py =================================================================== --- trunk/cumin/python/cumin/messaging/binding.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/messaging/binding.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -134,7 +134,7 @@ super(BindingSelector, self).__init__(app, name, binding) - self.adapter = BindingData(app) + self.table.adapter = BindingData(app) frame = "main.messaging.broker.binding" col = ObjectLinkColumn \ @@ -156,6 +156,8 @@ self.remove = BindingSelectionRemove(app, self) + self.add_search_filter(binding.bindingKey) + class Exchange(ObjectLinkColumn): def render_header_content(self, session): return "Exchange" Modified: trunk/cumin/python/cumin/messaging/broker.py =================================================================== --- trunk/cumin/python/cumin/messaging/broker.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/messaging/broker.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -65,7 +65,7 @@ super(BrokerSelector, self).__init__(app, name, broker) - self.adapter = BrokerData(app) + self.table.adapter = BrokerData(app) self.group = SessionAttribute(self, "group") Modified: trunk/cumin/python/cumin/messaging/brokerlink.py =================================================================== --- trunk/cumin/python/cumin/messaging/brokerlink.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/messaging/brokerlink.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -73,6 +73,8 @@ self.add_attribute_column(cls.transport) self.add_attribute_column(cls.durable) + self.add_search_filter(cls.host) + self.remove = BrokerLinkSelectionRemove(app, self) # Address column XXX @@ -100,6 +102,8 @@ self.add_attribute_column(cls.tag) self.add_attribute_column(cls.excludes) + self.add_search_filter(cls.src) + self.remove = RouteSelectionRemove(app, self) class RouteSelectionRemove(ObjectSelectorTask): Modified: trunk/cumin/python/cumin/messaging/connection.py =================================================================== --- trunk/cumin/python/cumin/messaging/connection.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/messaging/connection.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -74,6 +74,8 @@ self.add_attribute_column(cls.bytesFromClient) self.add_attribute_column(cls.bytesToClient) + self.add_search_filter(cls.address) + self.close = ConnectionSelectionClose(app, self) class ConnectionSelectionClose(ObjectSelectorTask): @@ -194,6 +196,8 @@ self.add_attribute_column(cls.framesOutstanding) self.add_attribute_column(cls.clientCredit) + self.add_search_filter(cls.name) + self.close = SessionSelectionClose(app, self) self.detach = SessionSelectionDetach(app, self) Modified: trunk/cumin/python/cumin/messaging/exchange.py =================================================================== --- trunk/cumin/python/cumin/messaging/exchange.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/messaging/exchange.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -79,6 +79,7 @@ self.add_attribute_column(cls.byteRoutes) self.add_reference_filter(vhost, cls.vhostRef) + self.add_search_filter(cls.name) self.remove = ExchangeSelectionRemove(app, self) Modified: trunk/cumin/python/cumin/messaging/queue.py =================================================================== --- trunk/cumin/python/cumin/messaging/queue.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/messaging/queue.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -79,6 +79,7 @@ self.add_attribute_column(cls.byteDepth) self.add_reference_filter(vhost, cls.vhostRef) + self.add_search_filter(cls.name) self.remove = QueueSelectionRemove(app, self) self.purge = QueueSelectionPurge(app, self) Modified: trunk/cumin/python/cumin/messaging/subscription.py =================================================================== --- trunk/cumin/python/cumin/messaging/subscription.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/messaging/subscription.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -21,6 +21,8 @@ self.add_attribute_column(cls.arguments) self.add_attribute_column(cls.delivered) + self.add_search_filter(cls.name) + class SubscriptionFrame(ObjectFrame): def __init__(self, app, name): cls = app.model.org_apache_qpid_broker.Subscription Modified: trunk/cumin/python/cumin/model.py =================================================================== --- trunk/cumin/python/cumin/model.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/model.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -1927,57 +1927,6 @@ def get_object_name(self, coll): return coll.Name -class CuminNegotiator(RemoteClass): - def __init__(self, model): - super(CuminNegotiator, self).__init__(model, "negotiator", - Negotiator, NegotiatorStats) - - prop = CuminProperty(self, "Name") - prop.title = "Name" - - prop = CuminProperty(self, "Machine") - prop.title = "Machine" - - prop = CuminProperty(self, "MyAddress") - prop.title = "Address" - - prop = CuminProperty(self, "DaemonStartTime") - prop.title = "Start Time" - - stat = CuminStat(self, "MonitorSelfAge") - stat.title = "Age" - stat.category = "general" - - stat = CuminStat(self, "MonitorSelfCPUUsage") - stat.title = "CPU Usage" - stat.category = "general" - - stat = CuminStat(self, "MonitorSelfImageSize") - stat.title = "Image Size" - stat.category = "general" - - stat = CuminStat(self, "MonitorSelfRegisteredSocketCount") - stat.title = "Registered Socket Count" - stat.category = "general" - - stat = CuminStat(self, "MonitorSelfResidentSetSize") - stat.title = "Resident Set Size" - stat.category = "general" - - action = GetStartedAction(self, "GetStarted") - action.navigable = False - - def init(self): - super(CuminNegotiator, self).init() - - #self.frame = self.model.frame.grid.pool.negotiator - - def get_title(self, session): - return "Negotiator" - - def get_object_name(self, neg): - return neg.Name - class CuminSubject(CuminClass): def __init__(self, model): super(CuminSubject, self).__init__(model, "subject", Subject) Modified: trunk/cumin/python/cumin/objectselector.py =================================================================== --- trunk/cumin/python/cumin/objectselector.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/objectselector.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -73,7 +73,7 @@ self.add_filter(attribute, this, that) - def add_like_filter(self, attribute, this, type=SqlLikeFilter.BEGINS): + def add_like_filter(self, attribute, this, type=SqlLikeFilter.CONTAINS): assert isinstance(attribute, Attribute), attribute assert isinstance(this, RosemaryAttribute), this @@ -88,8 +88,14 @@ for this, vattr, type in self.like_specs: value = vattr.get(session) - pre = type == SqlLikeFilter.CONTAINS and "%%" or "" - values[this.name] = "%s%s%%" % (pre, value) + # if the user typed a %, don't add another + pre = "" + post = "%" + if not "%" in value: + pre = type == SqlLikeFilter.CONTAINS and "%%" or "" + else: + post = "" + values[this.name] = "%s%s%s" % (pre, value, post) return values @@ -119,11 +125,12 @@ else: self.field = ObjectSqlField(self.table.adapter, self.attr) -class ObjectSelector(ObjectTable, Form): +class ObjectSelector(Form): def __init__(self, app, name, cls): - super(ObjectSelector, self).__init__(app, name, cls) + super(ObjectSelector, self).__init__(app, name) - self.init_ids(app, cls) + self.update_enabled = False + self.cls = cls self.switches = ObjectSelectorSwitches(app, "switches") self.add_child(self.switches) @@ -137,6 +144,9 @@ self.links = ObjectSelectorLinks(app, "links") self.add_child(self.links) + self.table = ObjectSelectorTable(app, "table", cls) + self.add_child(self.table) + self.tasks = list() def init(self): @@ -150,6 +160,38 @@ self.buttons.add_child(button) button.init() + def add_search_filter(self, this): + search = StringInput(self.app, "search") + search.param.default = "" + + self.table.add_like_filter(search.param, this) + self.filters.add_child(search) + + def add_filter(self, attribute, this, that=None): + self.table.add_filter(attribute, this, that) + + def add_column(self, col): + self.table.add_column(col) + + def add_attribute_column(self, attr): + self.table.add_attribute_column(attr) + + def add_reference_filter(self, attribute, ref): + self.table.add_reference_filter(attribute, ref) + + def insert_column(self, index, column): + self.table.insert_column(index, column) + + def render_title(self, session): + return self.table.render_title(session) + +class ObjectSelectorTable(ObjectTable): + def __init__(self, app, name, cls): + super(ObjectSelectorTable, self).__init__(app, name, cls) + + self.update_enabled = True + self.init_ids(app, cls) + def init_ids(self, app, cls): item = IntegerParameter(app, "item") @@ -345,3 +387,12 @@ def render_item_class(self, session, item): return "item" + +class MonitorSelfStatColumn(ObjectTableColumn): + def render_header_content(self, session): + title = self.field.get_title(session) + parts = title.split() + if len(parts) > 2: + if parts[0] == "Monitor" and parts[1] == "Self": + return " ".join(parts[2:]) + return title Modified: trunk/cumin/python/cumin/objectselector.strings =================================================================== --- trunk/cumin/python/cumin/objectselector.strings 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/objectselector.strings 2010-12-17 18:51:54 UTC (rev 4440) @@ -7,8 +7,18 @@ div.ObjectSelectorFilters { float: right; + padding-bottom: 0.25em; } +div.ObjectSelectorFilters ul { + float: left; + margin: 0; +} + +div.ObjectSelectorFilters input { + font-size: 0.75em; +} + div.ObjectSelectorButtons { background-color: #e7e7f7; padding: 0.35em 0.75em; @@ -31,14 +41,22 @@ [ObjectSelector.html] <div id="{id}" class="{class}"> {links} - - {filters} - {switches} - <form method="post" action="?"> + <form id="{id}.form" method="post" action="?"> + {filters} {buttons} + {table} + + <div>{hidden_inputs}</div> + </form> + <script type="text/javascript"> + cumin.restoreTableCheckboxes(); + </script> +</div> + +[ObjectSelectorTable.html] <table id="{id}" class="{class}"> <style type="text/css"> {css} @@ -48,14 +66,6 @@ <tbody>{rows}</tbody> </table> - <div>{hidden_inputs}</div> - </form> - - <script type="text/javascript"> - cumin.restoreTableCheckboxes(); - </script> -</div> - [ObjectSelectorControl.html] <div class="{class}"> <ul>{widgets}</ul> @@ -67,3 +77,9 @@ <ul>{widgets}</ul> </div> + +[ObjectSelectorFilters.html] +<div class="{class}"> + <ul>{widgets}</ul> + <input type="submit" value="Search"/> +</div><div style="clear:both;"></div> Modified: trunk/cumin/python/cumin/qmfadapter.py =================================================================== --- trunk/cumin/python/cumin/qmfadapter.py 2010-12-17 16:39:40 UTC (rev 4439) +++ trunk/cumin/python/cumin/qmfadapter.py 2010-12-17 18:51:54 UTC (rev 4440) @@ -94,6 +94,10 @@ self.fields_by_attr = dict() + def add_like_filter(self, attr): + #TODO: add sql LIKE filtering to QMF results + pass + class QmfField(DataAdapterField): def __init__(self, adapter, column): python_type = column.type.python_type @@ -104,7 +108,6 @@ self.adapter.columns.append(column) - class ObjectQmfField(QmfField): def __init__(self, adapter, attr): assert isinstance(adapter, ObjectQmfAdapter), adapter

1 0

r4439 - trunk/mint/python/mint
by tmckay＠fedoraproject.org 17 Dec '10

17 Dec '10

Author: tmckay Date: 2010-12-17 16:39:40 +0000 (Fri, 17 Dec 2010) New Revision: 4439 Modified: trunk/mint/python/mint/update.py Log: Add a test for None value in check for forced write. Scale testing revealed errors generated when objects arrive with save_time or update_time equal to None. Modified: trunk/mint/python/mint/update.py =================================================================== --- trunk/mint/python/mint/update.py 2010-12-17 16:01:37 UTC (rev 4438) +++ trunk/mint/python/mint/update.py 2010-12-17 16:39:40 UTC (rev 4439) @@ -380,7 +380,9 @@ # force a write if it's been too long, even if the values match if object_columns \ - or obj._save_time < obj._qmf_update_time - timedelta(hours=1): + or (obj._save_time != None and \ + obj._qmf_update_time != None and \ + obj._save_time < obj._qmf_update_time - timedelta(hours=1)): object_columns.append(cls.sql_table._qmf_update_time) sql = cls.sql_update_object.emit(object_columns)

1 0

r4438 - branches
by tmckay＠fedoraproject.org 17 Dec '10

17 Dec '10

Author: tmckay Date: 2010-12-17 16:01:37 +0000 (Fri, 17 Dec 2010) New Revision: 4438 Added: branches/scale_testing/ Log: Create a separate branch for instrumentation code for scale verification. May eventually be merged back into trunk. Copied: branches/scale_testing (from rev 4437, trunk)

1 0

r4437 - /
by tmckay＠fedoraproject.org 17 Dec '10

17 Dec '10

Author: tmckay Date: 2010-12-17 15:58:39 +0000 (Fri, 17 Dec 2010) New Revision: 4437 Added: branches/ Log: Add a branch directory to the repository.

1 0

r4436 - in trunk/cumin: bin python/cumin python/cumin/grid
by tmckay＠fedoraproject.org 15 Dec '10

15 Dec '10

Author: tmckay Date: 2010-12-15 14:11:58 +0000 (Wed, 15 Dec 2010) New Revision: 4436 Modified: trunk/cumin/bin/cumin-web trunk/cumin/python/cumin/config.py trunk/cumin/python/cumin/grid/submission.py trunk/cumin/python/cumin/main.py trunk/cumin/python/cumin/model.py Log: Add RequestMemory and RequestDisk fields to job submission form, with defaults. Add RequestDisk field to VM job submission form, with default. RequestMemory is already set to an expression for VM jobs. These fields are necessary for scheduling to partitionable slots. BZ 654305 Modified: trunk/cumin/bin/cumin-web =================================================================== --- trunk/cumin/bin/cumin-web 2010-12-14 19:45:52 UTC (rev 4435) +++ trunk/cumin/bin/cumin-web 2010-12-15 14:11:58 UTC (rev 4436) @@ -32,6 +32,12 @@ cumin.user = values.web.user cumin.update_interval = values.web.update_interval + # set default values for form inputs + cumin.set_form_defaults(values.web.request_memory, + values.web.request_memory_vm, + values.web.request_disk, + values.web.request_disk_vm) + cumin.check() cumin.init() Modified: trunk/cumin/python/cumin/config.py =================================================================== --- trunk/cumin/python/cumin/config.py 2010-12-14 19:45:52 UTC (rev 4435) +++ trunk/cumin/python/cumin/config.py 2010-12-15 14:11:58 UTC (rev 4436) @@ -33,6 +33,15 @@ param = ConfigParameter(web, "user", str) + param = ConfigParameter(web, "request-memory", int) + param.default = 512 # MB + param = ConfigParameter(web, "request-memory-vm", int) + param.default = 512 # MB + param = ConfigParameter(web, "request-disk", int) + param.default = 1024 #MB + param = ConfigParameter(web, "request-disk-vm", int) + param.default = 5 * 1024 #MB + data = CuminConfigSection(self, "data") data.log_file.default = os.path.join(self.home, "log", "data.log") Modified: trunk/cumin/python/cumin/grid/submission.py =================================================================== --- trunk/cumin/python/cumin/grid/submission.py 2010-12-14 19:45:52 UTC (rev 4435) +++ trunk/cumin/python/cumin/grid/submission.py 2010-12-15 14:11:58 UTC (rev 4436) @@ -326,6 +326,8 @@ stdout=None, stderr=None, usrlog=None, + request_mem_MB=None, + request_disk_KB=None, attrs={}): ad = dict() @@ -345,6 +347,8 @@ put("Out", stdout) put("Err", stderr) put("UserLog", usrlog) + put("RequestMemory", request_mem_MB) + put("RequestDisk", request_disk_KB) ad.update(standard_job_attributes) ad.update(attrs) @@ -416,6 +420,12 @@ self.usrlog.help = "User log file" self.add_extra_field(self.usrlog) + self.request_mem_MB = self.RequestMemField(app, "requestmem") + self.add_extra_field(self.request_mem_MB) + + self.request_disk_MB = self.RequestDiskField(app, "requestdisk") + self.add_extra_field(self.request_disk_MB) + #self.options = self.OptionsField(app, "options") #self.add_extra_field(self.options) @@ -423,6 +433,8 @@ self.add_extra_field(self.attrs) def process_display(self, session): + self.request_mem_MB.set(session, self.app.form_defaults.request_memory) + self.request_disk_MB.set(session, self.app.form_defaults.request_disk) self.scheduler.validate(session) def process_submit(self, session): @@ -441,6 +453,8 @@ stdout = self.stdout.get(session) stderr = self.stderr.get(session) usrlog = self.usrlog.get(session) + request_mem_MB = self.request_mem_MB.get(session) + request_disk_KB = self.request_disk_MB.get(session) * 1024 tokens = shlex.split(command) @@ -459,6 +473,8 @@ stdout=stdout, stderr=stderr, usrlog=usrlog, + request_mem_MB=request_mem_MB, + request_disk_KB=request_disk_KB, attrs=attrs) self.task.exit_with_redirect(session, scheduler) @@ -545,6 +561,22 @@ def render_title(self, session): return "User Log" + class RequestMemField(IntegerField): + def render_title(self, session): + return "Request Memory" + + def render_help(self, session): + return "Requested memory size (MB), required by " \ + "partitionable slots." + + class RequestDiskField(IntegerField): + def render_title(self, session): + return "Request Disk" + + def render_help(self, session): + return "Requested disk space (MB), required by " \ + "partitionable slots." + # class OptionsField(CheckboxField): # def __init__(self, app, name): # super(SubmissionAddForm.OptionsField, self).__init__(app, name) @@ -573,7 +605,7 @@ return "Submit VM job" def do_invoke(self, session, scheduler, invoc, - description, image, memory, attrs={}): + description, image, memory, attrs={}, request_disk_KB=0): ad = dict() # General @@ -588,6 +620,7 @@ "ceiling(ifThenElse(JobVMMemory =!= undefined," + \ "JobVMMemory, " + \ "ImageSize / 1024.000000))" + ad["RequestDisk"] = request_disk_KB # VM @@ -652,10 +685,11 @@ self.add_field(self.image) self.memory = self.MemoryField(app, "memory") - self.memory.help = "In megabytes" self.memory.required = True - self.memory.input.param.default = 512 self.add_extra_field(self.memory) + + self.request_disk_MB = self.RequestDiskField(app, "requestdisk") + self.add_extra_field(self.request_disk_MB) self.scheduler = JobSchedulerField(app, "scheduler", self.pool_name) self.add_extra_field(self.scheduler) @@ -665,6 +699,8 @@ def process_display(self, session): self.scheduler.validate(session) + self.memory.set(session, self.app.form_defaults.request_memory_vm) + self.request_disk_MB.set(session, self.app.form_defaults.request_disk_vm) def process_submit(self, session): self.validate(session) @@ -676,13 +712,15 @@ description = self.description.get(session) image = self.image.get(session) memory = self.memory.get(session) + request_disk_KB = self.request_disk_MB.get(session) * 1024 self.task.invoke(session, scheduler, description, image, memory, - attrs=attrs) + attrs=attrs, + request_disk_KB=request_disk_KB) self.task.exit_with_redirect(session, scheduler) @@ -698,8 +736,15 @@ return "Memory" def render_help(self, session): - return "The VM's memory size in megabytes" + return "The VM's memory size (MB)" + class RequestDiskField(IntegerField): + def render_title(self, session): + return "Request Disk" + + def render_help(self, session): + return "The VM's image size (MB), required by partitionable slots." + class DagJobSubmit(Task): def __init__(self, app): super(DagJobSubmit, self).__init__(app) Modified: trunk/cumin/python/cumin/main.py =================================================================== --- trunk/cumin/python/cumin/main.py 2010-12-14 19:45:52 UTC (rev 4435) +++ trunk/cumin/python/cumin/main.py 2010-12-15 14:11:58 UTC (rev 4436) @@ -54,6 +54,8 @@ self.operator_email = None self.update_interval = 10 + self.form_defaults = self.CuminFormDefaults() + # self.model.sql_logging_enabled = True def check(self): @@ -124,6 +126,23 @@ self.server.stop() self.session.stop() + def set_form_defaults(self, + request_memory, + request_memory_vm, + request_disk, + request_disk_vm): + self.form_defaults.request_memory = request_memory + self.form_defaults.request_memory_vm = request_memory_vm + self.form_defaults.request_disk = request_disk + self.form_defaults.request_disk_vm = request_disk_vm + + class CuminFormDefaults(object): + def __init__(self): + self.request_memory = 0 + self.request_memory_vm = 0 + self.request_disk = 0 + self.request_disk_vm = 0 + class CuminModule(object): def __init__(self, app, name): self.app = app Modified: trunk/cumin/python/cumin/model.py =================================================================== --- trunk/cumin/python/cumin/model.py 2010-12-14 19:45:52 UTC (rev 4435) +++ trunk/cumin/python/cumin/model.py 2010-12-15 14:11:58 UTC (rev 4436) @@ -1614,6 +1614,14 @@ prop.group = "Other" prop.writable = False + prop = AdProperty(self, "RequestMemory") + prop.description = "Required for scheduling to partitionable slots." + prop.group = "Other" + + prop = AdProperty(self, "RequestDisk") + prop.description = "Required for scheduling to partitionable slots." + prop.group = "Other" + class JobStatusProperty(AdProperty): def render_status(self, session, status): return JobStatusInfo.get_status_string(status)

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

cumin-developers December 2010