February 2011 - cumin-developers - Fedora Mailing-Lists

r4511 - branches/scale_testing/mech_scripts

by tmckay＠fedoraproject.org

Author: tmckay Date: 2011-02-09 14:41:07 +0000 (Wed, 09 Feb 2011) New Revision: 4511 Modified: branches/scale_testing/mech_scripts/test_driver.py Log: Skip directory files in links, retry connect up to 10 times on timeout Modified: branches/scale_testing/mech_scripts/test_driver.py =================================================================== --- branches/scale_testing/mech_scripts/test_driver.py 2011-02-09 14:02:40 UTC (rev 4510) +++ branches/scale_testing/mech_scripts/test_driver.py 2011-02-09 14:41:07 UTC (rev 4511) @@ -28,6 +28,7 @@ self.machine = None self.link_file = None self.visited = {} + self.file_list = [] self.file_init = False self.file_number = 0 @@ -38,9 +39,11 @@ if not self.file_init: self.file_init = True if os.path.isdir(self.link_file): - self.file_list = os.listdir(self.link_file) - for i in range(len(self.file_list)): - self.file_list[i] = os.path.join(self.link_file, self.file_list[i]) + file_list = os.listdir(self.link_file) + for i in file_list: + p = os.path.join(self.link_file, i) + if not os.path.isdir(p): + self.file_list.append(p) else: self.file_list = [self.link_file] @@ -149,9 +152,21 @@ if link in self.visited and self.visited[link] and if_modified_header: # value for link will be the time it was visited. headers={"If-Modified-Since": self.visited[link]} - conn.request('GET', link, headers=headers) else: - conn.request('GET', link) + headers={} + + stat = False + for i in range(10): + try: + conn.request('GET', link, headers=headers) + stat = True + break + except: + pass + + if not stat: + continue + resp = conn.getresponse() data = resp.read() get_latency += time_time() - time_for_get

13 years, 2 months

1
0
0 / 0

r4510 - trunk/cumin/model

by eallen＠fedoraproject.org

Author: eallen Date: 2011-02-09 14:02:40 +0000 (Wed, 09 Feb 2011) New Revision: 4510 Modified: trunk/cumin/model/cumin.xml trunk/cumin/model/rosemary.xml Log: Added schema entries for quotas for compatibility with the ObjectSelector class Modified: trunk/cumin/model/cumin.xml =================================================================== --- trunk/cumin/model/cumin.xml 2011-02-08 23:28:56 UTC (rev 4509) +++ trunk/cumin/model/cumin.xml 2011-02-09 14:02:40 UTC (rev 4510) @@ -47,5 +47,11 @@ <property name="Name" type="sstr"/> <property name="Quota" type="sstr"/> </class> + + <class name="Limit" storage="none"> + <property name="Name" type="sstr"/> + <property name="Usage" type="sstr"/> + <property name="Allowance" type="sstr"/> + </class> </package> </model> Modified: trunk/cumin/model/rosemary.xml =================================================================== --- trunk/cumin/model/rosemary.xml 2011-02-08 23:28:56 UTC (rev 4509) +++ trunk/cumin/model/rosemary.xml 2011-02-09 14:02:40 UTC (rev 4510) @@ -29,6 +29,16 @@ <title>Dynamic Quota</title> </property> </class> + + <class name="Limit"> + <property name="Usage"> + <title>Current Usage</title> + </property> + + <property name="Allowance"> + <title>Max Allowance</title> + </property> + </class> </package> <package name="org.apache.qpid.broker">

13 years, 2 months

1
0
0 / 0

r4509 - in trunk/cumin/python/cumin: . grid messaging

by eallen＠fedoraproject.org

Author: eallen Date: 2011-02-08 23:28:56 +0000 (Tue, 08 Feb 2011) New Revision: 4509 Modified: trunk/cumin/python/cumin/grid/job.py trunk/cumin/python/cumin/grid/limit.py trunk/cumin/python/cumin/grid/negotiator.py trunk/cumin/python/cumin/grid/pool.py trunk/cumin/python/cumin/grid/quota.py trunk/cumin/python/cumin/grid/quota.strings trunk/cumin/python/cumin/grid/submission.py trunk/cumin/python/cumin/messaging/binding.py trunk/cumin/python/cumin/model.py trunk/cumin/python/cumin/objectselector.py trunk/cumin/python/cumin/objectselector.strings trunk/cumin/python/cumin/qmfadapter.py trunk/cumin/python/cumin/widgets.py Log: Consolidated Job Summaries, Quotas, and Limits to use the same base classes for qmf exception handling, csv exporting, results sorting and paging. Modified: trunk/cumin/python/cumin/grid/job.py =================================================================== --- trunk/cumin/python/cumin/grid/job.py 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/grid/job.py 2011-02-08 23:28:56 UTC (rev 4509) @@ -1,22 +1,24 @@ import logging +from datetime import datetime -from wooly import * -from wooly.widgets import * -from wooly.forms import * -from wooly.resources import * -from wooly.tables import * -from time import time -from cumin.widgets import * -from cumin.model import FetchJobAd, FetchJobOutput -from cumin.objecttask import * -from cumin.parameters import * -from cumin.stat import * -from cumin.formats import * -from cumin.util import * -from cumin.qmfadapter import * -from parsley.stringex import partition +from cumin.objectframe import ObjectView, ObjectFrameTaskForm, ObjectFrame,\ + ObjectFrameTask +from cumin.qmfadapter import ObjectQmfAdapter +from cumin.objectselector import ObjectTableColumn, ObjectLinkColumn,\ + ObjectQmSelectorfTable, ObjectCheckboxColumn,\ + ObjectSelectorTask, ObjectSelectorTaskForm, ObjectQmfSelector +from cumin.widgets import StaticColumnHeader, Wait, CuminForm,\ + EditablePropertyRenderer, StateSwitch +from cumin.util import JobStatusInfo, strip_string_quotes, parse +from cumin.model import QmfCall +from cumin.formats import fmt_datetime, fmt_link -import main +from wooly import Widget, Parameter, Attribute +from wooly.util import StringCatalog, Writer, escape_amp, escape_entity +from wooly.forms import Form, FormButton, StringField +from wooly.widgets import ModeSet, PropertySet, TemplateRenderer +from wooly.template import WidgetTemplate +from wooly.parameters import ListParameter, IntegerParameter, DictParameter strings = StringCatalog(__file__) log = logging.getLogger("cumin.job") @@ -101,35 +103,26 @@ return self.frame.get_title(session) class JobSummariesAdapter(ObjectQmfAdapter): - def get_qmf_response(self, values): - try: - submission = values["obj"] - except: - return None + def __init__(self, app, cls, submission): + super(JobSummariesAdapter, self).__init__(app, cls) + self.submission = submission + + def get_qmf_results(self, values): + session = values['session'] + submission = self.submission.get(session) + return self.app.model.get_submission_job_summaries(submission) - def get_data(self, values, options): - results = self.get_qmf_response(values) + def do_get_data(self, values): + results = self.get_qmf_results(values) + summaries = results.data - if not results: + if summaries is None or len(summaries) == 0: return [] - rows = self.process_list_results(results.data) + return summaries - # sort the entire results set - rows = self.sort_rows(rows, options) - - # return only the current page - first_index = options.offset - last_index = len(rows) - if options.limit: - last_index = options.offset + options.limit - - page = rows[first_index:last_index] - - return page - def process_record(self, key, record): field_data = list() for column in self.columns: @@ -147,7 +140,7 @@ parts = gjid.split("#") if len(parts) == 3: val = parts[1] - except Exception, e: + except Exception: pass if not val: try: @@ -160,9 +153,8 @@ return field_data def get_count(self, values): - submission = values["obj"] - results = self.app.model.get_submission_job_summaries(submission) - return results.data and len(results.data) or 0 + data = self.do_get_data(values) + return len(data) class NonSortableObjectTableColumn(ObjectTableColumn): def __init__(self, app, name, attr): @@ -207,14 +199,13 @@ self.header = DynamicColumnHeader(app, "dynamic_header", selector_method, self) self.add_child(self.header) -class JobSelector(ObjectSelector): +class JobSelector(ObjectQmfSelector): def __init__(self, app, name, submission): cls = app.model.com_redhat_cumin_grid.JobSummary - self.defer_enabled = True super(JobSelector, self).__init__(app, name, cls) - self.table.adapter = JobSummariesAdapter(app, cls) + self.table.adapter = JobSummariesAdapter(app, cls, submission) self.submission = submission frame = "main.grid.pool.submission.job" @@ -242,6 +233,10 @@ def create_table(self, app, name, cls): return JobSelectorTable(app, name, cls) + def get_qmf_results(self, session): + values = self.get_data_values(session) + return self.table.adapter.get_qmf_results(values) + def render_dynamic_header(self, session): values = self.get_data_values(session) count = self.table.adapter.get_count(values) @@ -249,12 +244,6 @@ return count <= max_sort - def do_render(self, session): - if self.table.is_exception(session): - return self.table.render(session) - - return super(JobSelector, self).do_render(session) - def render_title(self, session): return "Jobs" @@ -269,12 +258,7 @@ submission = self.parent.parent.submission.get(session) return frame.get_href(session, submission._id, job_id) -class JobSelectorTable(ObjectSelectorTable): - def __init__(self, app, name, cls): - super(JobSelectorTable, self).__init__(app, name, cls) - - self.error_tmpl = WidgetTemplate(self, "error_html") - +class JobSelectorTable(ObjectQmSelectorfTable): def init_ids(self, app, cls): item = Parameter(app, "item") @@ -285,52 +269,6 @@ (app, "id", cls.JobId, self.ids) self.add_column(self.checkbox_column) - def get_data_values(self, session): - values = super(JobSelectorTable, self).get_data_values(session) - - submission = self.parent.submission.get(session) - - if submission: - values['obj'] = submission - values['args'] = () - return values - - def get_data(self, session): - values = self.get_data_values(session) - options = self.get_data_options(session) - - return self.adapter.get_data(values, options) - - def is_exception(self, session): - values = self.get_data_values(session) - qmf_response = self.adapter.get_qmf_response(values) - - if not qmf_response: - return True - - return qmf_response.exception and True or False - - def do_render(self, session): - # determine if there is an error getting the data - # and show an error message - values = self.get_data_values(session) - qmf_response = self.adapter.get_qmf_response(values) - if not qmf_response: - m = "" - elif qmf_response.exception: - e = qmf_response.exception - m = len(e.args) > 0 and e.args[0] or "" - if (not qmf_response) or qmf_response.exception: - writer = Writer() - msg = "The data is not available at this time. <br/>%s" % m - self.error_tmpl.render(writer, session, msg) - return writer.to_string() - - return super(JobSelectorTable, self).do_render(session) - - def render_error_msg(self, session, msg): - return msg - class JobObjectSelectorTask(ObjectSelectorTask): def __init__(self, app, selector, verb): super(JobObjectSelectorTask, self).__init__(app, selector) @@ -800,8 +738,6 @@ user_file = None err_file = None - id = self.frame.id.get(session) - scheduler = self.frame.get_scheduler(session, id) ads, error = self.do_get_items(session) for ad in ads: if ad['name'] == "Out": Modified: trunk/cumin/python/cumin/grid/limit.py =================================================================== --- trunk/cumin/python/cumin/grid/limit.py 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/grid/limit.py 2011-02-08 23:28:56 UTC (rev 4509) @@ -1,31 +1,31 @@ import logging -from wooly import * -from wooly.widgets import * -from wooly.forms import * -from wooly.resources import * -from wooly.tables import * -from cumin.stat import * -from cumin.widgets import * -from cumin.parameters import * -from cumin.formats import * -from cumin.util import * +from cumin.formats import fmt_link +from cumin.model import QmfCall +from cumin.objectselector import ObjectTableColumn, ObjectQmfSelector,\ + ObjectQmfTable +from cumin.objectframe import ObjectFrame, ObjectFrameTask, ObjectFrameTaskForm +from cumin.qmfadapter import ObjectQmfAdapter +from cumin.widgets import CuminView -from job import * +from wooly.forms import StringField, StringInput, IntegerField +from wooly.util import StringCatalog +from wooly.widgets import TabbedModeSet strings = StringCatalog(__file__) log = logging.getLogger("cumin.limit") -class LimitDataSet(object): - def __init__(self, app, collector): - self.app = app +class LimitAdapter(ObjectQmfAdapter): + def __init__(self, app, cls, collector): + super(LimitAdapter, self).__init__(app, cls) + self.collector = collector def get_negotiator(self, session): collector = self.collector.get(session) cls = self.app.model.com_redhat_grid.Negotiator - # TODO: why is there more than 1 negotiator? + # mis-configured pool if there is more than one negotiator negs = cls.get_selection(session.cursor, Pool=collector.Pool) try: youngest = negs[0] @@ -39,88 +39,96 @@ return negotiator - def get_items(self, session): + def get_count(self, values): + data = self.do_get_data(values) + return len(data) + + def get_qmf_results(self, values): + # used to get the cached limits, and to determine + # if there was an exception while getting the data + + session = values['session'] negotiator = self.get_negotiator(session) - if not negotiator: - return + return self.app.model.get_negotiator_limits(negotiator) - limits = self.app.model.get_negotiator_limits(negotiator) + def do_get_data(self, values): + results = self.get_qmf_results(values) + limits = results.data if limits is None or len(limits) == 0: - return () - #limits = {'a':{'CURRENT': 1, 'MAX': 2}} + return {} + #limits = {'a':{'CURRENT': 11, 'MAX': 22}, + # 'b':{'CURRENT': 3, 'MAX': 44}} - keys = limits.keys() - keys.sort() + return limits - return [{"name": x, - "curr": limits[x]["CURRENT"], - "max": limits[x]["MAX"]} - for x in keys] + def process_record(self, key, record): + return [key, record["CURRENT"], record["MAX"]] -class LimitSet(CuminItemTable): - def __init__(self, app, name, collector): - super(LimitSet, self).__init__(app, name) +class LimitTable(ObjectQmfTable): + def __init__(self, app, name, cls): + super(LimitTable, self).__init__(app, name, cls) - self.data = LimitDataSet(app, collector) - - self.defer_enabled = True - - col = self.NameColumn(app, "name") + col = self.NameColumn(app, "name", cls.Name) self.add_column(col) - self.set_default_column(col) - col = self.CurrentColumn(app, "curr") - col.align = "right" + col = self.UsageColumn(app, "curr", cls.Usage) self.add_column(col) - col = self.MaxColumn(app, "max") - col.align = "right" + col = self.MaxColumn(app, "max", cls.Allowance) self.add_column(col) - def do_get_items(self, session): - return self.data.get_items(session) + class NameColumn(ObjectTableColumn): + def render_cell_content(self, session, data): + limit_name = super(LimitTable.NameColumn, self).render_cell_content(session, data) + if session.page == self.app.export_page: + return limit_name - def render_title(self, session): - return "Limits" - - class NameColumn(NonSortableTableColumn): - def render_title(self, session): - return "Name" - - def render_content(self, session, data): - limit_name = data["name"] - try: - limit_max = int(data["max"]) - except: - limit_max = "0" - negotiator = self.parent.data.get_negotiator(session) + limit_max = data[2] + negotiator = self.parent.adapter.get_negotiator(session) self.frame.limit.id.set(session, negotiator._id) self.frame.limit.set_limit.form.limit_name.set(session, limit_name) self.frame.limit.set_limit.form.limit_max.set(session, limit_max) href = self.frame.limit.set_limit.get_href(session) return fmt_link(href, limit_name) - class CurrentColumn(NonSortableTableColumn): - def render_title(self, session): - return "Current Usage" + class UsageColumn(ObjectTableColumn): + def render_text_align(self, session): + return "right" - class MaxColumn(NonSortableTableColumn): - def render_title(self, session): - return "Max Allowance" + class MaxColumn(ObjectTableColumn): + def render_text_align(self, session): + return "right" - def render_content(self, session, data): + def render_cell_content(self, session, data): + limit_max = super(LimitTable.MaxColumn, self).render_cell_content(session, data) try: - limit_max = int(data["max"]) + limit_max = int(limit_max) except: - limit_max = "0" + limit_max = 0 return limit_max - class Limits(Attribute): - def get_default(self, session): - return dict() +class LimitSelector(ObjectQmfSelector): + def __init__(self, app, name, collector): + cls = app.model.com_redhat_cumin_grid.Limit + super(LimitSelector, self).__init__(app, name, cls) + + self.table.adapter = LimitAdapter(app, cls, collector) + + self.enable_csv_export(collector) + + def create_table(self, app, name, cls): + return LimitTable(app, name, cls) + + def render_title(self, session): + return "Limits" + + def get_qmf_results(self, session): + values = self.get_data_values(session) + return self.table.adapter.get_qmf_results(values) + class LimitFrame(ObjectFrame): def __init__(self, app, name): cls = app.model.com_redhat_grid.Negotiator Modified: trunk/cumin/python/cumin/grid/negotiator.py =================================================================== --- trunk/cumin/python/cumin/grid/negotiator.py 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/grid/negotiator.py 2011-02-08 23:28:56 UTC (rev 4509) @@ -69,11 +69,10 @@ self.add_attribute(self.results) def get_config_info(self, session): - negotiator = self.negotiator.get(session) - info = self.info.get(session) if not info: + negotiator = self.negotiator.get(session) info = self.get_group_info(session, negotiator) self.info.set(session, info) @@ -84,13 +83,7 @@ groups = results.data info = dict() - if not groups: - if (not results.status) and (not results.exception): - results.exception = Exception("Loading") - if not results.exception: - if not groups: - results.exception = Exception(results.status) try: groups = self.split_group_names(groups) except Exception, e: @@ -106,11 +99,9 @@ return info def get_results(self, session): - self.get_config_info(session) - return self.results.get(session) + return self.get_config_info(session) def has_child(self, session, group): - negotiator = self.negotiator.get(session) info = self.get_config_info(session) try: @@ -126,7 +117,6 @@ return info[group]['has_child'] def get_parent(self, session, group): - negotiator = self.negotiator.get(session) info = self.get_config_info(session) try: @@ -144,7 +134,6 @@ return info[group]['parent'] def get_siblings(self, session, node): - negotiator = self.negotiator.get(session) info = self.get_config_info(session) siblings = list() Modified: trunk/cumin/python/cumin/grid/pool.py =================================================================== --- trunk/cumin/python/cumin/grid/pool.py 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/grid/pool.py 2011-02-08 23:28:56 UTC (rev 4509) @@ -1,35 +1,23 @@ import logging -from wooly import * -from wooly.forms import * -from wooly.resources import * -from wooly.sql import * -from wooly.tables import * -from wooly.widgets import * +from cumin.widgets import Session, StateSwitch +from wooly.util import StringCatalog +from cumin.objectselector import ObjectSelector, ObjectLinkColumn +from rosemary.sqlfilter import SqlComparisonFilter +from cumin.objectframe import ObjectFrame +from submission import SubmissionFrame, PoolSubmissionJoinSelector +from slot import SlotFrame, SlotSelector, SlotMap, SlotMapPage +from scheduler import SchedulerFrame, SchedulerSelector +from negotiator import NegotiatorFrame, NegotiatorSelector,\ + NegotiatorEditDynamicQuota +from limit import LimitFrame, LimitSelector +from quota import QuotaSelector +from wooly import Attribute, Widget +from cumin.parameters import CollectorGridAttribute, RosemaryObjectParameter +from collector import CollectorGeneralStatSet +from cumin.stat import StatFlashChart, StatSet, FlashFullPage -from cumin.objectframe import * -from cumin.objectselector import * -from cumin.stat import * -from cumin.widgets import * -from cumin.parameters import * -from cumin.formats import * -from cumin.util import * -from collector import * -from job import * -from limit import * -from negotiator import * -from quota import * -from scheduler import * -from slot import * -from submission import * -from submitter import * - -from cumin.widgets import Session - -import cumin.model -import main - strings = StringCatalog(__file__) log = logging.getLogger("cumin.pool") @@ -105,7 +93,7 @@ self.quotas = QuotaSelector(app, "quotas", self.negotiator_attribute, self) self.view.add_tab(self.quotas) - self.limits = LimitSet(app, "limits", self.object) + self.limits = LimitSelector(app, "limits", self.object) self.view.add_tab(self.limits) class NegotiatorAttribute(Attribute): Modified: trunk/cumin/python/cumin/grid/quota.py =================================================================== --- trunk/cumin/python/cumin/grid/quota.py 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/grid/quota.py 2011-02-08 23:28:56 UTC (rev 4509) @@ -1,11 +1,15 @@ -from wooly.util import StringCatalog, Writer import logging -from cumin.widgets import StaticColumnHeader + +from negotiator import GroupHelper + from cumin.formats import fmt_link -from cumin.objectselector import ObjectTable, ObjectSelector, ObjectTableColumn -from wooly.template import WidgetTemplate +from cumin.objectselector import ObjectQmfSelector, ObjectTableColumn,\ + ObjectQmfTable from cumin.qmfadapter import ObjectQmfAdapter -from negotiator import GroupHelper +from cumin.widgets import StaticColumnHeader + +from wooly.template import WidgetTemplate +from wooly.util import StringCatalog, Writer from wooly import Parameter, Widget strings = StringCatalog(__file__) @@ -53,12 +57,10 @@ def get_count(self, values): return len(values['items']) -class QuotaTable(ObjectTable): +class QuotaTable(ObjectQmfTable): def __init__(self, app, name, cls): super(QuotaTable, self).__init__(app, name, cls) - self.update_enabled = False - limit = Widget(app, "limit") self.header.replace_child(limit) @@ -111,13 +113,14 @@ return values -class QuotaSelector(ObjectSelector): +class QuotaSelector(ObjectQmfSelector): def __init__(self, app, name, negotiator, frame): cls = app.model.com_redhat_cumin_grid.Quota super(QuotaSelector, self).__init__(app, name, cls) self.update_enabled = True + self.table.update_enabled = False self.negotiator = negotiator self.table.negotiator = negotiator @@ -144,51 +147,16 @@ self.enable_csv_export(negotiator) - self.error_tmpl = WidgetTemplate(self, "error_html") - self.loading_tmpl = WidgetTemplate(self, "deferred_html") - def create_table(self, app, name, cls): return QuotaTable(app, name, cls) def render_title(self, session): return "Quotas" - def do_render(self, session): - # determine if there is an error getting the data - # and show an error message - qmf_response = None - tmpl = self.error_tmpl - m = "" + def get_qmf_results(self, session): + negotiator = self.negotiator.get(session) + return self.app.model.get_negotiator_group_names(negotiator) - try: - qmf_response = self.group_helper.get_results(session) - if not qmf_response: - m = "" - elif qmf_response.exception: - e = qmf_response.exception - m = len(e.args) > 0 and e.args[0] or "" - if m == "Loading": - tmpl = self.loading_tmpl - else: - neg = self.negotiator.get(session) - neg_name = neg and " for negotiator %s" % neg.Name or "" - m = "%s%s" % (m, neg_name) - except AssertionError: - m = "Missing Negotiator" - - if (not qmf_response) or qmf_response.exception: - writer = Writer() - if "is unknown" in m: - m = "Please try again later" - err_msg = "The data is not available at this time. <br/>%s" % m - tmpl.render(writer, session, err_msg) - return writer.to_string() - - return super(QuotaSelector, self).do_render(session) - - def render_error_msg(self, session, msg): - return msg - class ExpandColumn(ObjectTableColumn): def __init__(self, app, name, attr, expand, group_helper): super(QuotaSelector.ExpandColumn, self).__init__(app, name, attr) Modified: trunk/cumin/python/cumin/grid/quota.strings =================================================================== --- trunk/cumin/python/cumin/grid/quota.strings 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/grid/quota.strings 2011-02-08 23:28:56 UTC (rev 4509) @@ -5,25 +5,3 @@ color: red; } -[QuotaSelector.deferred_html] -<div id="{id}"> - <div class="deferredSpacer">Loading...</div> -</div> -<script type="text/javascript"> -<![CDATA[ -(function() { - window.addEvent("domready", function () { - setTimeout( function () { window.location.reload(); }, 5000 ); - }); -}()) -]]> -</script> - -[QuotaSelector.error_html] -<div id="{id}"> - <div class="TaskInvocationSet"> - <ul> - <li>{error_msg}</li> - </ul> - </div> -</div> Modified: trunk/cumin/python/cumin/grid/submission.py =================================================================== --- trunk/cumin/python/cumin/grid/submission.py 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/grid/submission.py 2011-02-08 23:28:56 UTC (rev 4509) @@ -1,18 +1,23 @@ import logging import shlex - from random import choice +import os -from rosemary.sqlquery import * -from parsley.stringex import partition, rpartition +from job import JobSelector, JobFrame +from cumin.objectframe import ObjectFrame +from cumin.sqladapter import ObjectSqlAdapter +from cumin.objectselector import ObjectSelector, ObjectLinkColumn,\ + ObjectTableColumn +from cumin.task import TaskLink, Task, ObjectTaskForm +from cumin.parameters import ObjectAttribute -from cumin.objectframe import * -from cumin.objectselector import * -from cumin.objecttask import * -from cumin.widgets import * -from cumin.util import * +from rosemary.sqlfilter import SqlFilter +from rosemary.sqlquery import SqlInnerJoin -from job import JobSelector, JobFrame +from wooly import Attribute, Parameter +from wooly.forms import StringField, ScalarField, FormError, OptionInputSet,\ + MultilineStringField, FormField, IntegerField +from wooly.parameters import IntegerParameter log = logging.getLogger("cumin.grid.submission") Modified: trunk/cumin/python/cumin/messaging/binding.py =================================================================== --- trunk/cumin/python/cumin/messaging/binding.py 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/messaging/binding.py 2011-02-08 23:28:56 UTC (rev 4509) @@ -10,6 +10,7 @@ from cumin.widgets import * import main +from cumin.objectframe import ObjectFrameTaskForm strings = StringCatalog(__file__) log = logging.getLogger("cumin.messaging.exchange") Modified: trunk/cumin/python/cumin/model.py =================================================================== --- trunk/cumin/python/cumin/model.py 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/model.py 2011-02-08 23:28:56 UTC (rev 4509) @@ -73,7 +73,10 @@ return session def get_negotiator_group_names(self, negotiator): - assert negotiator + if not negotiator: + store = NegotiatorGroupNamesStore(self, None) + store.exception = Exception("Missing Negotiator") + return store self.lock.acquire() @@ -132,7 +135,10 @@ self.lock.release() def get_negotiator_limits(self, negotiator): - assert negotiator + if not negotiator: + store = NegotiatorLimitStore(self, None) + store.exception = Exception("Missing Negotiator") + return store self.lock.acquire() @@ -146,9 +152,7 @@ self.limits_by_negotiator[negotiator._qmf_agent_id] = store - sleep(1) - - return store.data + return store finally: self.lock.release() Modified: trunk/cumin/python/cumin/objectselector.py =================================================================== --- trunk/cumin/python/cumin/objectselector.py 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/objectselector.py 2011-02-08 23:28:56 UTC (rev 4509) @@ -1,14 +1,24 @@ -from objectframe import * -from objecttask import * -from qmfadapter import * -from sqladapter import * -from util import * -from wooly import * -from wooly.datatable import * -from wooly.forms import * -from wooly.widgets import * +from cumin.objectframe import ObjectFrame +from cumin.qmfadapter import ObjectQmfAdapter, ObjectQmfField +from cumin.sqladapter import ObjectSqlAdapter, ObjectSqlField +from cumin.util import Identifiable +from rosemary.model import RosemaryClass, RosemaryAttribute, RosemaryReference +from rosemary.sqlfilter import SqlLikeFilter +from wooly.util import StringCatalog, Writer, escape_entity +from wooly.datatable import DataTable, DataTableColumn +from wooly import Attribute, Widget, SessionAttribute, Parameter +from wooly.forms import Form, StringInput, FoldingFieldSubmitForm +from cumin.objecttask import ObjectTaskButton, ObjectTask +from wooly.parameters import IntegerParameter, ListParameter +from wooly.table import CheckboxColumnCell, CheckboxColumnHeader,\ + CheckboxColumnInput, LinkColumn +from wooly.widgets import WidgetSet, ItemSet +import wooly +from wooly.template import WidgetTemplate + + strings = StringCatalog(__file__) class ObjectTable(DataTable): @@ -486,3 +496,110 @@ page.modes.show_child(export_session, self.exporter) page.set_parameters(export_session, session, self.object_attributes, self.file_name) return escape_entity(export_session.marshal()) + +class QmfDetails(Widget): + def __init__(self, app, name, status): + super(QmfDetails, self).__init__(app, name) + + self.status = status + + def do_render(self, session): + if self.status.get(session): + return super(QmfDetails, self).do_render(session) + return "" + + def render_details(self, session): + status = self.status.get(session) + return status + + def render_details_path(self, session): + return self.parent.show_details.path + + def render_style(self, session): + show = self.parent.show_details.get(session) + return show and "block" or "none" + + def render_show(self, session): + show = self.parent.show_details.get(session) + return show and "hide" or "show" + +class ObjectQmfSelector(ObjectSelector): + def __init__(self, app, name, cls): + super(ObjectQmfSelector, self).__init__(app, name, cls) + + self.update_enabled = True + self.table.update_enabled = False + + self.show_details = Parameter(app, "sd") + self.add_parameter(self.show_details) + + self.status_msg = Attribute(app, "status") + self.add_attribute(self.status_msg) + + self.details = QmfDetails(app, "details", self.status_msg) + self.add_child(self.details) + + self.error_tmpl = WidgetTemplate(self, "error_html") + self.loading_tmpl = WidgetTemplate(self, "deferred_html") + + def init(self): + super(ObjectQmfSelector, self).init() + + assert self.get_qmf_results + + def do_render(self, session): + tmpl = self.error_tmpl + title = self.render_title(session) + + store = self.get_qmf_results(session) + self.status_msg.set(session, store.status) + error = None + + if store.exception: + error = store.exception + if "is unknown" in str(error): + error = "%s are not available at this time." % title + self.status_msg.set(session, store.exception) + else: + if store.status is None: + error = "Loading" + tmpl = self.loading_tmpl + else: + values = self.table.get_data_values(session) + count = self.table.adapter.get_count(values) + if count == 0: + error = "There are no %s" % title + + if store.status == "OK": + details = "The call to get the data suceeded, but no results were returned." + else: + details = "The call status is <span>%s</span>" % store.status + self.status_msg.set(session, details) + + if error: + writer = Writer() + tmpl.render(writer, session, "%s" % error) + return writer.to_string() + + return super(ObjectQmfSelector, self).do_render(session) + + def render_error_msg(self, session, msg): + return msg + +class ObjectQmfTable(ObjectTable): + def get_data_values(self, session): + values = super(ObjectQmfTable, self).get_data_values(session) + + # session is needed to retrieve the qmf data + values['session'] = session + + return values + +class ObjectQmSelectorfTable(ObjectSelectorTable): + def get_data_values(self, session): + values = super(ObjectQmSelectorfTable, self).get_data_values(session) + + # session is needed to retrieve the qmf data + values['session'] = session + + return values Modified: trunk/cumin/python/cumin/objectselector.strings =================================================================== --- trunk/cumin/python/cumin/objectselector.strings 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/objectselector.strings 2011-02-08 23:28:56 UTC (rev 4509) @@ -87,3 +87,79 @@ [ExportButton.html] <a href="{href}"><img src="resource?name=csv.png" alt="save as csv" title="Save table as CSV file" /></a> + +[ObjectQmfSelector.css] +div.TaskInvocationSet ul { + list-style-type: none; +} + +[ObjectQmfSelector.deferred_html] +<div id="{id}"> + <div class="deferredSpacer">Loading...</div> +</div> +<script type="text/javascript"> +<![CDATA[ +(function() { + + if (typeof window.reUpdated == "undefined") { + window.reUpdated = true; + window.addEvent("domready", function () { + setInterval( function () { + wooly.updateNow(); + }, 5000 ); + }); + } + +}()) +]]> +</script> + +[ObjectQmfSelector.error_html] +<div id="{id}"> + <div class="TaskInvocationSet"> + <ul> + <li>{error_msg} {details}</li> + </ul> + </div> +</div> + +[QmfDetails.css] +div.QmfDetails { + margin: 1em; +} +div.QmfDetails span.DetailsLink { + font-size: 80%; +} + +div.QmfDetails div.Details span { + font-style: italic; + font-weight: bold; +} + +[QmfDetails.html] +<div id="{id}" class="{class}"> + <span class="DetailsLink"><a onclick="showDetails(); return false;">[<span id="{id}_link">{show}</span> details]</a></span> + <div id="{id}_details" class="Details" style="display: {style};">{details}</div> +</div> +<script type="text/javascript"> +<![CDATA[ + function showDetails() { + var updateUrl = wooly.branchIntervalUpdate(); + var branch = updateUrl["session"]; + + var details = $("{id}_details"); + var show_hide = $("{id}_link"); + if (details.style.display == "block") { + show_hide.innerHTML = "show"; + details.style.display = "none"; + delete branch["{details_path}"]; + } else { + show_hide.innerHTML = "hide"; + details.style.display = "block"; + branch["{details_path}"] = "1"; + } + //updateUrl["session"] = branch; + wooly.restartIntervalUpdate(updateUrl.marshal()); + } +]]> +</script> Modified: trunk/cumin/python/cumin/qmfadapter.py =================================================================== --- trunk/cumin/python/cumin/qmfadapter.py 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/qmfadapter.py 2011-02-08 23:28:56 UTC (rev 4509) @@ -1,8 +1,6 @@ -from rosemary.sqlquery import * -from wooly.datatable import * - -from util import * from operator import itemgetter +from cumin.model import QmfCall +from wooly.datatable import DataAdapter, DataAdapterField class QmfAdapter(DataAdapter): def __init__(self, app, method): @@ -24,6 +22,21 @@ return results and len(results) or 0 def get_data(self, values, options): + data = self.do_get_data(values) + + if isinstance(data, dict): + rows = self.process_results(data) + elif isinstance(data, list): + rows = self.process_list_results(data) + else: + rows = [] + + rows = self.sort_rows(rows, options) + rows = self.limit_rows(rows, options) + + return rows + + def do_get_data(self, values): obj = values['obj'] args = values['args'] @@ -33,17 +46,8 @@ if results.error: results.data = self.default - if isinstance(results.data, dict): - rows = self.process_results(results.data) - elif isinstance(results.data, list): - rows = self.process_list_results(results.data) - else: - rows = [] + return results.data - rows = self.sort_rows(rows, options) - - return rows[options.offset:options.offset + options.limit] - def process_results(self, results): """ take the dict response from the qmf call and return a list of lists """ @@ -86,6 +90,17 @@ return sorted(rows, key=itemgetter(sort_field.index), reverse=rev) + def limit_rows(self, rows, options): + # return only the current page + first_index = options.offset + last_index = len(rows) + if options.limit: + last_index = options.offset + options.limit + + page = rows[first_index:last_index] + + return page + class ObjectQmfAdapter(QmfAdapter): def __init__(self, app, cls): super(ObjectQmfAdapter, self).__init__(app, cls._name) Modified: trunk/cumin/python/cumin/widgets.py =================================================================== --- trunk/cumin/python/cumin/widgets.py 2011-02-08 14:59:27 UTC (rev 4508) +++ trunk/cumin/python/cumin/widgets.py 2011-02-08 23:28:56 UTC (rev 4509) @@ -21,6 +21,7 @@ from wooly import Session from wooly.widgets import Link from wooly.tables import SqlTable +from cumin.objectframe import ObjectFrameTask strings = StringCatalog(__file__) @@ -949,7 +950,8 @@ self.add_parameter(self.param) def do_get_items(self, session): - return list(BrokerGroup.select()) #XXX avoid list()ing this? + return [] + #return list(BrokerGroup.select()) #XXX avoid list()ing this? def render_item_value(self, session, group): return group.id

13 years, 2 months

1
0
0 / 0

r4508 - branches/scale_testing/cumin/bin

by tmckay＠fedoraproject.org

Author: tmckay Date: 2011-02-08 14:59:27 +0000 (Tue, 08 Feb 2011) New Revision: 4508 Modified: branches/scale_testing/cumin/bin/cumin-data Log: Minor tweaks to stats collecting hanging around in workspace. Modified: branches/scale_testing/cumin/bin/cumin-data =================================================================== --- branches/scale_testing/cumin/bin/cumin-data 2011-02-08 14:41:19 UTC (rev 4507) +++ branches/scale_testing/cumin/bin/cumin-data 2011-02-08 14:59:27 UTC (rev 4508) @@ -12,7 +12,6 @@ def main(): config, values, opts, args = get_configuration(CuminDataOptionParser(), CuminDataConfig) - setup_logging(opts) model_dir = os.path.join(config.home, "model") @@ -59,7 +58,7 @@ print "[Starred columns are the number of events per second]" # set up for scale stats here - if opts.scale_stats: + if opts.scale_stats or opts.object_stats: stats.init_stats_path(mint, config.home) if opts.print_stats or \ @@ -75,6 +74,7 @@ if opts.object_stats: stats.print_values_by_class(mint) + stats.write_values_by_class(mint, time.localtime()) if opts.scale_stats: stats.write_values_by_class(mint, time.localtime()) @@ -84,14 +84,9 @@ else: while True: sleep(86400) - except Exception as inst: - print inst + except Exception: + print_exc() - finally: - # close the scale stats CSV file here: - if opts.scale_stats: - pass - mint.stop() if __name__ == "__main__":

13 years, 2 months

1
0
0 / 0

r4507 - branches/scale_testing/wooly/python/wooly

by tmckay＠fedoraproject.org

Author: tmckay Date: 2011-02-08 14:41:19 +0000 (Tue, 08 Feb 2011) New Revision: 4507 Modified: branches/scale_testing/wooly/python/wooly/pages.py branches/scale_testing/wooly/python/wooly/server.py Log: Add Cache-Control: max-age=N header to static pages, extend link logging to record presence of Cache-Control header on response. Modified: branches/scale_testing/wooly/python/wooly/pages.py =================================================================== --- branches/scale_testing/wooly/python/wooly/pages.py 2011-02-08 14:38:05 UTC (rev 4506) +++ branches/scale_testing/wooly/python/wooly/pages.py 2011-02-08 14:41:19 UTC (rev 4507) @@ -258,6 +258,9 @@ def get_content_type(self, session): return "text/css" + def get_cache_control(self, session): + return "max-age=86400" + def get_css(self): if not self.__css: writer = Writer() @@ -297,6 +300,9 @@ def get_content_type(self, session): return "text/javascript" + def get_cache_control(self, session): + return "max-age=86400" + def get_javascript(self): if not self.__javascript: writer = Writer() @@ -344,6 +350,9 @@ def get_last_modified(self, session): return self.then + def get_cache_control(self, session): + return "max-age=86400" + def get_content_type(self, session): name = self.rname.get(session) base, ext = os.path.splitext(name) Modified: branches/scale_testing/wooly/python/wooly/server.py =================================================================== --- branches/scale_testing/wooly/python/wooly/server.py 2011-02-08 14:38:05 UTC (rev 4506) +++ branches/scale_testing/wooly/python/wooly/server.py 2011-02-08 14:41:19 UTC (rev 4507) @@ -80,7 +80,7 @@ return then - def log_link(self, link, lastrequested): + def log_link(self, link, lastrequested, cache_control_on): import shutil open_mode = "a+" if os.path.exists(self.link_log_dest): @@ -105,12 +105,16 @@ f.write("True,") else: f.write("False,") + if cache_control_on: + f.write("True,") + else: + f.write("False,") f.write(link+"\n") + def service_request(self, env, response, scale_times=None): begin_service = datetime.utcnow() log.info("Request %s %s", env["REQUEST_METHOD"], env["REQUEST_URI"]) - #self.log_link(env["REQUEST_URI"], self.get_last_requested(env) != None) page = self.get_page(env) @@ -132,6 +136,17 @@ headers = () content = "" + #ccindex = -1 + #for cc in range(len(headers)): + # if headers[cc][0] == "Cache-Control" and "max-age" in headers[cc][1]: + # ccindex = cc + + #ccPresent = ccindex != -1 + #self.log_link(env["REQUEST_URI"], self.get_last_requested(env) != None, ccPresent) + ## Remove cache control for link logging so that we get the flag, but not the behavior. + #if ccPresent: + # del headers[ccindex] + response(status, headers) log.info("Response %s", status) @@ -250,7 +265,8 @@ ((self.server.host, self.server.port), self.server.service_request, request_queue_size=32, - numthreads=32) + numthreads=32, + max=32) self.wsgi_server.environ["wsgi.version"] = (1, 1)

13 years, 2 months

1
0
0 / 0

r4506 - branches/scale_testing/mech_scripts

by tmckay＠fedoraproject.org

Author: tmckay Date: 2011-02-08 14:38:05 +0000 (Tue, 08 Feb 2011) New Revision: 4506 Modified: branches/scale_testing/mech_scripts/test.py branches/scale_testing/mech_scripts/test_driver.py Log: Extend link files with cache_control notation so that links can be skipped to emulate caching behavior. Modified: branches/scale_testing/mech_scripts/test.py =================================================================== --- branches/scale_testing/mech_scripts/test.py 2011-02-07 16:44:47 UTC (rev 4505) +++ branches/scale_testing/mech_scripts/test.py 2011-02-08 14:38:05 UTC (rev 4506) @@ -51,7 +51,7 @@ os.system("python process_times.py " + subdir + " --csvfile=" + sys.argv[3] + " --threads=" + threads) - shutil.copyfile("./config.cfg.save", config) + shutil.copyfile("./.config.cfg.save", config) if __name__ == '__main__': main() Modified: branches/scale_testing/mech_scripts/test_driver.py =================================================================== --- branches/scale_testing/mech_scripts/test_driver.py 2011-02-07 16:44:47 UTC (rev 4505) +++ branches/scale_testing/mech_scripts/test_driver.py 2011-02-08 14:38:05 UTC (rev 4506) @@ -85,11 +85,13 @@ parser.add_option('--file', dest='file', type='str', default="./links") parser.add_option('--iter', dest='iter', type='int', default=1) parser.add_option('--info', dest='info', action="store_true", default=False) + parser.add_option('--noskip', dest='skip', action="store_false", default=True) # combine these to create an output file name for web server internal stats parser.add_option('--group', dest='group', type='str', default="headers") parser.add_option('--thread', dest='thread', type='str', default="") parser.add_option('--outdir', dest='outdir', type='str', default=".") + if type(self.script_options) == str: self.script_options = self.script_options.split() @@ -100,6 +102,7 @@ self.link_file = os.path.abspath(opts.file) self.iteration_limit = opts.iter self.print_info = opts.info + self.skip = opts.skip self.group = opts.group self.thread = opts.thread self.outdir = opts.outdir @@ -122,13 +125,25 @@ link_info = link.split(',') # Check for initial Boolean, indicates whether "If_Modified" header was used - if len(link_info) == 2: - if_modified_header = link_info[0] = "True" + if len(link_info) == 3: + # Newest file type. If_Modified header, then Cache-Control + if_modified_header = link_info[0] == "True" + cache_control = link_info[1] == "True" + link = link_info[2].strip() + elif len(link_info) == 2: + if_modified_header = link_info[0] == "True" + cache_control = False link = link_info[1].strip() else: if_modified_header = True + cache_control = False link = link_info[0].strip() - + + # Skip cached pages if skip flag is set, let's us see times for + # only the dynamic pages + if self.skip and cache_control: + continue + time_for_get = time_time() conn = httplib.HTTPConnection(self.machine) if link in self.visited and self.visited[link] and if_modified_header: @@ -166,7 +181,10 @@ # Record some custom stuff self.custom_timers['Page File'] = get_latency - self.custom_timers['Avg Get'] = get_latency / total_gets + if total_gets == 0: + self.custom_timers['Avg Get'] = 0 + else: + self.custom_timers['Avg Get'] = get_latency / total_gets self.stand_alone_stats['Total Gets'] = total_gets self.stand_alone_stats['Total Size kb'] = total_size / 1024.0

13 years, 2 months

1
0
0 / 0

r4505 - in trunk/cumin: model python/cumin python/cumin/grid

by eallen＠fedoraproject.org

Author: eallen Date: 2011-02-07 16:44:47 +0000 (Mon, 07 Feb 2011) New Revision: 4505 Added: trunk/cumin/python/cumin/grid/quota.py trunk/cumin/python/cumin/grid/quota.strings Modified: trunk/cumin/model/cumin.xml trunk/cumin/model/rosemary.xml trunk/cumin/python/cumin/grid/negotiator.py trunk/cumin/python/cumin/grid/negotiator.strings trunk/cumin/python/cumin/grid/pool.py trunk/cumin/python/cumin/model.py trunk/cumin/python/cumin/session.py Log: Use an ObjectAdapter and ObjectSelector for quotas to facilitate csv export, exception handling, and feedback to user. Modified: trunk/cumin/model/cumin.xml =================================================================== --- trunk/cumin/model/cumin.xml 2011-02-04 17:46:17 UTC (rev 4504) +++ trunk/cumin/model/cumin.xml 2011-02-07 16:44:47 UTC (rev 4505) @@ -41,5 +41,11 @@ <property name="GlobalJobId" type="sstr"/> <property name="JobId" type="sstr"/> </class> + + <class name="Quota" storage="none"> + <property name="id" type="sstr"/> + <property name="Name" type="sstr"/> + <property name="Quota" type="sstr"/> + </class> </package> </model> Modified: trunk/cumin/model/rosemary.xml =================================================================== --- trunk/cumin/model/rosemary.xml 2011-02-04 17:46:17 UTC (rev 4504) +++ trunk/cumin/model/rosemary.xml 2011-02-07 16:44:47 UTC (rev 4505) @@ -19,6 +19,16 @@ <title>Job Status</title> </property> </class> + + <class name="Quota"> + <property name="id"> + <title /> + </property> + + <property name="Quota"> + <title>Dynamic Quota</title> + </property> + </class> </package> <package name="org.apache.qpid.broker"> Modified: trunk/cumin/python/cumin/grid/negotiator.py =================================================================== --- trunk/cumin/python/cumin/grid/negotiator.py 2011-02-04 17:46:17 UTC (rev 4504) +++ trunk/cumin/python/cumin/grid/negotiator.py 2011-02-07 16:44:47 UTC (rev 4505) @@ -1,22 +1,16 @@ -from wooly import * -from wooly.forms import * -from wooly.resources import * -from wooly.tables import * -from wooly.widgets import * +from wooly.util import StringCatalog, Writer +import logging +from cumin.objectframe import ObjectFrame, ObjectFrameTaskForm, ObjectFrameTask +from cumin.objectselector import ObjectSelector, ObjectLinkColumn,\ + MonitorSelfStatColumn +from wooly import Widget, Attribute, Parameter +from parsley.stringex import rpartition +from cumin.parameters import RosemaryObjectParameter +from wooly.parameters import ListParameter +from wooly.template import WidgetTemplate +from cumin.stat import StatFlashChart +from cumin.model import QmfCall -from cumin.formats import * -from cumin.model import FetchRawConfigSet -from cumin.objectframe import * -from cumin.objectselector import * -from cumin.parameters import * -from cumin.stat import * -from cumin.util import * -from cumin.widgets import * - -from parsley.stringex import partition, rpartition - -from daemon import * - strings = StringCatalog(__file__) log = logging.getLogger("cumin.grid.negotiator") @@ -26,8 +20,6 @@ super(NegotiatorFrame, self).__init__(app, name, cls) - #self.edit_dynamic_quota = NegotiatorEditDynamicQuota(app, self) - class NegotiatorSelector(ObjectSelector): def __init__(self, app, name, pool): cls = app.model.com_redhat_grid.Negotiator @@ -51,10 +43,6 @@ self.add_column(stat) stat = MonitorSelfStatColumn(app, cls.MonitorSelfImageSize.name, cls.MonitorSelfImageSize) self.add_column(stat) - #stat = MonitorSelfStatColumn(app, cls.MonitorSelfRegisteredSocketCount.name, cls.MonitorSelfRegisteredSocketCount) - #self.add_column(stat) - #stat = MonitorSelfStatColumn(app, cls.MonitorSelfResidentSetSize.name, cls.MonitorSelfResidentSetSize) - #self.add_column(stat) stat = MonitorSelfStatColumn(app, cls.MonitorSelfTime.name, cls.MonitorSelfTime) self.add_column(stat) @@ -65,229 +53,6 @@ self.enable_csv_export(pool) -class QmfGroupColumn(ItemTableColumn): - def __init__(self, app, name, getter, negotiator, task): - super(QmfGroupColumn, self).__init__(app, name) - - self.header_class = StaticTableColumnHeader - - self.title = None - self.getter = getter - #self.align = "right" - self.user = False - self.negotiator = negotiator - self.task = task - self.user_task = None - - def render_title(self, session, *args): - return self.title - - def render_content(self, session, group): - data = self.getter(session) - for x in data: - if x[0] == group: - return self.render_data(session, x) - - return "" - - def render_data(self, session, data): - href = self.task.get_href(session) - content = data[2][2] and str(data[1]) or "NOT SET" - return fmt_link(href, content, "", "", self.fmt_hover(data[0])) - - def fmt_hover(self, group): - return "Edit the %s" % self.title - -class NegotiatorOverview(ItemTable): - def __init__(self, app, name, negotiator, frame): - super(NegotiatorOverview, self).__init__(app, name) - - self.negotiator = negotiator - - self.update_enabled = True - #self.defer_enabled = True - - self.group_helper = GroupHelper(app, "groups", negotiator) - self.add_child(self.group_helper) - - self.expand = Parameter(app, "expand") - self.add_parameter(self.expand) - - col = self.ExpandColumn(app, "expand_col", self.expand) - col.header_class = ItemTableColumnHeader - col.width = "20px" - self.add_column(col) - - col = self.GroupColumn(app, "group") - self.add_column(col) - self.set_default_column(col) - - task = frame.edit_dynamic_quota - col = self.DynamicColumn(app, "dynamic", None, negotiator, task) - col.title = "Dynamic Quota" - self.add_column(col) - - col = self.SpacerColumn(app, "spacer") - self.add_column(col) - - self.error_tmpl = WidgetTemplate(self, "error_html") - self.loading_tmpl = WidgetTemplate(self, "deferred_html") - - def render_title(self, session): - return "Quotas" - - def render_class(self, session): - return "class=\"mobjects\"" - - def render_deferred_content(self, session): - return "Loading..." - - def do_render(self, session): - tmpl = self.error_tmpl - msg = None - - try: - info = self.group_helper.get_config_info(session) - except AssertionError: - msg = "Missing negotiator" - except Exception, e: - msg = len(e.args) > 0 and e.args[0] or "" - if msg == "Loading": - tmpl = self.loading_tmpl - else: - neg = self.negotiator.get(session) - neg_name = neg and " for negotiator %s" % neg.Name or "" - msg = "%s%s" % (e, neg_name) - log.exception(e) - else: - return super(NegotiatorOverview, self).do_render(session) - - writer = Writer() - if "is unknown" in msg: - msg = "Please try again later" - err_msg = "The data is not available at this time. <br/>%s" % msg - tmpl.render(writer, session, err_msg) - return writer.to_string() - - def render_error_msg(self, session, msg): - return msg - - def do_get_items(self, session): - info = self.group_helper.get_config_info(session) - - names = list(info) - names = sorted(names) - if self.reversed.get(session): - names = reversed(names) - - expanded = self.expand.get(session) - - # make a list of groups to show based on what is expanded - parent = expanded and self.group_helper.get_parent(session, expanded) or None - items = list() - for name in names: - # always show top level groups - if "." not in name: - items.append(name) - # add the current expanded item - elif expanded == name: - items.append(name) - # add the ancestors - elif expanded and expanded.startswith(name+"."): - items.append(name) - # add the direct children of expanded item - elif expanded and name.startswith(expanded+"."): - sub_name = name[len(expanded) + 1:] - if "." not in sub_name: - items.append(name) - # and direct children of expanded item's parent - elif parent and name.startswith(parent+"."): - sub_name = name[len(parent) + 1:] - if "." not in sub_name: - items.append(name) - - self.group_helper.get_config_for_groups(session, "GROUP_QUOTA_DYNAMIC", items) - - return items - - class ExpandColumn(ItemTableColumn): - def __init__(self, app, name, expand): - super(NegotiatorOverview.ExpandColumn, self).__init__(app, name) - - self.expand = expand - - def render_title(self, session, *args): - return "" - - def get_class_list(self, session): - return ["expand"] - - def render_content(self, session, group): - if not self.parent.group_helper.has_child(session, group): - return "" - - info = self.parent.group_helper.get_config_info(session) - # return a " or - depending on current expand - expand = self.expand.get(session) - if expand and expand.startswith(group): - state = "-" - else: - state = "+" - - parent = self.parent.group_helper.get_parent(session, group) - return self.render_item_link(session, group, parent, state) - - def render_item_link(self, session, group, parent, state): - branch = session.branch() - - next_expand = state == "+" and group or parent - self.expand.set(branch, next_expand) - - hover = state == "-" and "Collapse" or "Expand" - cls = "action" - - return fmt_link(branch.marshal(), state, class_=cls, link_title=hover) - - class GroupColumn(NonSortableTableColumn): - def render_title(self, session, *args): - return "Name" - - def render_content(self, session, group): - # if a user - if "." in group: - parts = group.split('.') - indent = len(parts) - 1 - user = parts[indent] - return "<span style='padding-left: %dem;'>%s</span>" % (indent, user) - - return group - - class DynamicColumn(QmfGroupColumn): - def render_content(self, session, group): - value = self.parent.group_helper.get_config_value(session, group, "GROUP_QUOTA_DYNAMIC") - self.task.form.group_leader.set(session, group) - - try: - fval = float(value) - except ValueError: - return value - except TypeError: - if isinstance(value, Exception): - content = "<span class='QuotaError'>error</span>" - val = len(value.args) > 0 and value.args[0] or "" - return fmt_link("#", content, "", "", val) - - href = self.task.get_href(session) - content = "%s%%" % str(round(fval * 100.0, 2)) - return fmt_link(href, content, "", "", self.fmt_hover("")) - - class SpacerColumn(ItemTableColumn): - def render_title(self, session, *args): - return "" - - def render_content(self, session, _): - return "" - class GroupHelper(Widget): def __init__(self, app, name, negotiator): super(GroupHelper, self).__init__(app, name) @@ -300,37 +65,50 @@ self.loading = Attribute(app, "loading") self.add_attribute(self.loading) + self.results = Attribute(app, "results") + self.add_attribute(self.results) + def get_config_info(self, session): negotiator = self.negotiator.get(session) info = self.info.get(session) if not info: - results = self.app.model.get_negotiator_group_names(negotiator) - groups = results.data + info = self.get_group_info(session, negotiator) + self.info.set(session, info) + return info + + def get_group_info(self, session, negotiator): + results = self.app.model.get_negotiator_group_names(negotiator) + groups = results.data + info = dict() + + if not groups: + if (not results.status) and (not results.exception): + results.exception = Exception("Loading") + + if not results.exception: if not groups: - if (not results.status) and (not results.exception): - raise Exception("Loading") - if results.exception: - raise results.exception - else: - if not groups: - raise Exception(results.status) - try: - groups = self.split_group_names(groups) - except Exception, e: - log.exception(e) - groups = [] + results.exception = Exception(results.status) + try: + groups = self.split_group_names(groups) + except Exception, e: + log.exception(e) + groups = [] - info = dict() + if groups: for group in groups: info[group] = dict() - self.info.set(session, info) + self.results.set(session, results) return info + def get_results(self, session): + self.get_config_info(session) + return self.results.get(session) + def has_child(self, session, group): negotiator = self.negotiator.get(session) info = self.get_config_info(session) @@ -440,44 +218,6 @@ val = min(1.0, val) return val -class GroupAddForm(ObjectFrameTaskForm): - def __init__(self, app, name, task): - super(GroupAddForm, self).__init__(app, name, task) - - self.defer_enabled = True - - self.group_helper = GroupHelper(app, "groups", self.object) - self.add_child(self.group_helper) - - self.group_name = NameField(app, "name") - self.add_field(self.group_name) - - def render_title(self, session): - return "Add Group" - - def process_submit(self, session): - negotiator = self.object.get(session) - group_name = self.group_name.get(session) - original_groups = self.group_helper.get_group_names(session) - - if self.is_valid(group_name): - if group_name not in original_groups: - original_groups.append(group_name) - new_groups = ", ".join(original_groups) - self.task.invoke(session, negotiator, "GROUP_NAMES", new_groups) - self.task.reconfig(negotiator) - - info = self.group_helper.get_config_info(session) - info[group_name] = dict() - - self.task.exit_with_redirect(session) - - def is_valid(self, group): - ret = False - if len(group) and not "." in group and not " " in group: - ret = True - return ret - class GroupForm(ObjectFrameTaskForm): def __init__(self, app, name, task): super(GroupForm, self).__init__(app, name, task) @@ -515,107 +255,6 @@ def render_form_class(self, session): return "" -class EditPrioFactorForm(GroupForm): - def __init__(self, app, name, task): - super(EditPrioFactorForm, self).__init__(app, name, task) - - self.field_tmpl = WidgetTemplate(self, "field_html") - - factor = Parameter(app, "factor") - self.add_parameter(factor) - - self.factors = ListParameter(app, "factors", factor) - self.add_parameter(self.factors) - - def render_title(self, session): - return "Edit Group Priority Factor" - - def render_form_class(self, session): - return "PrioFactorForm" - - def render_data_col_header(self, session): - return "Priority Factor" - - def render_chart(self, session): - return None - - def render_groups(self, session): - writer = Writer() - - groups = self.group_helper.get_priority_factor(session) - for group in groups: - self.field_tmpl.render(writer, session, group) - - return writer.to_string() - - def render_factor_name(self, session, group): - return self.factors.path - - def render_factor_value(self, session, group): - return group[2][2] and group[1] or "--NOT SET--" - - def process_submit(self, session): - negotiator = self.object.get(session) - factors = self.factors.get(session) - group_names = self.group_names.get(session) - original_values = self.original_values.get(session) - - changed = False - for group, new_value, original_value in zip(group_names, factors, original_values): - if new_value != original_value: - if self.is_valid_factor(new_value): - self.task.invoke(session, negotiator, "GROUP_PRIO_FACTOR_"+group, new_value) - changed = True - if changed: - self.task.reconfig(negotiator) - self.task.exit_with_redirect(session) - - def is_valid_factor(self, value): - try: - factor = float(value) - except: - return False - return factor >= 1.0 - -class UserPrioFactorForm(EditPrioFactorForm): - def __init__(self, app, name, task): - super(UserPrioFactorForm, self).__init__(app, name, task) - - self.group = Parameter(app, "group") - self.add_parameter(self.group) - - def render_title(self, session): - return "Edit User Priority Factor" - - def render_groups(self, session): - writer = Writer() - - group = self.group.get(session) - users = list(self.group_helper.get_priority_factor(session, [group], True)) - for user in users: - self.field_tmpl.render(writer, session, user) - - return writer.to_string() - - def render_form_class(self, session): - return "UserPrioFactorForm" - - def process_submit(self, session): - negotiator = self.object.get(session) - factors = self.factors.get(session) - group = self.group.get(session) - original_values = self.original_values.get(session) - - changed = False - for user, new_value, original_value in zip([group], factors, original_values): - if new_value != original_value: - if self.is_valid_factor(new_value): - self.task.invoke(session, negotiator, "GROUP_PRIO_FACTOR_"+user, new_value) - changed = True - if changed: - self.task.reconfig(negotiator) - self.task.exit_with_redirect(session) - class EditDynamicQuotaForm(GroupForm): def __init__(self, app, name, task): super(EditDynamicQuotaForm, self).__init__(app, name, task) @@ -721,176 +360,6 @@ return str(pri / 100) -class EditStaticQuotaForm(GroupForm): - def __init__(self, app, name, task): - super(EditStaticQuotaForm, self).__init__(app, name, task) - - self.field_tmpl = WidgetTemplate(self, "field_html") - - quota = Parameter(app, "quota") - self.add_parameter(quota) - - self.quotas = ListParameter(app, "quotas", quota) - self.add_parameter(self.quotas) - - def render_title(self, session): - return "Edit Static Group Quota" - - def render_submit_content(self, session): - return "Change quota" - - def render_form_class(self, session): - return "StaticQuotaForm" - - def render_data_col_header(self, session): - return "Slots" - - def render_chart(self, session): - return None - - def render_groups(self, session): - writer = Writer() - - groups = self.group_helper.get_static_quota(session) - for group in groups: - self.field_tmpl.render(writer, session, group) - - return writer.to_string() - - def render_quota_name(self, session, group): - return self.quotas.path - - def render_quota_value(self, session, group): - return group[1] - - def process_submit(self, session): - negotiator = self.object.get(session) - quotas = self.quotas.get(session) - group_names = self.group_names.get(session) - original_values = self.original_values.get(session) - - changed = False - for group, new_value, original_value in zip(group_names, quotas, original_values): - if new_value != original_value: - self.task.invoke(session, negotiator, "GROUP_QUOTA_"+group, new_value) - changed = True - if changed: - self.task.reconfig(negotiator) - self.task.exit_with_redirect(session) - -class EditRegroupForm(GroupForm): - def __init__(self, app, name, task): - super(EditRegroupForm, self).__init__(app, name, task) - - self.field_tmpl = WidgetTemplate(self, "field_html") - - self.regroup = DictParameter(app, "regroup") - self.add_parameter(self.regroup) - - original_regroup = Parameter(app, "or") - self.add_parameter(original_regroup) - - self.original_regroups = ListParameter(app, "original_regroups", original_regroup) - self.add_parameter(self.original_regroups) - - def render_title(self, session): - return "Edit Autoregroup" - - def render_form_class(self, session): - return "StaticQuotaForm" - - def render_data_col_header(self, session): - return "Autoregroup" - - def render_chart(self, session): - return None - - def render_groups(self, session): - writer = Writer() - - regroups = list(self.group_helper.get_regroups(session)) - autoregroup = self.group_helper.get_autoregroup(session) - autoregroup[0][0] = "Default" - regroups.append(autoregroup[0]) - for regroup in regroups: - self.field_tmpl.render(writer, session, regroup) - - return writer.to_string() - - def render_regroup_selected_TRUE(self, session, regroup): - if regroup[2][2]: - return (regroup[1] == 'TRUE') and "checked=\"checked\"" or "" - - def render_regroup_selected_FALSE(self, session, regroup): - if regroup[2][2]: - return (regroup[1] == 'FALSE') and "checked=\"checked\"" or "" - - def render_regroup_selected_UNDEFINED(self, session, regroup): - if not regroup[2][2]: - return "checked=\"checked\"" - - def render_regroup_path(self, session, regroup): - return self.regroup.get_instance_key(regroup[0]) - - def render_original_regroup_path(self, session, regroup): - return self.original_regroups.path - - def render_regroup_value(self, session, regroup): - return regroup[2][2] and regroup[1] or "UNDEFINED" - - def process_submit(self, session): - negotiator = self.object.get(session) - regroup = self.regroup.get(session) - group_names = self.group_names.get(session) - original_regroups = self.original_regroups.get(session) - - changed = False - for group, original_value in zip(group_names, original_regroups): - if regroup[group] != original_value: - if group == "Default": - self.task.invoke(session, negotiator, "AUTOREGROUP", regroup[group]) - else: - self.task.invoke(session, negotiator, "GROUP_AUTOREGROUP_"+group, regroup[group]) - changed = True - if changed: - self.task.reconfig(negotiator) - self.task.exit_with_redirect(session) - -class UserRegroupForm(EditRegroupForm): - def __init__(self, app, name, task): - super(UserRegroupForm, self).__init__(app, name, task) - - self.group = Parameter(app, "group") - self.add_parameter(self.group) - - def render_groups(self, session): - writer = Writer() - - group = self.group.get(session) - regroups = list(self.group_helper.get_regroups(session, [group], True)) - for regroup in regroups: - self.field_tmpl.render(writer, session, regroup) - - return writer.to_string() - - def render_form_class(self, session): - return "UserRegroupForm" - - def process_submit(self, session): - negotiator = self.object.get(session) - regroup = self.regroup.get(session) - group = self.group.get(session) - original_regroups = self.original_regroups.get(session) - - changed = False - for user, original_value in zip([group], original_regroups): - if regroup[user] != original_value: - self.task.invoke(session, negotiator, "GROUP_AUTOREGROUP_"+user, regroup[user]) - changed = True - if changed: - self.task.reconfig(negotiator) - self.task.exit_with_redirect(session) - class PriorityPieChart(StatFlashChart): def __init__(self, app, name, negotiator, group_helper, group_leader): super(PriorityPieChart, self).__init__(app, name, negotiator) @@ -962,36 +431,6 @@ action = QmfCall(self.app) action.execute(negotiator, "Reconfig") -class NegotiatorGroupAdd(NegotiatorGroupTask): - def __init__(self, app, frame): - super(NegotiatorGroupAdd, self).__init__(app, frame) - - self.form = GroupAddForm(app, self.name, self) - - def get_title(self, session): - return "" - - def get_description(self, session): - return "Add group" - -class NegotiatorEditRegroup(NegotiatorGroupTask): - def __init__(self, app, frame): - super(NegotiatorEditRegroup, self).__init__(app, frame) - - self.form = EditRegroupForm(app, self.name, self) - - def get_title(self, session): - return "Edit autoregroup" - -class NegotiatorEditPrioFactor(NegotiatorGroupTask): - def __init__(self, app, frame): - super(NegotiatorEditPrioFactor, self).__init__(app, frame) - - self.form = EditPrioFactorForm(app, self.name, self) - - def get_title(self, session): - return "Edit priority factor" - class NegotiatorEditDynamicQuota(NegotiatorGroupTask): def __init__(self, app, frame): super(NegotiatorEditDynamicQuota, self).__init__(app, frame) @@ -1010,39 +449,3 @@ negotiator = self.frame.negotiator_attribute.get(osession) self.form.negotiator.set(session, negotiator) - -class NegotiatorEditStaticQuota(NegotiatorGroupTask): - def __init__(self, app, frame): - super(NegotiatorEditStaticQuota, self).__init__(app, frame) - - self.form = EditStaticQuotaForm(app, self.name, self) - - def get_title(self, session): - return "Edit static quota" - -class NegotiatorUserTask(NegotiatorGroupTask): - def do_enter(self, session, osession): - #try: - # negotiator, group = ng - #except: - # raise Exception("Must supply group for user forms") - super(NegotiatorUserTask, self).do_enter(session, osession) - #self.form.group.set(session, group) - -class NegotiatorUserPrioFactor(NegotiatorUserTask): - def __init__(self, app, frame): - super(NegotiatorUserPrioFactor, self).__init__(app, frame) - - self.form = UserPrioFactorForm(app, self.name, self) - - def get_title(self, session): - return "Edit user priority factor" - -class NegotiatorUserRegroup(NegotiatorUserTask): - def __init__(self, app, frame): - super(NegotiatorUserRegroup, self).__init__(app, frame) - - self.form = UserRegroupForm(app, self.name, self) - - def get_title(self, session): - return "Edit user autoregroup" Modified: trunk/cumin/python/cumin/grid/negotiator.strings =================================================================== --- trunk/cumin/python/cumin/grid/negotiator.strings 2011-02-04 17:46:17 UTC (rev 4504) +++ trunk/cumin/python/cumin/grid/negotiator.strings 2011-02-07 16:44:47 UTC (rev 4505) @@ -1,85 +1,3 @@ -[NegotiatorOverview.css] -div.GroupTable ul.actions { - display: inline; -} -div.GroupTable div.sactions { - padding: 0.75em; -} - -th.ExpandColumn { - width: 2em; -} -td.ExpandColumn a.action { - width: 1em; - display: block; - text-align: center; -} - -th.GroupColumn { - width: 50%; -} - -th.SpacerColumn { - width: 25%; -} - -table.mobjects th.DynamicColumn { - white-space: nowrap; - text-align: right; -} -table.mobjects td.DynamicColumn { - text-align: right; - padding-right: 2em; -} - -span.QuotaError { - font-style: italic; - color: red; -} - -[NegotiatorOverview.html] -<div id="{id}" class="CuminTable GroupTable"> - <table {class}> - <thead><tr>{headers}</tr></thead> - <tbody>{items}</tbody> - </table> -</div> - -[NegotiatorOverview.deferred_html] -<div id="{id}"> - <div class="deferredSpacer">Loading...</div> -</div> -<script type="text/javascript"> -<![CDATA[ -(function() { - window.addEvent("domready", function () { - setTimeout( function () { window.location.reload(); }, 5000 ); - }); -}()) -]]> -</script> - -[NegotiatorOverview.error_html] -<div id="{id}"> - <div class="TaskInvocationSet"> - <ul> - <li>{error_msg}</li> - </ul> - </div> -</div> - -[GroupForm.css] -div.deferredSpacer { - height: 10em; - background-color: #FFFFFF; - padding: 1em; -} - -div.invalid { - font-size: .8em; - color: red; -} - [GroupForm.html] <form id="{id}" class="mform {form_class}" method="post" action="?"> <div class="head"> @@ -283,196 +201,3 @@ <div class="deferredSpacer">Loading...</div> </div> </form> - -[EditStaticQuotaForm.css] -form.StaticQuotaForm { - width: 30em; -} - -[EditStaticQuotaForm.field_html] -<tr> - <td> - {group_name} - </td> - <td> - <input class="validate-numeric" type="text" name="{quota_name}" value="{quota_value}" - size="6" tabindex="100"/> -  - <input type="hidden" name="{group_name_path}" value="{group_name}" /> - <input type="hidden" name="{original_value_path}" value="{quota_value}" /> - </td> -</tr> - -[EditStaticQuotaForm.deferred_html] -<form id="{id}" class="mform StaticQuotaForm" method="post" action="?"> - <div class="head"> - <h1>{title}</h1> - </div> - <div class="body"> - <div class="deferredSpacer">Loading...</div> - </div> -</form> -<script type="text/javascript"> -<![CDATA[ -cumin.initializeGroupForm = function () { - var elem = $("{id}").elements[0]; - elem.focus(); - - var myFormValidator = new FormValidator($('{id}'), { - evaluateOnSubmit: false, - evaluateFieldsOnBlur: false - }); -} -]]> -</script> - -[EditPrioFactorForm.css] -form.mform.PrioFactorForm { - width: 32em; -} -form.mform.UserPrioFactorForm { - width: 40em; -} -form.mform.UserRegroupForm { - width: 48em; -} - -[EditPrioFactorForm.field_html] -<tr> - <td> - {group_name} - </td> - <td> - <input class="validate-numeric OneOrMore" type="text" name="{factor_name}" value="{factor_value}" - size="10" tabindex="100"/> <div class="invalid"></div> -  - <input type="hidden" name="{group_name_path}" value="{group_name}" /> - <input type="hidden" name="{original_value_path}" value="{factor_value}" /> - </td> -</tr> - -[EditPrioFactorForm.deferred_html] -<form id="{id}" class="mform PrioFactorForm" method="post" action="?"> - <div class="head"> - <h1>{title}</h1> - </div> - <div class="body"> - <div class="deferredSpacer">Loading...</div> - </div> -</form> -<script type="text/javascript"> -<![CDATA[ -cumin.initializeGroupForm = function () { - var elem = $("{id}").elements[0]; - elem.focus(); - - var myFormValidator = new FormValidator($('{id}'), { - evaluateOnSubmit: false, - evaluateFieldsOnBlur: false, - onElementFail: function (el, validators) { - var span = el.getParent("td").getElement("div"); - if (validators[0] == "validate-numeric") - span.set('text', "Must be a number."); - else - span.set('text', "Must be 1.0 or greater."); - }, - onElementPass: function (el) { - el.getParent("td").getElement("div").set('text', ""); - } - }); - - myFormValidator.add('OneOrMore', { - errorMsg: 'Values must be 1 or greater.', - test: function(element) { - var val = element.get('value').toFloat(); - return (val >= 1); - } - }); -} -]]> -</script> - -[EditRegroupForm.field_html] -<tr> - <td> - {group_name} - <input type="hidden" name="{group_name_path}" value="{group_name}" /> - </td> - <td> - <input type="radio" name="{regroup_path}" {regroup_selected_TRUE} id="regroup_{group_name}_TRUE" value="TRUE" /> <label for="regroup_{group_name}_TRUE">TRUE</label> - <input type="radio" name="{regroup_path}" {regroup_selected_FALSE} id="regroup_{group_name}_FALSE" value="FALSE" /> <label for="regroup_{group_name}_FALSE">FALSE</label> - <input type="radio" name="{regroup_path}" {regroup_selected_UNDEFINED} id="regroup_{group_name}_UNDEFINED" value="UNDEFINED" /> <label for="regroup_{group_name}_UNDEFINED">NOT SET</label> - <input type="hidden" name="{original_regroup_path}" value="{regroup_value}" /> - </td> -</tr> - -[EditRegroupForm.deferred_html] -<form id="{id}" class="mform StaticQuotaForm" method="post" action="?"> - <div class="head"> - <h1>{title}</h1> - </div> - <div class="body"> - <div class="deferredSpacer">Loading...</div> - </div> -</form> - -[AddGroupForm.css] -form.mform.AddForm { - width: 30em; -} -form.mform.AddForm div.label, form.mform.AddForm input { - float: left; - margin: 1em 0 1em 1em; -} - -[AddGroupForm.html] -<form id="{id}" class="mform AddForm" method="post" action="?"> - <div class="head"> - <h1>{title}</h1> - </div> - <div class="body"> - <div class="label">Group Name</div> - <input class="validate-nodot" type="text" name="{group_name_path}" value="" size="20" maxlength="40" /> - <div style="clear: left;"></div> - <div class="invalid"></div> - </div> - <div class="foot"> - {submit} - {cancel} - </div> - <div>{hidden_inputs}</div> -</form> -<script type="text/javascript"> -<![CDATA[ -window.addEvent('domready', function () { - var elem = $("{id}").elements[0]; - elem.focus(); - - var myFormValidator = new FormValidator($('{id}'), { - evaluateOnSubmit: false, - evaluateFieldsOnBlur: false, - onElementFail: function (el, validators) { - var span = el.getParent("div").getElement("div.invalid"); - span.set('text', "Must not contain a ."); - }, - onElementPass: function (el) { - el.getParent("div").getElement("div.invalid").set('text', ""); - } - }); - - myFormValidator.add('validate-nodot', { - errorMsg: '', - test: function(element) { - var bool = element.get('value').test(/\./,'i'); - return !bool; - } - }); -}); -]]> -</script> - -[ExpandColumn.css] -th.expand { - width: 1em; - padding: 0; -} Modified: trunk/cumin/python/cumin/grid/pool.py =================================================================== --- trunk/cumin/python/cumin/grid/pool.py 2011-02-04 17:46:17 UTC (rev 4504) +++ trunk/cumin/python/cumin/grid/pool.py 2011-02-07 16:44:47 UTC (rev 4505) @@ -19,6 +19,7 @@ from job import * from limit import * from negotiator import * +from quota import * from scheduler import * from slot import * from submission import * @@ -101,7 +102,7 @@ self.add_attribute(self.negotiator_attribute) self.edit_dynamic_quota = NegotiatorEditDynamicQuota(app, self) - self.quotas = NegotiatorOverview(app, "quotas", self.negotiator_attribute, self) + self.quotas = QuotaSelector(app, "quotas", self.negotiator_attribute, self) self.view.add_tab(self.quotas) self.limits = LimitSet(app, "limits", self.object) Added: trunk/cumin/python/cumin/grid/quota.py =================================================================== --- trunk/cumin/python/cumin/grid/quota.py (rev 0) +++ trunk/cumin/python/cumin/grid/quota.py 2011-02-07 16:44:47 UTC (rev 4505) @@ -0,0 +1,278 @@ +from wooly.util import StringCatalog, Writer +import logging +from cumin.widgets import StaticColumnHeader +from cumin.formats import fmt_link +from cumin.objectselector import ObjectTable, ObjectSelector, ObjectTableColumn +from wooly.template import WidgetTemplate +from cumin.qmfadapter import ObjectQmfAdapter +from negotiator import GroupHelper +from wooly import Parameter, Widget + +strings = StringCatalog(__file__) +log = logging.getLogger("cumin.grid.quota") + +class QmfGroupColumn(ObjectTableColumn): + def __init__(self, app, name, attr, getter, negotiator, task): + super(QmfGroupColumn, self).__init__(app, name, attr) + + self.header = StaticColumnHeader(app, "header") + self.replace_child(self.header) + + self.title = None + self.getter = getter + #self.align = "right" + self.user = False + self.negotiator = negotiator + self.task = task + self.user_task = None + + def render_title(self, session, *args): + return self.title + + def render_content(self, session, group): + data = self.getter(session) + for x in data: + if x[0] == group: + return self.render_data(session, x) + + return "" + + def render_data(self, session, data): + href = self.task.get_href(session) + content = data[2][2] and str(data[1]) or "NOT SET" + return fmt_link(href, content, "", "", self.fmt_hover(data[0])) + + def fmt_hover(self, group): + return "Edit the %s" % self.title + +class QuotaAdapter(ObjectQmfAdapter): + def get_data(self, values, options): + items = values['items'] + return items + + def get_count(self, values): + return len(values['items']) + +class QuotaTable(ObjectTable): + def __init__(self, app, name, cls): + super(QuotaTable, self).__init__(app, name, cls) + + self.update_enabled = False + + limit = Widget(app, "limit") + self.header.replace_child(limit) + + page = Widget(app, "page") + self.header.replace_child(page) + + def get_data_values(self, session): + values = super(QuotaTable, self).get_data_values(session) + + info = self.parent.group_helper.get_config_info(session) + expanded = self.parent.expand.get(session) + + names = list(info) + names = sorted(names) + + # get all the values for csv export + if session.page == self.app.export_page: + items = names + else: + # make a list of groups to show based on what is expanded + parent = expanded and self.parent.group_helper.get_parent(session, expanded) or None + items = list() + for name in names: + # always show top level groups + if "." not in name: + items.append(name) + # add the current expanded item + elif expanded == name: + items.append(name) + # add the ancestors + elif expanded and expanded.startswith(name+"."): + items.append(name) + # add the direct children of expanded item + elif expanded and name.startswith(expanded+"."): + sub_name = name[len(expanded) + 1:] + if "." not in sub_name: + items.append(name) + # and direct children of expanded item's parent + elif parent and name.startswith(parent+"."): + sub_name = name[len(parent) + 1:] + if "." not in sub_name: + items.append(name) + + self.parent.group_helper.get_config_for_groups(session, "GROUP_QUOTA_DYNAMIC", items) + + values['items'] = items + + negotiator = self.parent.negotiator.get(session) + values["obj"] = negotiator + + return values + +class QuotaSelector(ObjectSelector): + def __init__(self, app, name, negotiator, frame): + cls = app.model.com_redhat_cumin_grid.Quota + + super(QuotaSelector, self).__init__(app, name, cls) + + self.update_enabled = True + + self.negotiator = negotiator + self.table.negotiator = negotiator + self.table.adapter = QuotaAdapter(app, cls) + + self.group_helper = GroupHelper(app, "groups", negotiator) + self.add_child(self.group_helper) + + self.expand = Parameter(app, "expand") + self.add_parameter(self.expand) + + col = self.ExpandColumn(app, "id", cls.id, self.expand, self.group_helper) + col.header_class = StaticColumnHeader + col.width = "20px" + self.add_column(col) + + col = self.GroupColumn(app, "group", cls.Name) + col.width = "80%" + self.add_column(col) + + task = frame.edit_dynamic_quota + col = self.DynamicColumn(app, "dynamic", cls.Quota, None, negotiator, task, self.group_helper) + self.add_column(col) + + self.enable_csv_export(negotiator) + + self.error_tmpl = WidgetTemplate(self, "error_html") + self.loading_tmpl = WidgetTemplate(self, "deferred_html") + + def create_table(self, app, name, cls): + return QuotaTable(app, name, cls) + + def render_title(self, session): + return "Quotas" + + def do_render(self, session): + # determine if there is an error getting the data + # and show an error message + qmf_response = None + tmpl = self.error_tmpl + m = "" + + try: + qmf_response = self.group_helper.get_results(session) + if not qmf_response: + m = "" + elif qmf_response.exception: + e = qmf_response.exception + m = len(e.args) > 0 and e.args[0] or "" + if m == "Loading": + tmpl = self.loading_tmpl + else: + neg = self.negotiator.get(session) + neg_name = neg and " for negotiator %s" % neg.Name or "" + m = "%s%s" % (m, neg_name) + except AssertionError: + m = "Missing Negotiator" + + if (not qmf_response) or qmf_response.exception: + writer = Writer() + if "is unknown" in m: + m = "Please try again later" + err_msg = "The data is not available at this time. <br/>%s" % m + tmpl.render(writer, session, err_msg) + return writer.to_string() + + return super(QuotaSelector, self).do_render(session) + + def render_error_msg(self, session, msg): + return msg + + class ExpandColumn(ObjectTableColumn): + def __init__(self, app, name, attr, expand, group_helper): + super(QuotaSelector.ExpandColumn, self).__init__(app, name, attr) + + self.expand = expand + self.group_helper = group_helper + + def get_class_list(self, session): + return ["expand"] + + def render_header_content(self, session): + return "" + + def render_cell_content(self, session, group): + if not self.group_helper.has_child(session, group): + return "" + + self.group_helper.get_config_info(session) + # return a " or - depending on current expand + expand = self.expand.get(session) + if expand and expand.startswith(group): + state = "-" + else: + state = "+" + + parent = self.group_helper.get_parent(session, group) + return self.render_item_link(session, group, parent, state) + + def render_item_link(self, session, group, parent, state): + branch = session.branch() + + next_expand = state == "+" and group or parent + self.expand.set(branch, next_expand) + + hover = state == "-" and "Collapse" or "Expand" + cls = "action" + + return fmt_link(branch.marshal(), state, class_=cls, link_title=hover) + + class GroupColumn(ObjectTableColumn): + def __init__(self, app, name, attr): + super(QuotaSelector.GroupColumn, self).__init__(app, name, attr) + + self.header = StaticColumnHeader(app, "header") + self.replace_child(self.header) + + def render_cell_content(self, session, group): + if session.page == self.app.export_page: + return group + + # if a user + if "." in group: + parts = group.split('.') + indent = len(parts) - 1 + user = parts[indent] + return "<span style='padding-left: %dem;'>%s</span>" % (indent, user) + + return group + + class DynamicColumn(QmfGroupColumn): + def __init__(self, app, name, attr, getter, negotiator, task, group_helper): + super(QuotaSelector.DynamicColumn, self).__init__(app, name, attr, getter, negotiator, task) + + self.group_helper = group_helper + + def render_cell_content(self, session, group): + value = self.group_helper.get_config_value(session, group, "GROUP_QUOTA_DYNAMIC") + self.task.form.group_leader.set(session, group) + + try: + fval = float(value) + except ValueError: + return value + except TypeError: + if isinstance(value, Exception): + content = "<span class='QuotaError'>error</span>" + val = len(value.args) > 0 and value.args[0] or "" + if session.page == self.app.export_page: + return val + return fmt_link("#", content, "", "", val) + + if session.page == self.app.export_page: + return value + + href = self.task.get_href(session) + content = "%s%%" % str(round(fval * 100.0, 2)) + return fmt_link(href, content, "", "", self.fmt_hover("")) Added: trunk/cumin/python/cumin/grid/quota.strings =================================================================== --- trunk/cumin/python/cumin/grid/quota.strings (rev 0) +++ trunk/cumin/python/cumin/grid/quota.strings 2011-02-07 16:44:47 UTC (rev 4505) @@ -0,0 +1,29 @@ +[QuotaSelector.css] + +span.QuotaError { + font-style: italic; + color: red; +} + +[QuotaSelector.deferred_html] +<div id="{id}"> + <div class="deferredSpacer">Loading...</div> +</div> +<script type="text/javascript"> +<![CDATA[ +(function() { + window.addEvent("domready", function () { + setTimeout( function () { window.location.reload(); }, 5000 ); + }); +}()) +]]> +</script> + +[QuotaSelector.error_html] +<div id="{id}"> + <div class="TaskInvocationSet"> + <ul> + <li>{error_msg}</li> + </ul> + </div> +</div> Modified: trunk/cumin/python/cumin/model.py =================================================================== --- trunk/cumin/python/cumin/model.py 2011-02-04 17:46:17 UTC (rev 4504) +++ trunk/cumin/python/cumin/model.py 2011-02-07 16:44:47 UTC (rev 4505) @@ -1,21 +1,19 @@ -from decimal import * -from mint import * -from rosemary.model import * -from struct import unpack, calcsize -from threading import Thread, Lock -from types import * -from wooly import * -from wooly.parameters import * -from wooly.widgets import * -from wooly.datatable import * +import logging +import os +from threading import Lock, Thread +from datetime import datetime, timedelta +from time import sleep, mktime +from decimal import Decimal +from types import NoneType -from formats import * -from parameters import * -from sqladapter import * -from util import * +from cumin.formats import fmt_datetime, fmt_dict, fmt_none_brief, fmt_none,\ + fmt_duration +from cumin.util import calc_rate, JobStatusInfo, secs, wait +from cumin.sqladapter import SqlAdapter +from rosemary.sqlfilter import SqlComparisonFilter +from rosemary.sqlquery import SqlQueryOptions +from rosemary.model import RosemaryModel -import wooly - log = logging.getLogger("cumin.model") class CuminModel(RosemaryModel): @@ -107,15 +105,15 @@ for group in needed_groups: added += store.add_group_config(group, config) if added > 0: - store.update(None) - sleep(1) + store.update_new(None) + #sleep(1) except KeyError: store = NegotiatorGroupConfigValuesStore(self, negotiator, needed_groups, config) store.start_updates() self.group_config_values_by_negotiator[negotiator._qmf_agent_id] = store - sleep(1) + #sleep(1) return store finally: @@ -1011,16 +1009,13 @@ self.id = id def get_collector(self): - for collector in Collector.selectBy(Pool=self.id): - return collector + return None def get_grid(self): - for grid in Grid.selectBy(Pool=self.id): - return grid + return None def get_negotiator(self): - for negotiator in Negotiator.selectBy(Pool=self.id): - return negotiator + return None class ObjectStore(object): def __init__(self, model): @@ -1134,6 +1129,7 @@ self.status = status try: self.data = data["Value"] + #self.data = "MSG, GRID, MGMT, RT, GRID.SUB_1, GRID.SUB_1.SUB_A" except KeyError: pass @@ -1175,6 +1171,20 @@ raw_configs = action.execute(self.negotiator, self.configs[config], config+"_") self.data[config] = raw_configs + def update_new(self, cursor): + for config in self.configs: + action = FetchRawConfigSet(self.model.app) + new_configs = list() + for group in self.configs[config]: + if group not in self.data[config] or \ + self.data[config][group].error or \ + not self.data[config][group].data: + new_configs.append(group) + + raw_configs = action.execute(self.negotiator, new_configs, config+"_") + for group in raw_configs: + self.data[config][group] = raw_configs[group] + def delete(self): del self.model.group_config_values_by_negotiator[self.negotiator._qmf_agent_id] Modified: trunk/cumin/python/cumin/session.py =================================================================== --- trunk/cumin/python/cumin/session.py 2011-02-04 17:46:17 UTC (rev 4504) +++ trunk/cumin/python/cumin/session.py 2011-02-07 16:44:47 UTC (rev 4505) @@ -2,7 +2,8 @@ from util import * from mint.util import make_agent_id -from qmf.console import Console, Session, ClassKey, ObjectId +from qmf.console import Console, Session, ObjectId +from rosemary.model import RosemaryObject log = logging.getLogger("cumin.session")

13 years, 3 months

1
0
0 / 0

r4504 - in branches/scale_testing/mech_scripts: . test_config_single

by tmckay＠fedoraproject.org

Author: tmckay Date: 2011-02-04 17:46:17 +0000 (Fri, 04 Feb 2011) New Revision: 4504 Added: branches/scale_testing/mech_scripts/test.py branches/scale_testing/mech_scripts/test_config_single/ branches/scale_testing/mech_scripts/test_config_single/test_100thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_105thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_10thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_110thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_15thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_1thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_20thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_25thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_30thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_35thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_40thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_45thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_50thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_55thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_5thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_60thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_65thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_70thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_75thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_80thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_85thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_8thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_90thread.cfg branches/scale_testing/mech_scripts/test_config_single/test_95thread.cfg Modified: branches/scale_testing/mech_scripts/process_times.py branches/scale_testing/mech_scripts/test_driver.py Log: Add wrapper script and configs, page size change check. Modified: branches/scale_testing/mech_scripts/process_times.py =================================================================== --- branches/scale_testing/mech_scripts/process_times.py 2011-02-03 15:48:13 UTC (rev 4503) +++ branches/scale_testing/mech_scripts/process_times.py 2011-02-04 17:46:17 UTC (rev 4504) @@ -58,6 +58,8 @@ end_proc = 7 send_headers = 8 busy_threads = 9 + data_size = 10 # we don't really do anything with data_size right now, we just want to see it + last_field = 10 # start iteration count at 1, bump for blank lines in inputs. # this is used for calculating averages from the sums later @@ -67,6 +69,8 @@ sums = get_zero_list() longest = [""]+sums # this will be the worst case get, with a label count = 0 + page_size = -1 + line_count = 0 # Create a zero time delta max = datetime.utcnow() @@ -75,17 +79,28 @@ p = os.path.abspath(fname) f = open(p, "r") for line in f: + line_count += 1 - # Look for iteration marker - if len(line.strip()) == 0: + # Look for page end marker, will contain total size for that page for new stuff. + # Old stuff might still have a blank line + times = line.strip().split(",") + if times[0] == "page_break" or times[0] == "": + last_page_size = page_size + if len(times) > 1: + page_size = int(times[1]) + else: + page_size = 0 + + # Page size should change once on the second iteration, because caching is employed. + # Don't report page size change the first time + if iterations == 3 and last_page_size > 0 and abs(last_page_size - page_size) > 2: + print "******** Page size changed! File " + fname + ", Line " + str(line_count) + " **************" iterations += 1 continue - times = line.strip().split(",") - # handle files I collected before.... - if len(times) < busy_threads+1: - for i in range(len(times), busy_threads+1): + if len(times) < last_field+1: + for i in range(len(times), last_field+1): times.append("0") skip_line = False @@ -125,12 +140,12 @@ for i in range(len(sums)): avg.append(sums[i] / count) - print "**************** " + fname + " ****************" +# print "**************** " + fname + " ****************" # print "longest:" # dump(longest) - dump(["average values"]+avg) +# dump(["average values"]+avg) # Blank lines in the input file represent an iteration. # Get average values for a page by counting and dividing by the number of iterations. @@ -146,7 +161,7 @@ except: pass - dump(["per page"]+page_avg) +# dump(["per page"]+page_avg) return page_avg Added: branches/scale_testing/mech_scripts/test.py =================================================================== --- branches/scale_testing/mech_scripts/test.py (rev 0) +++ branches/scale_testing/mech_scripts/test.py 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,58 @@ +#!/usr/bin/python + +import os +import shutil +import subprocess +import re +import sys + +def main(): + if len(sys.argv) < 4: + print "usage: test.py config_file_dir project_name csv_output_file" + return + + # Get list of config files that we are going to substitute for config.cfg in multimechanize project + file_list = [] + if os.path.isdir(sys.argv[1]): + for i in os.listdir(sys.argv[1]): + file_list.append(os.path.join(sys.argv[1], i)) + else: + file_list = [sys.argv[1]] + + project = sys.argv[2] + proj_dir = os.path.join("projects", sys.argv[2]) + if not os.path.isdir(proj_dir): + print "Project name is not a directory:", project + return + + # This is the config file for multimechanize that we're going to copy over + # We can save it first and put it back later just to be nice + config = os.path.join(proj_dir, "config.cfg") + shutil.copyfile(config, "./.config.cfg.save") + + + print file_list + + + for file in file_list: + shutil.copyfile(file, config) + os.system("python multi-mechanize.py " + project) + + # Figure out what subdirectory to process. Name convention, same as config file with .cfg removed + subdir = os.path.basename(file).split(".")[0] + + # Figure out what the threadcount arg should be for process times + try: + threads = re.findall("[0123456789]+", subdir)[0] + except: + threads = "0" + + # run process times + os.system("python process_times.py " + subdir + " --csvfile=" + sys.argv[3] + " --threads=" + threads) + + + shutil.copyfile("./config.cfg.save", config) + +if __name__ == '__main__': + main() + Added: branches/scale_testing/mech_scripts/test_config_single/test_100thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_100thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_100thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 100 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_100thread Added: branches/scale_testing/mech_scripts/test_config_single/test_105thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_105thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_105thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 105 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_105thread Added: branches/scale_testing/mech_scripts/test_config_single/test_10thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_10thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_10thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 10 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_10thread Added: branches/scale_testing/mech_scripts/test_config_single/test_110thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_110thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_110thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 110 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_110thread Added: branches/scale_testing/mech_scripts/test_config_single/test_15thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_15thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_15thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 15 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_15thread Added: branches/scale_testing/mech_scripts/test_config_single/test_1thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_1thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_1thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,11 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + + +[user_group-1] +threads: 1 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_1thread Added: branches/scale_testing/mech_scripts/test_config_single/test_20thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_20thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_20thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 20 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_20thread Added: branches/scale_testing/mech_scripts/test_config_single/test_25thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_25thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_25thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 25 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_25thread Added: branches/scale_testing/mech_scripts/test_config_single/test_30thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_30thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_30thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 30 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_30thread Added: branches/scale_testing/mech_scripts/test_config_single/test_35thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_35thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_35thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 35 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_35thread Added: branches/scale_testing/mech_scripts/test_config_single/test_40thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_40thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_40thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 40 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_40thread Added: branches/scale_testing/mech_scripts/test_config_single/test_45thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_45thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_45thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 45 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_45thread Added: branches/scale_testing/mech_scripts/test_config_single/test_50thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_50thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_50thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 50 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_50thread Added: branches/scale_testing/mech_scripts/test_config_single/test_55thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_55thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_55thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,11 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 55 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_55thread + Added: branches/scale_testing/mech_scripts/test_config_single/test_5thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_5thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_5thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,11 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + + +[user_group-1] +threads: 5 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_5thread Added: branches/scale_testing/mech_scripts/test_config_single/test_60thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_60thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_60thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 60 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_60thread Added: branches/scale_testing/mech_scripts/test_config_single/test_65thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_65thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_65thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,11 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 65 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_65thread + Added: branches/scale_testing/mech_scripts/test_config_single/test_70thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_70thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_70thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 70 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_70thread Added: branches/scale_testing/mech_scripts/test_config_single/test_75thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_75thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_75thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 75 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_75thread \ No newline at end of file Added: branches/scale_testing/mech_scripts/test_config_single/test_80thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_80thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_80thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 80 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_80thread Added: branches/scale_testing/mech_scripts/test_config_single/test_85thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_85thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_85thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 85 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_85thread Added: branches/scale_testing/mech_scripts/test_config_single/test_8thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_8thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_8thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 8 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_8thread Added: branches/scale_testing/mech_scripts/test_config_single/test_90thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_90thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_90thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 90 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_90thread Added: branches/scale_testing/mech_scripts/test_config_single/test_95thread.cfg =================================================================== --- branches/scale_testing/mech_scripts/test_config_single/test_95thread.cfg (rev 0) +++ branches/scale_testing/mech_scripts/test_config_single/test_95thread.cfg 2011-02-04 17:46:17 UTC (rev 4504) @@ -0,0 +1,10 @@ +[global] +run_time: 30 +rampup: 0 +console_logging: off +results_ts_interval: 1 + +[user_group-1] +threads: 95 +script: test_driver.py +options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_95thread Modified: branches/scale_testing/mech_scripts/test_driver.py =================================================================== --- branches/scale_testing/mech_scripts/test_driver.py 2011-02-03 15:48:13 UTC (rev 4503) +++ branches/scale_testing/mech_scripts/test_driver.py 2011-02-04 17:46:17 UTC (rev 4504) @@ -32,6 +32,7 @@ self.file_init = False self.file_number = 0 self.file_iterations = 0 + self.last_page_total = 0 def next_file(self): if not self.file_init: @@ -51,23 +52,25 @@ self.file_number += 1 return name - def handle_custom_headers(self, curr_file, link, resp, insert_blank): + def handle_custom_headers(self, curr_file, link, resp, data_size): fname = os.path.join(self.outdir, self.group + self.thread + "_" + os.path.basename(curr_file)) - if os.path.isfile(fname): - f = open(fname, "a") - if insert_blank: - f.write("\n") - else: - f = open(fname, "w") + f = open(fname, "a+") f.write(link) for i in ("page_before_accept", "page_after_accept", "page_put_time", "page_after_get", "page_begin_service", "page_begin_proc", "page_end_proc","page_send_headers","page_busy_threads"): - f.write(","+resp.getheader(i,"0")) + f.write(","+resp.getheader(i,"0")) + f.write(","+str(data_size)) f.write("\n") f.close() + def write_page_end(self, curr_file, total_size): + fname = os.path.join(self.outdir, self.group + self.thread + "_" + os.path.basename(curr_file)) + f = open(fname, "a+") + f.write("page_break,"+str(total_size)+"\n") + f.close() + def run(self): try: # use this to get an idea of tets overhead. @@ -101,7 +104,12 @@ self.thread = opts.thread self.outdir = opts.outdir if self.outdir != "." and not os.path.isdir(self.outdir): - os.mkdir(self.outdir) + try: + # in a multi thread environment, we may + # actually collide here in the creation. + os.mkdir(self.outdir) + except: + pass curr_file = self.next_file() f = open(curr_file, "r") @@ -109,7 +117,7 @@ print curr_file total_gets = total_size = get_latency = 0 - first_link = True + for link in f: link_info = link.split(',') @@ -150,9 +158,12 @@ # write custom headers to a file -- too much overhead to # embed in the test. parse(datetime) is apparently expensive - self.handle_custom_headers(curr_file, link, resp, first_link) - first_link = False + # include the length of the data received, too, so we can look for changes + self.handle_custom_headers(curr_file, link, resp, len(data)) + # Indicate page break and include total data size for this page in headers file + self.write_page_end(curr_file, total_size) + # Record some custom stuff self.custom_timers['Page File'] = get_latency self.custom_timers['Avg Get'] = get_latency / total_gets

13 years, 3 months

1
0
0 / 0

r4503 - trunk/cumin/python/cumin

by eallen＠fedoraproject.org

Author: eallen Date: 2011-02-03 15:48:13 +0000 (Thu, 03 Feb 2011) New Revision: 4503 Modified: trunk/cumin/python/cumin/model.py Log: Removing more debug code Modified: trunk/cumin/python/cumin/model.py =================================================================== --- trunk/cumin/python/cumin/model.py 2011-02-03 15:09:05 UTC (rev 4502) +++ trunk/cumin/python/cumin/model.py 2011-02-03 15:48:13 UTC (rev 4503) @@ -1133,8 +1133,7 @@ def completion(status, data): self.status = status try: - #self.data = data["Value"] - self.data = "MSG, GRID, MGMT, RT, GRID.SUB_1, GRID.SUB_1.SUB_A" + self.data = data["Value"] except KeyError: pass

13 years, 3 months

1
0
0 / 0

r4502 - trunk/cumin/python/cumin

by eallen＠fedoraproject.org

Author: eallen Date: 2011-02-03 15:09:05 +0000 (Thu, 03 Feb 2011) New Revision: 4502 Modified: trunk/cumin/python/cumin/model.py Log: Remove debugging code Modified: trunk/cumin/python/cumin/model.py =================================================================== --- trunk/cumin/python/cumin/model.py 2011-02-02 21:26:13 UTC (rev 4501) +++ trunk/cumin/python/cumin/model.py 2011-02-03 15:09:05 UTC (rev 4502) @@ -1174,12 +1174,6 @@ for config in self.configs: action = FetchRawConfigSet(self.model.app) raw_configs = action.execute(self.negotiator, self.configs[config], config+"_") - for group in raw_configs: - qmfc = raw_configs[group] - qmfc.data = {'Value': 0.1} - qmfc.error = None - qmfc.got_data = True - qmfc.status = "OK" self.data[config] = raw_configs def delete(self):

13 years, 3 months

1
0
0 / 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

cumin-developers February 2011