Author: eallen
Date: 2011-02-21 20:11:08 +0000 (Mon, 21 Feb 2011)
New Revision: 4541
Modified:
trunk/cumin/python/cumin/charts.py
trunk/cumin/python/cumin/grid/slot.py
trunk/cumin/python/cumin/grid/slot.strings
trunk/cumin/resources/slots.swf
Log:
Implement BZ 647500 - Categorize slots into more natural categories.
- Updated action script code to handle categories
- Updated png generation code to match the new categories
- Updated json generation code to group slots into new categories
- Updated hover popup for swf and png to display the new categories
- Updated legend for sef and png to show the new categories
Modified: trunk/cumin/python/cumin/charts.py
===================================================================
--- trunk/cumin/python/cumin/charts.py 2011-02-21 19:30:33 UTC (rev 4540)
+++ trunk/cumin/python/cumin/charts.py 2011-02-21 20:11:08 UTC (rev 4541)
@@ -22,6 +22,18 @@
self.surface = None
self.gap = 3
+ def plot_colored_shape(self, interior, shape, width, height):
+ surface = ImageSurface(FORMAT_ARGB32, int(width), int(height))
+ cr = Context(surface)
+ cr.set_line_width(1)
+
+ cr.set_source_rgba(1.0, 1.0, 1.0, 1.0)
+ cr.rectangle(0, 0, width - 1, height - 1)
+ cr.fill()
+
+ self._plot_shape(cr, interior, shape, 0, 0, width, height)
+ return surface
+
def plot_colored_rect(self, interior, width, height):
surface = ImageSurface(FORMAT_ARGB32, int(width), int(height))
cr = Context(surface)
@@ -41,19 +53,33 @@
return surface
def _plot_square(self, cr, colors, state, x, y, width, height):
+ if state == "Unclaimed":
+ shape = "box"
+ elif state == "Claimed":
+ shape = "solid"
+ elif state in ("Owner", "Unavailable"):
+ shape = "diagonal"
+ elif state in ("Matched", "Preempting",
"Preempting/Matched"):
+ shape = "filled_triangle"
+ else:
+ shape = "box"
+
+ self._plot_shape(cr, colors, shape, x, y, width, height)
+
+ def _plot_shape(self, cr, colors, shape, x, y, width, height):
cr.set_source_rgb(*colors)
#cr.move_to(x, y)
cr.rectangle(x, y, width - self.gap, height - self.gap)
- if state == "Unclaimed": # leave empty
+ if shape == "box": # leave empty
cr.stroke()
- elif state == "Claimed": # solid fill
+ elif shape == "solid": # solid fill
cr.fill()
- elif state in ("Owner", "Unavailable"): # diagonal line
+ elif shape == "diagonal": # diagonal line
cr.move_to(x, y)
cr.line_to(x + width - self.gap, y + height - self.gap)
cr.stroke()
- elif state in ("Matched", "Preempting",
"Preempting/Matched"): # triangle
+ elif shape == "filled_triangle": # triangle
cr.stroke()
cr.move_to(x + width - self.gap, y)
cr.line_to(x, y + height - self.gap)
@@ -61,8 +87,8 @@
cr.close_path()
cr.fill()
- def plot_slots(self, slots, zl):
- count = len(slots)
+ def plot_shapes(self, shape_info, zl):
+ count = len(shape_info)
slot_size = self.slot_size(count, zl)
x = 0
y = 0
@@ -74,8 +100,9 @@
cr.set_line_width( zl == 1 and 1 or 2 )
- for slot in slots:
- interior, state = slot[:2]
+ for slot in shape_info:
+ interior = slot["color"]
+ shape = slot["shape"]
if col >= self.cols:
x = 0
@@ -83,7 +110,7 @@
col = 0
self.rows = self.rows + 1
- self._plot_square(cr, interior, state, x, y, slot_size, slot_size)
+ self._plot_shape(cr, interior, shape, x, y, slot_size, slot_size)
x = x + slot_size
col = col + 1
Modified: trunk/cumin/python/cumin/grid/slot.py
===================================================================
--- trunk/cumin/python/cumin/grid/slot.py 2011-02-21 19:30:33 UTC (rev 4540)
+++ trunk/cumin/python/cumin/grid/slot.py 2011-02-21 20:11:08 UTC (rev 4541)
@@ -1,6 +1,5 @@
import logging
-from wooly import *
from wooly.widgets import *
from wooly.forms import *
from wooly.resources import *
@@ -48,8 +47,15 @@
self.where_exprs.default = list()
- def render_sql_order_by(self, session):
- return "Order by \"System\",\"Name\""
+ def sort_records(self, records):
+ try:
+ for rec in records:
+ rec["category"], rec["category_group"] =
SlotMapPage.get_cat_and_group(rec["State"], rec["Activity"])
+ recs = sorted(records, key=lambda x:"%s%s%s" %
+ (x["category_group"], x["category"],
x["Name"]))
+ except Exception:
+ return records
+ return recs
class SlotView(CuminView):
def __init__(self, app, name, slot):
@@ -111,6 +117,28 @@
class SlotMapPage(Page):
""" handles the slots.vis request """
+ categories = {("Owner", None): ("Unavailable",
"Owner"),
+ ("Unclaimed", None): ("Available",
"Unclaimed"),
+ ("Matched", None): ("Starting",
"Transitioning"),
+ ("Claimed", "Idle"): ("Starting",
"Transitioning"),
+ ("Claimed", "Busy"): ("Running",
"Busy"),
+ ("Claimed", "Retiring"): ("Running",
"Busy"),
+ ("Claimed", "Suspended"): ("Paused",
"Busy"),
+ ("Preempting", None): ("Stopping",
"Transitioning"),
+ ("Unknown", None): ("Unknown",
"Unknown")}
+
+ shapes = {"Unavailable": {"shape": "filled_triangle",
"color": (.6,.6,.6)},
+ "Available": {"shape": "box",
"color": (.6,.6,.6)},
+ "Stopping": {"shape": "diagonal",
"color": (.8,0,0)},
+ "Starting": {"shape": "diagonal",
"color": (0,0,.8)},
+ "Paused": {"shape": "solid",
"color": (.4,.4,1)},
+ "Running": {"shape": "solid",
"color": (0,.8,0)},
+ "Busy": {"shape": "solid", "color":
(0,.8,0)},
+ "Transitioning": {"shape": "diagonal",
"color": (.8,0,.8)},
+ "Unclaimed": {"shape": "box",
"color": (.6,.6,.6)},
+ "Owner": {"shape": "filled_triangle",
"color": (.6,.6,.6)},
+ "Unknown": {"shape": "solid",
"color": (1,0,0)}}
+
interiors = {"Idle": (.7,.7,.7),
"Busy": (0.0, 0.4, 0.0),
"Suspended": (1,0,0),
@@ -120,7 +148,6 @@
"Retiring": (.8,.2,.8),
"Unknown": (.8,.8,.8)}
- shapes = ["rectangle", "circle", "circle"]
max_width = 400
max_png_age = 30
max_visible_slots = 200 # start drill-down mode after this number
@@ -134,12 +161,9 @@
self.zoom_level.default = 1
self.add_parameter(self.zoom_level)
- self.activity = Parameter(app, "act")
- self.add_parameter(self.activity)
+ self.category = Parameter(app, "category")
+ self.add_parameter(self.category)
- self.state = Parameter(app, "state")
- self.add_parameter(self.state)
-
self.json = Parameter(app, "json")
self.add_parameter(self.json)
@@ -152,70 +176,67 @@
self.groups = ListParameter(app, "group", group)
self.add_parameter(self.groups)
- self.cache = ImageCache()
self.slots = SlotDataSet(app)
+ @classmethod
+ def get_categories(cls):
+ return cls.categories
+
+ @classmethod
+ def get_unique_category_groups(cls):
+ categories = cls.categories
+ d = dict()
+ for x in categories:
+ _, group = categories[x]
+ if not group == "Unknown":
+ d[group] = None
+ return d.keys()
+
def get_content_type(self, session):
return self.json.get(session) and "text/plain" or
"image/png"
def get_cache_control(self, session):
return "no-cache"
- def render_activity(self, session, activity):
+ def render_category(self, session, category):
map = HeatMapChart()
- surface = map.plot_colored_rect(self.interiors[activity], 12, 12)
+ surface = map.plot_colored_shape(self.shapes[category]["color"],
+ self.shapes[category]["shape"], 20,
20)
writer = Writer()
surface.write_to_png(writer)
return writer.to_string()
- def render_state(self, session, state):
- map = HeatMapChart()
- surface = map.plot_state(state, 12, 12)
- writer = Writer()
- surface.write_to_png(writer)
- return writer.to_string()
-
def do_render(self, session):
- activity = self.activity.get(session)
- if activity:
- return self.render_activity(session, activity)
+ # render the legend icon
+ category = self.category.get(session)
+ if category:
+ return self.render_category(session, category)
- state = self.state.get(session)
- if state:
- return self.render_state(session, state)
-
+ # generate the json data for the flash chart
if self.json.get(session):
return self.render_json(session)
zl = self.zoom_level.get(session)
- # determine if cached copy is recent enough
- cached_png, args = self.get_cached(session, zl)
- if cached_png:
- self.set_cookie(session, args)
- return cached_png
-
map = HeatMapChart(self.max_width, self.max_width)
cursor = self.slots.execute(session)
- records = cursor.fetchall()
+ records = cursor_to_rows(cursor)
if len(records) == 0:
return
- columns = [x[0] for x in cursor.description]
- activity = columns.index("Activity")
- state = columns.index("State")
+ records = self.slots.sort_records(records)
+ shape_info = list()
+ for rec in records:
+ cat = rec["category"]
+ shape_info.append(self.shapes[cat])
+ size = map.plot_shapes(shape_info, zl)
- interiors = self.interiors.copy()
- interiors[None] = interiors["Unknown"]
- slots = [(interiors[x[activity]], x[state]) for x in records]
- size = map.plot_slots(slots, zl)
+ args = (size, map.width, map.height, len(shape_info), map.rows, map.cols)
+ cookie = "|".join((str(x) for x in args))
+ session.set_cookie("slot_info", cookie)
- args = (size, map.width, map.height, len(slots), map.rows, map.cols)
- self.set_cookie(session, args)
- self.cache_it(session, zl, map, args)
-
writer = Writer()
map.write(writer)
return writer.to_string()
@@ -231,16 +252,35 @@
d[atr].append(i)
return d
- def get_activityStates(self, records, plist):
- """ for each unique combination of activity:state, return a
dictionary entry
- that counts the number of records with that combo """
+ @classmethod
+ def get_cat_and_group(cls, state, activity):
+ act = activity
+ if state in ("Owner", "Unclaimed", "Matched",
"Preempting", "Unknown"):
+ # activity doesn't matter for these states
+ act = None
+ try:
+ cat, cat_group = cls.categories[(state, act)]
+ except Exception, e:
+ log.exception(e)
+ cat, cat_group = ("%s:%s" % (state, activity),
"Unknown")
+ return cat, cat_group
+
+ def get_cats(self, records, plist):
d = dict()
for i in plist:
- activityState = "%s:%s" % (records[i]["Activity"],
records[i]["State"])
- if not activityState in d:
- d[activityState] = 0
- d[activityState] += 1
+ state = records[i]["State"]
+ activity = records[i]["Activity"]
+
+ cat, cat_group = SlotMapPage.get_cat_and_group(state, activity)
+ if not cat_group in d:
+ d[cat_group] = dict()
+ if not cat in d[cat_group]:
+ d[cat_group][cat] = 0
+ d[cat_group]["_count"] = 0
+ d[cat_group]["_count"] += 1
+ d[cat_group][cat] += 1
+
return d
def render_json(self, session):
@@ -250,6 +290,7 @@
if len(records) == 0:
return "[]"
+ records = self.slots.sort_records(records)
groups = self.groups.get(session)
slot_count = len(records)
@@ -257,10 +298,11 @@
root.name = self.object_description
root.value = self.object_param.get(session)._id
root.slots = slot_count
- root.vis = self.json.get(session)
+ json = self.json.get(session)
+ root.vis = json
+ root.shape_info = self.shapes
root.activity_colors = self.interiors
expanded = self.expanded.get(session)
- json = self.json.get(session)
leaves = True
(root.tree, root.back) = self.treeify(records, range(slot_count), groups, 0,
expanded, leaves, slot_count, json)
@@ -273,11 +315,13 @@
if level == len(groups):
if leaves:
for i in sorted(plist, key=lambda x:"%s%s%s" %
- (records[x]["Activity"],
records[x]["State"], records[x]["Name"])):
+ (records[x]["category_group"],
records[x]["category"], records[x]["Name"])):
el = Element()
el.job_id = records[i]["JobId"] and
records[i]["JobId"] or ""
el.activity = records[i]["Activity"] and
records[i]["Activity"] or "Unknown"
el.state = records[i]["State"] and
records[i]["State"] or "Unknown"
+ el.category = records[i]["category"]
+ el.category_group = records[i]["category_group"]
el.value = records[i]["Name"] and
records[i]["Name"] or ""
el.load_avg = records[i]["LoadAvg"] and
round(records[i]["LoadAvg"], 2) or 0
el.name = "slot"
@@ -287,20 +331,19 @@
else:
if not expanded:
# display summary info for all the slots under this grouping
- activityStates = self.get_activityStates(records, plist)
- for activityState in sorted(activityStates):
+ category_groups = self.get_cats(records, plist)
+ for category_group in sorted(category_groups):
el = Element()
el.name = "slot_info"
- el.a_s = activityState
- el.count = activityStates[activityState]
- el.value = "%s.%s" %
(records[plist[0]][groups[level-1]], activityState) # uid
+ el.category = category_group
+ el.count = category_groups[category_group]["_count"]
+ el.value = "%s.%s" %
(records[plist[0]][groups[level-1]], category_group) # uid
level_list.append(el)
return (level_list, False)
# not a leaf
group = groups[level]
level_dict = self.get_info_array(records, plist, group)
- first = True
for key in sorted(level_dict):
el = Element()
el.name = group
@@ -323,27 +366,13 @@
return (level_list, False)
- def get_cached(self, session, zl):
- filename = self.gen_filename(session)
- return self.cache.find_recent(filename, self.max_png_age)
-
- def cache_it(self, session, zl, map, args):
- filename = self.gen_filename(session)
- writer = self.cache.create_cache_file(filename, args)
- map.write(writer)
-
- def gen_filename(self, session):
- return session.marshal()
-
- def set_cookie(self, session, args):
- cookie = "|".join((str(x) for x in args))
- session.set_cookie("slot_info", cookie)
-
class SlotMap(Widget):
def __init__(self, app, name):
super(SlotMap, self).__init__(app, name)
- self.slot_info = self.SlotInfo(app, "slot_info")
+ self.slots = SlotDataSet(app)
+
+ self.slot_info = self.SlotInfo(app, "slot_info", self.slots)
self.add_child(self.slot_info)
self.slot_legend = self.SlotLegend(app, "slot_legend")
@@ -351,8 +380,6 @@
self.slot_clip_size = 400
- self.slots = SlotDataSet(app)
-
def render_image_href(self, session):
raise Exception("Not implemented")
@@ -375,18 +402,27 @@
def render_slot_clip_size(self, session):
return self.slot_clip_size
- class SlotLegend(Widget):
+ class SlotLegend(ItemSet):
def __init__(self, app, name):
super(SlotMap.SlotLegend, self).__init__(app, name)
- self.states = SlotStates(app, "slot_states")
- self.add_child(self.states)
+ categories = SlotMapPage.get_unique_category_groups()
+ for cat in sorted(categories):
+ acat = SlotCategories(app, cat)
+ self.add_child(acat)
- self.activities = SlotActivities(app, "slot_activities")
- self.add_child(self.activities)
+ def do_get_items(self, session):
+ return self.children
+ def render_category(self, session, cat):
+ return cat.name
+
+ def render_category_items(self, session, cat):
+ return cat.render(session)
+
class SlotInfo(ItemSet):
- display_names = {"JobId": ("Job ID", "",
""),
+ display_names = {"category": ("", "",
""),
+ "JobId": ("Job ID", "",
""),
"System": ("System", "",
""),
"Machine": ("Machine", "",
""),
"State": ("State", "",
""),
@@ -394,7 +430,7 @@
"Name": ("Name", "",
""),
"_id": ("Slot_ID", "slotInfo_id",
"hidden_row")}
- def __init__(self, app, name):
+ def __init__(self, app, name, slot_data):
super(SlotMap.SlotInfo, self).__init__(app, name)
self.index = IntegerParameter(app, "i")
@@ -403,26 +439,25 @@
self.slot = Attribute(app, "slot")
self.add_attribute(self.slot)
+ self.slot_data = slot_data
+
self.info_div_tmpl = WidgetTemplate(self, "bg_html")
def do_render(self, session):
index = self.index.get(session)
if index is not None:
- self.parent.slots.limit.set(session, 1)
- self.parent.slots.offset.set(session, index)
+ #self.parent.slots.limit.set(session, 1)
+ #self.parent.slots.offset.set(session, index)
- cursor = self.parent.slots.execute(session)
+ cursor = self.slot_data.execute(session)
+ records = cursor_to_rows(cursor)
- data = dict()
- names = [x[0] for x in cursor.description]
- values = cursor.fetchone()
+ if len(records):
+ records = self.slot_data.sort_records(records)
- for name, value in zip(names, values):
- data[name] = value
+ self.slot.set(session, records[index])
- self.slot.set(session, data)
-
writer = Writer()
self.info_div_tmpl.render(writer, session)
return writer.to_string()
@@ -432,7 +467,9 @@
def do_get_items(self, session):
slot = self.slot.get(session)
- return ((self.display_names[x], slot[x])
+ return ((
+ x == "category" and
(slot["category_group"],"","title_row") or
self.display_names[x],
+ slot[x])
for x in slot if x in self.display_names)
def render_slot_info_url(self, session):
@@ -453,46 +490,27 @@
def render_item_row_class(self, session, item):
return item[0][2] and "class='%s'" % item[0][2] or
""
-class SlotStates(ItemSet):
- states = ("Unclaimed", "Claimed", "Unavailable",
"Matched", "Preempting")
-
+class SlotCategories(ItemSet):
def do_get_items(self, session):
- return self.states
+ cat_group = self.name
+ categories = SlotMapPage.get_categories()
- def render_item_size(self, session, activity):
- return 12
+ d = dict()
+ for x in categories:
+ cat, group = categories[x]
+ if group is cat_group:
+ d[cat] = None
- def render_item_title(self, session, state):
- return state
+ return sorted(d.keys())
- def render_item_href(self, session, state):
- params = list()
- params.append("state=%s" % state)
+ def render_item_size(self, session, cat):
+ return 20
- return "poolslots.vis?" + ";".join(params)
+ def render_item_title(self, session, cat):
+ return cat
-class SlotActivities(ItemSet):
- activities = (("Idle", "clear"),
- ("Busy", "green"),
- ("Suspended", "red"),
- ("Vacating", "orange"),
- ("Killing", "blue"),
- ("Benchmarking", "yellow"),
- ("Retiring", "purple"),
- ("Unknown", "grey"))
-
- def do_get_items(self, session):
- return self.activities
-
- def render_item_size(self, session, activity):
- return 12
-
- def render_item_title(self, session, activity):
- return activity[0]
-
- def render_item_href(self, session, activity):
+ def render_item_href(self, session, cat):
params = list()
- params.append("act=%s" % activity[0])
+ params.append("category=%s" % cat)
return "poolslots.vis?" + ";".join(params)
-
Modified: trunk/cumin/python/cumin/grid/slot.strings
===================================================================
--- trunk/cumin/python/cumin/grid/slot.strings 2011-02-21 19:30:33 UTC (rev 4540)
+++ trunk/cumin/python/cumin/grid/slot.strings 2011-02-21 20:11:08 UTC (rev 4541)
@@ -303,12 +303,18 @@
text-align: right;
padding-right: 1em;
}
-
td.slot_info_value {
font-weight: normal;
height: 1.1em;
}
+tr.title_row td.slot_info_title, tr.title_row td.slot_info_value {
+ padding-bottom: 0.5em;
+ border-bottom: 1px solid black;
+ font-style: italic;
+ font-weight: bold;
+}
+
html > body .outerpair1 {
background: url(resource?name=upperrightfade.png) right top no-repeat;
}
@@ -359,65 +365,53 @@
</tr>
[SlotLegend.css]
-div.slot_legend {
- margin-top: 0.1em;
- margin-left: auto;
- margin-right: auto;
- width: 20em;
+div.SlotLegend {
+ -moz-border-radius: 0.5em;
+ background-color: #FCFCCC;
+ border: 1px solid #BABA00;
+ margin: 0 0 0 8em;
+ padding: 0 0 1em 2em;
+ width: 11em;
}
-
-div.slot_legend h3 {
- margin-top: 0;
+div.SlotLegend dl {
+ margin: auto;
+ padding: 0;
}
-
-div.slot_states, div.slot_activities {
- float: left;
- margin-left: 1.3em;
+div.SlotLegend dt {
+ margin: 0;
+ padding: 1em 0 0 0;
+ font-weight: bold;
}
-
-div.slot_states ul, div.slot_activities ul {
+div.SlotLegend dd {
+ margin: 0 0 0 1em;
padding: 0;
+}
+div.SlotLegend dd img {
margin: 0;
- font-size: 0.9em;
- list-style: none;
+ position: relative;
+ top: 6px;
}
-div.slot_states li, div.slot_activities li {
- padding-bottom: 0.25em;
-}
-
-div.slot_states li:hover, div.slot_activities li:hover {
- color: red;
-}
-
[SlotLegend.html]
-<div class="slot_legend">
- {slot_states}
-
- {slot_activities}
-
- <div style="clear:both"><!-- --></div>
+<div id="{id}" class="{class}">
+ <dl>
+ {items}
+ </dl>
</div>
-[SlotStates.html]
-<div class="slot_states">
- <h3>States</h3>
+[SlotLegend.item_html]
+ <dt>{category}</dt>
+ {category_items}
- <ul>{items}</ul>
-</div>
+[SlotCategories.html]
+{items}
-[SlotActivities.html]
-<div class="slot_activities">
- <h3>Activities</h3>
-
- <ul>{items}</ul>
-</div>
-
-[SlotStates.item_html]
-<li onmouseover="vis_treemap_over('state', '{item_title}')"
onmouseout="vis_treemap_out('state', '{item_title}')">
+[SlotCategories.item_html]
+<dd onmouseover="vis_treemap_over('{item_title}')"
onmouseout="vis_treemap_out('{item_title}')">
<img src="{item_href}" width="{item_size}"
height="{item_size}" title="{item_title}"
alt="{item_title}"/>
+
{item_title}
-</li>
+</dd>
[SlotActivities.item_html]
<li onmouseover="vis_treemap_over('activity',
'{item_title}')" onmouseout="vis_treemap_out('activity',
'{item_title}')">
Modified: trunk/cumin/resources/slots.swf
===================================================================
(Binary files differ)