Author: croberts
Date: 2012-04-17 16:04:46 +0000 (Tue, 17 Apr 2012)
New Revision: 5305
Modified:
branches/play_plumage/cumin/etc/cumin.conf
branches/play_plumage/cumin/model/plumage/plumage.xml
branches/play_plumage/cumin/model/plumage/rosemary.xml
branches/play_plumage/cumin/python/cumin/config.py
branches/play_plumage/cumin/python/cumin/model.py
branches/play_plumage/mint/python/mint/plumage/main.py
branches/play_plumage/mint/python/mint/plumage/session.py
branches/play_plumage/rosemary/python/rosemary/model.py
Log:
Updating to pull actual data from ODS defined by config entry plumage_host and
plumage_port (default is localhost:27017). The database name and collection are defined
in plumage/rosemary.xml. For this update, plumage.xml has also been changed to use the
same field names as the plumage database.
Modified: branches/play_plumage/cumin/etc/cumin.conf
===================================================================
--- branches/play_plumage/cumin/etc/cumin.conf 2012-04-16 19:47:45 UTC (rev 5304)
+++ branches/play_plumage/cumin/etc/cumin.conf 2012-04-17 16:04:46 UTC (rev 5305)
@@ -189,6 +189,8 @@
expire-threshold: 90d
# vacuum-enabled: True
vacuum-interval: 1h
+# plumage_host: localhost
+# plumage_port: 27017
# *************** Some parameter explanations by section ***************
Modified: branches/play_plumage/cumin/model/plumage/plumage.xml
===================================================================
--- branches/play_plumage/cumin/model/plumage/plumage.xml 2012-04-16 19:47:45 UTC (rev
5304)
+++ branches/play_plumage/cumin/model/plumage/plumage.xml 2012-04-17 16:04:46 UTC (rev
5305)
@@ -1,24 +1,15 @@
<?xml version="1.0" encoding="utf-8"?>
<schema package="com.redhat.grid.plumage"
package_type="generic">
<class name="OSUtil">
- <property name="uuid" unique="y"
- type="uuid" access="RC" desc="UUID of System
Image"/>
-
- <property name="osName"
- type="sstr" access="RO" desc="Operating System Name"
/>
-
- <statistic name="update_time" type="absTime"
unit="uint32" />
- <statistic name="machine" type="sstr" optional="y"
default=""/>
- <statistic name="slot" type="sstr" optional="y"
default="" />
- <statistic name="architecture" type="sstr"
optional="y" default=""/>
+ <statistic name="ts" type="absTime" unit="uint32"
/>
+ <statistic name="m" type="sstr" optional="y"
default=""/>
+ <statistic name="n" type="sstr" optional="y"
default="" />
+ <statistic name="ar" type="sstr" optional="y"
default=""/>
<statistic name="os" type="sstr" optional="y"
default=""/>
- <statistic name="kbidle" type="uint32" uint="uint32"
optional="y" default="0"/>
- <statistic name="lastactive" type="uint32"
uint="uint32" optional="y" default="0"/>
- <statistic name="state" type="sstr" optional="y"
default=""/>
+ <statistic name="ki" type="uint32" uint="uint32"
optional="y" default="0"/>
+ <statistic name="la" type="uint32" uint="uint32"
optional="y" default="0"/>
+ <statistic name="st" type="sstr" optional="y"
default=""/>
<statistic name="cpu" type="uint32" uint="uint32"
optional="y" default="0" />
<statistic name="mem" type="uint32" uint="uint32"
optional="y" default="0"/>
- <statistic name="avail" type="uint32" unit="uint32"
optional="y" default="0"/>
- <statistic name="used" type="uint32" unit="uint32"
optional="y" default="0"/>
-
</class>
</schema>
Modified: branches/play_plumage/cumin/model/plumage/rosemary.xml
===================================================================
--- branches/play_plumage/cumin/model/plumage/rosemary.xml 2012-04-16 19:47:45 UTC (rev
5304)
+++ branches/play_plumage/cumin/model/plumage/rosemary.xml 2012-04-17 16:04:46 UTC (rev
5305)
@@ -1,28 +1,23 @@
<model>
<package name="com.redhat.grid.plumage" persistent="time">
<class name="OSUtil">
- <source>some stuff goes here about a query</source>
+ <source>
+ <database>condor_stats</database>
+ <collection>samples.machine</collection>
+ </source>
- <statistic name="update_time" timestamp="y">
+ <statistic name="ts" timestamp="y">
</statistic>
- <statistic name="used">
- <title>Used</title>
- </statistic>
-
- <statistic name="avail">
- <title>Available</title>
- </statistic>
-
- <statistic name="machine">
+ <statistic name="m">
<title>Machine</title>
</statistic>
- <statistic name="slot">
+ <statistic name="n">
<title>Slot</title>
</statistic>
- <statistic name="architecture">
+ <statistic name="ar">
<title>Architecture</title>
</statistic>
@@ -30,15 +25,15 @@
<title>OS</title>
</statistic>
- <statistic name="kbidle">
+ <statistic name="ki">
<title>KB idle</title>
</statistic>
- <statistic name="lastactive">
+ <statistic name="la">
<title>Last active</title>
</statistic>
- <statistic name="state">
+ <statistic name="st">
<title>State</title>
</statistic>
Modified: branches/play_plumage/cumin/python/cumin/config.py
===================================================================
--- branches/play_plumage/cumin/python/cumin/config.py 2012-04-16 19:47:45 UTC (rev 5304)
+++ branches/play_plumage/cumin/python/cumin/config.py 2012-04-17 16:04:46 UTC (rev 5305)
@@ -83,7 +83,7 @@
def __init__(self, section_name="report", strict_section=False):
super(CuminReportConfig, self).__init__()
- data = CuminDataConfigSection(self, section_name, strict_section)
+ data = CuminReportConfigSection(self, section_name, strict_section)
_common_data_section(self, data, section_name)
class CuminMasterConfig(CuminConfig):
@@ -224,8 +224,11 @@
super(CuminReportConfigSection, self).__init__(config, name,
strict_section)
- param = ConfigParameter(self, "plumage_server", str)
+ param = ConfigParameter(self, "plumage_host", str)
param.default = "localhost"
+
+ param = ConfigParameter(self, "plumage_port", int)
+ param.default = 27017
class CuminOptionParser(OptionParser,object):
def __init__(self):
Modified: branches/play_plumage/cumin/python/cumin/model.py
===================================================================
--- branches/play_plumage/cumin/python/cumin/model.py 2012-04-16 19:47:45 UTC (rev 5304)
+++ branches/play_plumage/cumin/python/cumin/model.py 2012-04-17 16:04:46 UTC (rev 5305)
@@ -423,10 +423,10 @@
class AggregateSamplesSqlAdapter(SamplesSqlAdapter):
def __init__(self, app, cls, sig):
super(AggregateSamplesSqlAdapter, self).__init__(app, cls, sig)
- self.querymap = {"used":"sum(case when state = 'Claimed'
then 1 else 0 end) as value",
- "avail":"sum(case when state = 'Claimed'
or state = 'Available' then 1 else 0 end) as value",
- "total":"sum(case when state = 'Claimed'
or state = 'Available' or state = 'Unavailable' then 1 else 0 end) as
value",
- "unavail":"sum(case when state =
'Unavailable' then 1 else 0 end) as value"}
+ self.querymap = {"used":"sum(case when st = 'Claimed' then
1 else 0 end) as value",
+ "avail":"sum(case when st = 'Claimed' or
st = 'Unclaimed' then 1 else 0 end) as value",
+ "total":"sum(case when st = 'Claimed' or
st = 'Unclaimed' or st = 'Owner' then 1 else 0 end) as value",
+ "unavail":"sum(case when st = 'Owner'
then 1 else 0 end) as value"}
self.max_samples = 60
Modified: branches/play_plumage/mint/python/mint/plumage/main.py
===================================================================
--- branches/play_plumage/mint/python/mint/plumage/main.py 2012-04-16 19:47:45 UTC (rev
5304)
+++ branches/play_plumage/mint/python/mint/plumage/main.py 2012-04-17 16:04:46 UTC (rev
5305)
@@ -29,6 +29,9 @@
self.vacuum_enabled = True
self.vacuum_thread = VacuumThread(self)
+
+ self.plumage_host = "localhost"
+ self.plumage_port = 27017
self.print_event_level = 0
Modified: branches/play_plumage/mint/python/mint/plumage/session.py
===================================================================
--- branches/play_plumage/mint/python/mint/plumage/session.py 2012-04-16 19:47:45 UTC (rev
5304)
+++ branches/play_plumage/mint/python/mint/plumage/session.py 2012-04-17 16:04:46 UTC (rev
5305)
@@ -5,9 +5,16 @@
from time import sleep
from update import ObjectUpdate
from random import randrange
+from datetime import timedelta, datetime
+from pymongo import Connection
+from pymongo.collection import Collection
+from pymongo.database import Database
+
log = logging.getLogger("mint.plumage.session")
+UTC_DIFF = datetime.utcnow() - datetime.now()
+
class PlumageSession(object):
def __init__(self, app, server_url):
self.app = app
@@ -66,66 +73,68 @@
super(PlumageSessionThread, self).__init__(app)
self.server_url = server_url
self.cls = cls
-
+
+ self.connection = Connection(app.plumage_host, app.plumage_port) #Needs to be
from config
+ self.database = Database(self.connection, self.cls.database)
+ self.collection = Collection(self.database, self.cls.collection)
+
def run(self):
- uuid = 1
- while True:
- if self.stop_requested:
- break
- # We create objects here. Tag them with the right class,
- # probably specified to us from a config option (with a corresponding
- # query specification in the xml)
- #cls = self.app.model.com_redhat_grid_plumage.OSUtil
+ # We create objects here. Tag them with the right class,
+ # probably specified to us from a config option (with a corresponding
+ # query specification in the xml)
- class OSUtil(object):
- pass
-
- # We need to get some data, and we need to turn it into some kind
- # of update object.
- os = OSUtil()
-
- os.uuid = str(uuid)
- os.osName = "Linux"
- os.update_time = time.time() * 1000000000
-
- # random, 16 slots
- os.avail = randrange(0, 17)
-
- # some number of available used
- os.used = randrange(0, os.avail+1)
-
- machines = ["croberts.redhat.com", "tmckay.redhat.com",
"shadowman.redhat.com"]
- os.machine = machines[randrange(0,3)]
- os.slot = "%d@%s" % (randrange(1, 16), os.machine)
-
- architectures = ["X86_64", "X86", "Itanium"]
- os.architecture = architectures[randrange(0,3)]
-
- oses = ["LINUX", "WINDOWS", "MAC"]
- os.os = oses[randrange(0,3)]
-
- os.kbidle = randrange(0, 100)
-
- os.lastactive = randrange(0, 100)
-
- states = ["Claimed", "Available",
"Unavailable"]
- os.state = states[randrange(0,3)]
+ class OSUtil(object):
+ pass
+
+ itemSet = self.collection.find().sort("ts", -1)
+
+ for item in itemSet:
+ record = OSUtil()
+ for name, value in item.iteritems():
+ if isinstance(value, datetime):
+ #required since MongoDB uses UTC for everything, here we get our
sanity back
+ setattr(record, name, value - UTC_DIFF)
+ else:
+ setattr(record, name, item[name])
- os.cpu = randrange(0,65)
-
- os.mem = randrange(0, 100000)
-
- obj = ObjectUpdate(self.app.model, os, self.cls)
+ obj = ObjectUpdate(self.app.model, record, self.cls)
self.app.update_thread.enqueue(obj)
- sleep(30)
-
- # five machines
- uuid = (uuid + 1) % 6
- if uuid == 0:
- uuid = 1
-
+# If you'd rather fake the data, use the following instead
+# while True:
+# if self.stop_requested:
+# break
+# # We need to get some data, and we need to turn it into some kind
+# # of update object.
+# os = OSUtil()
+#
+# os.ts = time.time() * 1000000000
+#
+# machines = ["croberts.redhat.com", "tmckay.redhat.com",
"shadowman.redhat.com"]
+# os.m = machines[randrange(0,3)]
+# os.n = "%d@%s" % (randrange(1, 16), os.m)
+#
+# architectures = ["X86_64", "X86", "Itanium"]
+# os.ar = architectures[randrange(0,3)]
+#
+# oses = ["LINUX", "WINDOWS", "MAC"]
+# os.os = oses[randrange(0,3)]
+#
+# os.ki = randrange(0, 100)
+#
+# os.la = randrange(0, 100)
+#
+# states = ["Claimed", "Unclaimed", "Owner"]
+# os.st = states[randrange(0,3)]
+#
+# os.cpu = randrange(0,65)
+#
+# os.mem = randrange(0, 100000)
+#
+# obj = ObjectUpdate(self.app.model, os, self.cls)
+# self.app.update_thread.enqueue(obj)
+
'''
class MintConsole(Console):
def __init__(self, model):
Modified: branches/play_plumage/rosemary/python/rosemary/model.py
===================================================================
--- branches/play_plumage/rosemary/python/rosemary/model.py 2012-04-16 19:47:45 UTC (rev
5304)
+++ branches/play_plumage/rosemary/python/rosemary/model.py 2012-04-17 16:04:46 UTC (rev
5305)
@@ -757,6 +757,8 @@
# sample table
self.add_unique_to_samples = True
self.source = ""
+ self.database = ""
+ self.collection = ""
def init(self):
super(Generic_class, self).init()
@@ -768,7 +770,10 @@
# Todo: this could be optimized if we use the shortest
# set of unique column groups but probably there is only one
- self.signature_cols = self._unique_col_groups[self.col_group_list[0]]
+ if self._unique_col_groups is not None and len(self._unique_col_groups) > 0:
+ self.signature_cols = self._unique_col_groups[self.col_group_list[0]]
+ else:
+ self.signature_cols = []
# Set up a query that we can use on a unique
# column set
@@ -780,6 +785,8 @@
super(Generic_class, self).extend(elem)
self.source = elem.findtext("source")
+ self.database = elem.findtext("source/database")
+ self.collection = elem.findtext("source/collection")
def load_class_components(self, elem):
for child in elem.findall("property"):
@@ -841,6 +848,7 @@
self.optional = None
self.unique = None
self.formatter = None
+ self.storage = None
self.title = None
self.description = None