r4504 - in branches/scale_testing/mech_scripts: . test_config_single

Author: tmckay
Date: 2011-02-04 17:46:17 +0000 (Fri, 04 Feb 2011)
New Revision: 4504

Added:
   branches/scale_testing/mech_scripts/test.py
   branches/scale_testing/mech_scripts/test_config_single/
   branches/scale_testing/mech_scripts/test_config_single/test_100thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_105thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_10thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_110thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_15thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_1thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_20thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_25thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_30thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_35thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_40thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_45thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_50thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_55thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_5thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_60thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_65thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_70thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_75thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_80thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_85thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_8thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_90thread.cfg
   branches/scale_testing/mech_scripts/test_config_single/test_95thread.cfg
Modified:
   branches/scale_testing/mech_scripts/process_times.py
   branches/scale_testing/mech_scripts/test_driver.py
Log:
Add wrapper script and configs, page size change check.



Modified: branches/scale_testing/mech_scripts/process_times.py
===================================================================
--- branches/scale_testing/mech_scripts/process_times.py	2011-02-03 15:48:13 UTC (rev
4503)
+++ branches/scale_testing/mech_scripts/process_times.py	2011-02-04 17:46:17 UTC (rev
4504)
@@ -58,6 +58,8 @@
     end_proc = 7
     send_headers = 8
     busy_threads = 9
+    data_size = 10 # we don't really do anything with data_size right now, we just
want to see it
+    last_field = 10
 
     # start iteration count at 1, bump for blank lines in inputs.
     # this is used for calculating averages from the sums later
@@ -67,6 +69,8 @@
     sums = get_zero_list()
     longest = [""]+sums # this will be the worst case get, with a label
     count = 0
+    page_size = -1
+    line_count = 0
 
     # Create a zero time delta
     max = datetime.utcnow()
@@ -75,17 +79,28 @@
     p = os.path.abspath(fname)
     f = open(p, "r")
     for line in f:
+        line_count += 1
 
-        # Look for iteration marker
-        if len(line.strip()) == 0:
+        # Look for page end marker, will contain total size for that page for new stuff.
+        # Old stuff might still have a blank line
+        times = line.strip().split(",")
+        if times[0] == "page_break" or times[0] == "":
+            last_page_size = page_size
+            if len(times) > 1:
+                page_size = int(times[1])
+            else:
+                page_size = 0
+
+            # Page size should change once on the second iteration, because caching is
employed.
+            # Don't report page size change the first time
+            if iterations == 3 and last_page_size > 0 and abs(last_page_size -
page_size) > 2:
+                print "******** Page size changed! File " + fname + ",
Line " + str(line_count) + " **************"
             iterations += 1
             continue
 
-        times = line.strip().split(",")
-
         # handle files I collected before....
-        if len(times) < busy_threads+1:
-            for i in range(len(times), busy_threads+1):
+        if len(times) < last_field+1:
+            for i in range(len(times), last_field+1):
                 times.append("0")
 
         skip_line = False
@@ -125,12 +140,12 @@
         for i in range(len(sums)):
             avg.append(sums[i] / count)
 
-    print "**************** " + fname + " ****************"
+#    print "**************** " + fname + " ****************"
         
 #    print "longest:"    
 #    dump(longest)
 
-    dump(["average values"]+avg)
+#    dump(["average values"]+avg)
 
     # Blank lines in the input file represent an iteration.
     # Get average values for a page by counting and dividing by the number of
iterations.
@@ -146,7 +161,7 @@
     except:
         pass
 
-    dump(["per page"]+page_avg)
+#    dump(["per page"]+page_avg)
 
     return page_avg
 

Added: branches/scale_testing/mech_scripts/test.py
===================================================================
--- branches/scale_testing/mech_scripts/test.py	                        (rev 0)
+++ branches/scale_testing/mech_scripts/test.py	2011-02-04 17:46:17 UTC (rev 4504)
@@ -0,0 +1,58 @@
+#!/usr/bin/python
+
+import os
+import shutil
+import subprocess
+import re
+import sys
+
+def main():
+    if len(sys.argv) < 4:
+        print "usage: test.py config_file_dir project_name csv_output_file"
+        return
+
+    # Get list of config files that we are going to substitute for config.cfg in
multimechanize project
+    file_list = []
+    if os.path.isdir(sys.argv[1]):
+        for i in os.listdir(sys.argv[1]):
+            file_list.append(os.path.join(sys.argv[1], i))
+    else:
+        file_list = [sys.argv[1]]
+
+    project = sys.argv[2]
+    proj_dir = os.path.join("projects", sys.argv[2])
+    if not os.path.isdir(proj_dir):
+        print "Project name is not a directory:", project
+        return
+
+    # This is the config file for multimechanize that we're going to copy over
+    # We can save it first and put it back later just to be nice
+    config = os.path.join(proj_dir, "config.cfg")
+    shutil.copyfile(config, "./.config.cfg.save")
+
+
+    print file_list
+
+
+    for file in file_list:
+        shutil.copyfile(file, config)
+        os.system("python multi-mechanize.py " + project)
+
+        # Figure out what subdirectory to process.  Name convention, same as config file
with .cfg removed
+        subdir = os.path.basename(file).split(".")[0]
+
+        # Figure out what the threadcount arg should be for process times
+        try:
+            threads = re.findall("[0123456789]+", subdir)[0]
+        except:
+            threads = "0"
+
+        # run process times
+        os.system("python process_times.py " + subdir + " --csvfile="
+ sys.argv[3] + " --threads=" + threads)
+
+
+    shutil.copyfile("./config.cfg.save", config)
+
+if __name__ == '__main__':
+    main()
+

Added: branches/scale_testing/mech_scripts/test_config_single/test_100thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_100thread.cfg	            
           (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_100thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 100
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_100thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_105thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_105thread.cfg	            
           (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_105thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 105
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_105thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_10thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_10thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_10thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 10
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_10thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_110thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_110thread.cfg	            
           (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_110thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 110
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_110thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_15thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_15thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_15thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 15
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_15thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_1thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_1thread.cfg	              
         (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_1thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,11 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+
+[user_group-1]
+threads: 1
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_1thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_20thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_20thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_20thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 20
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_20thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_25thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_25thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_25thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 25
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_25thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_30thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_30thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_30thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 30
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_30thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_35thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_35thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_35thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 35
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_35thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_40thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_40thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_40thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 40
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_40thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_45thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_45thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_45thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 45
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_45thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_50thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_50thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_50thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 50
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_50thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_55thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_55thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_55thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,11 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 55
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_55thread
+

Added: branches/scale_testing/mech_scripts/test_config_single/test_5thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_5thread.cfg	              
         (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_5thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,11 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+
+[user_group-1]
+threads: 5
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_5thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_60thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_60thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_60thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 60
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_60thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_65thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_65thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_65thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,11 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 65
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_65thread
+

Added: branches/scale_testing/mech_scripts/test_config_single/test_70thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_70thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_70thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 70
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_70thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_75thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_75thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_75thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 75
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_75thread
\ No newline at end of file

Added: branches/scale_testing/mech_scripts/test_config_single/test_80thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_80thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_80thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 80
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_80thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_85thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_85thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_85thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 85
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_85thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_8thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_8thread.cfg	              
         (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_8thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 8
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit --outdir=./test_8thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_90thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_90thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_90thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 90
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_90thread

Added: branches/scale_testing/mech_scripts/test_config_single/test_95thread.cfg
===================================================================
--- branches/scale_testing/mech_scripts/test_config_single/test_95thread.cfg	             
          (rev 0)
+++ branches/scale_testing/mech_scripts/test_config_single/test_95thread.cfg	2011-02-04
17:46:17 UTC (rev 4504)
@@ -0,0 +1,10 @@
+[global]
+run_time: 30
+rampup: 0
+console_logging: off
+results_ts_interval: 1
+
+[user_group-1]
+threads: 95
+script: test_driver.py
+options: --machine=grid0.lab.bos.redhat.com:45672 --file=./limit
--outdir=./test_95thread

Modified: branches/scale_testing/mech_scripts/test_driver.py
===================================================================
--- branches/scale_testing/mech_scripts/test_driver.py	2011-02-03 15:48:13 UTC (rev 4503)
+++ branches/scale_testing/mech_scripts/test_driver.py	2011-02-04 17:46:17 UTC (rev 4504)
@@ -32,6 +32,7 @@
         self.file_init = False
         self.file_number = 0
         self.file_iterations = 0
+        self.last_page_total = 0
 
     def next_file(self):
         if not self.file_init:
@@ -51,23 +52,25 @@
             self.file_number += 1        
         return name
 
-    def handle_custom_headers(self, curr_file, link, resp, insert_blank):
+    def handle_custom_headers(self, curr_file, link, resp, data_size):
         fname = os.path.join(self.outdir, self.group + self.thread + "_" +
os.path.basename(curr_file))
-        if os.path.isfile(fname):
-            f = open(fname, "a")
-            if insert_blank:
-                f.write("\n")
-        else:
-            f = open(fname, "w")
+        f = open(fname, "a+")
 
         f.write(link)
         for i in ("page_before_accept", "page_after_accept",
"page_put_time",
                   "page_after_get", "page_begin_service",
"page_begin_proc",
                  
"page_end_proc","page_send_headers","page_busy_threads"):
-            f.write(","+resp.getheader(i,"0"))                  
+            f.write(","+resp.getheader(i,"0"))
+        f.write(","+str(data_size))
         f.write("\n")
         f.close()
 
+    def write_page_end(self, curr_file, total_size):
+        fname = os.path.join(self.outdir, self.group + self.thread + "_" +
os.path.basename(curr_file))
+        f = open(fname, "a+")
+        f.write("page_break,"+str(total_size)+"\n")
+        f.close()
+
     def run(self):
         try:
             # use this to get an idea of tets overhead.
@@ -101,7 +104,12 @@
                 self.thread = opts.thread
                 self.outdir = opts.outdir
                 if self.outdir != "." and not os.path.isdir(self.outdir):
-                    os.mkdir(self.outdir)
+                    try:
+                        # in a multi thread environment, we may
+                        # actually collide here in the creation.
+                        os.mkdir(self.outdir)
+                    except:
+                        pass
                   
             curr_file = self.next_file()
             f = open(curr_file, "r")
@@ -109,7 +117,7 @@
                 print curr_file
 
             total_gets = total_size = get_latency = 0
-            first_link = True
+
             for link in f:
                 link_info = link.split(',')
 
@@ -150,9 +158,12 @@
 
                 # write custom headers to a file -- too much overhead to 
                 # embed in the test.  parse(datetime) is apparently expensive
-                self.handle_custom_headers(curr_file, link, resp, first_link)
-                first_link = False
+                # include the length of the data received, too, so we can look for
changes
+                self.handle_custom_headers(curr_file, link, resp, len(data))
 
+            # Indicate page break and include total data size for this page in headers
file
+            self.write_page_end(curr_file, total_size)
+
             # Record some custom stuff
             self.custom_timers['Page File'] = get_latency
             self.custom_timers['Avg Get'] = get_latency / total_gets