From: martyntaylor mtaylor@redhat.com
--- src/app/models/task.rb | 4 +- src/app/services/data_service.rb | 270 ++++++++++++++++++++++--------- src/spec/factories/pool.rb | 2 +- src/spec/factories/user.rb | 5 + src/spec/services/data_service_spec.rb | 219 ++++++++++++++++++++++++-- 5 files changed, 402 insertions(+), 98 deletions(-)
diff --git a/src/app/models/task.rb b/src/app/models/task.rb index feec26a..8058089 100644 --- a/src/app/models/task.rb +++ b/src/app/models/task.rb @@ -118,7 +118,7 @@ class Task < ActiveRecord::Base errors.add("created_at", "Task started but does not have the creation time set") if time_started and created_at.nil? # Removed check on time_started exisiting. if time_ended does. This can now occur, when the task fails before is starts. e.g. When Over Qutoa #errors.add("time_started", "Task ends but does not have the start time set") if time_ended and time_started.nil? - errors.add("time_ended", "Tasks ends before it's started") unless time_ended.nil? or time_started.nil? or time_ended > time_started - errors.add("time_started", "Tasks starts before it's created") unless time_started.nil? or created_at.nil? or time_started > created_at + errors.add("time_ended", "Tasks ends before it's started") unless time_ended.nil? or time_started.nil? or time_ended >= time_started + errors.add("time_started", "Tasks starts before it's created") unless time_started.nil? or created_at.nil? or time_started >= created_at end end diff --git a/src/app/services/data_service.rb b/src/app/services/data_service.rb index d741574..c57f6b8 100644 --- a/src/app/services/data_service.rb +++ b/src/app/services/data_service.rb @@ -1,63 +1,44 @@ class DataService
+ # Structures for holding graph data QoSDataPoint = Struct.new(:time, :average, :max, :min) - QuotaUsagePoint = Struct.new(:used, :max) - TotalQuotaUsagePoint = Struct.new(:name, :no_instances)
- # This will return array of data points between start and end, if there is a data point where the interval start + interval end - # is greater than the end time, it will be ignored - # Example: - # start = 12.30, end = 12.32, interval = 45secs - # Intervals: 12.30.00 - 12.30.45, 12.30.45 - 12.31.30 will be returned. Interval 12.31.30 - 12.32.15 will not - def self.qos_task_submission_stats(start_time, end_time, interval_length, parent, action) + QuotaUsagePoint = Struct.new(:used, :max)
- instances = [] + TotalQuotaUsagePoint = Struct.new(:name, :no_instances)
- if parent.class == Provider - cloud_accounts = CloudAccount.find(:all, :conditions => {:provider_id => parent.id}) - cloud_accounts.each do |cloud_account| - instances.concat(instances) - end - elsif parent.class == Pool || parent.class == CloudAccount - instances = parent.instances - else - return nil - end + QoSFailureRatePoint = Struct.new(:time, :failure_rate)
- return calculate_qos_task_submission_stats(start_time, end_time, interval_length, instances, action) + def self.qos_task_submission_stats(parent, start_time, end_time, interval_length, action) + return qos_time_stats(parent, start_time, end_time, interval_length, {:action => action}, TASK_SUBMISSION_TIMES) end
- def self.tasks_submissions_mean_max_min(time, tasks) - - first_pass = true - - total_time = nil - maximum_time = nil - minimum_time = nil + def self.qos_task_completion_stats(parent, start_time, end_time, interval_length, action) + return qos_time_stats(parent, start_time, end_time, interval_length, {:action => action}, TASK_COMPLETION_TIMES) + end
- tasks.each do |task| + def self.qos_task_submission_mean_max_min(parent, start_time, end_time, action) + return qos_times_mean_max_min(parent, start_time, end_time, action, TASK_SUBMISSION_TIMES) + end
- if(first_pass == true) - total_time = task.submission_time - maximum_time = task.submission_time - minimum_time = task.submission_time - first_pass = false - else - total_time += task.submission_time + def self.qos_task_completion_mean_max_min(parent, start_time, end_time, action) + return qos_times_mean_max_min(parent, start_time, end_time, action, TASK_COMPLETION_TIMES) + end
- if task.submission_time > maximum_time - maximum_time = task.submission_time - end + def self.qos_instance_runtime_stats(parent, start_time, end_time, interval_length) + return qos_time_stats(parent, start_time, end_time, interval_length, nil, INSTANCE_RUN_TIMES) + end
- if task.submission_time< minimum_time - minimum_time = task.submission_time - end - end + def self.qos_instance_runtime_mean_max_min(parent, start_time, end_time) + return qos_times_mean_max_min(parent, start_time, end_time, nil, INSTANCE_RUN_TIMES) + end
- end - average_time = total_time / tasks.length + def self.qos_failure_rate(parent, start_time, end_time, failure_code) + return failure_rate(parent, start_time, end_time, failure_code) + end
- return QoSDataPoint.new(time, average_time, maximum_time, minimum_time) + def self.qos_failure_rate_stats(parent, start_time, end_time, interval_length, failure_code) + qos_time_stats(parent, start_time, end_time, interval_length, {:failure_code => failure_code}, FAILURE_RATE) end
# Returns the Used and Maximum Resource Usage @@ -86,29 +67,6 @@ class DataService return nil end
- def self.overall_usage(parent) - usage_points = [] - Quota::RESOURCE_NAMES.each do |resource_name| - usage_points << quota_utilisation(parent, resource_name) - end - - worst_case = nil - usage_points.each do |usage_point| - if worst_case - if worst_case.max == Quota::NO_LIMIT - worst_case = usage_point - elsif usage_point.max == Quota::NO_LIMIT - # DO Nothing - elsif ((100 / worst_case.max) * worst_case.used) < ((100 / usage_point.max) * usage_point.used) - worst_case = usage_point - end - else - worst_case = usage_point - end - end - return worst_case - end - def self.total_quota_utilisation(provider) data_points = [] free_instances = 0 @@ -125,29 +83,183 @@ class DataService return data_points end
+ ##################### + ## PRIVATE METHODS ## + ##################### private - def self.calculate_qos_task_submission_stats(start_time, end_time, interval_length, instances, action)
+ TASK_SUBMISSION_TIMES = "TASK_SUBMISSION_TIMES" + + TASK_COMPLETION_TIMES = "TASK_COMPLETION_TIMES" + + INSTANCE_RUN_TIMES = "INSTANCE_RUN_TIMES" + + FAILURE_RATE = "FAILURE_RATE" + + def self.qos_time_stats(parent, start_time, end_time, interval_length, params, compare_field) data = [] until start_time > (end_time - interval_length) do interval_time = start_time + interval_length
- tasks = Task.find(:all, :conditions => { :time_submitted => start_time..interval_time, + case compare_field + when FAILURE_RATE + data << failure_rate(parent, start_time, interval_time, params[:failure_code]) + when INSTANCE_RUN_TIMES + data << qos_times_mean_max_min(parent, start_time, interval_time, nil, compare_field) + when TASK_COMPLETION_TIMES + data << qos_times_mean_max_min(parent, start_time, interval_time, params[:action], compare_field) + when TASK_SUBMISSION_TIMES + data << qos_times_mean_max_min(parent, start_time, interval_time, params[:action], compare_field) + end + + start_time = interval_time + end + return data + end + + # Calculates the mean, max and min times, for the tasks state time, e.g. submission, completion, etc... + def self.qos_times_mean_max_min(parent, start_time, end_time, action, compare_field) + first_pass = true + + total_time = nil + maximum_time = nil + minimum_time = nil + + case compare_field + when TASK_SUBMISSION_TIMES + list = get_compare_tasks(parent, compare_field, start_time, end_time, action) + when TASK_COMPLETION_TIMES + list = get_compare_tasks(parent, compare_field, start_time, end_time, action) + when INSTANCE_RUN_TIMES + list = get_compare_instances(parent, compare_field, start_time, end_time) + else + return nil + end + + list.each do |l| + case compare_field + when TASK_SUBMISSION_TIMES + compare_time = l.submission_time + when TASK_COMPLETION_TIMES + compare_time = l.runtime + when INSTANCE_RUN_TIMES + compare_time = l.total_state_time(Instance::STATE_RUNNING) + else + return nil + end + + if(first_pass == true) + total_time = compare_time + maximum_time = compare_time + minimum_time = compare_time + first_pass = false + else + total_time += compare_time + + if compare_time > maximum_time + maximum_time = compare_time + end + + if compare_time < minimum_time + minimum_time = compare_time + end + end + end + + if total_time == nil + average_time = nil + elsif total_time == 0 + average_time = 0 + else + average_time = total_time / list.length + end + + return QoSDataPoint.new(start_time, average_time, maximum_time, minimum_time) + end + + def self.get_parent_instances(parent) + instances = [] + + if parent.class == Provider + cloud_accounts = CloudAccount.find(:all, :conditions => {:provider_id => parent.id}) + cloud_accounts.each do |cloud_account| + instances.concat(cloud_account.instances) + end + elsif parent.class == Pool || parent.class == CloudAccount + instances = parent.instances + else + return nil + end + + return instances + end + + def self.get_compare_tasks(parent, compare_field, start_time, end_time, action) + instances = get_parent_instances(parent) + case compare_field + when TASK_SUBMISSION_TIMES + return Task.find(:all, :conditions => {:time_submitted => start_time...end_time, :time_started => start_time..Time.now, :failure_code => nil, :action => action, - :task_target_id => instances - }) - if tasks.length > 0 - data << tasks_submissions_mean_max_min(start_time, tasks) + :task_target_id => instances }) + when TASK_COMPLETION_TIMES + return Task.find(:all, :conditions => {:time_started => start_time...end_time, + :time_ended => start_time..Time.now, + :failure_code => nil, + :action => action, + :task_target_id => instances, + :state => Task::STATE_FINISHED }) else - data << QoSDataPoint.new(start_time, 0, 0, 0) - end + return nil + end + end
- start_time = interval_time + # returns the failure rate of instance starts for instances associated with the parent, (pool/cloudaccount) given the failure code + def self.failure_rate(parent, start_time, end_time, failure_code) + tasks = Task.find(:all, :conditions => { :created_at => start_time...end_time, + :task_target_id => parent.instances }) + + failed_tasks = tasks.find_all{ |task| task.failure_code == failure_code} + if tasks.length > 0 + failure_rate = (100 / tasks.length) * failed_tasks.length + return QoSFailureRatePoint.new(start_time, failure_rate) end + return 0 + end
- return data + def self.overall_usage(parent) + usage_points = [] + Quota::RESOURCE_NAMES.each do |resource_name| + usage_points << quota_utilisation(parent, resource_name) + end + + worst_case = nil + usage_points.each do |usage_point| + if worst_case + if worst_case.max == Quota::NO_LIMIT + worst_case = usage_point + elsif usage_point.max == Quota::NO_LIMIT + # DO Nothing + elsif ((worst_case.max / 100) * worst_case.used) < ((usage_point.max / 100) * usage_point.used) + worst_case = usage_point + end + else + worst_case = usage_point + end + end + return worst_case + end + + def self.get_compare_instances(parent, compare_field, start_time, end_time) + instances = get_parent_instances(parent) + case compare_field + when INSTANCE_RUN_TIMES + return instances.find(:all, :conditions => {:time_last_pending => start_time...end_time, + :time_last_running => start_time..Time.now}) + else + return nil + end end
-end +end \ No newline at end of file diff --git a/src/spec/factories/pool.rb b/src/spec/factories/pool.rb index 4ce7c89..d85b6a7 100644 --- a/src/spec/factories/pool.rb +++ b/src/spec/factories/pool.rb @@ -1,6 +1,6 @@ Factory.define :pool do |p| p.name 'mypool' - p.owner { |owner| owner.association(:user, :login => 'pool_owner', :email => 'pool_owner@example.com') } + p.association :owner, :factory => :pool_user end
Factory.define :tpool, :parent => :pool do |p| diff --git a/src/spec/factories/user.rb b/src/spec/factories/user.rb index 0cc33c7..778aa30 100644 --- a/src/spec/factories/user.rb +++ b/src/spec/factories/user.rb @@ -15,3 +15,8 @@ end
Factory.define :provider_admin_user, :parent => :user do |u| end + +Factory.define :pool_user, :parent => :user do |u| + u.sequence(:login) { |n| "pool_user#{n}" } + u.email { |e| "#{e.login}@example.com" } +end \ No newline at end of file diff --git a/src/spec/services/data_service_spec.rb b/src/spec/services/data_service_spec.rb index 56bbc41..d80f0b5 100644 --- a/src/spec/services/data_service_spec.rb +++ b/src/spec/services/data_service_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper'
describe DataService do
- it "should calculate the total instance quota usage for a provider with a numbner of cloud accounts" do + it "should calculate the total instance quota usage for a provider with a number of cloud accounts" do client = mock('DeltaCloud', :null_object => true) provider = Factory.build(:mock_provider) provider.stub!(:connect).and_return(client) @@ -73,20 +73,27 @@ describe DataService do end
it "should calculate the average, max and min task submission times" do - tasks = [] - instance = Factory :instance + user = Factory :user + pool = Factory(:pool, :owner => user) + instance = Factory(:instance, :pool_id => pool.id)
+ start_time = Time.utc(2010,"jan",1,20,15,1) for i in 1..10 do - time = Time.utc(2010,"jan",1,20,15,1) - task = Task.new(:instance => instance, :type => "InstanceTask", :state => Task::STATE_PENDING, :failure_code => nil) - task.time_submitted = time - time += i - task.time_started = time - task.save - tasks << task + task = InstanceTask.new(:instance => instance, + :state => Task::STATE_PENDING, + :failure_code => nil, + :task_target_id => instance.id, + :type => "InstanceTask", + :action => InstanceTask::ACTION_CREATE) + task.save! + + task.created_at = start_time + task.time_submitted = start_time + task.time_started = start_time + i + task.save! end
- data_point = DataService.tasks_submissions_mean_max_min(Time.now, tasks) + data_point = DataService.qos_task_submission_mean_max_min(pool, start_time, Time.now, InstanceTask::ACTION_CREATE)
data_point.average.should == 5.5 data_point.min.should == 1 @@ -94,7 +101,8 @@ describe DataService do end
it "should create data points for the average, max and min task submission times between two times at given intervals" do - pool = Factory :pool + user = Factory :user + pool = Factory(:pool, :owner => user) instance = Factory(:instance, :pool_id => pool.id)
expected_averages = [ 20, 40, 60, 80, 100] @@ -106,7 +114,7 @@ describe DataService do
generate_tasks(start_time, interval_length, instance, expected_averages)
- data_points = DataService.qos_task_submission_stats(start_time, end_time, interval_length, pool, InstanceTask::ACTION_CREATE) + data_points = DataService.qos_task_submission_stats(pool, start_time, end_time, interval_length, InstanceTask::ACTION_CREATE)
for i in 0...data_points.length average_time = expected_averages[i] @@ -147,7 +155,7 @@ describe DataService do generate_tasks(start_time, interval_length, instance, expected_average) end
- data_points = DataService.qos_task_submission_stats(start_time, end_time, interval_length, pool, InstanceTask::ACTION_CREATE) + data_points = DataService.qos_task_submission_stats(pool, start_time, end_time, interval_length, InstanceTask::ACTION_CREATE)
for i in 0...data_points.length dp = data_points[i] @@ -157,6 +165,186 @@ describe DataService do end end
+ it "should create data points for mean, max, min instance runtimes" do + interval_length = 1000 + + start_time = Time.utc(2010,"jan",1,20,15,1) + start_time1 = start_time + interval_length + start_time2 = start_time1 + interval_length + start_times = [start_time, start_time1, start_time2] + + end_time = start_time2 + interval_length + + runtime1 = [5, 10, 15, 20, 25] + runtime2 = [10, 20, 30, 40, 50] + runtime3 = [100, 200, 300, 400, 500] + runtimes = [runtime1, runtime2, runtime3] + + user = Factory :pool_user + pool = Factory(:pool, :owner => user) + cloud_account = Factory :mock_cloud_account + + for i in 0..2 do + runtimes[i].each do |runtime| + instance = Factory(:instance, :pool => pool, :cloud_account => cloud_account, :state => Instance::STATE_STOPPED) + instance.save! + + instance.time_last_pending = start_times[i] + (interval_length / 2) + instance.time_last_running = start_times[i] + (interval_length / 2) + instance.acc_running_time = runtime + instance.save! + end + end + + stats = DataService.qos_instance_runtime_stats(cloud_account, start_time, end_time, interval_length) + stats[0].should == DataService::QoSDataPoint.new(start_times[0], 15, 25, 5) + stats[1].should == DataService::QoSDataPoint.new(start_times[1], 30, 50, 10) + stats[2].should == DataService::QoSDataPoint.new(start_times[2], 300, 500, 100) + + end + + it "should generate the mean max and min instance runtimes of instances for a given cloud account or pool" do + user = Factory :pool_user + pool = Factory(:pool, :owner => user) + + cloud_account = Factory :mock_cloud_account + + start_time = Time.utc(2010,"jan",1,20,15,1) + [50, 100, 150, 200, 250].each do |runtime| + instance = Factory(:new_instance, :pool => pool, :cloud_account => cloud_account) + instance.time_last_pending = start_time + instance.time_last_running = start_time + instance.acc_running_time = runtime + instance.save! + end + + expected_results = DataService::QoSDataPoint.new(start_time, 150, 250, 50) + results = DataService.qos_instance_runtime_mean_max_min(pool, start_time, Time.now) + results.should == expected_results + end + + it "should calculate the average time it takes a provider to complete a task between two times" do + user = Factory :pool_user + pool = Factory(:pool, :owner => user) + cloud_account = Factory(:mock_cloud_account) + instance = Factory(:instance, :pool => pool, :cloud_account => cloud_account) + + start_time = Time.utc(2010,"jan",1,20,15,1) + task_completion_times = [10, 20, 30, 40, 50] + total_time = 0 + + task_completion_times.each do |time| + task = InstanceTask.new(:instance => instance, + :type => "InstanceTask", + :state => Task::STATE_FINISHED, + :failure_code => nil, + :action => InstanceTask::ACTION_CREATE, + :task_target_id => instance.id) + task.save! + + task.created_at = start_time + task.time_started = start_time + task.time_ended = start_time + time + task.save! + + total_time += time + end + + expected_average_time = total_time / task_completion_times.length + average_time = DataService.qos_task_completion_mean_max_min(cloud_account.provider, start_time, Time.now, InstanceTask::ACTION_CREATE) + + average_time[:average].should == expected_average_time + average_time[:min].should == 10 + average_time[:max].should == 50 + end + + it "should calculate the correct failure rate of instances starts for a particular pool or cloud account" do + start_time = Time.utc(2010,"jan",1,20,15,1) + user = Factory :pool_user + pool = Factory(:pool, :owner => user) + cloud_account = Factory :mock_cloud_account + instance = Factory(:instance, :pool => pool, :cloud_account => cloud_account) + + failures = 5 + non_failures = 20 + + for i in 1..failures + task = InstanceTask.new(:instance => instance, + :type => "InstanceTask", + :state => Task::STATE_FAILED, + :failure_code => Task::FAILURE_OVER_POOL_QUOTA, + :action => InstanceTask::ACTION_CREATE, + :task_target_id => instance.id) + task.created_at = Time.now + task.save! + end + + for i in 1..non_failures + task = InstanceTask.new(:instance => instance, + :type => "InstanceTask", + :state => Task::STATE_FINISHED, + :failure_code => nil, + :action => InstanceTask::ACTION_CREATE, + :task_target_id => instance.id) + task.created_at = Time.now + task.save! + end + + date = DataService.failure_rate(pool, start_time, Time.now, Task::FAILURE_OVER_POOL_QUOTA) + date.failure_rate.should == (100 / (non_failures + failures)) * failures + end + + it "should create data points for failure rates of instances between two times at given intervals" do + interval_length = 1000 + + start_time = Time.utc(2010,"jan",1,20,15,1) + start_time1 = start_time + interval_length + start_time2 = start_time1 + interval_length + start_times = [start_time, start_time1, start_time2] + + end_time = start_time2 + interval_length + + failures = [5, 10, 15] + number_of_instances = 20 + + user = Factory :pool_user + pool = Factory(:pool, :owner => user) + cloud_account = Factory :mock_cloud_account + instance = Factory(:instance, :pool => pool, :cloud_account => cloud_account) + + for i in 0..2 + for j in 1..failures[i] + task = InstanceTask.new(:instance => instance, + :type => "InstanceTask", + :state => Task::STATE_FAILED, + :failure_code => Task::FAILURE_OVER_POOL_QUOTA, + :action => InstanceTask::ACTION_CREATE, + :task_target_id => instance.id) + task.created_at = start_times[i] + task.time_submitted = start_times[i] + task.save! + end + + non_failures = number_of_instances - failures[i] + for j in 1..non_failures + task = InstanceTask.new(:instance => instance, + :type => "InstanceTask", + :state => Task::STATE_FINISHED, + :failure_code => nil, + :action => InstanceTask::ACTION_CREATE, + :task_target_id => instance.id) + task.created_at = start_times[i] + task.time_submitted = start_times[i] + task.save! + end + end + + data = DataService.qos_failure_rate_stats(pool, start_time, end_time, interval_length, Task::FAILURE_OVER_POOL_QUOTA) + data[0].should == DataService::QoSFailureRatePoint.new(start_time, 25) + data[1].should == DataService::QoSFailureRatePoint.new(start_time1, 50) + data[2].should == DataService::QoSFailureRatePoint.new(start_time2, 75) + end + def generate_tasks(start_time, interval_length, instance, expected_averages) interval_time = start_time expected_averages.each do |avg| @@ -178,5 +366,4 @@ describe DataService do interval_time += interval_length end end - -end +end \ No newline at end of file