[Toaster] [PATCH 1/2] toaster: time data for tasks differ from buildstats
Damian, Alexandru
alexandru.damian at intel.com
Fri Mar 14 10:35:15 PDT 2014
Sorry, I meant - did not take in, the patch doesn't apply on the latest
tree.
Please wait until next submission is merged and rebase.
Alex
On Fri, Mar 14, 2014 at 5:33 PM, Damian, Alexandru <
alexandru.damian at intel.com> wrote:
> And, taken in.
>
>
> On Tue, Mar 11, 2014 at 3:15 PM, Marius Avram <marius.avram at intel.com>wrote:
>
>> The BuildInfoHelper class was measuring separately the time for a build.
>> This was causing many difference in the time measured by buildstats class
>> and
>> the time measured here. The main problem is toaster does database
>> operations
>> which delay the arrival of events.
>>
>> The simplest solution, which is implemented in this fix, is to keep the
>> time
>> of creation for every BaseTask event and calculate the duration by making
>> a
>> difference between the creation time of TaskSucceeded and TaskStarted.
>>
>> [YOCTO #5485]
>>
>> Signed-off-by: Marius Avram <marius.avram at intel.com>
>> ---
>> bitbake/lib/bb/build.py | 2 +
>> bitbake/lib/bb/ui/buildinfohelper.py | 71
>> ++++++++++++++++++++--------------
>> bitbake/lib/bb/ui/toasterui.py | 6 ++-
>> 3 files changed, 50 insertions(+), 29 deletions(-)
>>
>> diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
>> index 52e4149..5cb4c06 100644
>> --- a/bitbake/lib/bb/build.py
>> +++ b/bitbake/lib/bb/build.py
>> @@ -30,6 +30,7 @@ import sys
>> import logging
>> import shlex
>> import glob
>> +import time
>> import bb
>> import bb.msg
>> import bb.process
>> @@ -75,6 +76,7 @@ class TaskBase(event.Event):
>> self.taskfile = d.getVar("FILE", True)
>> self.taskname = self._task
>> self.logfile = logfile
>> + self.time = time.time()
>> event.Event.__init__(self)
>> self._message = "recipe %s: task %s: %s" % (d.getVar("PF",
>> True), t, self.getDisplayName())
>>
>> diff --git a/bitbake/lib/bb/ui/buildinfohelper.py
>> b/bitbake/lib/bb/ui/buildinfohelper.py
>> index d7b526a..31b85a2 100644
>> --- a/bitbake/lib/bb/ui/buildinfohelper.py
>> +++ b/bitbake/lib/bb/ui/buildinfohelper.py
>> @@ -125,9 +125,9 @@ class ORMWrapper(object):
>> task_object.sstate_result = Task.SSTATE_FAILED
>>
>> # mark down duration if we have a start time
>> - if 'start_time' in task_information.keys():
>> - duration = datetime.datetime.now() -
>> task_information['start_time']
>> - task_object.elapsed_time = duration.total_seconds()
>> + if 'end_time' in task_information.keys():
>> + duration = task_information['end_time'] -
>> task_information['start_time']
>> + task_object.elapsed_time = duration
>>
>> task_object.save()
>> return task_object
>> @@ -457,6 +457,31 @@ class BuildInfoHelper(object):
>> return " ".join(sorted(ret))
>>
>>
>> + def _get_real_recipe(self, event):
>> + # Will be used by events derived from TaskBase. Some of this
>> + # events have a missing namespace in front of their taskfile
>> name,
>> + # which is called virtual:native:. To avoid duplication of
>> entries
>> + # in the database it will append the namespace where it is needed
>> + # and get the correct recipe and identifier
>> +
>> + identifier = event.taskfile + ":" + event.taskname
>> + if not identifier in self.internal_state['taskdata']:
>> + if isinstance(event, bb.build.TaskBase):
>> + # we do a bit of guessing
>> + candidates = [x for x in
>> self.internal_state['taskdata'].keys() if x.endswith(identifier)]
>> + if len(candidates) == 1:
>> + identifier = candidates[0]
>> +
>> + if isinstance(event, bb.build.TaskBase):
>> + assert identifier in self.internal_state['taskdata']
>> + identifierlist = identifier.split(":")
>> + realtaskfile = ":".join(identifierlist[0:len(identifierlist)-1])
>> + recipe_information =
>> self._get_recipe_information_from_taskfile(realtaskfile)
>> + recipe =
>> self.orm_wrapper.get_update_recipe_object(recipe_information, True)
>> +
>> + return (recipe, identifier)
>> +
>> +
>> ################################
>> ## external available methods to store information
>>
>> @@ -497,15 +522,13 @@ class BuildInfoHelper(object):
>>
>> self.orm_wrapper.update_build_object(self.internal_state['build'], errors,
>> warnings, taskfailures)
>>
>> def store_started_task(self, event):
>> - assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted,
>> bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
>> + assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted,
>> bb.runqueue.runQueueTaskStarted,
>> + bb.build.TaskStarted, bb.runqueue.runQueueTaskSkipped))
>> assert 'taskfile' in vars(event)
>> localfilepath = event.taskfile.split(":")[-1]
>> assert localfilepath.startswith("/")
>>
>> - identifier = event.taskfile + ":" + event.taskname
>> -
>> - recipe_information =
>> self._get_recipe_information_from_taskfile(event.taskfile)
>> - recipe =
>> self.orm_wrapper.get_update_recipe_object(recipe_information)
>> + (recipe, identifier) = self._get_real_recipe(event)
>>
>> task_information = self._get_task_information(event, recipe)
>> task_information['outcome'] = Task.OUTCOME_NA
>> @@ -529,13 +552,14 @@ class BuildInfoHelper(object):
>> self.task_order += 1
>> task_information['order'] = self.task_order
>>
>> - task_obj =
>> self.orm_wrapper.get_update_task_object(task_information)
>> -
>> - self.internal_state['taskdata'][identifier] = {
>> - 'start_time': datetime.datetime.now(),
>> - 'outcome': task_information['outcome'],
>> - }
>> + self.internal_state['taskdata'][identifier] = {}
>> + self.internal_state['taskdata'][identifier]['outcome'] =
>> task_information['outcome']
>> + # This would apply only in the case of TaskStarted
>> + if 'time' in vars(event):
>> + self.internal_state['taskdata'][identifier]['start_time'] =
>> event.time
>>
>> + # If task object already exists it will only update the
>> start_time field
>> + task_obj =
>> self.orm_wrapper.get_update_task_object(task_information)
>>
>> def store_tasks_stats(self, event):
>> for (taskfile, taskname, taskstats) in event.data:
>> @@ -562,24 +586,15 @@ class BuildInfoHelper(object):
>> localfilepath = event.taskfile.split(":")[-1]
>> assert localfilepath.startswith("/")
>>
>> - identifier = event.taskfile + ":" + event.taskname
>> - if not identifier in self.internal_state['taskdata']:
>> - if isinstance(event, bb.build.TaskBase):
>> - # we do a bit of guessing
>> - candidates = [x for x in
>> self.internal_state['taskdata'].keys() if x.endswith(identifier)]
>> - if len(candidates) == 1:
>> - identifier = candidates[0]
>> + (recipe, identifier) = self._get_real_recipe(event)
>>
>> - assert identifier in self.internal_state['taskdata']
>> - identifierlist = identifier.split(":")
>> - realtaskfile = ":".join(identifierlist[0:len(identifierlist)-1])
>> - recipe_information =
>> self._get_recipe_information_from_taskfile(realtaskfile)
>> - recipe =
>> self.orm_wrapper.get_update_recipe_object(recipe_information, True)
>> task_information = self._get_task_information(event,recipe)
>> -
>> - task_information['start_time'] =
>> self.internal_state['taskdata'][identifier]['start_time']
>> task_information['outcome'] =
>> self.internal_state['taskdata'][identifier]['outcome']
>>
>> + if 'time' in vars(event):
>> + task_information['start_time'] =
>> self.internal_state['taskdata'][identifier]['start_time']
>> + task_information['end_time'] = event.time
>> +
>> if 'logfile' in vars(event):
>> task_information['logfile'] = event.logfile
>>
>> diff --git a/bitbake/lib/bb/ui/toasterui.py
>> b/bitbake/lib/bb/ui/toasterui.py
>> index e469d93..58abfd3 100644
>> --- a/bitbake/lib/bb/ui/toasterui.py
>> +++ b/bitbake/lib/bb/ui/toasterui.py
>> @@ -109,7 +109,11 @@ def main(server, eventHandler, params ):
>> if isinstance(event, bb.event.BuildStarted):
>> buildinfohelper.store_started_build(event)
>>
>> - if isinstance(event, (bb.build.TaskStarted,
>> bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
>> + if isinstance(event, bb.build.TaskStarted):
>> + buildinfohelper.store_started_task(event)
>> + continue
>> +
>> + if isinstance(event, (bb.build.TaskSucceeded,
>> bb.build.TaskFailedSilent)):
>> buildinfohelper.update_and_store_task(event)
>> continue
>>
>> --
>> 1.7.9.5
>>
>> --
>> _______________________________________________
>> toaster mailing list
>> toaster at yoctoproject.org
>> https://lists.yoctoproject.org/listinfo/toaster
>>
>
>
>
> --
> Alex Damian
> Yocto Project
> SSG / OTC
>
--
Alex Damian
Yocto Project
SSG / OTC
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.yoctoproject.org/pipermail/toaster/attachments/20140314/9079bed2/attachment-0001.html>
More information about the toaster
mailing list