[Toaster] [PATCH 1/2] toaster: time data for tasks differ from buildstats
Damian, Alexandru
alexandru.damian at intel.com
Mon Mar 17 08:13:22 PDT 2014
Marius, can you please rebase and re-submit ?
Thanks,
Alex
On Fri, Mar 14, 2014 at 5:35 PM, Damian, Alexandru <
alexandru.damian at intel.com> wrote:
> Sorry, I meant - did not take in, the patch doesn't apply on the latest
> tree.
>
> Please wait until next submission is merged and rebase.
>
> Alex
>
>
> On Fri, Mar 14, 2014 at 5:33 PM, Damian, Alexandru <
> alexandru.damian at intel.com> wrote:
>
>> And, taken in.
>>
>>
>> On Tue, Mar 11, 2014 at 3:15 PM, Marius Avram <marius.avram at intel.com>wrote:
>>
>>> The BuildInfoHelper class was measuring separately the time for a build.
>>> This was causing many difference in the time measured by buildstats
>>> class and
>>> the time measured here. The main problem is toaster does database
>>> operations
>>> which delay the arrival of events.
>>>
>>> The simplest solution, which is implemented in this fix, is to keep the
>>> time
>>> of creation for every BaseTask event and calculate the duration by
>>> making a
>>> difference between the creation time of TaskSucceeded and TaskStarted.
>>>
>>> [YOCTO #5485]
>>>
>>> Signed-off-by: Marius Avram <marius.avram at intel.com>
>>> ---
>>> bitbake/lib/bb/build.py | 2 +
>>> bitbake/lib/bb/ui/buildinfohelper.py | 71
>>> ++++++++++++++++++++--------------
>>> bitbake/lib/bb/ui/toasterui.py | 6 ++-
>>> 3 files changed, 50 insertions(+), 29 deletions(-)
>>>
>>> diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
>>> index 52e4149..5cb4c06 100644
>>> --- a/bitbake/lib/bb/build.py
>>> +++ b/bitbake/lib/bb/build.py
>>> @@ -30,6 +30,7 @@ import sys
>>> import logging
>>> import shlex
>>> import glob
>>> +import time
>>> import bb
>>> import bb.msg
>>> import bb.process
>>> @@ -75,6 +76,7 @@ class TaskBase(event.Event):
>>> self.taskfile = d.getVar("FILE", True)
>>> self.taskname = self._task
>>> self.logfile = logfile
>>> + self.time = time.time()
>>> event.Event.__init__(self)
>>> self._message = "recipe %s: task %s: %s" % (d.getVar("PF",
>>> True), t, self.getDisplayName())
>>>
>>> diff --git a/bitbake/lib/bb/ui/buildinfohelper.py
>>> b/bitbake/lib/bb/ui/buildinfohelper.py
>>> index d7b526a..31b85a2 100644
>>> --- a/bitbake/lib/bb/ui/buildinfohelper.py
>>> +++ b/bitbake/lib/bb/ui/buildinfohelper.py
>>> @@ -125,9 +125,9 @@ class ORMWrapper(object):
>>> task_object.sstate_result = Task.SSTATE_FAILED
>>>
>>> # mark down duration if we have a start time
>>> - if 'start_time' in task_information.keys():
>>> - duration = datetime.datetime.now() -
>>> task_information['start_time']
>>> - task_object.elapsed_time = duration.total_seconds()
>>> + if 'end_time' in task_information.keys():
>>> + duration = task_information['end_time'] -
>>> task_information['start_time']
>>> + task_object.elapsed_time = duration
>>>
>>> task_object.save()
>>> return task_object
>>> @@ -457,6 +457,31 @@ class BuildInfoHelper(object):
>>> return " ".join(sorted(ret))
>>>
>>>
>>> + def _get_real_recipe(self, event):
>>> + # Will be used by events derived from TaskBase. Some of this
>>> + # events have a missing namespace in front of their taskfile
>>> name,
>>> + # which is called virtual:native:. To avoid duplication of
>>> entries
>>> + # in the database it will append the namespace where it is
>>> needed
>>> + # and get the correct recipe and identifier
>>> +
>>> + identifier = event.taskfile + ":" + event.taskname
>>> + if not identifier in self.internal_state['taskdata']:
>>> + if isinstance(event, bb.build.TaskBase):
>>> + # we do a bit of guessing
>>> + candidates = [x for x in
>>> self.internal_state['taskdata'].keys() if x.endswith(identifier)]
>>> + if len(candidates) == 1:
>>> + identifier = candidates[0]
>>> +
>>> + if isinstance(event, bb.build.TaskBase):
>>> + assert identifier in self.internal_state['taskdata']
>>> + identifierlist = identifier.split(":")
>>> + realtaskfile = ":".join(identifierlist[0:len(identifierlist)-1])
>>> + recipe_information =
>>> self._get_recipe_information_from_taskfile(realtaskfile)
>>> + recipe =
>>> self.orm_wrapper.get_update_recipe_object(recipe_information, True)
>>> +
>>> + return (recipe, identifier)
>>> +
>>> +
>>> ################################
>>> ## external available methods to store information
>>>
>>> @@ -497,15 +522,13 @@ class BuildInfoHelper(object):
>>>
>>> self.orm_wrapper.update_build_object(self.internal_state['build'], errors,
>>> warnings, taskfailures)
>>>
>>> def store_started_task(self, event):
>>> - assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted,
>>> bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
>>> + assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted,
>>> bb.runqueue.runQueueTaskStarted,
>>> + bb.build.TaskStarted, bb.runqueue.runQueueTaskSkipped))
>>> assert 'taskfile' in vars(event)
>>> localfilepath = event.taskfile.split(":")[-1]
>>> assert localfilepath.startswith("/")
>>>
>>> - identifier = event.taskfile + ":" + event.taskname
>>> -
>>> - recipe_information =
>>> self._get_recipe_information_from_taskfile(event.taskfile)
>>> - recipe =
>>> self.orm_wrapper.get_update_recipe_object(recipe_information)
>>> + (recipe, identifier) = self._get_real_recipe(event)
>>>
>>> task_information = self._get_task_information(event, recipe)
>>> task_information['outcome'] = Task.OUTCOME_NA
>>> @@ -529,13 +552,14 @@ class BuildInfoHelper(object):
>>> self.task_order += 1
>>> task_information['order'] = self.task_order
>>>
>>> - task_obj =
>>> self.orm_wrapper.get_update_task_object(task_information)
>>> -
>>> - self.internal_state['taskdata'][identifier] = {
>>> - 'start_time': datetime.datetime.now(),
>>> - 'outcome': task_information['outcome'],
>>> - }
>>> + self.internal_state['taskdata'][identifier] = {}
>>> + self.internal_state['taskdata'][identifier]['outcome'] =
>>> task_information['outcome']
>>> + # This would apply only in the case of TaskStarted
>>> + if 'time' in vars(event):
>>> + self.internal_state['taskdata'][identifier]['start_time'] =
>>> event.time
>>>
>>> + # If task object already exists it will only update the
>>> start_time field
>>> + task_obj =
>>> self.orm_wrapper.get_update_task_object(task_information)
>>>
>>> def store_tasks_stats(self, event):
>>> for (taskfile, taskname, taskstats) in event.data:
>>> @@ -562,24 +586,15 @@ class BuildInfoHelper(object):
>>> localfilepath = event.taskfile.split(":")[-1]
>>> assert localfilepath.startswith("/")
>>>
>>> - identifier = event.taskfile + ":" + event.taskname
>>> - if not identifier in self.internal_state['taskdata']:
>>> - if isinstance(event, bb.build.TaskBase):
>>> - # we do a bit of guessing
>>> - candidates = [x for x in
>>> self.internal_state['taskdata'].keys() if x.endswith(identifier)]
>>> - if len(candidates) == 1:
>>> - identifier = candidates[0]
>>> + (recipe, identifier) = self._get_real_recipe(event)
>>>
>>> - assert identifier in self.internal_state['taskdata']
>>> - identifierlist = identifier.split(":")
>>> - realtaskfile = ":".join(identifierlist[0:len(identifierlist)-1])
>>> - recipe_information =
>>> self._get_recipe_information_from_taskfile(realtaskfile)
>>> - recipe =
>>> self.orm_wrapper.get_update_recipe_object(recipe_information, True)
>>> task_information = self._get_task_information(event,recipe)
>>> -
>>> - task_information['start_time'] =
>>> self.internal_state['taskdata'][identifier]['start_time']
>>> task_information['outcome'] =
>>> self.internal_state['taskdata'][identifier]['outcome']
>>>
>>> + if 'time' in vars(event):
>>> + task_information['start_time'] =
>>> self.internal_state['taskdata'][identifier]['start_time']
>>> + task_information['end_time'] = event.time
>>> +
>>> if 'logfile' in vars(event):
>>> task_information['logfile'] = event.logfile
>>>
>>> diff --git a/bitbake/lib/bb/ui/toasterui.py
>>> b/bitbake/lib/bb/ui/toasterui.py
>>> index e469d93..58abfd3 100644
>>> --- a/bitbake/lib/bb/ui/toasterui.py
>>> +++ b/bitbake/lib/bb/ui/toasterui.py
>>> @@ -109,7 +109,11 @@ def main(server, eventHandler, params ):
>>> if isinstance(event, bb.event.BuildStarted):
>>> buildinfohelper.store_started_build(event)
>>>
>>> - if isinstance(event, (bb.build.TaskStarted,
>>> bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
>>> + if isinstance(event, bb.build.TaskStarted):
>>> + buildinfohelper.store_started_task(event)
>>> + continue
>>> +
>>> + if isinstance(event, (bb.build.TaskSucceeded,
>>> bb.build.TaskFailedSilent)):
>>> buildinfohelper.update_and_store_task(event)
>>> continue
>>>
>>> --
>>> 1.7.9.5
>>>
>>> --
>>> _______________________________________________
>>> toaster mailing list
>>> toaster at yoctoproject.org
>>> https://lists.yoctoproject.org/listinfo/toaster
>>>
>>
>>
>>
>> --
>> Alex Damian
>> Yocto Project
>> SSG / OTC
>>
>
>
>
> --
> Alex Damian
> Yocto Project
> SSG / OTC
>
--
Alex Damian
Yocto Project
SSG / OTC
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.yoctoproject.org/pipermail/toaster/attachments/20140317/ee955250/attachment-0001.html>
More information about the toaster
mailing list