[Toaster] [PATCH 1/2] toaster: time data for tasks differ from buildstats
Damian, Alexandru
alexandru.damian at intel.com
Fri Mar 14 10:33:12 PDT 2014
And, taken in.
On Tue, Mar 11, 2014 at 3:15 PM, Marius Avram <marius.avram at intel.com>wrote:
> The BuildInfoHelper class was measuring separately the time for a build.
> This was causing many difference in the time measured by buildstats class
> and
> the time measured here. The main problem is toaster does database
> operations
> which delay the arrival of events.
>
> The simplest solution, which is implemented in this fix, is to keep the
> time
> of creation for every BaseTask event and calculate the duration by making a
> difference between the creation time of TaskSucceeded and TaskStarted.
>
> [YOCTO #5485]
>
> Signed-off-by: Marius Avram <marius.avram at intel.com>
> ---
> bitbake/lib/bb/build.py | 2 +
> bitbake/lib/bb/ui/buildinfohelper.py | 71
> ++++++++++++++++++++--------------
> bitbake/lib/bb/ui/toasterui.py | 6 ++-
> 3 files changed, 50 insertions(+), 29 deletions(-)
>
> diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
> index 52e4149..5cb4c06 100644
> --- a/bitbake/lib/bb/build.py
> +++ b/bitbake/lib/bb/build.py
> @@ -30,6 +30,7 @@ import sys
> import logging
> import shlex
> import glob
> +import time
> import bb
> import bb.msg
> import bb.process
> @@ -75,6 +76,7 @@ class TaskBase(event.Event):
> self.taskfile = d.getVar("FILE", True)
> self.taskname = self._task
> self.logfile = logfile
> + self.time = time.time()
> event.Event.__init__(self)
> self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True),
> t, self.getDisplayName())
>
> diff --git a/bitbake/lib/bb/ui/buildinfohelper.py
> b/bitbake/lib/bb/ui/buildinfohelper.py
> index d7b526a..31b85a2 100644
> --- a/bitbake/lib/bb/ui/buildinfohelper.py
> +++ b/bitbake/lib/bb/ui/buildinfohelper.py
> @@ -125,9 +125,9 @@ class ORMWrapper(object):
> task_object.sstate_result = Task.SSTATE_FAILED
>
> # mark down duration if we have a start time
> - if 'start_time' in task_information.keys():
> - duration = datetime.datetime.now() -
> task_information['start_time']
> - task_object.elapsed_time = duration.total_seconds()
> + if 'end_time' in task_information.keys():
> + duration = task_information['end_time'] -
> task_information['start_time']
> + task_object.elapsed_time = duration
>
> task_object.save()
> return task_object
> @@ -457,6 +457,31 @@ class BuildInfoHelper(object):
> return " ".join(sorted(ret))
>
>
> + def _get_real_recipe(self, event):
> + # Will be used by events derived from TaskBase. Some of this
> + # events have a missing namespace in front of their taskfile name,
> + # which is called virtual:native:. To avoid duplication of entries
> + # in the database it will append the namespace where it is needed
> + # and get the correct recipe and identifier
> +
> + identifier = event.taskfile + ":" + event.taskname
> + if not identifier in self.internal_state['taskdata']:
> + if isinstance(event, bb.build.TaskBase):
> + # we do a bit of guessing
> + candidates = [x for x in
> self.internal_state['taskdata'].keys() if x.endswith(identifier)]
> + if len(candidates) == 1:
> + identifier = candidates[0]
> +
> + if isinstance(event, bb.build.TaskBase):
> + assert identifier in self.internal_state['taskdata']
> + identifierlist = identifier.split(":")
> + realtaskfile = ":".join(identifierlist[0:len(identifierlist)-1])
> + recipe_information =
> self._get_recipe_information_from_taskfile(realtaskfile)
> + recipe =
> self.orm_wrapper.get_update_recipe_object(recipe_information, True)
> +
> + return (recipe, identifier)
> +
> +
> ################################
> ## external available methods to store information
>
> @@ -497,15 +522,13 @@ class BuildInfoHelper(object):
>
> self.orm_wrapper.update_build_object(self.internal_state['build'], errors,
> warnings, taskfailures)
>
> def store_started_task(self, event):
> - assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted,
> bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
> + assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted,
> bb.runqueue.runQueueTaskStarted,
> + bb.build.TaskStarted, bb.runqueue.runQueueTaskSkipped))
> assert 'taskfile' in vars(event)
> localfilepath = event.taskfile.split(":")[-1]
> assert localfilepath.startswith("/")
>
> - identifier = event.taskfile + ":" + event.taskname
> -
> - recipe_information =
> self._get_recipe_information_from_taskfile(event.taskfile)
> - recipe =
> self.orm_wrapper.get_update_recipe_object(recipe_information)
> + (recipe, identifier) = self._get_real_recipe(event)
>
> task_information = self._get_task_information(event, recipe)
> task_information['outcome'] = Task.OUTCOME_NA
> @@ -529,13 +552,14 @@ class BuildInfoHelper(object):
> self.task_order += 1
> task_information['order'] = self.task_order
>
> - task_obj =
> self.orm_wrapper.get_update_task_object(task_information)
> -
> - self.internal_state['taskdata'][identifier] = {
> - 'start_time': datetime.datetime.now(),
> - 'outcome': task_information['outcome'],
> - }
> + self.internal_state['taskdata'][identifier] = {}
> + self.internal_state['taskdata'][identifier]['outcome'] =
> task_information['outcome']
> + # This would apply only in the case of TaskStarted
> + if 'time' in vars(event):
> + self.internal_state['taskdata'][identifier]['start_time'] =
> event.time
>
> + # If task object already exists it will only update the
> start_time field
> + task_obj =
> self.orm_wrapper.get_update_task_object(task_information)
>
> def store_tasks_stats(self, event):
> for (taskfile, taskname, taskstats) in event.data:
> @@ -562,24 +586,15 @@ class BuildInfoHelper(object):
> localfilepath = event.taskfile.split(":")[-1]
> assert localfilepath.startswith("/")
>
> - identifier = event.taskfile + ":" + event.taskname
> - if not identifier in self.internal_state['taskdata']:
> - if isinstance(event, bb.build.TaskBase):
> - # we do a bit of guessing
> - candidates = [x for x in
> self.internal_state['taskdata'].keys() if x.endswith(identifier)]
> - if len(candidates) == 1:
> - identifier = candidates[0]
> + (recipe, identifier) = self._get_real_recipe(event)
>
> - assert identifier in self.internal_state['taskdata']
> - identifierlist = identifier.split(":")
> - realtaskfile = ":".join(identifierlist[0:len(identifierlist)-1])
> - recipe_information =
> self._get_recipe_information_from_taskfile(realtaskfile)
> - recipe =
> self.orm_wrapper.get_update_recipe_object(recipe_information, True)
> task_information = self._get_task_information(event,recipe)
> -
> - task_information['start_time'] =
> self.internal_state['taskdata'][identifier]['start_time']
> task_information['outcome'] =
> self.internal_state['taskdata'][identifier]['outcome']
>
> + if 'time' in vars(event):
> + task_information['start_time'] =
> self.internal_state['taskdata'][identifier]['start_time']
> + task_information['end_time'] = event.time
> +
> if 'logfile' in vars(event):
> task_information['logfile'] = event.logfile
>
> diff --git a/bitbake/lib/bb/ui/toasterui.py
> b/bitbake/lib/bb/ui/toasterui.py
> index e469d93..58abfd3 100644
> --- a/bitbake/lib/bb/ui/toasterui.py
> +++ b/bitbake/lib/bb/ui/toasterui.py
> @@ -109,7 +109,11 @@ def main(server, eventHandler, params ):
> if isinstance(event, bb.event.BuildStarted):
> buildinfohelper.store_started_build(event)
>
> - if isinstance(event, (bb.build.TaskStarted,
> bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
> + if isinstance(event, bb.build.TaskStarted):
> + buildinfohelper.store_started_task(event)
> + continue
> +
> + if isinstance(event, (bb.build.TaskSucceeded,
> bb.build.TaskFailedSilent)):
> buildinfohelper.update_and_store_task(event)
> continue
>
> --
> 1.7.9.5
>
> --
> _______________________________________________
> toaster mailing list
> toaster at yoctoproject.org
> https://lists.yoctoproject.org/listinfo/toaster
>
--
Alex Damian
Yocto Project
SSG / OTC
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.yoctoproject.org/pipermail/toaster/attachments/20140314/00adcef6/attachment.html>
More information about the toaster
mailing list