[Toaster] [PATCH][v2] toaster: create Build object earlier in bitbake processing
Brian Avery
avery.brian at gmail.com
Fri Feb 19 08:54:17 PST 2016
I wouldn't remove the ParseStarted event from the event list. Earlier
(before you or I worked on it) toaster would show transitions in the
progress bar before the build started so that users got feedback.
Things like
1) checking out layers
2) parsing
3) then the build %
It would be nice at some point to restore this and we'd need
Parsestarted for that.
-b
On Fri, Feb 19, 2016 at 4:17 AM, Elliot Smith <elliot.smith at intel.com> wrote:
> If a build fails because of a bitbake error occurring before the
> BuildStarted event fires, we do not generate a Build object
> for command-line builds. This means that failed command-line builds
> don't appear in Toaster at all.
>
> To resolve, split build creation into two steps:
>
> 1. Just before buildTargets() is invoked on the XMLRPC server: create
> the base Build object. Note that as soon as a Toaster-triggered
> build starts, targets are added to it; but this event is the earliest
> point when task and targets are available for command-line builds.
> (This requires a new TargetsAcquired event to be fired by the XMLRPC
> server when the buildTargets() command is called.)
>
> 2. BuildStarted event: add any layer information to either type of build
> (command-line or Toaster-triggered).
>
> Note that the build_name property cannot be set until BuildStarted,
> as it is not available until then, which could cause problems
> for creating Build objects earlier; however, this property is
> redundant, as it's never used anywhere in Toaster, so it has been
> removed (along with any functions which refer to it).
>
> [YOCTO #8440]
>
> Signed-off-by: Elliot Smith <elliot.smith at intel.com>
> ---
> bitbake/lib/bb/event.py | 7 +++
> bitbake/lib/bb/server/xmlrpc.py | 3 +
> bitbake/lib/bb/ui/buildinfohelper.py | 73 ++++++++++------------
> bitbake/lib/bb/ui/toasterui.py | 21 ++++---
> .../orm/migrations/0005_remove_build_build_name.py | 18 ++++++
> bitbake/lib/toaster/orm/models.py | 1 -
> .../fixtures/toastergui-unittest-data.xml | 4 --
> 7 files changed, 73 insertions(+), 54 deletions(-)
> create mode 100644 bitbake/lib/toaster/orm/migrations/0005_remove_build_build_name.py
>
> diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
> index 5ffe89e..e94cdb8 100644
> --- a/bitbake/lib/bb/event.py
> +++ b/bitbake/lib/bb/event.py
> @@ -444,6 +444,13 @@ class MultipleProviders(Event):
> """
> return self._candidates
>
> +class TargetsAcquired(Event):
> + """Target we are aiming at known"""
> + def __init__(self, task, targetsList):
> + Event.__init__(self)
> + self.task = task
> + self.targetsList = targetsList
> +
> class ParseStarted(OperationStarted):
> """Recipe parsing for the runqueue has begun"""
> def __init__(self, total):
> diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py
> index 1ceca51..5f735ab 100644
> --- a/bitbake/lib/bb/server/xmlrpc.py
> +++ b/bitbake/lib/bb/server/xmlrpc.py
> @@ -112,6 +112,9 @@ class BitBakeServerCommands():
> """
> Run a cooker command on the server
> """
> + if 'buildTargets' in command[0]:
> + bb.event.fire(bb.event.TargetsAcquired(command[2], command[1]), self.server.readonly)
> +
> return self.cooker.command.runCommand(command, self.server.readonly)
>
> def getEventHandle(self):
> diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
> index 81abede..46380fa 100644
> --- a/bitbake/lib/bb/ui/buildinfohelper.py
> +++ b/bitbake/lib/bb/ui/buildinfohelper.py
> @@ -127,13 +127,12 @@ class ORMWrapper(object):
> # pylint: disable=bad-continuation
> # we do not follow the python conventions for continuation indentation due to long lines here
>
> - def create_build_object(self, build_info, brbe, project_id):
> + def create_build_object(self, build_info, brbe, project_id = None):
> assert 'machine' in build_info
> assert 'distro' in build_info
> assert 'distro_version' in build_info
> assert 'started_on' in build_info
> assert 'cooker_log_path' in build_info
> - assert 'build_name' in build_info
> assert 'bitbake_version' in build_info
>
> prj = None
> @@ -161,7 +160,6 @@ class ORMWrapper(object):
> build.distro=build_info['distro']
> build.distro_version=build_info['distro_version']
> build.cooker_log_path=build_info['cooker_log_path']
> - build.build_name=build_info['build_name']
> build.bitbake_version=build_info['bitbake_version']
> build.save()
>
> @@ -174,7 +172,6 @@ class ORMWrapper(object):
> started_on=build_info['started_on'],
> completed_on=build_info['started_on'],
> cooker_log_path=build_info['cooker_log_path'],
> - build_name=build_info['build_name'],
> bitbake_version=build_info['bitbake_version'])
>
> logger.debug(1, "buildinfohelper: build is created %s" % build)
> @@ -854,7 +851,6 @@ class BuildInfoHelper(object):
> build_info['started_on'] = timezone.now()
> build_info['completed_on'] = timezone.now()
> build_info['cooker_log_path'] = build_log_path
> - build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
> build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
> build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0]
> return build_info
> @@ -919,26 +915,6 @@ class BuildInfoHelper(object):
>
> return recipe_info
>
> - def _get_path_information(self, task_object):
> - assert isinstance(task_object, Task)
> - build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/"
> - build_stats_path = []
> -
> - for t in self.internal_state['targets']:
> - buildname = self.internal_state['build'].build_name
> - pe, pv = task_object.recipe.version.split(":",1)
> - if len(pe) > 0:
> - package = task_object.recipe.name + "-" + pe + "_" + pv
> - else:
> - package = task_object.recipe.name + "-" + pv
> -
> - build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir,
> - buildname=buildname,
> - package=package))
> -
> - return build_stats_path
> -
> -
> ################################
> ## external available methods to store information
> @staticmethod
> @@ -962,17 +938,43 @@ class BuildInfoHelper(object):
> except NotExisting as nee:
> logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee)
>
> -
> - def store_started_build(self, event, build_log_path):
> - assert '_pkgs' in vars(event)
> + def store_new_build(self, build_log_path):
> + """
> + create a skeletal build object (or retrieve an existing one) as soon as
> + bitbake starts trying to do the build; we use the buildTargets()
> + command on the XMLRPC server as the indicator of a build start
> + """
> build_information = self._get_build_information(build_log_path)
> + self.internal_state['build'] = self.orm_wrapper.create_build_object(build_information, self.brbe)
>
> - # Update brbe and project as they can be changed for every build
> - self.project = build_information['project']
> + def store_targets(self, event):
> + """
> + store targets for the current build, if that build was started from
> + the command line; targets for non-cli builds are irrelevant, as we
> + create them from the BuildRequest anyway
> +
> + event: a TargetsAcquired event with a task property (e.g. "build")
> + and a targetsList property (e.g. ["zlib", "dropbear"])
> + """
> + if self.internal_state['build'].project.is_default:
> + targets = map(lambda target: target + ':' + event.task, event.targetsList)
> +
> + target_information = {
> + 'targets': targets,
> + 'build': self.internal_state['build']
> + }
> +
> + self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
>
> - build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project)
> + def update_build(self, event):
> + """
> + update the current build with layer and config data once it
> + actually starts
> +
> + event: a BuildStarted event
> + """
>
> - self.internal_state['build'] = build_obj
> + build_obj = self.internal_state['build']
>
> # save layer version information for this build
> if not 'lvs' in self.internal_state:
> @@ -983,13 +985,6 @@ class BuildInfoHelper(object):
>
> del self.internal_state['lvs']
>
> - # create target information
> - target_information = {}
> - target_information['targets'] = event._pkgs
> - target_information['build'] = build_obj
> -
> - self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
> -
> # Save build configuration
> data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]
>
> diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
> index 32b1889..1584872 100644
> --- a/bitbake/lib/bb/ui/toasterui.py
> +++ b/bitbake/lib/bb/ui/toasterui.py
> @@ -93,14 +93,14 @@ def _close_build_log(build_log):
> logger.removeHandler(build_log)
>
> _evt_list = [ "bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", "logging.LogRecord",
> - "bb.build.TaskFailed", "bb.build.TaskBase", "bb.event.ParseStarted",
> + "bb.build.TaskFailed", "bb.build.TaskBase",
> "bb.event.ParseProgress", "bb.event.ParseCompleted", "bb.event.CacheLoadStarted",
> "bb.event.CacheLoadProgress", "bb.event.CacheLoadCompleted", "bb.command.CommandFailed",
> "bb.command.CommandExit", "bb.command.CommandCompleted", "bb.cooker.CookerExit",
> "bb.event.MultipleProviders", "bb.event.NoProvider", "bb.runqueue.sceneQueueTaskStarted",
> "bb.runqueue.runQueueTaskStarted", "bb.runqueue.runQueueTaskFailed", "bb.runqueue.sceneQueueTaskFailed",
> "bb.event.BuildBase", "bb.build.TaskStarted", "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent",
> - "bb.event.MetadataEvent"]
> + "bb.event.MetadataEvent", "bb.event.TargetsAcquired"]
>
> def main(server, eventHandler, params):
> # set to a logging.FileHandler instance when a build starts;
> @@ -186,19 +186,20 @@ def main(server, eventHandler, params):
> # pylint: disable=protected-access
> # the code will look into the protected variables of the event; no easy way around this
>
> - # we treat ParseStarted as the first event of toaster-triggered
> - # builds; that way we get the Build Configuration included in the log
> - # and any errors that occur before BuildStarted is fired
> - if isinstance(event, bb.event.ParseStarted):
> + # start of build: this event is fired just before the buildTargets()
> + # command is invoked on the XMLRPC server
> + if isinstance(event, bb.event.TargetsAcquired):
> if not (build_log and build_log_file_path):
> build_log, build_log_file_path = _open_build_log(log_dir)
> + buildinfohelper.store_new_build(build_log_file_path)
> + buildinfohelper.store_targets(event)
> continue
>
> + # when the build proper starts, we extract information about
> + # any layers and config data
> if isinstance(event, bb.event.BuildStarted):
> - if not (build_log and build_log_file_path):
> - build_log, build_log_file_path = _open_build_log(log_dir)
> -
> - buildinfohelper.store_started_build(event, build_log_file_path)
> + buildinfohelper.update_build(event)
> + continue
>
> if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
> buildinfohelper.update_and_store_task(event)
> diff --git a/bitbake/lib/toaster/orm/migrations/0005_remove_build_build_name.py b/bitbake/lib/toaster/orm/migrations/0005_remove_build_build_name.py
> new file mode 100644
> index 0000000..7159793
> --- /dev/null
> +++ b/bitbake/lib/toaster/orm/migrations/0005_remove_build_build_name.py
> @@ -0,0 +1,18 @@
> +# -*- coding: utf-8 -*-
> +from __future__ import unicode_literals
> +
> +from django.db import migrations, models
> +
> +
> +class Migration(migrations.Migration):
> +
> + dependencies = [
> + ('orm', '0004_provides'),
> + ]
> +
> + operations = [
> + migrations.RemoveField(
> + model_name='build',
> + name='build_name',
> + ),
> + ]
> diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py
> index ab6940f..39eb2b1 100644
> --- a/bitbake/lib/toaster/orm/models.py
> +++ b/bitbake/lib/toaster/orm/models.py
> @@ -370,7 +370,6 @@ class Build(models.Model):
> completed_on = models.DateTimeField()
> outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS)
> cooker_log_path = models.CharField(max_length=500)
> - build_name = models.CharField(max_length=100)
> bitbake_version = models.CharField(max_length=50)
>
> @staticmethod
> diff --git a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
> index 2d83ff8..a554e62 100644
> --- a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
> +++ b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
> @@ -39,7 +39,6 @@
> <field type="DateTimeField" name="completed_on">2016-02-14T18:46:20.114530+00:00</field>
> <field type="IntegerField" name="outcome">0</field>
> <field type="CharField" name="cooker_log_path"></field>
> - <field type="CharField" name="build_name">a</field>
> <field type="CharField" name="bitbake_version"></field>
> </object>
> <object pk="2" model="orm.build">
> @@ -51,7 +50,6 @@
> <field type="DateTimeField" name="completed_on">2016-02-13T18:46:20.114530+00:00</field>
> <field type="IntegerField" name="outcome">0</field>
> <field type="CharField" name="cooker_log_path"></field>
> - <field type="CharField" name="build_name">b</field>
> <field type="CharField" name="bitbake_version"></field>
> </object>
> <object pk="3" model="orm.build">
> @@ -63,7 +61,6 @@
> <field type="DateTimeField" name="completed_on">2016-02-12T18:46:20.114530+00:00</field>
> <field type="IntegerField" name="outcome">1</field>
> <field type="CharField" name="cooker_log_path"></field>
> - <field type="CharField" name="build_name">c</field>
> <field type="CharField" name="bitbake_version"></field>
> </object>
> <object pk="4" model="orm.build">
> @@ -75,7 +72,6 @@
> <field type="DateTimeField" name="completed_on">2016-02-11T18:46:20.114530+00:00</field>
> <field type="IntegerField" name="outcome">0</field>
> <field type="CharField" name="cooker_log_path"></field>
> - <field type="CharField" name="build_name">d</field>
> <field type="CharField" name="bitbake_version"></field>
> </object>
> <object pk="1" model="orm.target">
> --
> Elliot Smith
> Software Engineer
> Intel OTC
>
> ---------------------------------------------------------------------
> Intel Corporation (UK) Limited
> Registered No. 1134945 (England)
> Registered Office: Pipers Way, Swindon SN3 1RJ
> VAT No: 860 2173 47
>
> This e-mail and any attachments may contain confidential material for
> the sole use of the intended recipient(s). Any review or distribution
> by others is strictly prohibited. If you are not the intended
> recipient, please contact the sender and delete all copies.
>
> --
> _______________________________________________
> toaster mailing list
> toaster at yoctoproject.org
> https://lists.yoctoproject.org/listinfo/toaster
More information about the toaster
mailing list