[Toaster] [PATCH] toaster: create Build object earlier in bitbake processing
Michael Wood
michael.g.wood at intel.com
Thu Mar 10 06:02:51 PST 2016
Thanks sent to bitbake devel & pushed to toaster-next
On 08/03/16 14:51, Elliot Smith wrote:
> If a build fails because of a bitbake error occurring before the
> BuildStarted event fires, we do not generate a Build object
> for command-line builds. This means that failed command-line builds
> don't appear in Toaster at all.
>
> To resolve, split build creation into two steps:
>
> 1. Just before buildTargets() is invoked on the XMLRPC server: create
> the base Build object. Note that as soon as a Toaster-triggered
> build starts, targets are added to it; but this event is the earliest
> point when task and targets are available for command-line builds.
> (This requires a new TargetsAcquired event to be fired by the XMLRPC
> server when the buildTargets() command is called.)
>
> 2. BuildStarted event: add any layer information to either type of build
> (command-line or Toaster-triggered).
>
> Note that the build_name property cannot be set until BuildStarted,
> as it is not available until then, which could cause problems
> for creating Build objects earlier; however, this property is
> redundant, as it's never used anywhere in Toaster, so it has been
> removed (along with any functions which refer to it).
>
> [YOCTO #8440]
>
> Signed-off-by: Elliot Smith <elliot.smith at intel.com>
> ---
> bitbake/lib/bb/event.py | 7 +++
> bitbake/lib/bb/server/xmlrpc.py | 3 +
> bitbake/lib/bb/ui/buildinfohelper.py | 73 ++++++++++------------
> bitbake/lib/bb/ui/toasterui.py | 17 ++---
> .../orm/migrations/0006_remove_build_build_name.py | 18 ++++++
> bitbake/lib/toaster/orm/models.py | 1 -
> .../fixtures/toastergui-unittest-data.xml | 4 --
> 7 files changed, 71 insertions(+), 52 deletions(-)
> create mode 100644 bitbake/lib/toaster/orm/migrations/0006_remove_build_build_name.py
>
> diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
> index 5ffe89e..e94cdb8 100644
> --- a/bitbake/lib/bb/event.py
> +++ b/bitbake/lib/bb/event.py
> @@ -444,6 +444,13 @@ class MultipleProviders(Event):
> """
> return self._candidates
>
> +class TargetsAcquired(Event):
> + """Target we are aiming at known"""
> + def __init__(self, task, targetsList):
> + Event.__init__(self)
> + self.task = task
> + self.targetsList = targetsList
> +
> class ParseStarted(OperationStarted):
> """Recipe parsing for the runqueue has begun"""
> def __init__(self, total):
> diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py
> index 1ceca51..5f735ab 100644
> --- a/bitbake/lib/bb/server/xmlrpc.py
> +++ b/bitbake/lib/bb/server/xmlrpc.py
> @@ -112,6 +112,9 @@ class BitBakeServerCommands():
> """
> Run a cooker command on the server
> """
> + if 'buildTargets' in command[0]:
> + bb.event.fire(bb.event.TargetsAcquired(command[2], command[1]), self.server.readonly)
> +
> return self.cooker.command.runCommand(command, self.server.readonly)
>
> def getEventHandle(self):
> diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
> index 7fedb76..7805086 100644
> --- a/bitbake/lib/bb/ui/buildinfohelper.py
> +++ b/bitbake/lib/bb/ui/buildinfohelper.py
> @@ -134,13 +134,12 @@ class ORMWrapper(object):
> # pylint: disable=bad-continuation
> # we do not follow the python conventions for continuation indentation due to long lines here
>
> - def create_build_object(self, build_info, brbe, project_id):
> + def create_build_object(self, build_info, brbe, project_id = None):
> assert 'machine' in build_info
> assert 'distro' in build_info
> assert 'distro_version' in build_info
> assert 'started_on' in build_info
> assert 'cooker_log_path' in build_info
> - assert 'build_name' in build_info
> assert 'bitbake_version' in build_info
>
> prj = None
> @@ -168,7 +167,6 @@ class ORMWrapper(object):
> build.distro=build_info['distro']
> build.distro_version=build_info['distro_version']
> build.cooker_log_path=build_info['cooker_log_path']
> - build.build_name=build_info['build_name']
> build.bitbake_version=build_info['bitbake_version']
> build.save()
>
> @@ -181,7 +179,6 @@ class ORMWrapper(object):
> started_on=build_info['started_on'],
> completed_on=build_info['started_on'],
> cooker_log_path=build_info['cooker_log_path'],
> - build_name=build_info['build_name'],
> bitbake_version=build_info['bitbake_version'])
>
> logger.debug(1, "buildinfohelper: build is created %s" % build)
> @@ -875,7 +872,6 @@ class BuildInfoHelper(object):
> build_info['started_on'] = timezone.now()
> build_info['completed_on'] = timezone.now()
> build_info['cooker_log_path'] = build_log_path
> - build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
> build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
> build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0]
> return build_info
> @@ -940,26 +936,6 @@ class BuildInfoHelper(object):
>
> return recipe_info
>
> - def _get_path_information(self, task_object):
> - assert isinstance(task_object, Task)
> - build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/"
> - build_stats_path = []
> -
> - for t in self.internal_state['targets']:
> - buildname = self.internal_state['build'].build_name
> - pe, pv = task_object.recipe.version.split(":",1)
> - if len(pe) > 0:
> - package = task_object.recipe.name + "-" + pe + "_" + pv
> - else:
> - package = task_object.recipe.name + "-" + pv
> -
> - build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir,
> - buildname=buildname,
> - package=package))
> -
> - return build_stats_path
> -
> -
> ################################
> ## external available methods to store information
> @staticmethod
> @@ -983,17 +959,43 @@ class BuildInfoHelper(object):
> except NotExisting as nee:
> logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee)
>
> -
> - def store_started_build(self, event, build_log_path):
> - assert '_pkgs' in vars(event)
> + def store_new_build(self, build_log_path):
> + """
> + create a skeletal build object (or retrieve an existing one) as soon as
> + bitbake starts trying to do the build; we use the buildTargets()
> + command on the XMLRPC server as the indicator of a build start
> + """
> build_information = self._get_build_information(build_log_path)
> + self.internal_state['build'] = self.orm_wrapper.create_build_object(build_information, self.brbe)
>
> - # Update brbe and project as they can be changed for every build
> - self.project = build_information['project']
> + def store_targets(self, event):
> + """
> + store targets for the current build, if that build was started from
> + the command line; targets for non-cli builds are irrelevant, as we
> + create them from the BuildRequest anyway
> +
> + event: a TargetsAcquired event with a task property (e.g. "build")
> + and a targetsList property (e.g. ["zlib", "dropbear"])
> + """
> + if self.internal_state['build'].project.is_default:
> + targets = map(lambda target: target + ':' + event.task, event.targetsList)
> +
> + target_information = {
> + 'targets': targets,
> + 'build': self.internal_state['build']
> + }
> +
> + self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
>
> - build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project)
> + def update_build(self, event):
> + """
> + update the current build with layer and config data once it
> + actually starts
> +
> + event: a BuildStarted event
> + """
>
> - self.internal_state['build'] = build_obj
> + build_obj = self.internal_state['build']
>
> # save layer version information for this build
> if not 'lvs' in self.internal_state:
> @@ -1004,13 +1006,6 @@ class BuildInfoHelper(object):
>
> del self.internal_state['lvs']
>
> - # create target information
> - target_information = {}
> - target_information['targets'] = event._pkgs
> - target_information['build'] = build_obj
> -
> - self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
> -
> # Save build configuration
> data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]
>
> diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
> index eee8d14..ee40110 100644
> --- a/bitbake/lib/bb/ui/toasterui.py
> +++ b/bitbake/lib/bb/ui/toasterui.py
> @@ -119,6 +119,7 @@ _evt_list = [
> "bb.event.RecipeParsed",
> "bb.event.SanityCheck",
> "bb.event.SanityCheckPassed",
> + "bb.event.TargetsAcquired",
> "bb.event.TreeDataPreparationCompleted",
> "bb.event.TreeDataPreparationStarted",
> "bb.runqueue.runQueueTaskCompleted",
> @@ -231,19 +232,19 @@ def main(server, eventHandler, params):
> # pylint: disable=protected-access
> # the code will look into the protected variables of the event; no easy way around this
>
> - # we treat ParseStarted as the first event of toaster-triggered
> - # builds; that way we get the Build Configuration included in the log
> - # and any errors that occur before BuildStarted is fired
> - if isinstance(event, bb.event.ParseStarted):
> + # start of build: this event is fired just before the buildTargets()
> + # command is invoked on the XMLRPC server
> + if isinstance(event, bb.event.TargetsAcquired):
> if not (build_log and build_log_file_path):
> build_log, build_log_file_path = _open_build_log(log_dir)
> + buildinfohelper.store_new_build(build_log_file_path)
> + buildinfohelper.store_targets(event)
> continue
>
> + # when the build proper starts, we extract information about
> + # any layers and config data
> if isinstance(event, bb.event.BuildStarted):
> - if not (build_log and build_log_file_path):
> - build_log, build_log_file_path = _open_build_log(log_dir)
> -
> - buildinfohelper.store_started_build(event, build_log_file_path)
> + buildinfohelper.update_build(event)
> continue
>
> if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
> diff --git a/bitbake/lib/toaster/orm/migrations/0006_remove_build_build_name.py b/bitbake/lib/toaster/orm/migrations/0006_remove_build_build_name.py
> new file mode 100644
> index 0000000..6036359
> --- /dev/null
> +++ b/bitbake/lib/toaster/orm/migrations/0006_remove_build_build_name.py
> @@ -0,0 +1,18 @@
> +# -*- coding: utf-8 -*-
> +from __future__ import unicode_literals
> +
> +from django.db import migrations, models
> +
> +
> +class Migration(migrations.Migration):
> +
> + dependencies = [
> + ('orm', '0005_task_field_separation'),
> + ]
> +
> + operations = [
> + migrations.RemoveField(
> + model_name='build',
> + name='build_name',
> + ),
> + ]
> diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py
> index 182d355..0551240 100644
> --- a/bitbake/lib/toaster/orm/models.py
> +++ b/bitbake/lib/toaster/orm/models.py
> @@ -370,7 +370,6 @@ class Build(models.Model):
> completed_on = models.DateTimeField()
> outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS)
> cooker_log_path = models.CharField(max_length=500)
> - build_name = models.CharField(max_length=100)
> bitbake_version = models.CharField(max_length=50)
>
> @staticmethod
> diff --git a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
> index 2d83ff8..a554e62 100644
> --- a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
> +++ b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
> @@ -39,7 +39,6 @@
> <field type="DateTimeField" name="completed_on">2016-02-14T18:46:20.114530+00:00</field>
> <field type="IntegerField" name="outcome">0</field>
> <field type="CharField" name="cooker_log_path"></field>
> - <field type="CharField" name="build_name">a</field>
> <field type="CharField" name="bitbake_version"></field>
> </object>
> <object pk="2" model="orm.build">
> @@ -51,7 +50,6 @@
> <field type="DateTimeField" name="completed_on">2016-02-13T18:46:20.114530+00:00</field>
> <field type="IntegerField" name="outcome">0</field>
> <field type="CharField" name="cooker_log_path"></field>
> - <field type="CharField" name="build_name">b</field>
> <field type="CharField" name="bitbake_version"></field>
> </object>
> <object pk="3" model="orm.build">
> @@ -63,7 +61,6 @@
> <field type="DateTimeField" name="completed_on">2016-02-12T18:46:20.114530+00:00</field>
> <field type="IntegerField" name="outcome">1</field>
> <field type="CharField" name="cooker_log_path"></field>
> - <field type="CharField" name="build_name">c</field>
> <field type="CharField" name="bitbake_version"></field>
> </object>
> <object pk="4" model="orm.build">
> @@ -75,7 +72,6 @@
> <field type="DateTimeField" name="completed_on">2016-02-11T18:46:20.114530+00:00</field>
> <field type="IntegerField" name="outcome">0</field>
> <field type="CharField" name="cooker_log_path"></field>
> - <field type="CharField" name="build_name">d</field>
> <field type="CharField" name="bitbake_version"></field>
> </object>
> <object pk="1" model="orm.target">
More information about the toaster
mailing list