[Toaster] [PATCH] toaster: create Build object earlier in bitbake processing
Elliot Smith
elliot.smith at intel.com
Tue Mar 8 06:51:29 PST 2016
If a build fails because of a bitbake error occurring before the
BuildStarted event fires, we do not generate a Build object
for command-line builds. This means that failed command-line builds
don't appear in Toaster at all.
To resolve, split build creation into two steps:
1. Just before buildTargets() is invoked on the XMLRPC server: create
the base Build object. Note that as soon as a Toaster-triggered
build starts, targets are added to it; but this event is the earliest
point when task and targets are available for command-line builds.
(This requires a new TargetsAcquired event to be fired by the XMLRPC
server when the buildTargets() command is called.)
2. BuildStarted event: add any layer information to either type of build
(command-line or Toaster-triggered).
Note that the build_name property cannot be set until BuildStarted,
as it is not available until then, which could cause problems
for creating Build objects earlier; however, this property is
redundant, as it's never used anywhere in Toaster, so it has been
removed (along with any functions which refer to it).
[YOCTO #8440]
Signed-off-by: Elliot Smith <elliot.smith at intel.com>
---
bitbake/lib/bb/event.py | 7 +++
bitbake/lib/bb/server/xmlrpc.py | 3 +
bitbake/lib/bb/ui/buildinfohelper.py | 73 ++++++++++------------
bitbake/lib/bb/ui/toasterui.py | 17 ++---
.../orm/migrations/0006_remove_build_build_name.py | 18 ++++++
bitbake/lib/toaster/orm/models.py | 1 -
.../fixtures/toastergui-unittest-data.xml | 4 --
7 files changed, 71 insertions(+), 52 deletions(-)
create mode 100644 bitbake/lib/toaster/orm/migrations/0006_remove_build_build_name.py
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index 5ffe89e..e94cdb8 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -444,6 +444,13 @@ class MultipleProviders(Event):
"""
return self._candidates
+class TargetsAcquired(Event):
+ """Target we are aiming at known"""
+ def __init__(self, task, targetsList):
+ Event.__init__(self)
+ self.task = task
+ self.targetsList = targetsList
+
class ParseStarted(OperationStarted):
"""Recipe parsing for the runqueue has begun"""
def __init__(self, total):
diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py
index 1ceca51..5f735ab 100644
--- a/bitbake/lib/bb/server/xmlrpc.py
+++ b/bitbake/lib/bb/server/xmlrpc.py
@@ -112,6 +112,9 @@ class BitBakeServerCommands():
"""
Run a cooker command on the server
"""
+ if 'buildTargets' in command[0]:
+ bb.event.fire(bb.event.TargetsAcquired(command[2], command[1]), self.server.readonly)
+
return self.cooker.command.runCommand(command, self.server.readonly)
def getEventHandle(self):
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 7fedb76..7805086 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -134,13 +134,12 @@ class ORMWrapper(object):
# pylint: disable=bad-continuation
# we do not follow the python conventions for continuation indentation due to long lines here
- def create_build_object(self, build_info, brbe, project_id):
+ def create_build_object(self, build_info, brbe, project_id = None):
assert 'machine' in build_info
assert 'distro' in build_info
assert 'distro_version' in build_info
assert 'started_on' in build_info
assert 'cooker_log_path' in build_info
- assert 'build_name' in build_info
assert 'bitbake_version' in build_info
prj = None
@@ -168,7 +167,6 @@ class ORMWrapper(object):
build.distro=build_info['distro']
build.distro_version=build_info['distro_version']
build.cooker_log_path=build_info['cooker_log_path']
- build.build_name=build_info['build_name']
build.bitbake_version=build_info['bitbake_version']
build.save()
@@ -181,7 +179,6 @@ class ORMWrapper(object):
started_on=build_info['started_on'],
completed_on=build_info['started_on'],
cooker_log_path=build_info['cooker_log_path'],
- build_name=build_info['build_name'],
bitbake_version=build_info['bitbake_version'])
logger.debug(1, "buildinfohelper: build is created %s" % build)
@@ -875,7 +872,6 @@ class BuildInfoHelper(object):
build_info['started_on'] = timezone.now()
build_info['completed_on'] = timezone.now()
build_info['cooker_log_path'] = build_log_path
- build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0]
return build_info
@@ -940,26 +936,6 @@ class BuildInfoHelper(object):
return recipe_info
- def _get_path_information(self, task_object):
- assert isinstance(task_object, Task)
- build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/"
- build_stats_path = []
-
- for t in self.internal_state['targets']:
- buildname = self.internal_state['build'].build_name
- pe, pv = task_object.recipe.version.split(":",1)
- if len(pe) > 0:
- package = task_object.recipe.name + "-" + pe + "_" + pv
- else:
- package = task_object.recipe.name + "-" + pv
-
- build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir,
- buildname=buildname,
- package=package))
-
- return build_stats_path
-
-
################################
## external available methods to store information
@staticmethod
@@ -983,17 +959,43 @@ class BuildInfoHelper(object):
except NotExisting as nee:
logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee)
-
- def store_started_build(self, event, build_log_path):
- assert '_pkgs' in vars(event)
+ def store_new_build(self, build_log_path):
+ """
+ create a skeletal build object (or retrieve an existing one) as soon as
+ bitbake starts trying to do the build; we use the buildTargets()
+ command on the XMLRPC server as the indicator of a build start
+ """
build_information = self._get_build_information(build_log_path)
+ self.internal_state['build'] = self.orm_wrapper.create_build_object(build_information, self.brbe)
- # Update brbe and project as they can be changed for every build
- self.project = build_information['project']
+ def store_targets(self, event):
+ """
+ store targets for the current build, if that build was started from
+ the command line; targets for non-cli builds are irrelevant, as we
+ create them from the BuildRequest anyway
+
+ event: a TargetsAcquired event with a task property (e.g. "build")
+ and a targetsList property (e.g. ["zlib", "dropbear"])
+ """
+ if self.internal_state['build'].project.is_default:
+ targets = map(lambda target: target + ':' + event.task, event.targetsList)
+
+ target_information = {
+ 'targets': targets,
+ 'build': self.internal_state['build']
+ }
+
+ self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
- build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project)
+ def update_build(self, event):
+ """
+ update the current build with layer and config data once it
+ actually starts
+
+ event: a BuildStarted event
+ """
- self.internal_state['build'] = build_obj
+ build_obj = self.internal_state['build']
# save layer version information for this build
if not 'lvs' in self.internal_state:
@@ -1004,13 +1006,6 @@ class BuildInfoHelper(object):
del self.internal_state['lvs']
- # create target information
- target_information = {}
- target_information['targets'] = event._pkgs
- target_information['build'] = build_obj
-
- self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
-
# Save build configuration
data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index eee8d14..ee40110 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -119,6 +119,7 @@ _evt_list = [
"bb.event.RecipeParsed",
"bb.event.SanityCheck",
"bb.event.SanityCheckPassed",
+ "bb.event.TargetsAcquired",
"bb.event.TreeDataPreparationCompleted",
"bb.event.TreeDataPreparationStarted",
"bb.runqueue.runQueueTaskCompleted",
@@ -231,19 +232,19 @@ def main(server, eventHandler, params):
# pylint: disable=protected-access
# the code will look into the protected variables of the event; no easy way around this
- # we treat ParseStarted as the first event of toaster-triggered
- # builds; that way we get the Build Configuration included in the log
- # and any errors that occur before BuildStarted is fired
- if isinstance(event, bb.event.ParseStarted):
+ # start of build: this event is fired just before the buildTargets()
+ # command is invoked on the XMLRPC server
+ if isinstance(event, bb.event.TargetsAcquired):
if not (build_log and build_log_file_path):
build_log, build_log_file_path = _open_build_log(log_dir)
+ buildinfohelper.store_new_build(build_log_file_path)
+ buildinfohelper.store_targets(event)
continue
+ # when the build proper starts, we extract information about
+ # any layers and config data
if isinstance(event, bb.event.BuildStarted):
- if not (build_log and build_log_file_path):
- build_log, build_log_file_path = _open_build_log(log_dir)
-
- buildinfohelper.store_started_build(event, build_log_file_path)
+ buildinfohelper.update_build(event)
continue
if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
diff --git a/bitbake/lib/toaster/orm/migrations/0006_remove_build_build_name.py b/bitbake/lib/toaster/orm/migrations/0006_remove_build_build_name.py
new file mode 100644
index 0000000..6036359
--- /dev/null
+++ b/bitbake/lib/toaster/orm/migrations/0006_remove_build_build_name.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('orm', '0005_task_field_separation'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='build',
+ name='build_name',
+ ),
+ ]
diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py
index 182d355..0551240 100644
--- a/bitbake/lib/toaster/orm/models.py
+++ b/bitbake/lib/toaster/orm/models.py
@@ -370,7 +370,6 @@ class Build(models.Model):
completed_on = models.DateTimeField()
outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS)
cooker_log_path = models.CharField(max_length=500)
- build_name = models.CharField(max_length=100)
bitbake_version = models.CharField(max_length=50)
@staticmethod
diff --git a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
index 2d83ff8..a554e62 100644
--- a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
+++ b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
@@ -39,7 +39,6 @@
<field type="DateTimeField" name="completed_on">2016-02-14T18:46:20.114530+00:00</field>
<field type="IntegerField" name="outcome">0</field>
<field type="CharField" name="cooker_log_path"></field>
- <field type="CharField" name="build_name">a</field>
<field type="CharField" name="bitbake_version"></field>
</object>
<object pk="2" model="orm.build">
@@ -51,7 +50,6 @@
<field type="DateTimeField" name="completed_on">2016-02-13T18:46:20.114530+00:00</field>
<field type="IntegerField" name="outcome">0</field>
<field type="CharField" name="cooker_log_path"></field>
- <field type="CharField" name="build_name">b</field>
<field type="CharField" name="bitbake_version"></field>
</object>
<object pk="3" model="orm.build">
@@ -63,7 +61,6 @@
<field type="DateTimeField" name="completed_on">2016-02-12T18:46:20.114530+00:00</field>
<field type="IntegerField" name="outcome">1</field>
<field type="CharField" name="cooker_log_path"></field>
- <field type="CharField" name="build_name">c</field>
<field type="CharField" name="bitbake_version"></field>
</object>
<object pk="4" model="orm.build">
@@ -75,7 +72,6 @@
<field type="DateTimeField" name="completed_on">2016-02-11T18:46:20.114530+00:00</field>
<field type="IntegerField" name="outcome">0</field>
<field type="CharField" name="cooker_log_path"></field>
- <field type="CharField" name="build_name">d</field>
<field type="CharField" name="bitbake_version"></field>
</object>
<object pk="1" model="orm.target">
--
Elliot Smith
Software Engineer
Intel OTC
---------------------------------------------------------------------
Intel Corporation (UK) Limited
Registered No. 1134945 (England)
Registered Office: Pipers Way, Swindon SN3 1RJ
VAT No: 860 2173 47
This e-mail and any attachments may contain confidential material for
the sole use of the intended recipient(s). Any review or distribution
by others is strictly prohibited. If you are not the intended
recipient, please contact the sender and delete all copies.
More information about the toaster
mailing list