[Toaster] [PATCH 1/3] toaster.bbclass: improve how we gather buildstats for Toaster
Smith, Elliot
elliot.smith at intel.com
Mon Mar 7 11:26:04 PST 2016
No, I didn't, I'll fix it.
Elliot
On 7 March 2016 at 18:39, Michael Wood <michael.g.wood at intel.com> wrote:
> On 07/03/16 12:21, Elliot Smith wrote:
>
>> Clean up the code which gathers buildstats for Toaster,
>> and modify the field names so that the correct parts of the
>> buildstats files are used to derive the CPU usage values.
>>
>> Also derive elapsed time for the build here, rather than in
>> Toaster, as we have ready access to the data in the correct
>> format.
>>
>> [YOCTO #8842]
>>
>> Signed-off-by: Elliot Smith <elliot.smith at intel.com>
>> ---
>> meta/classes/toaster.bbclass | 70
>> ++++++++++++++++++++++++++++----------------
>> 1 file changed, 45 insertions(+), 25 deletions(-)
>>
>> diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass
>> index 51a4c74..b7e8773 100644
>> --- a/meta/classes/toaster.bbclass
>> +++ b/meta/classes/toaster.bbclass
>> @@ -202,25 +202,37 @@ python toaster_collect_task_stats() {
>> import bb.utils
>> import os
>> + toaster_statlist_file =
>> os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")
>> +
>> if not e.data.getVar('BUILDSTATS_BASE', True):
>> return # if we don't have buildstats, we cannot collect stats
>> + def stat_to_float(value):
>> + return float(value.strip('% \n\r'))
>> +
>> def _append_read_list(v):
>> lock =
>> bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)
>> - with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True),
>> "toasterstatlist"), "a") as fout:
>> + with open(toaster_statlist_file, "a") as fout:
>> taskdir =
>> e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}")
>> fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname,
>> os.path.join(taskdir, e.task), e.data.expand("${PN}")))
>> bb.utils.unlockfile(lock)
>> def _read_stats(filename):
>> - cpu_usage = 0
>> - disk_io = 0
>> - started = '0'
>> - ended = '0'
>> - pn = ''
>> + # seconds
>> + cpu_time_user = 0
>> + cpu_time_system = 0
>> +
>> + # bytes
>> + disk_io_read = 0
>> + disk_io_write = 0
>> +
>> + started = 0
>> + ended = 0
>> +
>> taskname = ''
>> +
>> statinfo = {}
>> with open(filename, 'r') as task_bs:
>> @@ -228,41 +240,49 @@ python toaster_collect_task_stats() {
>> k,v = line.strip().split(": ", 1)
>> statinfo[k] = v
>> - if "CPU usage" in statinfo:
>> - cpu_usage = str(statinfo["CPU usage"]).strip('% \n\r')
>> -
>> - if "IO write_bytes" in statinfo:
>> - disk_io = disk_io + int(statinfo["IO write_bytes"].strip('%
>> \n\r'))
>> -
>> - if "IO read_bytes" in statinfo:
>> - disk_io = disk_io + int(statinfo["IO read_bytes"].strip('%
>> \n\r'))
>> -
>> if "Started" in statinfo:
>> - started = str(statinfo["Started"]).strip('% \n\r')
>> + started = stat_to_float(statinfo["Started"])
>> if "Ended" in statinfo:
>> - ended = str(statinfo["Ended"]).strip('% \n\r')
>> + ended = stat_to_float(statinfo["Ended"])
>> - elapsed_time = float(ended) - float(started)
>> + if "Child rusage ru_utime" in statinfo:
>> + cpu_time_user = cpu_time_user +
>> stat_to_float(statinfo["Child rusage ru_utime"])
>> - cpu_usage = float(cpu_usage)
>> + if "Child rusage ru_stime" in statinfo:
>> + cpu_time_system = cpu_time_system +
>> stat_to_float(statinfo["Child rusage ru_stime"])
>> - return {'cpu_usage': cpu_usage, 'disk_io': disk_io,
>> 'elapsed_time': elapsed_time}
>> + if "IO write_bytes" in statinfo:
>> + write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r'))
>> + disk_io_write = disk_io_write + write_bytes
>> + if "IO read_bytes" in statinfo:
>> + read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r'))
>> + disk_io_read = disk_io_read + read_bytes
>> +
>> + return {
>> + 'stat_file': filename,
>> + 'cpu_time_user': cpu_time_user,
>> + 'cpu_time_system': cpu_time_system,
>> + 'disk_io_read': disk_io_read,
>> + 'disk_io_write': disk_io_write,
>> + 'started': started,
>> + 'ended': ended
>> + }
>> if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
>> _append_read_list(e)
>> pass
>> -
>> - if isinstance(e, bb.event.BuildCompleted) and
>> os.path.exists(os.path.join(e.data.getVar('BUILDSTATS_BASE', True),
>> "toasterstatlist")):
>> + if isinstance(e, bb.event.BuildCompleted) and
>> os.path.exists(toaster_statlist_file):
>> events = []
>> - with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True),
>> "toasterstatlist"), "r") as fin:
>> + with open(toaster_statlist_file, "r") as fin:
>> for line in fin:
>> (taskfile, taskname, filename, recipename) =
>> line.strip().split("::")
>> - events.append((taskfile, taskname,
>> _read_stats(filename), recipename))
>> + stats = _read_stats(filename)
>> + events.append((taskfile, taskname, stats, recipename))
>> bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events),
>> e.data)
>> - os.unlink(os.path.join(e.data.getVar('BUILDSTATS_BASE', True),
>> "toasterstatlist"))
>> + #os.unlink(toaster_statlist_file)
>>
>
> Did you mean to leave that 'unlink' commented out?
>
>> }
>> # dump relevant build history data as an event when the build is
>> completed
>>
>
> ---------------------------------------------------------------------
> Intel Corporation (UK) Limited
> Registered No. 1134945 (England)
> Registered Office: Pipers Way, Swindon SN3 1RJ
> VAT No: 860 2173 47
>
> This e-mail and any attachments may contain confidential material for
> the sole use of the intended recipient(s). Any review or distribution
> by others is strictly prohibited. If you are not the intended
> recipient, please contact the sender and delete all copies.
>
> --
> _______________________________________________
> toaster mailing list
> toaster at yoctoproject.org
> https://lists.yoctoproject.org/listinfo/toaster
>
--
Elliot Smith
Software Engineer
Intel Open Source Technology Centre
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.yoctoproject.org/pipermail/toaster/attachments/20160307/6abe33ba/attachment.html>
More information about the toaster
mailing list