[Toaster] [review-request][PATCH] toaster: Import external layers outside poky from toasterconf.json
Brian Avery
avery.brian at gmail.com
Mon Aug 31 09:35:03 PDT 2015
yes. this is the patch I am on. I'll try you approach later today.
Thanks for the clarifications :)
-b
On Mon, Aug 31, 2015 at 4:17 AM, sujith h <sujith.h at gmail.com> wrote:
> Hi Brian,
>
> I hope you are using this patch for testing.
>
> Thanks,
> Sujith H
>
> On Tue, Aug 25, 2015 at 3:23 PM, Sujith H <sujith.h at gmail.com> wrote:
>>
>> This patch helps to import external layers from toasterconf.json
>> which are already checked out. The branch of the checkedout layers
>> is HEAD. Hence no checkout happens when toaster is using the
>> configuration.
>> This essentially speeds up toaster, while using locally checked out
>> layers.
>>
>> Signed-off-by: Sujith H <sujith.h at gmail.com>
>> Signed-off-by: Sujith Haridasan <Sujith_Haridasan at mentor.com>
>> ---
>> .../toaster/bldcontrol/localhostbecontroller.py | 112
>> +++++++++++++--------
>> .../bldcontrol/management/commands/loadconf.py | 14 ++-
>> 2 files changed, 79 insertions(+), 47 deletions(-)
>>
>> diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
>> b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
>> index 231a7d3..837145b 100644
>> --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
>> +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
>> @@ -32,6 +32,8 @@ from toastermain import settings
>>
>> from bbcontroller import BuildEnvironmentController, ShellCmdException,
>> BuildSetupException
>>
>> +from orm.models import Layer_Version
>> +
>> import logging
>> logger = logging.getLogger("toaster")
>>
>> @@ -196,6 +198,21 @@ class
>> LocalhostBEController(BuildEnvironmentController):
>> #logger.debug("localhostbecontroller: using HEAD checkout in %s"
>> % local_checkout_path)
>> return local_checkout_path
>>
>> + def getHeadLayers(self, gitrepos, layers):
>> + """ HEAD layers are special cases. They don't need to be cloned.
>> """
>> +
>> + layerlist = []
>> + headlayers = [commit for giturl, commit in gitrepos.keys() if
>> commit == "HEAD"]
>> + if set(headlayers) == set(["HEAD"]):
>> + # Populate layerlist if all of them are HEAD
>> + for layer_version in
>> Layer_Version.objects.all().filter(local_path__startswith = "/"):
>> + if layer_version.commit == "HEAD":
>> + if layer_version.local_path not in layerlist:
>> + layerlist.append(layer_version.local_path)
>> + if "poky" in layer_version.local_path and
>> os.path.exists(os.path.join(os.path.dirname(layer_version.local_path),
>> "oe-init-build-env")):
>> + self.pokydirname =
>> os.path.dirname(layer_version.local_path)
>> + return layerlist
>> +
>>
>> def setLayers(self, bitbakes, layers):
>> """ a word of attention: by convention, the first layer for any
>> build will be poky! """
>> @@ -242,51 +259,58 @@ class
>> LocalhostBEController(BuildEnvironmentController):
>>
>> layerlist = []
>>
>> + # Verify HEAD layers
>> +
>> + layerlist = self.getHeadLayers(gitrepos,layers)
>> +
>> + # If the layerlist doesn't have already cloned layers then
>> checkout repositories
>> + if len(layerlist) == 0:
>> +
>> + # 3. checkout the repositories
>> + for giturl, commit in gitrepos.keys():
>> + localdirname = os.path.join(self.be.sourcedir,
>> self.getGitCloneDirectory(giturl, commit))
>> + logger.debug("localhostbecontroller: giturl %s:%s checking
>> out in current directory %s" % (giturl, commit, localdirname))
>> +
>> + # make sure our directory is a git repository
>> + if os.path.exists(localdirname):
>> + localremotes = self._shellcmd("git remote -v",
>> localdirname)
>> + if not giturl in localremotes:
>> + raise BuildSetupException("Existing git repository
>> at %s, but with different remotes ('%s', expected '%s'). Toaster will not
>> continue out of fear of damaging something." % (localdirname, ",
>> ".join(localremotes.split("\n")), giturl))
>> + else:
>> + if giturl in cached_layers:
>> + logger.debug("localhostbecontroller git-copying %s
>> to %s" % (cached_layers[giturl], localdirname))
>> + self._shellcmd("git clone \"%s\" \"%s\"" %
>> (cached_layers[giturl], localdirname))
>> + self._shellcmd("git remote remove origin",
>> localdirname)
>> + self._shellcmd("git remote add origin \"%s\"" %
>> giturl, localdirname)
>> + else:
>> + logger.debug("localhostbecontroller: cloning %s:%s
>> in %s" % (giturl, commit, localdirname))
>> + self._shellcmd("git clone \"%s\" --single-branch
>> --branch \"%s\" \"%s\"" % (giturl, commit, localdirname))
>> +
>> + # branch magic name "HEAD" will inhibit checkout
>> + if commit != "HEAD":
>> + logger.debug("localhostbecontroller: checking out
>> commit %s to %s " % (commit, localdirname))
>> + self._shellcmd("git fetch --all && git checkout \"%s\"
>> && git rebase \"origin/%s\"" % (commit, commit) , localdirname)
>> +
>> + # take the localdirname as poky dir if we can find the
>> oe-init-build-env
>> + if self.pokydirname is None and
>> os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
>> + logger.debug("localhostbecontroller: selected poky dir
>> name %s" % localdirname)
>> + self.pokydirname = localdirname
>> +
>> + # make sure we have a working bitbake
>> + if not os.path.exists(os.path.join(self.pokydirname,
>> 'bitbake')):
>> + logger.debug("localhostbecontroller: checking
>> bitbake into the poky dirname %s " % self.pokydirname)
>> + self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\"
>> " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname,
>> 'bitbake')))
>> +
>> + # verify our repositories
>> + for name, dirpath in gitrepos[(giturl, commit)]:
>> + localdirpath = os.path.join(localdirname, dirpath)
>> + logger.debug("localhostbecontroller: localdirpath
>> expected '%s'" % localdirpath)
>> + if not os.path.exists(localdirpath):
>> + raise BuildSetupException("Cannot find layer git
>> path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath,
>> giturl, commit))
>> +
>> + if name != "bitbake":
>> + layerlist.append(localdirpath.rstrip("/"))
>>
>> - # 3. checkout the repositories
>> - for giturl, commit in gitrepos.keys():
>> - localdirname = os.path.join(self.be.sourcedir,
>> self.getGitCloneDirectory(giturl, commit))
>> - logger.debug("localhostbecontroller: giturl %s:%s checking
>> out in current directory %s" % (giturl, commit, localdirname))
>> -
>> - # make sure our directory is a git repository
>> - if os.path.exists(localdirname):
>> - localremotes = self._shellcmd("git remote -v",
>> localdirname)
>> - if not giturl in localremotes:
>> - raise BuildSetupException("Existing git repository at
>> %s, but with different remotes ('%s', expected '%s'). Toaster will not
>> continue out of fear of damaging something." % (localdirname, ",
>> ".join(localremotes.split("\n")), giturl))
>> - else:
>> - if giturl in cached_layers:
>> - logger.debug("localhostbecontroller git-copying %s to
>> %s" % (cached_layers[giturl], localdirname))
>> - self._shellcmd("git clone \"%s\" \"%s\"" %
>> (cached_layers[giturl], localdirname))
>> - self._shellcmd("git remote remove origin",
>> localdirname)
>> - self._shellcmd("git remote add origin \"%s\"" %
>> giturl, localdirname)
>> - else:
>> - logger.debug("localhostbecontroller: cloning %s:%s in
>> %s" % (giturl, commit, localdirname))
>> - self._shellcmd("git clone \"%s\" --single-branch
>> --branch \"%s\" \"%s\"" % (giturl, commit, localdirname))
>> -
>> - # branch magic name "HEAD" will inhibit checkout
>> - if commit != "HEAD":
>> - logger.debug("localhostbecontroller: checking out commit
>> %s to %s " % (commit, localdirname))
>> - self._shellcmd("git fetch --all && git checkout \"%s\" &&
>> git rebase \"origin/%s\"" % (commit, commit) , localdirname)
>> -
>> - # take the localdirname as poky dir if we can find the
>> oe-init-build-env
>> - if self.pokydirname is None and
>> os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
>> - logger.debug("localhostbecontroller: selected poky dir
>> name %s" % localdirname)
>> - self.pokydirname = localdirname
>> -
>> - # make sure we have a working bitbake
>> - if not os.path.exists(os.path.join(self.pokydirname,
>> 'bitbake')):
>> - logger.debug("localhostbecontroller: checking bitbake
>> into the poky dirname %s " % self.pokydirname)
>> - self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " %
>> (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname,
>> 'bitbake')))
>> -
>> - # verify our repositories
>> - for name, dirpath in gitrepos[(giturl, commit)]:
>> - localdirpath = os.path.join(localdirname, dirpath)
>> - logger.debug("localhostbecontroller: localdirpath
>> expected '%s'" % localdirpath)
>> - if not os.path.exists(localdirpath):
>> - raise BuildSetupException("Cannot find layer git path
>> '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl,
>> commit))
>> -
>> - if name != "bitbake":
>> - layerlist.append(localdirpath.rstrip("/"))
>>
>> logger.debug("localhostbecontroller: current layer list %s " %
>> pformat(layerlist))
>>
>> diff --git
>> a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
>> b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
>> index 5022b59..21bc380 100644
>> --- a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
>> +++ b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
>> @@ -45,12 +45,14 @@ class Command(BaseCommand):
>> for i in ['bitbake', 'releases', 'defaultrelease', 'config',
>> 'layersources']:
>> assert i in data
>>
>> - def _read_git_url_from_local_repository(address):
>> + def _read_git_url_from_local_repository(address, path = None):
>> url = None
>> + if not path:
>> + path = os.path.dirname(filepath)
>> # we detect the remote name at runtime
>> import subprocess
>> (remote, remote_name) = address.split(":", 1)
>> - cmd = subprocess.Popen("git remote -v", shell=True, cwd =
>> os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE)
>> + cmd = subprocess.Popen("git remote -v", shell=True, cwd =
>> path, stdout=subprocess.PIPE, stderr = subprocess.PIPE)
>> (out,err) = cmd.communicate()
>> if cmd.returncode != 0:
>> logging.warning("Error while importing layer vcs_url: git
>> error: %s" % err)
>> @@ -121,7 +123,11 @@ class Command(BaseCommand):
>> logger.error("Local layer path %s must exists.
>> Are you trying to import a layer that does not exist ? Check your local
>> toasterconf.json" % lo.local_path)
>>
>> if layerinfo['vcs_url'].startswith("remote:"):
>> - lo.vcs_url =
>> _read_git_url_from_local_repository(layerinfo['vcs_url'])
>> + if not layerinfo['local_path'].startswith("/"):
>> + path = None
>> + else:
>> + path = layerinfo['local_path']
>> + lo.vcs_url =
>> _read_git_url_from_local_repository(layerinfo['vcs_url'], path)
>> if lo.vcs_url is None:
>> logger.error("The toaster config file
>> references the local git repo, but Toaster cannot detect it.\nYour local
>> configuration for layer %s is invalid. Make sure that the toasterconf.json
>> file is correct." % layerinfo['name'])
>>
>> @@ -138,6 +144,8 @@ class Command(BaseCommand):
>> commit = branch.name,
>> layer = lo)
>> lvo.dirpath = layerinfo['dirpath']
>> + if len(layerinfo['local_path']) > 1 and
>> layerinfo['local_path'].startswith("/") and branch.name == "HEAD":
>> + lvo.local_path = layerinfo['local_path']
>> lvo.save()
>> # set releases
>> for ri in data['releases']:
>> --
>> 1.9.1
>>
>
>
>
> --
> സുജിത് ഹരിദാസന്
> Bangalore
> <Project>Contributor to KDE project
> http://fci.wikia.com/wiki/Anti-DRM-Campaign
> <Blog> http://sujithh.info
More information about the toaster
mailing list