[Toaster] [review-request][PATCH] toaster: Import external layers outside poky from toasterconf.json
sujith h
sujith.h at gmail.com
Mon Aug 31 04:17:57 PDT 2015
Hi Brian,
I hope you are using this patch for testing.
Thanks,
Sujith H
On Tue, Aug 25, 2015 at 3:23 PM, Sujith H <sujith.h at gmail.com> wrote:
> This patch helps to import external layers from toasterconf.json
> which are already checked out. The branch of the checkedout layers
> is HEAD. Hence no checkout happens when toaster is using the configuration.
> This essentially speeds up toaster, while using locally checked out layers.
>
> Signed-off-by: Sujith H <sujith.h at gmail.com>
> Signed-off-by: Sujith Haridasan <Sujith_Haridasan at mentor.com>
> ---
> .../toaster/bldcontrol/localhostbecontroller.py | 112
> +++++++++++++--------
> .../bldcontrol/management/commands/loadconf.py | 14 ++-
> 2 files changed, 79 insertions(+), 47 deletions(-)
>
> diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
> b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
> index 231a7d3..837145b 100644
> --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
> +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
> @@ -32,6 +32,8 @@ from toastermain import settings
>
> from bbcontroller import BuildEnvironmentController, ShellCmdException,
> BuildSetupException
>
> +from orm.models import Layer_Version
> +
> import logging
> logger = logging.getLogger("toaster")
>
> @@ -196,6 +198,21 @@ class
> LocalhostBEController(BuildEnvironmentController):
> #logger.debug("localhostbecontroller: using HEAD checkout in %s"
> % local_checkout_path)
> return local_checkout_path
>
> + def getHeadLayers(self, gitrepos, layers):
> + """ HEAD layers are special cases. They don't need to be cloned.
> """
> +
> + layerlist = []
> + headlayers = [commit for giturl, commit in gitrepos.keys() if
> commit == "HEAD"]
> + if set(headlayers) == set(["HEAD"]):
> + # Populate layerlist if all of them are HEAD
> + for layer_version in
> Layer_Version.objects.all().filter(local_path__startswith = "/"):
> + if layer_version.commit == "HEAD":
> + if layer_version.local_path not in layerlist:
> + layerlist.append(layer_version.local_path)
> + if "poky" in layer_version.local_path and
> os.path.exists(os.path.join(os.path.dirname(layer_version.local_path),
> "oe-init-build-env")):
> + self.pokydirname =
> os.path.dirname(layer_version.local_path)
> + return layerlist
> +
>
> def setLayers(self, bitbakes, layers):
> """ a word of attention: by convention, the first layer for any
> build will be poky! """
> @@ -242,51 +259,58 @@ class
> LocalhostBEController(BuildEnvironmentController):
>
> layerlist = []
>
> + # Verify HEAD layers
> +
> + layerlist = self.getHeadLayers(gitrepos,layers)
> +
> + # If the layerlist doesn't have already cloned layers then
> checkout repositories
> + if len(layerlist) == 0:
> +
> + # 3. checkout the repositories
> + for giturl, commit in gitrepos.keys():
> + localdirname = os.path.join(self.be.sourcedir,
> self.getGitCloneDirectory(giturl, commit))
> + logger.debug("localhostbecontroller: giturl %s:%s checking
> out in current directory %s" % (giturl, commit, localdirname))
> +
> + # make sure our directory is a git repository
> + if os.path.exists(localdirname):
> + localremotes = self._shellcmd("git remote -v",
> localdirname)
> + if not giturl in localremotes:
> + raise BuildSetupException("Existing git repository
> at %s, but with different remotes ('%s', expected '%s'). Toaster will not
> continue out of fear of damaging something." % (localdirname, ",
> ".join(localremotes.split("\n")), giturl))
> + else:
> + if giturl in cached_layers:
> + logger.debug("localhostbecontroller git-copying %s
> to %s" % (cached_layers[giturl], localdirname))
> + self._shellcmd("git clone \"%s\" \"%s\"" %
> (cached_layers[giturl], localdirname))
> + self._shellcmd("git remote remove origin",
> localdirname)
> + self._shellcmd("git remote add origin \"%s\"" %
> giturl, localdirname)
> + else:
> + logger.debug("localhostbecontroller: cloning %s:%s
> in %s" % (giturl, commit, localdirname))
> + self._shellcmd("git clone \"%s\" --single-branch
> --branch \"%s\" \"%s\"" % (giturl, commit, localdirname))
> +
> + # branch magic name "HEAD" will inhibit checkout
> + if commit != "HEAD":
> + logger.debug("localhostbecontroller: checking out
> commit %s to %s " % (commit, localdirname))
> + self._shellcmd("git fetch --all && git checkout \"%s\"
> && git rebase \"origin/%s\"" % (commit, commit) , localdirname)
> +
> + # take the localdirname as poky dir if we can find the
> oe-init-build-env
> + if self.pokydirname is None and
> os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
> + logger.debug("localhostbecontroller: selected poky dir
> name %s" % localdirname)
> + self.pokydirname = localdirname
> +
> + # make sure we have a working bitbake
> + if not os.path.exists(os.path.join(self.pokydirname,
> 'bitbake')):
> + logger.debug("localhostbecontroller: checking
> bitbake into the poky dirname %s " % self.pokydirname)
> + self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\"
> " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname,
> 'bitbake')))
> +
> + # verify our repositories
> + for name, dirpath in gitrepos[(giturl, commit)]:
> + localdirpath = os.path.join(localdirname, dirpath)
> + logger.debug("localhostbecontroller: localdirpath
> expected '%s'" % localdirpath)
> + if not os.path.exists(localdirpath):
> + raise BuildSetupException("Cannot find layer git
> path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath,
> giturl, commit))
> +
> + if name != "bitbake":
> + layerlist.append(localdirpath.rstrip("/"))
>
> - # 3. checkout the repositories
> - for giturl, commit in gitrepos.keys():
> - localdirname = os.path.join(self.be.sourcedir,
> self.getGitCloneDirectory(giturl, commit))
> - logger.debug("localhostbecontroller: giturl %s:%s checking
> out in current directory %s" % (giturl, commit, localdirname))
> -
> - # make sure our directory is a git repository
> - if os.path.exists(localdirname):
> - localremotes = self._shellcmd("git remote -v",
> localdirname)
> - if not giturl in localremotes:
> - raise BuildSetupException("Existing git repository at
> %s, but with different remotes ('%s', expected '%s'). Toaster will not
> continue out of fear of damaging something." % (localdirname, ",
> ".join(localremotes.split("\n")), giturl))
> - else:
> - if giturl in cached_layers:
> - logger.debug("localhostbecontroller git-copying %s to
> %s" % (cached_layers[giturl], localdirname))
> - self._shellcmd("git clone \"%s\" \"%s\"" %
> (cached_layers[giturl], localdirname))
> - self._shellcmd("git remote remove origin",
> localdirname)
> - self._shellcmd("git remote add origin \"%s\"" %
> giturl, localdirname)
> - else:
> - logger.debug("localhostbecontroller: cloning %s:%s in
> %s" % (giturl, commit, localdirname))
> - self._shellcmd("git clone \"%s\" --single-branch
> --branch \"%s\" \"%s\"" % (giturl, commit, localdirname))
> -
> - # branch magic name "HEAD" will inhibit checkout
> - if commit != "HEAD":
> - logger.debug("localhostbecontroller: checking out commit
> %s to %s " % (commit, localdirname))
> - self._shellcmd("git fetch --all && git checkout \"%s\" &&
> git rebase \"origin/%s\"" % (commit, commit) , localdirname)
> -
> - # take the localdirname as poky dir if we can find the
> oe-init-build-env
> - if self.pokydirname is None and
> os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
> - logger.debug("localhostbecontroller: selected poky dir
> name %s" % localdirname)
> - self.pokydirname = localdirname
> -
> - # make sure we have a working bitbake
> - if not os.path.exists(os.path.join(self.pokydirname,
> 'bitbake')):
> - logger.debug("localhostbecontroller: checking bitbake
> into the poky dirname %s " % self.pokydirname)
> - self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " %
> (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname,
> 'bitbake')))
> -
> - # verify our repositories
> - for name, dirpath in gitrepos[(giturl, commit)]:
> - localdirpath = os.path.join(localdirname, dirpath)
> - logger.debug("localhostbecontroller: localdirpath
> expected '%s'" % localdirpath)
> - if not os.path.exists(localdirpath):
> - raise BuildSetupException("Cannot find layer git path
> '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl,
> commit))
> -
> - if name != "bitbake":
> - layerlist.append(localdirpath.rstrip("/"))
>
> logger.debug("localhostbecontroller: current layer list %s " %
> pformat(layerlist))
>
> diff --git
> a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
> b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
> index 5022b59..21bc380 100644
> --- a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
> +++ b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
> @@ -45,12 +45,14 @@ class Command(BaseCommand):
> for i in ['bitbake', 'releases', 'defaultrelease', 'config',
> 'layersources']:
> assert i in data
>
> - def _read_git_url_from_local_repository(address):
> + def _read_git_url_from_local_repository(address, path = None):
> url = None
> + if not path:
> + path = os.path.dirname(filepath)
> # we detect the remote name at runtime
> import subprocess
> (remote, remote_name) = address.split(":", 1)
> - cmd = subprocess.Popen("git remote -v", shell=True, cwd =
> os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE)
> + cmd = subprocess.Popen("git remote -v", shell=True, cwd =
> path, stdout=subprocess.PIPE, stderr = subprocess.PIPE)
> (out,err) = cmd.communicate()
> if cmd.returncode != 0:
> logging.warning("Error while importing layer vcs_url: git
> error: %s" % err)
> @@ -121,7 +123,11 @@ class Command(BaseCommand):
> logger.error("Local layer path %s must exists.
> Are you trying to import a layer that does not exist ? Check your local
> toasterconf.json" % lo.local_path)
>
> if layerinfo['vcs_url'].startswith("remote:"):
> - lo.vcs_url =
> _read_git_url_from_local_repository(layerinfo['vcs_url'])
> + if not layerinfo['local_path'].startswith("/"):
> + path = None
> + else:
> + path = layerinfo['local_path']
> + lo.vcs_url =
> _read_git_url_from_local_repository(layerinfo['vcs_url'], path)
> if lo.vcs_url is None:
> logger.error("The toaster config file
> references the local git repo, but Toaster cannot detect it.\nYour local
> configuration for layer %s is invalid. Make sure that the toasterconf.json
> file is correct." % layerinfo['name'])
>
> @@ -138,6 +144,8 @@ class Command(BaseCommand):
> commit = branch.name,
> layer = lo)
> lvo.dirpath = layerinfo['dirpath']
> + if len(layerinfo['local_path']) > 1 and
> layerinfo['local_path'].startswith("/") and branch.name == "HEAD":
> + lvo.local_path = layerinfo['local_path']
> lvo.save()
> # set releases
> for ri in data['releases']:
> --
> 1.9.1
>
>
--
സുജിത് ഹരിദാസന്
Bangalore
<Project>Contributor to KDE project
http://fci.wikia.com/wiki/Anti-DRM-Campaign
<Blog> http://sujithh.info
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.yoctoproject.org/pipermail/toaster/attachments/20150831/1eee6293/attachment-0001.html>
More information about the toaster
mailing list